refactor(standard): granularity, XDG compliance, and literate thinking medium
Some checks failed
Deploy-Agent-V15-Stdin / JOB-V15-STDIN (push) Failing after 2s

This commit is contained in:
2026-04-27 19:53:45 -04:00
parent 215fe0eae7
commit 41e25d091e
5 changed files with 432 additions and 456 deletions

View File

@@ -6,275 +6,196 @@
* Zero-to-One Setup (setup.org)
The ~setup.org~ file defines the automated installation and initialization sequence for the OpenCortex.
** The Installer Script (opencortex.sh)
#+begin_src bash :tangle (expand-file-name "../opencortex.sh" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
#!/bin/bash
set -e
* Phase A: Demand (Thinking)
** The Agnostic LLM Provider Registry
To fulfill the mandate of sovereignty and extensibility, the setup process must move away from a single hardcoded LLM provider (like OpenRouter).
PORT=9105
HOST="localhost"
RED='\033[0;31m'; GREEN='\033[0;32m'; BLUE='\033[0;34m'; YELLOW='\033[0;33m'; NC='\033[0m'
** Design Goals:
1. **Modular Adapters:** Each provider (Ollama, Groq, OpenAI, etc.) is a data-driven structure defining its required fields (API_KEY, BASE_URL) and its "ping" validation logic.
2. **Interactive Selection:** The user should be presented with a multi-select list of providers.
3. **Local-First Default:** If no cloud keys are provided, the system must default to a local Ollama/llama.cpp configuration.
4. **State Persistence:** Configuration is saved to `providers.lisp` in the XDG Config directory.
5. **Secret Splitting:** Sensitive keys go to `.env`, while metadata (models, URLs) lives in `state/providers.lisp`.
command_exists() { command -v "$1" >/dev/null 2>&1; }
* Phase B: Protocol (Success Criteria)
# Resolve symlinks to find the actual repository location
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
done
export SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
# Load environment variables if they exist
if [ -f "$SCRIPT_DIR/.env" ]; then
while IFS="=" read -r key value || [ -n "$key" ]; do
if [[ $key =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
val=$(echo "$value" | sed "s/^\"//;s/\"$//")
export "$key=$val"
fi
done < "$SCRIPT_DIR/.env"
[ -n "$ORG_AGENT_DAEMON_PORT" ] && PORT=$ORG_AGENT_DAEMON_PORT
[ -n "$DAEMON_HOST" ] && HOST=$DAEMON_HOST
fi
# --- 1. BOOTSTRAP ---
# If the script is run standalone, it clones the full repo and restarts itself.
if [ ! -d "$SCRIPT_DIR/.git" ] && [ ! -d "$HOME/.opencortex" ] && [[ ! "$(pwd)" =~ "opencortex" ]]; then
echo -e "${BLUE}=== OpenCortex: Zero-to-One Bootstrapper ===${NC}"
git clone ssh://git@10.10.10.201:2222/amr/opencortex.git ~/.opencortex
cd ~/.opencortex && git submodule update --init --recursive
exec ./opencortex.sh "$@"
fi
# --- 2. SETUP ---
setup_system() {
NON_INTERACTIVE=false
for arg in "$@"; do
if [ "$arg" == "--non-interactive" ]; then NON_INTERACTIVE=true; fi
done
echo -e "${BLUE}=== OpenCortex: Initializing System ===${NC}"
echo -e "${YELLOW}--- Installing System Dependencies ---${NC}"
if command_exists apt-get; then
sudo apt-get update && sudo apt-get install -y sbcl emacs-nox rlwrap netcat-openbsd curl git socat libssl-dev libncurses5-dev libffi-dev zlib1g-dev libsqlite3-dev
fi
if [ ! -d "$HOME/quicklisp" ]; then
curl -O https://beta.quicklisp.org/quicklisp.lisp
sbcl --non-interactive --load quicklisp.lisp --eval "(quicklisp-quickstart:install)" --eval "(ql-util:without-prompting (ql:add-to-init-file))"
rm quicklisp.lisp
fi
cd "$SCRIPT_DIR"
if [ ! -f .env ]; then
if [ "$NON_INTERACTIVE" = true ]; then
echo "Non-interactive mode: Using environment variables for .env creation."
cp .env.example .env
[ -n "$MEMEX_USER" ] && sed -i "s|MEMEX_USER=.*|MEMEX_USER=\"$MEMEX_USER\"|" .env
[ -n "$MEMEX_ASSISTANT" ] && sed -i "s|MEMEX_ASSISTANT=.*|MEMEX_ASSISTANT=\"$MEMEX_ASSISTANT\"|" .env
[ -n "$OPENROUTER_API_KEY" ] && sed -i "s|OPENROUTER_API_KEY=.*|OPENROUTER_API_KEY=\"$OPENROUTER_API_KEY\"|" .env
[ -n "$MEMEX_DIR" ] && sed -i "s|MEMEX_DIR=.*|MEMEX_DIR=\"$MEMEX_DIR\"|" .env
else
cp .env.example .env
echo -e "\n${YELLOW}--- Identity Configuration ---${NC}"
read -p "Your Name [User]: " user_name < /dev/tty
user_name=${user_name:-User}
sed -i "s|MEMEX_USER=.*|MEMEX_USER=\"$user_name\"|" .env
read -p "Agent Name [OpenCortex]: " agent_name < /dev/tty
agent_name=${agent_name:-OpenCortex}
sed -i "s|MEMEX_ASSISTANT=.*|MEMEX_ASSISTANT=\"$agent_name\"|" .env
echo -e "\n${YELLOW}--- LLM Configuration ---${NC}"
read -p "OpenRouter API Key: " openrouter_key < /dev/tty
[ -n "$openrouter_key" ] && sed -i "s|OPENROUTER_API_KEY=.*|OPENROUTER_API_KEY=\"$openrouter_key\"|" .env
echo -e "\n${YELLOW}--- Memex Folder Structure ---${NC}"
read -p "Memex Root [\$HOME/memex]: " memex_dir < /dev/tty
memex_dir=${memex_dir:-\$HOME/memex}
sed -i "s|MEMEX_DIR=.*|MEMEX_DIR=\"$memex_dir\"|" .env
fi
# Hydrate default paths
M_DIR=$(grep MEMEX_DIR .env | cut -d'"' -f2 | sed "s|\$HOME|$HOME|")
sed -i "s|SKILLS_DIR=.*|SKILLS_DIR=\"$SCRIPT_DIR/skills\"|" .env
sed -i "s|ZETTELKASTEN_DIR=.*|ZETTELKASTEN_DIR=\"$M_DIR/notes\"|" .env
mkdir -p "$M_DIR" "$M_DIR/notes" "$M_DIR/areas" "$M_DIR/resources" "$M_DIR/archives" "$M_DIR/system" "$M_DIR/inbox" "$M_DIR/daily" "$M_DIR/projects"
fi
mkdir -p library
for f in harness/*.org skills/*.org; do
emacs -Q --batch --eval "(require 'org)" --eval "(org-babel-tangle-file \"$f\")" >/dev/null 2>&1 || true
done
mkdir -p "$HOME/.local/bin"
ln -sf "$SCRIPT_DIR/opencortex.sh" "$HOME/.local/bin/opencortex"
for shell_config in "$HOME/.bashrc" "$HOME/.profile"; do
if [ -f "$shell_config" ]; then
if ! grep -q ".local/bin" "$shell_config"; then
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$shell_config"
fi
fi
done
export PATH="$HOME/.local/bin:$PATH"
echo -e "${YELLOW}--- Compiling and Loading OpenCortex ---${NC}"
sbcl --non-interactive --eval '(load (merge-pathnames "quicklisp/setup.lisp" (user-homedir-pathname)))' --eval '(push (truename (uiop:getenv "SCRIPT_DIR")) asdf:*central-registry*)' --eval "(ql:quickload '(:opencortex :croatoan))"
if [ $? -ne 0 ]; then
echo -e "${RED}✗ Compilation failed.${NC}"
exit 1
fi
if [ "$NON_INTERACTIVE" = true ]; then
echo "Setup complete (Non-interactive)."
exit 0
fi
echo -e "${YELLOW}--- Finalizing: Awakening the Brain ---${NC}"
"$SCRIPT_DIR/opencortex.sh" --boot > "$SCRIPT_DIR/brain.log" 2>&1 &
success=false
for i in {1..30}; do
if nc -z localhost $PORT 2>/dev/null; then success=true; break; fi
sleep 2
echo -n "."
done
if [ "$success" = true ]; then
echo -e "\n${GREEN}✓ Brain is alive on port $PORT.${NC}"
exit 0
else
echo -e "\n${RED}✗ Brain failed to wake up.${NC}"
exit 1
fi
}
# --- 3. COMMAND ROUTER ---
COMMAND=$1
[ -z "$COMMAND" ] && COMMAND="cli"
shift || true
DEFAULT_PORT=9105
DEFAULT_HOST="localhost"
TARGET_PORT=${PORT:-$DEFAULT_PORT}
TARGET_HOST=${HOST:-$DEFAULT_HOST}
# If uninitialized, force setup.
if [ ! -f "$SCRIPT_DIR/library/package.lisp" ] || [ ! -f "$SCRIPT_DIR/.env" ]; then
COMMAND="setup"
fi
case "$COMMAND" in
setup)
setup_system "$@"
;;
--boot|boot)
export SKILLS_DIR="${SCRIPT_DIR}/skills"
[ -z "$MEMEX_DIR" ] && export MEMEX_DIR="$HOME/memex"
if [ -f "$SCRIPT_DIR/.env" ]; then
export OPENROUTER_API_KEY=$(grep OPENROUTER_API_KEY "$SCRIPT_DIR/.env" | cut -d'"' -f2)
fi
exec sbcl --non-interactive --eval '(load (merge-pathnames "quicklisp/setup.lisp" (user-homedir-pathname)))' --eval '(setf *debugger-hook* (lambda (c h) (declare (ignore h)) (format *error-output* "FATAL LISP ERROR: ~a~%" c) (uiop:print-backtrace :stream *error-output*) (uiop:quit 1)))' --eval '(push (truename (uiop:getenv "SCRIPT_DIR")) asdf:*central-registry*)' --eval '(format t "--- Quickloading OpenCortex ---~%")' --eval "(ql:quickload '(:opencortex :croatoan))" --eval '(opencortex:main)'
;;
tui)
if ! nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then
echo -e "Brain is offline. Awakening..."
"$SCRIPT_DIR/opencortex.sh" --boot > "$SCRIPT_DIR/brain.log" 2>&1 &
for i in {1..15}; do
sleep 2
if nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then break; fi
echo -n "."
done
echo ""
fi
echo -e "Launching Croatoan TUI..."
export SKILLS_DIR="${SCRIPT_DIR}/skills"
[ -z "$MEMEX_DIR" ] && export MEMEX_DIR="$HOME/memex"
exec sbcl --eval '(load (merge-pathnames "quicklisp/setup.lisp" (user-homedir-pathname)))' --eval '(push (truename (uiop:getenv "SCRIPT_DIR")) asdf:*central-registry*)' --eval '(ql:quickload :opencortex/tui)' --eval '(opencortex.tui:main)'
;;
cli)
if ! nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then
echo -e "Brain is offline. Awakening..."
"$SCRIPT_DIR/opencortex.sh" --boot > "$SCRIPT_DIR/brain.log" 2>&1 &
for i in {1..15}; do
sleep 2
if nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then break; fi
echo -n "."
done
echo ""
fi
if command_exists socat; then
echo -e "Connected to OpenCortex on $TARGET_HOST:$TARGET_PORT (Channel: CLI)"
while true; do
read -p "User: " MESSAGE
if [ -z "$MESSAGE" ]; then continue; fi
if [ "$MESSAGE" = "/exit" ]; then break; fi
# Frame the message
PAYLOAD="(:TYPE :EVENT :META (:SOURCE :CLI) :PAYLOAD (:SENSOR :USER-INPUT :TEXT \"$MESSAGE\"))"
LEN=$(printf "%s" "$PAYLOAD" | wc -c)
HEXLEN=$(printf "%06x" $LEN)
# Send and read response
(printf "%s%s" "$HEXLEN" "$PAYLOAD" | nc -N $TARGET_HOST $TARGET_PORT) | while read -r LINE; do
CLEAN=$(echo "$LINE" | sed 's/^......//')
if [[ "$CLEAN" == *":TEXT"* ]]; then
TEXT=$(echo "$CLEAN" | sed -n 's/.*:TEXT "\([^"]*\)".*/\1/p')
echo -e "Agent: $TEXT"
fi
done
done
else
echo "Error: socat required for CLI interaction."
exit 1
fi
;;
*)
echo -e "Unknown command: $COMMAND"
echo "Available commands: setup, boot, tui, cli"
exit 1
;;
esac
** Test Suite Context
#+begin_src lisp :tangle (expand-file-name "setup-wizard-tests.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/tests"))
(defpackage :opencortex-setup-tests
(:use :cl :fiveam :opencortex)
(:export #:setup-suite))
#+end_src
** Metabolic Docker Infrastructure (Dockerfile)
#+begin_src dockerfile :tangle (expand-file-name "../infrastructure/docker/Dockerfile" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
FROM debian:bullseye-slim
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \
sbcl \
emacs-nox \
curl \
git \
socat \
netcat-openbsd \
libssl-dev \
libncurses5-dev \
libffi-dev \
zlib1g-dev \
libsqlite3-dev \
&& rm -rf /var/lib/apt/lists/*
# Install Quicklisp
RUN curl -O https://beta.quicklisp.org/quicklisp.lisp \
&& sbcl --non-interactive --load quicklisp.lisp --eval "(quicklisp-quickstart:install)" --eval "(ql-util:without-prompting (ql:add-to-init-file))" \
&& rm quicklisp.lisp
WORKDIR /app
COPY . .
# Initialize system in non-interactive mode
RUN mkdir -p /root/memex && ./opencortex.sh setup --non-interactive
EXPOSE 9105
CMD ["./opencortex.sh", "boot"]
#+begin_src lisp :tangle (expand-file-name "setup-wizard-tests.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/tests"))
(in-package :opencortex-setup-tests)
#+end_src
#+begin_src lisp :tangle (expand-file-name "setup-wizard-tests.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/tests"))
(def-suite setup-suite :description "Verification of the Lisp Setup Wizard")
#+end_src
#+begin_src lisp :tangle (expand-file-name "setup-wizard-tests.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/tests"))
(in-suite setup-suite)
#+end_src
** Persistence Tests
#+begin_src lisp :tangle (expand-file-name "setup-wizard-tests.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/tests"))
(test test-provider-registry-persistence
"Verify that multiple providers can be registered and saved."
(let ((opencortex::*providers* nil))
(opencortex:register-provider :ollama '(:url "http://localhost:11434" :model "llama3"))
(opencortex:register-provider :groq '(:key "gsk_123" :model "mixtral-8x7b"))
(is (equal "gsk_123" (getf (getf opencortex::*providers* :groq) :key)))))
#+end_src
** Fallback Tests
#+begin_src lisp :tangle (expand-file-name "setup-wizard-tests.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/tests"))
(test test-sovereign-fallback-logic
"Verify that the system identifies as healthy with only local providers."
(let ((opencortex::*providers* (list :ollama '(:url "http://localhost:11434"))))
(is (opencortex:system-ready-p))))
#+end_src
* Phase C: Implementation (Build)
** Package Context
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(in-package :opencortex)
#+end_src
** Global Provider Registry
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defvar *providers* nil "Global registry of configured LLM providers.")
#+end_src
** Provider Templates
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defvar *provider-templates*
'((:ollama . (:name "Ollama (Local)" :fields ((:url :label "URL") (:model :label "Model")) :default-url "http://localhost:11434" :default-model "llama3"))
(:openrouter . (:name "OpenRouter" :fields ((:key :label "API Key" :secret t) (:model :label "Model")) :default-model "anthropic/claude-3-opus-20240229"))
(:openai . (:name "OpenAI" :fields ((:key :label "API Key" :secret t) (:model :label "Model")) :default-model "gpt-4-turbo"))
(:groq . (:name "Groq" :fields ((:key :label "API Key" :secret t) (:model :label "Model")) :default-model "mixtral-8x7b-32768"))
(:gemini . (:name "Google Gemini" :fields ((:key :label "API Key" :secret t) (:model :label "Model")) :default-model "gemini-1.5-pro"))
(:anthropic . (:name "Anthropic" :fields ((:key :label "API Key" :secret t) (:model :label "Model")) :default-model "claude-3-5-sonnet-20240620")))
"Templates for supported LLM providers. Fields marked :secret go to .env.")
#+end_src
** XDG Configuration Utilities
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun get-oc-config-dir ()
"Resolves the OpenCortex configuration directory following XDG standards."
(let ((env (uiop:getenv "OC_CONFIG_DIR")))
(if (and env (> (length env) 0))
(uiop:ensure-directory-pathname env)
(merge-pathnames ".config/opencortex/" (user-homedir-pathname)))))
#+end_src
** Secret Persistence
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun save-secret (id key value)
"Appends a secret to the XDG config .env file and updates the current environment."
(let* ((env-key (format nil "~:@(~a_~a~)" id key))
(path (merge-pathnames ".env" (get-oc-config-dir))))
(ensure-directories-exist path)
(with-open-file (s path :direction :output :if-exists :append :if-does-not-exist :create)
(format s "~%~a=\"~a\"" env-key value))
(setf (uiop:getenv env-key) value)))
#+end_src
** Provider Metadata Persistence
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun save-providers ()
"Persist provider configuration to XDG config directory."
(let ((path (merge-pathnames "providers.lisp" (get-oc-config-dir))))
(ensure-directories-exist path)
(with-open-file (s path :direction :output :if-exists :supersede)
(format s ";;; OpenCortex Provider Metadata~%~s~%" *providers*))))
#+end_src
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun load-providers ()
"Load provider configuration from XDG config directory."
(let ((path (merge-pathnames "providers.lisp" (get-oc-config-dir))))
(when (uiop:file-exists-p path)
(with-open-file (s path)
(setf *providers* (read s))))))
#+end_src
** Registry API
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun register-provider (id config)
"Update the global provider registry."
(setf (getf *providers* id) config))
#+end_src
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun system-ready-p ()
"Predicate verifying if at least one provider is configured."
(and *providers* (> (length *providers*) 0)))
#+end_src
** User Interface Primitives
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun prompt-for (label &optional default)
"Interactively prompt the user for input with an optional default."
(format t "~a~@[ [~a]~]: " label default)
(finish-output)
(let ((input (read-line)))
(if (and (string= input "") default)
default
input)))
#+end_src
** Provider Configuration Loop
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun configure-provider (id)
"Guided configuration for a specific LLM provider template."
(let* ((template (cdr (assoc id *provider-templates*)))
(fields (getf template :fields))
(config nil))
(format t "~%--- Configuring ~a ---~%" (getf template :name))
(dolist (field-spec fields)
(let* ((field (first field-spec))
(label (getf (rest field-spec) :label))
(is-secret (getf (rest field-spec) :secret))
(default-key (intern (format nil "DEFAULT-~a" field) :keyword))
(default (getf template default-key))
(val (prompt-for label default)))
(if is-secret
(save-secret id field val)
(setf (getf config field) val))))
(register-provider id config)
(format t "✓ ~a metadata registered.~%" (getf template :name))))
#+end_src
** Main Setup Orchestrator
#+begin_src lisp :tangle (expand-file-name "setup-wizard.lisp" (concat (or (getenv "INSTALL_DIR") ".") "/harness"))
(defun run-setup-wizard ()
"Entry point for the interactive OpenCortex Lisp Setup Wizard."
(format t "=== OpenCortex: Advanced Setup Wizard ===~%")
;; 1. Identity
(let ((user (prompt-for "Your Name" "User"))
(agent (prompt-for "Agent Name" "OpenCortex")))
(format t "Welcome, ~a. I am ~a.~%" user agent))
;; 2. Providers
(format t "~%Available Providers:~%")
(loop for (id . data) in *provider-templates*
do (format t " ~a: ~a~%" id (getf data :name)))
(format t "~%Enter provider IDs to configure (comma separated, or 'all'): ")
(finish-output)
(let* ((input (read-line))
(ids (if (string= input "all")
(mapcar #'car *provider-templates*)
(mapcar (lambda (s) (intern (string-upcase (string-trim " " s)) :keyword))
(uiop:split-string input :separator ",")))))
(dolist (id ids)
(when (assoc id *provider-templates*)
(configure-provider id))))
(save-providers)
(format t "~%Setup complete. Running doctor check...~%")
(doctor-run-all))
#+end_src