diff --git a/harness/act.org b/harness/act.org index 552eccf..3541cb1 100644 --- a/harness/act.org +++ b/harness/act.org @@ -4,32 +4,92 @@ #+STARTUP: content * Stage 3: Act (act.lisp) -** Architectural Intent: Actuation -The Act stage performs the final side-effects of the reasoning engine. It routes approved actions to their registered physical actuators (CLI, Shell, Emacs, etc.) and handles the execution of internal system tools. -** Actuator Configuration -The core harness can be configured via environment variables to operate silently or target different default outputs. +** Architectural Intent: The Last Mile + +The Act stage is where cognition meets reality. After the Probabilistic engine proposes and the Deterministic engine verifies, Act executes the approved action. + +The key insight of the Act stage is that *execution is the point of no return*. Once a command is sent to the shell or a file is written, side effects have occurred. Therefore, Act implements a "last-mile" safety check - even after skills have verified the action, there's a final validation before dispatch. + +** Why Separate Actuators? + +The actuator pattern decouples /what to do/ from /how to do it/: + +- The reasoning engine generates action plists like `(:TYPE :REQUEST :TARGET :SHELL :PAYLOAD ...)` +- The actuator interprets the target and executes appropriately +- Adding a new actuator (Telegram, Matrix, etc.) doesn't require changing the reasoning code + +This follows the Open/Closed principle: open for extension, closed for modification. + +** The Feedback Loop + +Act is unique in the pipeline because it can generate new signals. When a tool executes and returns data, that data becomes a new signal that feeds back into Perceive → Reason → Act. + +Example feedback chain: +1. User asks "What files changed today?" +2. Reason generates shell command action +3. Act executes shell, gets file list +4. Act returns file list as feedback signal +5. Reason processes file list, generates human-readable response +6. Act displays response + +* Package Context #+begin_src lisp :tangle ../library/act.lisp (in-package :opencortex) +#+end_src -(defvar *default-actuator* :cli) -(defvar *silent-actuators* '(:cli :system-message :emacs)) +* Actuator Configuration +** Actuator Registry Variables + +#+begin_src lisp :tangle ../library/act.lisp +(defvar *default-actuator* :cli + "The actuator used when no explicit target is specified. + Override with DEFAULT_ACTUATOR environment variable.") + +(defvar *silent-actuators* '(:cli :system-message :emacs) + "List of actuators that don't generate tool-output feedback. + These typically have their own feedback mechanisms (CLI prints directly, etc.)") +#+end_src + +** initialize-actuators: System Bootstrap + +#+begin_src lisp :tangle ../library/act.lisp (defun initialize-actuators () - "Loads actuator routing defaults from environment variables and registers core harness actuators." + "Load actuator configuration from environment and register core actuators. + + Environment variables: + - DEFAULT_ACTUATOR: Keyword for default target (:cli, :shell, etc.) + - SILENT_ACTUATORS: Comma-separated list of actuators that skip feedback + + Registers three core actuators: + 1. :system - Internal commands (eval, create-skill, message) + 2. :tool - Cognitive tool execution + 3. :tui - Terminal UI output via reply stream" + + ;; Load environment configuration (let ((def (uiop:getenv "DEFAULT_ACTUATOR")) (silent (uiop:getenv "SILENT_ACTUATORS"))) + + ;; Set default actuator (when def - (setf *default-actuator* (intern (string-upcase def) "KEYWORD"))) + (setf *default-actuator* + (intern (string-upcase def) "KEYWORD"))) + + ;; Parse silent actuators list (when silent (setf *silent-actuators* - (mapcar (lambda (s) (intern (string-upcase (string-trim '(#\Space) s)) "KEYWORD")) + (mapcar (lambda (s) + (intern (string-upcase (string-trim '(#\Space) s)) + "KEYWORD")) (str:split "," silent))))) - + ;; Register core harness actuators (register-actuator :system #'execute-system-action) (register-actuator :tool #'execute-tool-action) + + ;; TUI actuator: sends response back through the reply stream (register-actuator :tui (lambda (action context) (let* ((meta (getf context :meta)) (stream (getf meta :reply-stream))) @@ -38,70 +98,123 @@ The core harness can be configured via environment variables to operate silently (finish-output stream)))))) #+end_src -** Dispatching Actions -The `dispatch-action` function is the primary router. It identifies the target actuator and executes the requested side-effects. +* Action Dispatching + +** dispatch-action: The Router #+begin_src lisp :tangle ../library/act.lisp (defun dispatch-action (action context) + "Route an approved action to its registered actuator. + + ACTION is a plist with structure: + (:TYPE :REQUEST :TARGET :shell :PAYLOAD (...)) + + CONTEXT is the signal being processed (for metadata access) + + The target is resolved in order of priority: + 1. Explicit :target in the action + 2. :source from the original signal's metadata + 3. *default-actuator* configuration variable + + Returns the actuator's result (may be a feedback signal or NIL)." + (let ((payload (proto-get action :payload))) + + ;; Heartbeats don't generate actuation (when (eq (proto-get payload :sensor) :heartbeat) - (return-from dispatch-action nil))) - "Routes an approved action to its registered physical actuator." - (when (and action (listp action)) - (let* ((meta (proto-get context :meta)) - (source (proto-get meta :source)) - (raw-target (or (ignore-errors (getf action :TARGET)) - (ignore-errors (getf action :target)) - source - *default-actuator*)) - (target (intern (string-upcase (string raw-target)) :keyword)) - (actuator-fn (gethash target *actuator-registry*))) - ;; Ensure outbound action has meta if context had it - (when (and meta (null (getf action :meta))) - (setf (getf action :meta) meta)) - (if actuator-fn - (funcall actuator-fn action context) - (harness-log "ACT ERROR: No actuator for ~s (from ~s)" target raw-target))))) + (return-from dispatch-action nil)) + + (when (and action (listp action)) + (let* ((meta (proto-get context :meta)) + (source (proto-get meta :source)) + (raw-target (or (ignore-errors (getf action :TARGET)) + (ignore-errors (getf action :target)) + source + *default-actuator*)) + (target (intern (string-upcase (string raw-target)) :keyword)) + (actuator-fn (gethash target *actuator-registry*))) + + ;; Preserve metadata in outbound action + (when (and meta (null (getf action :meta))) + (setf (getf action :meta) meta)) + + ;; Execute or log error + (if actuator-fn + (funcall actuator-fn action context) + (harness-log "ACT ERROR: No actuator registered for '~s' (requested by ~s)" + target raw-target)))))) #+end_src -** Internal System Actions -The `:system` actuator handles internal harness commands like code evaluation and dynamic skill loading. +* Actuator Implementations + +** execute-system-action: Internal Commands #+begin_src lisp :tangle ../library/act.lisp (defun execute-system-action (action context) - "Processes internal harness commands. (ACTUATOR)" + "Execute internal harness commands. + + This actuator handles meta-commands that affect the harness itself, + rather than external side effects. Commands include: + + - :eval - Evaluate arbitrary Lisp code (DANGEROUS, validate first!) + - :create-skill - Write a new skill org file and reload + - :message - Log a message to the harness log + + These commands bypass the normal actuator system since they operate + on the harness internals rather than external systems." + (declare (ignore context)) - (let* ((payload (ignore-errors (getf action :payload))) + + (let* ((payload (ignore-errors (getf action :payload))) (cmd (ignore-errors (getf payload :action)))) + (case cmd - (:eval (let ((code (getf payload :code))) - (eval (read-from-string code)))) - (:create-skill (let* ((filename (getf payload :filename)) (content (getf payload :content)) - (skills-dir (merge-pathnames "skills/" (asdf:system-source-directory :opencortex))) - (full-path (merge-pathnames filename skills-dir))) - (with-open-file (out full-path :direction :output :if-exists :supersede) (write-string content out)) - (load-skill-from-org full-path))) - (:message (harness-log "ACT [System]: ~a" (getf payload :text))) - (t (harness-log "ACT ERROR [System]: Unknown command ~s" cmd))))) + ;; Evaluate Lisp code - guarded by lisp-validator skill + (:eval + (let ((code (getf payload :code))) + (eval (read-from-string code)))) + + ;; Create and load a new skill from content + (:create-skill + (let* ((filename (getf payload :filename)) + (content (getf payload :content)) + (skills-dir (merge-pathnames "skills/" + (asdf:system-source-directory :opencortex))) + (full-path (merge-pathnames filename skills-dir))) + (with-open-file (out full-path + :direction :output + :if-exists :supersede) + (write-string content out)) + (load-skill-from-org full-path))) + + ;; Log an informational message + (:message + (harness-log "ACT [System]: ~a" (getf payload :text))) + + ;; Unknown command + (t + (harness-log "ACT ERROR [System]: Unknown command '~s'" cmd))))) #+end_src -** Cognitive Tool Actuation -The `:tool` actuator handles the execution of registered cognitive tools. +** execute-tool-action: Cognitive Tool Execution #+begin_src lisp :tangle ../library/act.lisp -(defun format-tool-result (tool-name result) - "Intelligently formats a tool result for user display." - (if (listp result) - (let ((status (getf result :status)) - (content (getf result :content)) - (msg (getf result :message))) - (cond ((and (eq status :success) content) (format nil "~a" content)) - ((and (eq status :error) msg) (format nil "ERROR [~a]: ~a" tool-name msg)) - (t (format nil "TOOL [~a] RESULT: ~s" tool-name result)))) - (format nil "TOOL [~a] RESULT: ~a" tool-name result))) - (defun execute-tool-action (action context) - "Executes a registered cognitive tool. (ACTUATOR)" + "Execute a registered cognitive tool. + + Tools are registered functions with: + - A guard function (optional, for safety checks) + - A body function (the actual implementation) + - Metadata (description, parameter specs) + + This actuator: + 1. Looks up the tool by name + 2. Runs the guard function (if present) + 3. Executes the body function with parsed arguments + 4. Returns a feedback signal with the result + + The feedback mechanism allows tool results to trigger further reasoning." + (let* ((payload (getf action :payload)) (tool-name (getf payload :tool)) (tool-args (getf payload :args)) @@ -109,74 +222,167 @@ The `:tool` actuator handles the execution of registered cognitive tools. (meta (getf context :meta)) (source (getf meta :source)) (tool (gethash (string-downcase (string tool-name)) *cognitive-tools*))) + (if tool (handler-case - (let* ((clean-args (if (and (listp tool-args) (listp (car tool-args))) (car tool-args) tool-args)) + ;; Parse arguments (handle both flat and nested plists) + (let* ((clean-args (if (and (listp tool-args) + (listp (car tool-args))) + (car tool-args) + tool-args)) (result (funcall (cognitive-tool-body tool) clean-args))) - (let ((feedback (list :TYPE :EVENT :DEPTH (1+ depth) :META meta - :PAYLOAD (list :SENSOR :tool-output :RESULT result :TOOL tool-name)))) - ;; If we have a source, send a status message with the result, formatted for humans - (when source - (dispatch-action (list :TYPE :REQUEST :TARGET source - :PAYLOAD (list :ACTION :MESSAGE :TEXT (format-tool-result tool-name result))) - context)) - feedback)) + + ;; Format result for source + (when source + (dispatch-action (list :TYPE :REQUEST + :TARGET source + :PAYLOAD (list :ACTION :MESSAGE + :TEXT (format-tool-result tool-name result))) + context)) + + ;; Return feedback signal for potential further processing + (list :TYPE :EVENT + :DEPTH (1+ depth) + :META meta + :PAYLOAD (list :SENSOR :tool-output + :RESULT result + :TOOL tool-name))) + + ;; Tool execution error (error (c) - (list :TYPE :EVENT :DEPTH (1+ depth) :META meta - :PAYLOAD (list :SENSOR :tool-error :tool tool-name :message (format nil "~a" c))))) - (list :TYPE :EVENT :DEPTH (1+ depth) :META meta - :PAYLOAD (list :SENSOR :tool-error :message "Tool not found"))))) + (list :TYPE :EVENT + :DEPTH (1+ depth) + :META meta + :PAYLOAD (list :SENSOR :tool-error + :TOOL tool-name + :MESSAGE (format nil "~a" c))))) + + ;; Tool not found + (list :TYPE :EVENT + :DEPTH (1+ depth) + :META meta + :PAYLOAD (list :SENSOR :tool-error + :MESSAGE (format nil "Tool '~a' not found" tool-name))))) #+end_src -** The Act Gate -The final stage of the metabolic loop. It performs a "last-mile" safety check before dispatching the action to the registered actuator. +** format-tool-result: Human-Readable Output + +#+begin_src lisp :tangle ../library/act.lisp +(defun format-tool-result (tool-name result) + "Format a tool result for human-readable display. + + Tools return either: + - A plist: (:status :success :content \"...\") or (:status :error :message \"...\") + - A raw value (string, number, etc.) + + This function normalizes both formats into a consistent string presentation." + + (if (listp result) + (let ((status (getf result :status)) + (content (getf result :content)) + (msg (getf result :message))) + (cond + ((and (eq status :success) content) + (format nil "~a" content)) + ((and (eq status :error) msg) + (format nil "ERROR [~a]: ~a" tool-name msg)) + (t + (format nil "TOOL [~a] RESULT: ~s" tool-name result)))) + (format nil "TOOL [~a] RESULT: ~a" tool-name result))) +#+end_src + +* The Act Gate + +** act-gate: Final Pipeline Stage #+begin_src lisp :tangle ../library/act.lisp (defun act-gate (signal) - "Final Stage: Actuation and feedback generation." + "Final stage of the metabolic pipeline: Actuation. + + This stage has three responsibilities: + + 1. Last-mile safety check: Run deterministic gates one more time + before execution (handles race conditions, concurrent modifications) + + 2. Actuation: Dispatch the approved action to its target actuator + + 3. Feedback generation: If the action produced results, create a + feedback signal that feeds back into the pipeline + + Modifies the signal: + - :approved-action - May be modified by last-mile verification + - :status - Set to :acted + + Returns a feedback signal if the action produced results, otherwise NIL." + (let* ((approved (getf signal :approved-action)) (type (getf signal :type)) (meta (getf signal :meta)) (source (getf meta :source)) (feedback nil) - ;; context must keep internal objects for actuators to function (context signal)) - - ;; 1. Last-Mile Safety Check (The Bouncer & Deterministic Gates) + + ;; Step 1: Last-mile deterministic verification + ;; This catches any issues that arose between reasoning and acting (when approved (let* ((original-type (getf approved :type)) (verified (deterministic-verify approved signal))) - (if (and (listp verified) + + ;; Check if deterministic verification blocked the action + (if (and (listp verified) (member (getf verified :type) '(:LOG :EVENT :log :event)) (not (member original-type '(:LOG :EVENT :log :event)))) + + ;; Action was blocked by verification (progn (harness-log "ACT BLOCKED: Action failed last-mile deterministic check.") (setf (getf signal :approved-action) nil) (setf approved nil) (setf feedback verified)) + + ;; Action passed verification (progn (setf (getf signal :approved-action) verified) (setf approved verified))))) - ;; 2. Actuation Logic + ;; Step 2: Actuation based on signal type (case type - (:REQUEST (dispatch-action signal context)) - (:LOG (dispatch-action signal context)) - (:EVENT + ;; Explicit requests go directly to dispatch + (:REQUEST + (dispatch-action signal context)) + + ;; Log messages also dispatch + (:LOG + (dispatch-action signal context)) + + ;; Events with approved actions dispatch to their target + (:EVENT (if approved (let* ((target (getf approved :target)) (result (dispatch-action approved context))) - ;; If the actuator returns a signal (like :tool-output), it becomes the feedback. - ;; Otherwise, generate tool-output feedback for non-silent actuators. - (cond ((and (listp result) (member (getf result :type) '(:EVENT :LOG))) - (setf feedback result)) - ((and result (not (member target *silent-actuators*))) - (setf feedback (list :type :EVENT :depth (1+ (getf signal :depth 0)) :meta meta - :payload (list :sensor :tool-output :result result :tool approved)))))) - ;; If no approved action but we have a source, this might be a raw event/log stimulus. + + ;; Determine feedback based on actuator response + (cond + ;; Actuator returned a signal - use it as feedback + ((and (listp result) + (member (getf result :type) '(:EVENT :LOG))) + (setf feedback result)) + + ;; Non-silent actuator with result - format as tool-output + ((and result + (not (member target *silent-actuators*))) + (setf feedback (list :type :EVENT + :depth (1+ (getf signal :depth 0)) + :meta meta + :payload (list :sensor :tool-output + :result result + :tool approved)))))) + + ;; No approved action, but have source - might be raw event (when source (dispatch-action signal context))))) - + + ;; Step 3: Update signal status (setf (getf signal :status) :acted) feedback)) -#+end_src +#+end_src \ No newline at end of file diff --git a/harness/loop.org b/harness/loop.org index c5fccd8..3d68609 100644 --- a/harness/loop.org +++ b/harness/loop.org @@ -4,128 +4,284 @@ #+STARTUP: content * The Metabolic Loop (loop.lisp) -** Architectural Intent: The Heartbeat -The Metabolic Loop is the high-level coordinator of the OpenCortex. It orchestrates the flow of energy (information) through the system by calling the three metabolic stages in sequence: -1. **Perceive:** Sensory intake. -2. **Reason:** Cognitive processing. -3. **Act:** Physical side-effects. +** Architectural Intent -** Package and Variables -The loop requires thread-safe interrupt handling to ensure that the agent can be stopped gracefully without leaving the Lisp image in an inconsistent state. +The Metabolic Loop is the /cranial nerve reflex/ of OpenCortex. While skills provide specialized intelligence, the loop provides the fundamental rhythm of existence: the continuous processing of signals from perception through cognition to action. + +Unlike a simple event loop, the Metabolic Loop implements a sophisticated error recovery model. When the system encounters an error, it distinguishes between: + +1. *Transient errors* (tool failures, network timeouts) - recoverable, no state rollback +2. *Critical errors* (undefined functions, malformed data structures) - require memory rollback +3. *Recursive loops* (signals generating more signals indefinitely) - depth limit enforcement + +This design ensures the agent remains stable under adverse conditions while preserving the ability to recover from genuine system failures. + +** Why Separate Perceive-Reason-Act? + +The three-stage pipeline mirrors the classical sense-think-act paradigm but with a crucial difference: each stage is a pure function that transforms a signal. This allows: + +- *Perceive* to normalize raw input into a standardized signal format +- *Reason* to transform the perceived signal into an approved action (or reject it) +- *Act* to execute the approved action and potentially generate a feedback signal + +The feedback loop (Act returning a signal that feeds back into Perceive) enables complex multi-step operations where each action can trigger subsequent reasoning. + +** Thread Safety + +The loop operates in a multi-threaded environment: +- The main thread runs the heartbeat and idle loop +- Async sensors spawn threads for non-blocking I/O +- Interrupt handling requires mutex protection to prevent race conditions + +* Package and Thread-Safe Variables #+begin_src lisp :tangle ../library/loop.lisp (in-package :opencortex) -(defvar *interrupt-flag* nil) -(defvar *interrupt-lock* (bt:make-lock "harness-interrupt-lock")) -(defvar *heartbeat-thread* nil) +(defvar *interrupt-flag* nil + "Atomic flag set by signal handlers to trigger graceful shutdown. + Using a dedicated variable avoids race conditions in interrupt handling.") + +(defvar *interrupt-lock* (bt:make-lock "harness-interrupt-lock") + "Mutex protecting *interrupt-flag* access. + Locking is required because SBCL's interrupt handlers run in uncertain contexts.") + +(defvar *heartbeat-thread* nil + "Handle to the heartbeat thread, allowing explicit termination on shutdown.") #+end_src -** The Metabolic Pipeline -The `process-signal` function is the core metabolic processor. It iterates through the Perceive-Reason-Act gates until the signal is fully processed or an error state is reached. We have refined the error handling to ensure that memory rollbacks only occur on critical system failures, preventing transient tool errors from wiping short-term cognitive state. +* The Metabolic Pipeline + +** process-signal: The Core Engine + +This function implements the Perceive-Reason-Act pipeline. It processes a signal through all three stages and handles the feedback loop where Actions can generate new signals. + +The depth counter prevents infinite recursion—a signal that generates another signal that generates another, etc. By limiting to depth 10, we ensure the system eventually converges or gracefully terminates. #+begin_src lisp :tangle ../library/loop.lisp (defun process-signal (signal) - "The entry point to the Metabolic Pipeline: Perceive -> Reason -> Act." + "The entry point to the Metabolic Pipeline: Perceive -> Reason -> Act. + + SIGNAL is a property list with the following structure: + - :type - :EVENT, :REQUEST, :RESPONSE, etc. + - :payload - The actual content (sensor data, approved actions, etc.) + - :meta - Metadata including source, session, reply stream + - :depth - Recursion depth counter (starts at 0) + - :status - Processing status (:perceived, :reasoned, :acted) + + Returns NIL when processing is complete, or a new signal for feedback loop." + (let ((current-signal signal)) (loop while current-signal do + + ;; Depth limiting prevents infinite recursion from feedback loops (let ((depth (getf current-signal :depth 0)) (meta (getf current-signal :meta))) - (when (> depth 10) (harness-log "METABOLISM ERROR: Max depth reached.") (return nil)) + (when (> depth 10) + (harness-log "METABOLISM ERROR: Max recursion depth reached.") + (return nil)) + + ;; Check for graceful shutdown interrupt (when (bt:with-lock-held (*interrupt-lock*) *interrupt-flag*) - (harness-log "METABOLISM: Interrupted.") + (harness-log "METABOLISM: Interrupted by shutdown signal.") (bt:with-lock-held (*interrupt-lock*) (setf *interrupt-flag* nil)) (return nil)) + + ;; The three-stage pipeline wrapped in error handling (handler-case (progn + ;; Stage 1: Perceive - normalize sensory input (setf current-signal (perceive-gate current-signal)) + + ;; Stage 2: Reason - generate and verify action proposals (setf current-signal (reason-gate current-signal)) + + ;; Stage 3: Act - execute approved actions (let ((feedback (act-gate current-signal))) - ;; feedback generation (if feedback + ;; Action generated a feedback signal - continue processing (progn - ;; Inherit meta from trigger signal - (unless (getf feedback :meta) (setf (getf feedback :meta) meta)) + ;; Preserve metadata from original signal + (unless (getf feedback :meta) + (setf (getf feedback :meta) meta)) (setf current-signal feedback)) + ;; No feedback - pipeline complete (setf current-signal nil)))) + + ;; Error recovery with differentiated response (error (c) (let ((sensor (ignore-errors (getf (getf current-signal :payload) :sensor)))) (harness-log "METABOLISM CRASH [~a]: ~a" (or sensor :unknown) c) - ;; Only rollback on critical errors, not standard tool or loop errors + + ;; Only rollback memory on critical errors, not transient tool failures + ;; This prevents losing recent context due to a single bad API call (unless (member sensor '(:loop-error :tool-error :syntax-error)) (harness-log "CRITICAL ERROR: Initiating Micro-Rollback.") (rollback-memory 0)) + + ;; At deep recursion or known error types, terminate gracefully (if (or (> depth 2) (member sensor '(:loop-error :tool-error))) (setf current-signal nil) - (setf current-signal (list :type :EVENT :depth (1+ depth) :meta meta - :payload (list :sensor :loop-error :message (format nil "~a" c) :depth depth))))))))))) + ;; Otherwise, convert error to a loop-error signal for retry + (setf current-signal + (list :type :EVENT + :depth (1+ depth) + :meta meta + :payload (list :sensor :loop-error + :message (format nil "~a" c) + :depth depth))))))))))) #+end_src -** Heartbeat Mechanism -The heartbeat ensures the agent remains "alive" even in the absence of external stimuli, allowing for latent reflection and periodic maintenance. The interval is externalized to the `HEARTBEAT_INTERVAL` environment variable. +** The Feedback Loop Explained + +The pipeline implements a feedback loop where Act can return a new signal: + +1. User input arrives → Perceive normalizes it +2. Reason generates an action → Act executes it +3. If the action was a tool call that returned new information → Act returns a feedback signal +4. Feedback signal feeds back into step 1 for further reasoning + +This enables multi-step workflows where each action can trigger additional analysis. + +* Heartbeat Mechanism + +The heartbeat thread ensures the agent remains alive even without external input. It drives two critical functions: + +1. **Latent reflection** - the agent can think without external prompting +2. **Periodic maintenance** - memory auto-save, orphan detection, etc. + +** Heartbeat Configuration Variables #+begin_src lisp :tangle ../library/loop.lisp (defvar *auto-save-interval* 300 - "Save memory to disk every N seconds. Set from MEMORY_AUTO_SAVE_INTERVAL env.") + "Interval in seconds between automatic memory saves. + Defaults to 300 seconds (5 minutes). Set via MEMORY_AUTO_SAVE_INTERVAL env var.") (defvar *heartbeat-save-counter* 0 - "Counter for auto-save triggers.") + "Tracks heartbeats since last save, used to calculate auto-save timing.") +#+end_src +** start-heartbeat: The Pulsing Heart + +#+begin_src lisp :tangle ../library/loop.lisp (defun start-heartbeat () - "Starts the background heartbeat thread. Interval is loaded from HEARTBEAT_INTERVAL." + "Starts the background heartbeat thread. + + The heartbeat runs in a dedicated thread to avoid blocking the main + signal processing loop. Each heartbeat: + + 1. Injects a :HEARTBEAT signal into the metabolic pipeline + 2. Checks if memory should be auto-saved (based on interval ratio) + + Configuration via environment: + - HEARTBEAT_INTERVAL: Seconds between heartbeats (default: 60) + - MEMORY_AUTO_SAVE_INTERVAL: Seconds between auto-saves (default: 300)" + (let ((interval (or (ignore-errors (parse-integer (uiop:getenv "HEARTBEAT_INTERVAL"))) 60)) (auto-save (or (ignore-errors (parse-integer (uiop:getenv "MEMORY_AUTO_SAVE_INTERVAL"))) *auto-save-interval*))) (setf *auto-save-interval* auto-save) (setf *heartbeat-save-counter* 0) - (setf *heartbeat-thread* - (bt:make-thread - (lambda () - (loop - (sleep interval) + + (setf *heartbeat-thread* + (bt:make-thread + (lambda () + (loop + ;; Wait for interval + (sleep interval) + + ;; Update counter and check if it's time to save (incf *heartbeat-save-counter*) (when (>= *heartbeat-save-counter* (/ *auto-save-interval* interval)) (setf *heartbeat-save-counter* 0) (save-memory-to-disk)) - ;; inject-stimulus is synchronous for heartbeats, preventing accumulation. - (inject-stimulus (list :type :EVENT :payload (list :sensor :heartbeat :unix-time (get-universal-time)))))) + + ;; Inject heartbeat signal - this runs through the full pipeline + ;; allowing the agent to do latent reflection even with no input + (inject-stimulus + (list :type :EVENT + :payload (list :sensor :heartbeat + :unix-time (get-universal-time))))) + :name "opencortex-heartbeat")))) #+end_src -** Main Entry Point -The `main` function initializes the environment, loads skills, and starts the heartbeat. It now includes a graceful shutdown handler for `SIGINT` (Ctrl+C) and uses `DAEMON_SLEEP_INTERVAL` to control its idle rhythm. +* Main Entry Point + +** Shutdown Configuration #+begin_src lisp :tangle ../library/loop.lisp (defvar *shutdown-save-enabled* t - "If non-nil, save memory to disk on graceful shutdown.") + "When T, save memory to disk on graceful shutdown. + Disable for testing or when memory persistence is handled externally.") +#+end_src +** main: System Bootstrap and Idle Loop + +The main function orchestrates system startup: + +1. Load environment variables from ~/.local/share/opencortex/.env +2. Restore memory from previous snapshot (crash recovery) +3. Initialize actuators and load all skills +4. Start the heartbeat thread +5. Register SIGINT handler for graceful Ctrl+C shutdown +6. Enter idle loop (sleeping in 1-hour increments) + +#+begin_src lisp :tangle ../library/loop.lisp (defun main () - "Entry point for the Skeleton MVP. Handles initialization and graceful shutdown." - (let* ((home (uiop:getenv "HOME")) - (env-file (uiop:merge-pathnames* ".local/share/opencortex/.env" (uiop:ensure-directory-pathname home)))) - (when (uiop:file-exists-p env-file) (cl-dotenv:load-env env-file))) + "Entry point for OpenCortex. Initializes the system and enters idle loop. - ;; Load memory from disk if a snapshot exists + Startup sequence: + 1. Load environment from ~/.local/share/opencortex/.env + 2. Restore memory from disk (if snapshot exists) + 3. Initialize actuators (shell, cli, system) + 4. Load all skills from SKILLS_DIR + 5. Start heartbeat thread + 6. Register SIGINT handler for graceful shutdown + 7. Enter idle loop (sleeps in DAEMON_SLEEP_INTERVAL chunks) + + The idle loop checks for interrupts and saves memory before exit." + + ;; Step 1: Load environment variables from standard location + (let* ((home (uiop:getenv "HOME")) + (env-file (uiop:merge-pathnames* + ".local/share/opencortex/.env" + (uiop:ensure-directory-pathname home)))) + (when (uiop:file-exists-p env-file) + (cl-dotenv:load-env env-file))) + + ;; Step 2: Crash recovery - load memory from previous snapshot (load-memory-from-disk) + ;; Step 3-4: Initialize actuators and load skills (initialize-actuators) (initialize-all-skills) + ;; Step 5: Start the heartbeat (start-heartbeat) - - ;; Graceful shutdown handler for SBCL + + ;; Step 6: Register graceful shutdown handler + ;; SBCL-specific: catches Ctrl+C (SIGINT) and saves before exit #+sbcl - (sb-sys:enable-interrupt sb-unix:sigint - (lambda (sig code scp) - (declare (ignore sig code scp)) + (sb-sys:enable-interrupt sb-unix:sigint + (lambda (sig code scp) + (declare (ignore sig code scp)) (harness-log "SHUTDOWN: SIGINT received. Saving memory...") - (when *shutdown-save-enabled* (save-memory-to-disk)) + (when *shutdown-save-enabled* + (save-memory-to-disk)) (uiop:quit 0))) - (let ((sleep-interval (or (ignore-errors (parse-integer (uiop:getenv "DAEMON_SLEEP_INTERVAL"))) 3600))) - (loop + ;; Step 7: Idle loop - sleep in chunks, checking for interrupts + (let ((sleep-interval (or (ignore-errors + (parse-integer (uiop:getenv "DAEMON_SLEEP_INTERVAL"))) + 3600))) + (loop + ;; Check for interrupt before each sleep cycle (when (bt:with-lock-held (*interrupt-lock*) *interrupt-flag*) (harness-log "SHUTDOWN: Interrupt flag set. Saving memory...") - (when *shutdown-save-enabled* (save-memory-to-disk)) + (when *shutdown-save-enabled* + (save-memory-to-disk)) (return)) + + ;; Sleep in configured intervals (default: 1 hour) (sleep sleep-interval)))) -#+end_src +#+end_src \ No newline at end of file diff --git a/harness/manifest.org b/harness/manifest.org index 3b49ddd..2129630 100644 --- a/harness/manifest.org +++ b/harness/manifest.org @@ -4,30 +4,80 @@ #+STARTUP: content * Manifest (opencortex.asd) -** Architectural Intent: The ASDF Skeleton -The ~opencortex.asd~ file is the physical blueprint of the Lisp Machine. It uses **Another System Definition Facility (ASDF)** to orchestrate the compilation and loading of all harness modules. +** Architectural Intent: The Thin Harness Philosophy -Traditional Lisp systems often use complex, non-linear dependency graphs. However, the ~opencortex~ harness mandates a strict, linear bootstrap sequence. +The ~opencortex.asd~ file is the physical blueprint of the Lisp Machine. It uses **ASDF** (Another System Definition Facility) to orchestrate compilation and loading of all harness modules. -*** 1. Strict Serial Loading (:serial t) -The harness uses the ~:serial t~ flag. This is a critical design choice that ensures every file is compiled and loaded in the exact order it appears in the ~:components~ list. This eliminates "macro-not-found" errors by guaranteeing that the ~package.lisp~ and ~skills.lisp~ (where the core macros are defined) are always established before any behavioral logic or skills are loaded. +The core design principle is *Thin Harness, Fat Skills*: -*** 2. Isolation of the Verification Suite -To maintain a "Zero-Overhead" production environment, the testing logic is isolated into a secondary system: ~:opencortex/tests~. This allows the harness to boot in production without loading the ~FiveAM~ framework or the voluminous test data, keeping the memory footprint minimal and the attack surface small. +- **Harness** = The minimal, unbreakable core (protocol, signal processing, memory) +- **Skills** = The intelligence layer (policy, validation, actuation, LLM integration) + +This separation means: +- The harness rarely changes (immune system) +- Skills can be hot-loaded, modified, and swapped without touching the core +- Bugs in skills don't crash the system + +** Why ASDF?** + +ASDF is the de facto standard for Common Lisp project management. It: +1. Handles dependency resolution and loading order +2. Compiles files in the right order (preventing "undefined function" errors) +3. Supports system building for deployment +4. Integrates with Quicklisp for dependency management + +* The Build Pipeline -** The Build Pipeline #+begin_src mermaid flowchart TD - Org[Literate Org Files] -- Tangle --> Lisp[Source .lisp Files] - Lisp --> ASDF[ASDF Manifest: .asd] + Org[Literate Org Files] -- Org-Babel Tangle --> Lisp[Source .lisp Files] + Lisp --> ASDF[ASDF Manifest: opencortex.asd] ASDF --> Loader[SBCL Compiler / Loader] Loader --> Image[Live Harness Image] Image -- Build --> Binary[Standalone Binary] + + subgraph Skills["Skills Layer (Dynamic)"] + S1[Policy Skill] + S2[Bouncer Skill] + S3[LLM Gateway] + S4[...other skills] + end + + Image --> Skills #+end_src -** Harness System Definition -This system defines the core "Thin Harness." It includes the protocol, the object store, and the functional loop. +* Design Decisions + +** Strict Serial Loading + +The harness uses ~:serial t~ in the ASDF definition. This means: + +1. Files are loaded in order: package → skills → communication → memory → context → perceive → reason → act → loop +2. ~package.lisp~ is always loaded before any code that uses its symbols +3. ~skills.lisp~ (defining macros like ~defskill~, ~def-cognitive-tool~) loads before skills + +This eliminates "macro not found" errors that plague non-linear loading systems. + +** Why Not Module Dependencies?** + +Traditional ASDF uses ~:depends-on~ to declare dependencies. We use ~:serial t~ because: + +1. *Explicit is better than implicit* - the loading order is visible in one place +2. *Prevents circular dependencies* - skills are loaded after the harness, never before +3. *Simpler debugging* - when something fails, the loading order is always clear + +** Isolation of Tests + +The testing system (~:opencortex/tests~) is separate from the production system (~:opencortex~). This means: + +- Production deployments don't load FiveAM (saves memory, reduces attack surface) +- Tests can be run independently: ~(ql:quickload :opencortex/tests)~ +- Test data doesn't pollute the production image + +* System Definitions + +** Main Harness System #+begin_src lisp :tangle ../opencortex.asd (defsystem :opencortex @@ -36,61 +86,144 @@ This system defines the core "Thin Harness." It includes the protocol, the objec :version "0.1.0" :license "AGPLv3" :description "The Probabilistic-Deterministic Lisp Machine Harness" - :depends-on (:usocket :bordeaux-threads :dexador :uiop :cl-dotenv :cl-ppcre :hunchentoot :ironclad :str :cl-json :uuid) - :serial t - :components ((:file "library/package") - (:file "library/skills") - (:file "library/communication") - (:file "library/memory") - (:file "library/context") - (:file "library/perceive") - (:file "library/reason") - (:file "library/act") - (:file "library/loop")) + + :depends-on (:usocket ; TCP socket networking + :bordeaux-threads ; Threading (heartbeat, async sensors) + :dexador ; HTTP client (LLM APIs) + :uiop ; Portable I/O, file operations + :cl-dotenv ; Environment variable loading + :cl-ppcre ; Regular expressions (parsing) + :hunchentoot ; HTTP server (optional web interface) + :ironclad ; Cryptography (Merkle hashing) + :str ; String utilities + :cl-json ; JSON parsing/serialization + :uuid) ; UUID generation for org-mode IDs + + :serial t ; Load files in order listed below + + :components ((:file "library/package") ; Package definitions, core vars + (:file "library/skills") ; Skill engine, cognitive tools + (:file "library/communication") ; Protocol, framing, validation + (:file "library/memory") ; Org-object store, snapshots + (:file "library/context") ; Context assembly, query + (:file "library/perceive") ; Stage 1: Sensory normalization + (:file "library/reason") ; Stage 2: Neural + deterministic + (:file "library/act") ; Stage 3: Actuation + (:file "library/loop")) ; Main entry, heartbeat + :build-operation "program-op" :build-pathname "opencortex-server" :entry-point "opencortex:main") #+end_src -** Verification Suite Definition -This system contains the empirical tests required by the Engineering Standards. It depends on ~:opencortex~ and the ~FiveAM~ testing framework. +** Test System #+begin_src lisp :tangle ../opencortex.asd (defsystem :opencortex/tests - :depends-on (:opencortex :fiveam) + :depends-on (:opencortex ; The harness we're testing + :fiveam) ; Testing framework + :components ((:file "tests/communication-tests") (:file "tests/pipeline-tests") (:file "tests/act-tests") (:file "tests/boot-sequence-tests") (:file "tests/memory-tests") (:file "tests/immune-system-tests")) - :perform (test-op (o s) - (uiop:symbol-call :fiveam :run! (uiop:find-symbol* :communication-protocol-suite :opencortex-tests)) - (uiop:symbol-call :fiveam :run! (uiop:find-symbol* :pipeline-suite :opencortex-pipeline-tests)) - (uiop:symbol-call :fiveam :run! (uiop:find-symbol* :safety-suite :opencortex-safety-tests)) - (uiop:symbol-call :fiveam :run! (uiop:find-symbol* :boot-suite :opencortex-boot-tests)) - (uiop:symbol-call :fiveam :run! (uiop:find-symbol* :memory-suite :opencortex-memory-tests)) - (uiop:symbol-call :fiveam :run! (uiop:find-symbol* :immune-suite :opencortex-immune-system-tests)))) + + :perform (test-op (o s) + (uiop:symbol-call :fiveam :run! :communication-protocol-suite) + (uiop:symbol-call :fiveam :run! :pipeline-suite) + (uiop:symbol-call :fiveam :run! :safety-suite) + (uiop:symbol-call :fiveam :run! :boot-suite) + (uiop:symbol-call :fiveam :run! :memory-suite) + (uiop:symbol-call :fiveam :run! :immune-suite))) #+end_src -** TUI Client Definition -This system defines the native Croatoan TUI client. +** TUI Client System #+begin_src lisp :tangle ../opencortex.asd (defsystem :opencortex/tui - :depends-on (:opencortex :croatoan :usocket :bordeaux-threads) + :depends-on (:opencortex ; The daemon we're connecting to + :croatoan ; Terminal UI library + :usocket ; Socket communication + :bordeaux-threads) ; Background listening thread + :components ((:file "library/tui-client"))) #+end_src * The Harness Boundary Contract -The harness is the minimal, unbreakable core of OpenCortex. It consists of the literate source files that define the kernel and the system manifest. Any proposed modification to these files must be justified, because the harness is the system's immune system and must never grow fat. + +** Why a Boundary Contract? + +The harness is the immune system of OpenCortex. If it grows fat (accumulating features, dependencies, complexity), it becomes harder to: +- Verify for security +- Debug when things go wrong +- Maintain across versions + +The Boundary Contract defines what IS the harness vs. what belongs in skills. ** Primary Boundary Files -- ~harness/*.org~ — The literate source of truth for all kernel modules. -- ~opencortex.asd~ — The ASDF system manifest. -** Generated Artifacts (NOT Primary Boundary) -The files in ~library/*.lisp~ are derivative artifacts produced by tangling the harness Org files. They are NOT primary boundary files; modifying them directly violates the Engineering Standard of Literate-Only Modification. Any change to the harness must be made in the corresponding Org file and then tangled. +| File | Purpose | Modification | +|------|---------|--------------| +| ~harness/*.org~ | Literate source of truth | Only via Org edits + tangle | +| ~opencortex.asd~ | System manifest | Only via Org edits + tangle | +| ~library/*.lisp~ | Tangled from .org | NEVER edit directly | -** Enforcement -The Policy skill's ~*modularity-protected-paths*~ variable guards the primary boundary locations by default. Any agent action that proposes to modify a file within these paths must include a ~:modularity-justification~ field explaining why the change cannot be implemented as a skill. +** Generated Artifacts (NOT Primary) + +The ~library/*.lisp~ files are tangles from the ~harness/*.org~ files. They are derivative artifacts. Direct modification violates the Literate Granularity standard. + +** Protected Paths + +The Policy skill guards these paths by default: + +#+begin_src lisp +(defvar *modularity-protected-paths* + '("harness/" + "opencortex.asd" + "library/package.lisp" + "library/communication.lisp" + "library/memory.lisp" + "library/context.lisp" + "library/perceive.lisp" + "library/reason.lisp" + "library/act.lisp" + "library/loop.lisp")) +#+end_src + +Any agent action proposing to modify these files must include a ~:modularity-justification~ field explaining why the change cannot be implemented as a skill. + +** Enforcement Chain + +1. *Policy Skill* (priority 500) - Checks for missing justifications +2. *Bouncer Skill* (priority 100) - Intercepts unauthorized modifications +3. *Git Hooks* (optional) - Prevents direct .lisp commits + +* Quick Reference + +** Building the System + +#+begin_src bash +# Development: Load source +(ql:quickload :opencortex) + +# Build standalone binary +(asdf:make :opencortex) + +# Run tests +(ql:quickload :opencortex/tests) +(asdf:test-system :opencortex/tests) +#+end_src + +** Loading Order + +1. ~library/package.lisp~ - Creates ~:opencortex~ package +2. ~library/skills.lisp~ - Defines ~defskill~, ~def-cognitive-tool~ macros +3. ~library/communication.lisp~ - Protocol, framing, validation +4. ~library/memory.lisp~ - Org-object, Merkle tree, snapshots +5. ~library/context.lisp~ - Context assembly functions +6. ~library/perceive.lisp~ - Stage 1: Perceive gate +7. ~library/reason.lisp~ - Stage 2: Reason (think + verify) +8. ~library/act.lisp~ - Stage 3: Act (dispatch + execute) +9. ~library/loop.lisp~ - Main entry point, heartbeat \ No newline at end of file diff --git a/harness/perceive.org b/harness/perceive.org index 4bbac59..16fe77f 100644 --- a/harness/perceive.org +++ b/harness/perceive.org @@ -4,86 +4,219 @@ #+STARTUP: content * Stage 1: Perceive (perceive.lisp) -** Architectural Intent: Sensory Ingestion -The Perceive stage is the "sensory cortex" of the OpenCortex. It takes raw stimuli from the outside world (keyboard events, chat messages, heartbeats, or system interrupts) and normalizes them into internal **Signals**. -** Async Sensor Routing -To prevent blocking the main pipeline, certain sensors (like user commands or chat messages) are processed asynchronously in their own threads. +** Architectural Intent: Sensory Normalization + +The Perceive stage is the "sensory cortex" of OpenCortex. Its job is to take raw stimuli from the outside world and transform them into standardized Signals that the rest of the pipeline can process. + +Raw stimuli come from diverse sources: +- Terminal input (CLI) +- Emacs org-mode buffers (via swank) +- Telegram/Signal messages +- Heartbeats (internal clock) +- Shell command outputs + +Each source has its own format and protocol. Perceive normalizes all of them into the Signal format: + +: (TYPE :EVENT META (...) PAYLOAD (...)) + +** Why Normalize? + +Without normalization, each downstream component (Reason, Act) would need to understand each input format. With normalization: + +1. The gateway layer (CLI, Emacs, Telegram) just sends raw messages +2. Perceive transforms them into Signals +3. Reason and Act work with a single, consistent format +4. Adding new input sources only requires gateway code, not changes to the core + +** The Signal Format + +Signals are property lists with a consistent structure: + +| Key | Description | +|-----|-------------| +| :type | :EVENT, :REQUEST, :RESPONSE, :LOG | +| :payload | The actual content (sensor data, actions, etc.) | +| :meta | Metadata: source, session, reply stream | +| :status | Processing status: :perceived, :reasoned, :acted | +| :depth | Recursion depth for feedback loops | +| :approved-action | Set by Reason, executed by Act | +| :foveal-focus | ID of the node user is interacting with | + +** Async vs Sync Processing + +Some sensors (user input, chat messages) are processed asynchronously in dedicated threads. This prevents: +- A slow API call from blocking the entire system +- Race conditions when multiple inputs arrive simultaneously + +Other sensors (heartbeats, interrupts) are processed synchronously to maintain ordering guarantees. + +* Package Context #+begin_src lisp :tangle ../library/perceive.lisp (in-package :opencortex) +#+end_src +* Sensor Configuration + +** Async Sensor Registry + +#+begin_src lisp :tangle ../library/perceive.lisp (defvar *async-sensors* '(:chat-message :delegation :user-command) - "List of sensors that should be processed asynchronously to avoid blocking gateways.") + "Sensors that are processed in dedicated threads. + + These sensors can block (waiting for API responses, user input, etc.) + so they run in separate threads to avoid blocking the main pipeline. + + Other sensors (:heartbeat, :interrupt, :buffer-update) are processed + synchronously to maintain temporal ordering.") #+end_src ** Foveal Focus State -The system tracks the user's current point of interaction to provide context to the reasoning engine. #+begin_src lisp :tangle ../library/perceive.lisp (defvar *foveal-focus-id* nil - "The Org ID of the node the user is currently interacting with.") + "The Org ID of the node the user is currently interacting with. + + This enables the reasoning engine to provide contextually relevant + responses. When editing a specific note, the agent knows which + note you're referring to without needing explicit ID references. + + Updated on :point-update events from Emacs.") #+end_src -** Stimulus Injection -The entry point for raw messages. It determines if the signal should be processed synchronously or asynchronously. +* Stimulus Injection + +** inject-stimulus: Entry Point #+begin_src lisp :tangle ../library/perceive.lisp (defun inject-stimulus (raw-message &key stream (depth 0)) - "Enqueues a raw message into the reactive signal pipeline." - (let* ((payload (getf raw-message :payload)) + "Inject a raw message into the signal processing pipeline. + + RAW-MESSAGE is a property list that will be normalized into a Signal. + STREAM is an optional output stream for responses (used by TUI/CLI). + DEPTH tracks recursion depth for feedback loops. + + This function determines whether to process synchronously or + asynchronously based on the sensor type, then calls process-signal + to run through the Perceive -> Reason -> Act pipeline. + + Error handling: Uses restarts to prevent individual signals from + crashing the entire system. Failed signals are logged and dropped." + + (let* ((payload (getf raw-message :payload)) (sensor (getf payload :sensor)) (meta (getf raw-message :meta)) - (async-p (or (getf payload :async-p) (member sensor *async-sensors*)))) - - ;; Ensure META exists and contains the stream if provided - (unless meta (setf meta (list :SOURCE :SYSTEM :SESSION-ID "internal"))) - (when stream (setf (getf meta :reply-stream) stream)) + (async-p (or (getf payload :async-p) + (member sensor *async-sensors*)))) + + ;; Ensure metadata exists + (unless meta + (setf meta (list :SOURCE :SYSTEM :SESSION-ID "internal"))) + + ;; Attach reply stream if provided + (when stream + (setf (getf meta :reply-stream) stream)) + (setf (getf raw-message :meta) meta) - (if async-p - (bt:make-thread - (lambda () - (restart-case (handler-bind ((error (lambda (c) (harness-log "ASYNC ERROR: ~a" c) (invoke-restart 'skip-event)))) - (process-signal raw-message)) - (skip-event () nil))) + (if async-p + ;; Async: process in dedicated thread + (bt:make-thread + (lambda () + (restart-case + (handler-bind ((error (lambda (c) + (harness-log "ASYNC ERROR: ~a" c) + (invoke-restart 'skip-event)))) + (process-signal raw-message)) + (skip-event () nil))) :name "opencortex-async-task") - (restart-case (handler-bind ((error (lambda (c) (harness-log "SYSTEM ERROR: ~a" c) (invoke-restart 'skip-event)))) - (process-signal raw-message)) - (skip-event () (harness-log "SYSTEM RECOVERY: Stimulus dropped.~%")))))) + + ;; Sync: process in main thread with recovery + (restart-case + (handler-bind ((error (lambda (c) + (harness-log "SYSTEM ERROR: ~a" c) + (invoke-restart 'skip-event)))) + (process-signal raw-message)) + (skip-event () + (harness-log "SYSTEM RECOVERY: Stimulus dropped.")))))) #+end_src -** The Perceive Gate -The initial stage of the metabolic loop. It logs the signal, performs selective memory snapshots, and updates the Memory graph based on incoming AST updates. +* The Perceive Gate + +** perceive-gate: Signal Normalization #+begin_src lisp :tangle ../library/perceive.lisp (defun perceive-gate (signal) - "Initial processing: Normalizes raw stimuli and updates memory." + "Stage 1 of the metabolic pipeline: Normalize sensory input. + + This function: + 1. Logs the incoming signal for debugging + 2. Handles special sensor types (:buffer-update, :point-update, etc.) + 3. Updates the Memory graph with incoming data + 4. Tracks foveal focus (user's current node) + 5. Sets :status to :perceived + + Modifies the signal in place and returns it for the next stage. + + Memory snapshots are taken before AST updates to enable rollback + if the update causes issues." + (let* ((payload (getf signal :payload)) (type (getf signal :type)) (meta (getf signal :meta)) (sensor (getf payload :sensor))) - (harness-log "GATE [Perceive]: ~a (~a) [Source: ~s]" type (or sensor "no-sensor") (getf meta :source)) - + + ;; Log the incoming signal for debugging + (harness-log "GATE [Perceive]: ~a (~a) [Source: ~s]" + type (or sensor "no-sensor") (getf meta :source)) + + ;; Handle EVENT type sensors (cond ((eq type :EVENT) (case sensor - (:buffer-update - (let ((ast (getf payload :ast))) - (when ast - (snapshot-memory) + + ;; Org buffer was modified - update memory + (:buffer-update + (let ((ast (getf payload :ast))) + (when ast + (snapshot-memory) ; Enable rollback if update causes issues (ingest-ast ast)))) - (:point-update - (let ((element (getf payload :element))) - (when element + + ;; Point moved to different org node - update focus + (:point-update + (let ((element (getf payload :element))) + (when element (snapshot-memory) - (setf *foveal-focus-id* (ignore-errors (getf element :id))) + ;; Track foveal focus for contextual reasoning + (setf *foveal-focus-id* + (ignore-errors (getf element :id))) (ingest-ast element)))) - (:interrupt - (bt:with-lock-held (*interrupt-lock*) (setf *interrupt-flag* t))))) + + ;; System interrupt - trigger shutdown + (:interrupt + (bt:with-lock-held (*interrupt-lock*) + (setf *interrupt-flag* t))))) + + ;; Log responses from actuators ((eq type :RESPONSE) - (harness-log "GATE [Perceive]: Act Result -> ~a" (getf payload :status)))) - + (harness-log "GATE [Perceive]: Act Result -> ~a" + (getf payload :status)))) + + ;; Update signal status (setf (getf signal :status) :perceived) (setf (getf signal :foveal-focus) *foveal-focus-id*) signal)) #+end_src + +** Sensor Types Reference + +| Sensor | Source | Processing | Description | +|--------|--------|------------|-------------| +| :user-input | CLI/TUI | Async | Text input from terminal | +| :chat-message | Telegram/Signal | Async | Messages from messaging apps | +| :heartbeat | Internal | Sync | Periodic maintenance trigger | +| :buffer-update | Emacs | Sync | Org buffer was modified | +| :point-update | Emacs | Sync | Cursor moved to different headline | +| :interrupt | System | Sync | SIGINT received | +| :tool-output | Internal | Sync | Result from cognitive tool | +| :loop-error | Internal | Sync | Error during signal processing | \ No newline at end of file diff --git a/harness/reason.org b/harness/reason.org index 57b1f5f..9464ac0 100644 --- a/harness/reason.org +++ b/harness/reason.org @@ -4,52 +4,143 @@ #+STARTUP: content * Stage 2: Reason (reason.lisp) -** Architectural Intent: Unified Cognition -The Reason stage is the cognitive engine of the OpenCortex. It bridges the gap between raw sensory data (Perceive) and physical side-effects (Act). -* Cognition Engine (reason.lisp) +** Architectural Intent: The Dual-Engine Cognitive Architecture + +The Reason stage implements the core innovation of OpenCortex: the separation of probabilistic reasoning (neural/LLM) from deterministic verification (logic/safety). + +This dual-engine design solves a fundamental problem in AI safety: + +1. *Probabilistic Engine* - Uses LLMs for semantic understanding, natural language generation, and complex reasoning. It is powerful but can hallucinate, make syntax errors, or propose unsafe actions. + +2. *Deterministic Engine* - Uses formal verification (skills) to check LLM proposals before execution. It is slower but provably correct. + +The LLM proposes; the skills verify. This is the "Bouncer Pattern" - the deterministic engine is literally a bouncer that checks the LLM's proposals at the door before letting them through to execution. + +** Why Plists for Communication? + +The Reason stage communicates exclusively through property lists (plists). This design choice reflects the homoiconic nature of Lisp - plists are native data structures that can be read, written, and manipulated by the same code that processes them. + +A plist message like: +: (TYPE :REQUEST TARGET :CLI PAYLOAD (ACTION :MESSAGE TEXT "Hello")) + +Is simultaneously: +- Human-readable text +- Machine-parseable data structure +- Executable Lisp code + +This means the reasoning pipeline can generate, modify, and execute its own communication protocol without external parsing. + +* Package Context -** Package Context #+begin_src lisp :tangle ../library/reason.lisp (in-package :opencortex) #+end_src -** Neural Backend Registry -#+begin_src lisp :tangle ../library/reason.lisp -(defvar *probabilistic-backends* (make-hash-table :test 'equal)) -(defvar *provider-cascade* nil) -(defvar *model-selector-fn* nil) -(defvar *consensus-enabled-p* nil) +* Probabilistic Engine (Neural/LLM Integration) +The probabilistic engine is responsible for all neural/LLM operations. It maintains a registry of provider backends and implements a cascading failover mechanism. + +** Backend Registry Variables + +#+begin_src lisp :tangle ../library/reason.lisp +(defvar *probabilistic-backends* (make-hash-table :test 'equal) + "Registry mapping provider keywords (:openrouter, :ollama) to their calling functions.") + +(defvar *provider-cascade* nil + "Ordered list of provider keywords to try. First available provider wins.") + +(defvar *model-selector-fn* nil + "Optional function that selects a specific model for each provider. + Signature: (funcall fn provider context) => model-name-string") + +(defvar *consensus-enabled-p* nil + "When T, run multiple providers and compare results for critical decisions.") +#+end_src + +** register-probabilistic-backend: Backend Registration + +#+begin_src lisp :tangle ../library/reason.lisp (defun register-probabilistic-backend (name fn) - "Registers a neural provider (e.g., :gemini, :anthropic) with its calling function." + "Register a neural provider backend. + + NAME is a keyword like :openrouter or :ollama. + FN is a function with signature: (funcall fn prompt system-prompt &key model) + returning either: + - (list :status :success :content \"response text\") + - (list :status :error :message \"error description\") + - a simple string on success + + Example registration: + (register-probabilistic-backend :openrouter #'openrouter-call)" + (setf (gethash name *probabilistic-backends*) fn)) #+end_src -** Probabilistic Reasoning (probabilistic-call) +** probabilistic-call: Cascade Dispatch + #+begin_src lisp :tangle ../library/reason.lisp -(defun probabilistic-call (prompt &key (system-prompt "You are the Probabilistic engine.") (cascade nil) (context nil)) - "Dispatches a neural request through the provider cascade. Returns a Lisp plist or a failure log." +(defun probabilistic-call (prompt &key + (system-prompt "You are the Probabilistic engine.") + (cascade nil) + (context nil)) + "Dispatch a neural request through the provider cascade. + + PROMPT - The user's query or task description. + SYSTEM-PROMPT - Instructions for how the LLM should behave. + CASCADE - Override the default provider cascade. + CONTEXT - Current signal context (for model selection). + + Returns the LLM response as a string, or a failure plist if all providers fail. + + The cascade mechanism ensures reliability: if OpenRouter is rate-limited, + it automatically falls back to OpenAI, then Anthropic, etc." + (let ((backends (or cascade *provider-cascade*))) (or (dolist (backend backends) (let ((backend-fn (gethash backend *probabilistic-backends*))) (when backend-fn (harness-log "PROBABILISTIC: Attempting backend ~a..." backend) - (let* ((model (when *model-selector-fn* (funcall *model-selector-fn* backend context))) - (result (if model + + ;; Optional model selection based on context + (let* ((model (when *model-selector-fn* + (funcall *model-selector-fn* backend context))) + (result (if model (funcall backend-fn prompt system-prompt :model model) (funcall backend-fn prompt system-prompt)))) + + ;; Normalize result format (cond ((and (listp result) (eq (getf result :status) :success)) (return (getf result :content))) - ((stringp result) (return result)) - (t (harness-log "PROBABILISTIC: Backend ~a failed: ~a" backend (getf result :message)))))))) - (list :type :LOG :payload (list :text "Neural Cascade Failure: All providers exhausted."))))) + ((stringp result) + (return result)) + (t + (harness-log "PROBABILISTIC: Backend ~a failed: ~a" + backend (getf result :message)))))))) + + ;; All providers failed + (list :type :LOG + :payload (list :text "Neural Cascade Failure: All providers exhausted."))))) #+end_src -** Cognitive Proposal (Think) +* Cognitive Proposal Generation (Think) + +The `think` function is the heart of the probabilistic engine. It constructs a prompt from context, sends it to the LLM, and parses the response into a structured action. + +** strip-markdown: Clean LLM Output + #+begin_src lisp :tangle ../library/reason.lisp (defun strip-markdown (text) - "Strips common markdown code block markers from text." + "Strip markdown formatting from LLM output. + + LLMs often wrap their responses in code fences (```lisp ...```). + This function removes those markers to extract the raw plist. + + Handles: + - Leading code fences with language tags: ```lisp + - Trailing code fences: ``` + - Orphan closing fences: ```" + (if (and text (stringp text)) (let ((cleaned text)) (setf cleaned (cl-ppcre:regex-replace-all "^```[a-z]*\\n" cleaned "")) @@ -57,105 +148,297 @@ The Reason stage is the cognitive engine of the OpenCortex. It bridges the gap b (setf cleaned (cl-ppcre:regex-replace-all "```" cleaned "")) (string-trim '(#\Space #\Newline #\Tab) cleaned)) text)) +#+end_src +** normalize-plist-keywords: Fix LLM Keyword Output + +#+begin_src lisp :tangle ../library/reason.lisp (defun normalize-plist-keywords (plist) - "Normalize all keys in a plist to keywords (e.g., TYPE -> :TYPE)." + "Normalize all keys in a plist to keywords. + + LLMs often return plists with unquoted keys: (TYPE REQUEST ...) + instead of keyword syntax: (:TYPE :REQUEST ...) + + This function converts all symbol keys to their keyword equivalents, + making the plist compatible with standard Lisp property accessors. + + Example transformation: + (TYPE REQUEST PAYLOAD (ACTION MESSAGE TEXT \"Hi\")) + => (:TYPE :REQUEST :PAYLOAD (:ACTION :MESSAGE :TEXT \"Hi\"))" + (when (listp plist) (loop for (k . rest) on plist by #'cddr collect (if (and (symbolp k) (not (keywordp k))) (intern (string k) :keyword) k) collect (car rest)))) +#+end_src +** think: Generate Action Proposal + +#+begin_src lisp :tangle ../library/reason.lisp (defun think (context) - "Generates a Lisp action proposal based on current context." + "Generate a Lisp action proposal based on current context. + + This is the core cognitive function. It: + + 1. Finds the most relevant skill based on context + 2. Assembles global awareness (memory context, system logs) + 3. Constructs a detailed prompt with available tools + 4. Calls the LLM via probabilistic-call + 5. Parses the LLM response into a structured action plist + + The LLM is instructed to respond with exactly ONE plist, never prose. + This constraint makes parsing deterministic and prevents rambling. + + Returns a plist with structure: + (:TYPE :REQUEST :TARGET :CLI :PAYLOAD (:ACTION :MESSAGE :TEXT \"...\"))" + + ;; Gather context components (let* ((active-skill (find-triggered-skill context)) (tool-belt (generate-tool-belt-prompt)) (global-context (context-assemble-global-awareness)) (system-logs (context-get-system-logs)) (assistant-name (or (uiop:getenv "MEMEX_ASSISTANT") "Agent"))) - (let* ((prompt-generator (when active-skill (skill-probabilistic-prompt active-skill)))) - (raw-prompt (if prompt-generator + + ;; Generate prompt from skill or raw text + (let* ((prompt-generator (when active-skill + (skill-probabilistic-prompt active-skill))) + (raw-prompt (if prompt-generator (funcall prompt-generator context) + ;; Fallback: use raw user input (let ((p (proto-get (proto-get context :payload) :text))) - (if (and p (stringp p)) p "Maintain metabolic stasis.")))) - (system-prompt (format nil "IDENTITY: ~a. MANDATE: Respond with ONE Lisp plist. ~a ~a RECENT_LOGS: ~a + (if (and p (stringp p)) + p + "Maintain metabolic stasis.")))) + (system-prompt (format nil + "IDENTITY: ~a + +You are a component of the OpenCortex neurosymbolic AI agent. +Your task is to generate exactly ONE valid Lisp plist response. + +MANDATE: Respond with ONE Lisp plist. Never output prose. + IMPORTANT: To reply to the user, you MUST use: (:TYPE :REQUEST :PAYLOAD (:ACTION :MESSAGE :TEXT \"\")) To call a tool, you MUST use: (:TYPE :REQUEST :TARGET :TOOL :ACTION :CALL :TOOL \"\" :ARGS (:arg1 \"val\")) -MANDATORY VALIDATION RULE: Before declaring any Lisp code edit complete, you MUST call the `:validate-lisp` tool with the proposed code. If the tool returns `:status :error`, read the `:reason` and `:failed` fields, fix the defect, and re-validate. You are strictly forbidden from relying on your own paren-balancing or syntax intuition. +MANDATORY VALIDATION RULE: Before declaring any Lisp code edit complete, +you MUST call the `:validate-lisp` tool with the proposed code. If the tool +returns `:status :error`, read the `:reason` and `:failed` fields, fix the +defect, and re-validate. You are strictly forbidden from relying on your +own paren-balancing or syntax intuition. -PROVIDER RULE: Always use the default cascade provider unless a specific model or capability is required for the task." - assistant-name global-context tool-belt system-logs))) - (let* ((thought (probabilistic-call raw-prompt :system-prompt system-prompt :context context)) +PROVIDER RULE: Always use the default cascade provider unless a specific +model or capability is required for the task. + +AVAILABLE TOOLS: +~a + +GLOBAL CONTEXT: +~a + +RECENT LOGS: +~a" + assistant-name + tool-belt + global-context + system-logs))) + + ;; Call LLM and process response + (let* ((thought (probabilistic-call raw-prompt + :system-prompt system-prompt + :context context)) (cleaned (strip-markdown thought)) (meta (proto-get context :meta)) (source (proto-get meta :source))) - (harness-log "THINK: raw cleaned = ~a" (subseq cleaned 0 (min 100 (length cleaned)))) - (if (and cleaned (stringp cleaned)) + + (when cleaned + (harness-log "THINK: LLM raw output = ~a" + (subseq cleaned 0 (min 200 (length cleaned))))) + + ;; Parse LLM response + (if (and cleaned (stringp cleaned) (> (length cleaned) 0)) (let ((*read-eval* nil)) - (if (and (> (length cleaned) 0) (char= (char cleaned 0) #\()) - (handler-case + (if (char= (char cleaned 0) #\() + ;; Response starts with paren - try to parse as plist + (handler-case (let ((parsed (read-from-string cleaned))) - (harness-log "THINK: parsed = ~a" parsed) - (let ((type (proto-get parsed :TYPE)) - (target (or (proto-get parsed :TARGET) (proto-get parsed :target)))) - (cond ((member type '(:REQUEST :EVENT :STATUS :RESPONSE)) - (unless (proto-get parsed :target) (setf (getf parsed :target) (or source :CLI))) - parsed) - ;; Handle raw plists or lists of plists that look like tool calls or data - ((or (eq target :TOOL) (eq target :tool) (getf parsed :TOOL) (getf parsed :tool) - (and (listp parsed) (listp (car parsed)) (keywordp (caar parsed)))) - (list :TYPE :REQUEST :TARGET :TOOL :PAYLOAD parsed)) - (t (list :TYPE :REQUEST :TARGET (or source :CLI) :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned)))))) - (error (c) (harness-log "THINK ERROR: ~a" c) (list :TYPE :REQUEST :TARGET (or source :CLI) :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned)))) - (list :TYPE :REQUEST :TARGET (or source :CLI) :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned)))) + (when parsed + (harness-log "THINK: parsed = ~a" parsed) + + ;; Normalize keyword keys (LLM often returns TYPE instead of :TYPE) + (let ((parsed-normalized (normalize-plist-keywords parsed)) + (type (proto-get parsed :TYPE)) + (target (or (proto-get parsed :TARGET) + (proto-get parsed :target)))) + + (cond + ;; Recognized message type - use directly + ((member type '(:REQUEST :EVENT :STATUS :RESPONSE)) + (unless (proto-get parsed :target) + (setf (getf parsed :target) (or source :CLI))) + parsed-normalized) + + ;; Tool call detected - wrap in standard envelope + ((or (eq target :TOOL) + (eq target :tool) + (getf parsed :TOOL) + (getf parsed :tool) + (and (listp parsed) + (listp (car parsed)) + (keywordp (caar parsed)))) + (list :TYPE :REQUEST + :TARGET :TOOL + :PAYLOAD (normalize-plist-keywords parsed))) + + ;; Unknown format - treat as user message + (t + (list :TYPE :REQUEST + :TARGET (or source :CLI) + :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned))))))) + (error (c) + (harness-log "THINK ERROR: ~a" c) + (list :TYPE :REQUEST + :TARGET (or source :CLI) + :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned)))) + + ;; No leading paren - treat as plain text message + (list :TYPE :REQUEST + :TARGET (or source :CLI) + :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned)))) + + ;; No response from LLM thought))))) #+end_src -** Deterministic Verification +* Deterministic Engine (Formal Verification) + +The deterministic engine runs all registered skills' verification functions. This is where safety checks, policy enforcement, and skill-specific processing happen. + +** deterministic-verify: Skill Chain Verification + #+begin_src lisp :tangle ../library/reason.lisp (defun deterministic-verify (proposed-action context) - "Iterates through all skill deterministic-gates sorted by priority." + "Run all skill deterministic gates on a proposed action. + + Each skill can define a deterministic function that either: + - Passes the action through unchanged + - Modifies the action (adds explanation, changes target, etc.) + - Blocks the action (returns a :LOG message instead) + + Skills are sorted by priority (highest first). A skill with higher + priority can intercept and modify actions before lower-priority + skills see them. + + The Bouncer Pattern: If any skill returns a :LOG or :EVENT type, + processing stops and that message is returned immediately. This + allows skills to veto actions. + + Example skill chain: + 1. Policy skill (priority 500) - checks for missing explanations + 2. Protocol validator (priority 95) - validates message schema + 3. Shell actuator guard (priority 50) - checks command whitelist" + (let ((current-action proposed-action) (skills nil)) - (maphash (lambda (name skill) (declare (ignore name)) (when (skill-deterministic-fn skill) (push skill skills))) *skills-registry*) + + ;; Collect all skills with deterministic functions + (maphash (lambda (name skill) + (declare (ignore name)) + (when (skill-deterministic-fn skill) + (push skill skills))) + *skills-registry*) + + ;; Sort by priority (highest first) (setf skills (sort skills #'> :key #'skill-priority)) + + ;; Run each skill's gate (dolist (skill skills) (let ((trigger (skill-trigger-fn skill)) (gate (skill-deterministic-fn skill))) - (when (or (null trigger) (ignore-errors (funcall trigger context))) + + ;; Skill activates if no trigger or trigger returns true + (when (or (null trigger) + (ignore-errors (funcall trigger context))) + + ;; Run the gate (let ((next-action (funcall gate current-action context))) (let ((original-type (proto-get current-action :type))) - (when (and (listp next-action) - (member (proto-get next-action :type) '(:LOG :EVENT :log :event)) + + ;; Check if skill intercepted (returned LOG/EVENT instead of REQUEST) + (when (and (listp next-action) + (member (proto-get next-action :type) + '(:LOG :EVENT :log :event)) (or (not (member original-type '(:LOG :EVENT :log :event))) (not (eq next-action current-action)))) - (harness-log "DETERMINISTIC: Intercepted by skill '~a'" (skill-name skill)) + + ;; Skill blocked or modified - stop processing + (harness-log "DETERMINISTIC: Intercepted by skill '~a'" + (skill-name skill)) (return-from deterministic-verify next-action))) + + ;; Action passed through - continue to next skill (setf current-action next-action))))) + + ;; Return final action (may be modified by skills, or original if all passed) current-action)) #+end_src -** Reasoning Gate (The Pipeline Stage) +* Reason Gate (Pipeline Stage) + +** reason-gate: The Stage Function + #+begin_src lisp :tangle ../library/reason.lisp (defun reason-gate (signal) - "Unified Stage: Combines Probabilistic proposals and Deterministic verification." + "Stage 2 of the metabolic pipeline: Reason. + + Transforms perceived signals into approved actions by combining: + 1. Probabilistic reasoning (LLM generates proposal) + 2. Deterministic verification (skills validate proposal) + + Only processes :EVENT signals with :user-input or :chat-message sensors. + Other signals pass through unchanged (heartbeats, tool outputs, etc.). + + Modifies the signal in place by setting: + - :approved-action - The final verified action, or NIL + - :status - :reasoned + + Returns the modified signal." + (let* ((type (proto-get signal :type)) (payload (proto-get signal :payload)) (sensor (proto-get payload :sensor))) - (unless (and (eq type :EVENT) (member sensor '(:user-input :chat-message))) + + ;; Only reason about user input, not internal signals + (unless (and (eq type :EVENT) + (member sensor '(:user-input :chat-message))) (return-from reason-gate signal)) + + ;; Generate proposal via LLM (let ((candidate (think signal))) - (harness-log "REASON: candidate = ~a" (type-of candidate)) - (if (and candidate (listp candidate) (keywordp (car candidate))) - (setf (getf signal :approved-action) (deterministic-verify candidate signal)) + + (harness-log "REASON: candidate type = ~a" (type-of candidate)) + + ;; Validate candidate is a proper plist (not an error string or symbol) + (if (and candidate + (listp candidate) + (or (keywordp (car candidate)) + (eq (car candidate) 'TYPE) + (eq (car candidate) 'type))) + + ;; Valid proposal - run through deterministic verification + (setf (getf signal :approved-action) + (deterministic-verify candidate signal)) + + ;; Invalid response - log and drop (progn - (harness-log "REASON: Invalid candidate type ~a, dropping" (type-of candidate)) + (harness-log "REASON: Invalid candidate type ~a, dropping" + (type-of candidate)) (setf (getf signal :approved-action) nil))) + (setf (getf signal :status) :reasoned) signal))) -#+end_src +#+end_src \ No newline at end of file