amr/passepartout
1
0
Fork 0
Code Issues 15 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

RELEASE: Finalize Semantic Restructuring v0.1.0
Some checks failed
Deploy-Agent-V15-Stdin / JOB-V15-STDIN (push) Failing after 3s
Details

Browse Source 
- Folders: literate->harness, src->library, system->environment, scripts->interfaces.
- Synchronized all :tangle paths and system definitions.
- Hardened .gitignore for binary and log artifacts.
- Consolidated all documentation into docs/.
This commit is contained in:
Amr Gharbeia
2026-04-21 12:41:50 -04:00
parent dd3873cd5e
commit 94a8a0ab0b
53 changed files with 475 additions and 760 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
harness/act.org Normal file
View File

@@ -0,0 +1,212 @@
#+TITLE: Stage 3: Act (act.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:act:
#+STARTUP: content
* Stage 3: Act (act.lisp)
** Architectural Intent: Actuation
The Act stage performs the final physical side-effects of the metabolic pipeline. It takes an approved **Action** (the result of the Reasoning stage) and routes it to the correct physical **Actuator**.
Actuators are the "hands" of the OpenCortex. They can be local (printing to a terminal), virtual (executing a shell command), or remote (sending a Matrix message). Crucially, the core microharness does not know *how* to talk to these services; it only knows how to *dispatch* to the registered actuator functions.
** Pipeline Initialization
#+begin_src lisp :tangle ../library/act.lisp
(in-package :opencortex)
#+end_src
* Actuator Configuration
** Default Actuator
#+begin_src lisp :tangle ../library/act.lisp
(defvar *default-actuator* :cli
"The fallback actuator used if a signal has no source or target metadata.")
#+end_src
** Silent Actuators
To prevent infinite feedback loops, certain actuators are flagged as "silent." Results from these actuators are logged but do not trigger a fresh metabolic cycle.
#+begin_src lisp :tangle ../library/act.lisp
(defvar *silent-actuators* '(:cli :system-message :emacs)
"List of actuators whose feedback should not re-enter the Reasoning stage.")
#+end_src
** Initialization Logic (initialize-actuators)
This function hydrates the actuator configuration from the environment and registers the core built-in actuators.
#+begin_src lisp :tangle ../library/act.lisp
(defun initialize-actuators ()
"Loads actuator routing defaults from environment variables and registers core harness actuators."
(let ((def (uiop:getenv "DEFAULT_ACTUATOR"))
(silent (uiop:getenv "SILENT_ACTUATORS")))
(when def
(setf *default-actuator* (intern (string-upcase def) "KEYWORD")))
(when silent
(setf *silent-actuators*
(mapcar (lambda (s) (intern (string-upcase (string-trim '(#\Space) s)) "KEYWORD"))
(str:split "," silent)))))
;; Register core harness actuators
(register-actuator :system #'execute-system-action)
(register-actuator :tool #'execute-tool-action)
(register-actuator :tui (lambda (action context)
(let* ((meta (getf context :meta))
(stream (getf meta :reply-stream)))
(when (and stream (open-stream-p stream))
(format stream "~a" (frame-message action))
(finish-output stream))))))
#+end_src
* Primary Routing
** Dispatching Logic (dispatch-action)
The primary router. It identifies the target actuator based on the Signal's `:META` source or the Action's `:TARGET`.
#+begin_src lisp :tangle ../library/act.lisp
(defun dispatch-action (action context)
"Routes an approved action to its registered physical actuator."
(let ((payload (proto-get action :payload)))
;; Optimization: Heartbeats are system events, not actions.
(when (eq (proto-get payload :sensor) :heartbeat)
(return-from dispatch-action nil)))
(when (and action (listp action))
(let* ((meta (proto-get context :meta))
(source (proto-get meta :source))
(raw-target (or (ignore-errors (getf action :TARGET))
(ignore-errors (getf action :target))
source
*default-actuator*))
(target (intern (string-upcase (string raw-target)) :keyword))
(actuator-fn (gethash target *actuator-registry*)))
;; Propagation: Ensure outbound action inherits metadata
(when (and meta (null (getf action :meta)))
(setf (getf action :meta) meta))
(if actuator-fn
(funcall actuator-fn action context)
(harness-log "ACT ERROR: No actuator for ~s (from ~s)" target raw-target)))))
#+end_src
* Built-in Actuators
** System Actuator (execute-system-action)
Handles meta-operations like hot-loading skills or evaluating raw Lisp within the image.
#+begin_src lisp :tangle ../library/act.lisp
(defun execute-system-action (action context)
"Processes internal harness commands. (ACTUATOR)"
(declare (ignore context))
(let* ((payload (ignore-errors (getf action :payload)))
(cmd (ignore-errors (getf payload :action))))
(case cmd
(:eval (let ((code (getf payload :code)))
(eval (read-from-string code))))
(:create-skill (let* ((filename (getf payload :filename)) (content (getf payload :content))
(skills-dir (merge-pathnames "skills/" (asdf:system-source-directory :opencortex)))
(full-path (merge-pathnames filename skills-dir)))
(with-open-file (out full-path :direction :output :if-exists :supersede) (write-string content out))
(load-skill-from-org full-path)))
(:message (harness-log "ACT [System]: ~a" (getf payload :text)))
(t (harness-log "ACT ERROR [System]: Unknown command ~s" cmd)))))
#+end_src
** Tool Result Formatting (format-tool-result)
A UI helper that distills technical LLM responses into human-readable text.
#+begin_src lisp :tangle ../library/act.lisp
(defun format-tool-result (tool-name result)
"Intelligently formats a tool result for user display."
(if (listp result)
(let ((status (getf result :status))
(content (getf result :content))
(msg (getf result :message)))
(cond ((and (eq status :success) content) (format nil "~a" content))
((and (eq status :error) msg) (format nil "ERROR [~a]: ~a" tool-name msg))
(t (format nil "TOOL [~a] RESULT: ~s" tool-name result))))
(format nil "TOOL [~a] RESULT: ~a" tool-name result)))
#+end_src
** Tool Actuator (execute-tool-action)
The engine for physical interaction. It executes a cognitive tool and generates feedback signals for the user.
#+begin_src lisp :tangle ../library/act.lisp
(defun execute-tool-action (action context)
"Executes a registered cognitive tool and generates feedback signals. (ACTUATOR)"
(let* ((payload (getf action :payload))
(tool-name (getf payload :tool))
(tool-args (getf payload :args))
(depth (getf context :depth 0))
(meta (getf context :meta))
(source (getf meta :source))
(tool (gethash (string-downcase (string tool-name)) *cognitive-tools*)))
(if tool
(handler-case
(let* ((clean-args (if (and (listp tool-args) (listp (car tool-args))) (car tool-args) tool-args))
(result (funcall (cognitive-tool-body tool) clean-args)))
(let ((feedback (list :TYPE :EVENT :DEPTH (1+ depth) :META meta
:PAYLOAD (list :SENSOR :tool-output :RESULT result :TOOL tool-name))))
;; UI Propagation: Send distilled text result back to the source client
(when source
(dispatch-action (list :TYPE :REQUEST :TARGET source
:PAYLOAD (list :ACTION :MESSAGE :TEXT (format-tool-result tool-name result)))
context))
feedback))
(error (c)
(list :TYPE :EVENT :DEPTH (1+ depth) :META meta
:PAYLOAD (list :SENSOR :tool-error :tool tool-name :message (format nil "~a" c)))))
(list :TYPE :EVENT :DEPTH (1+ depth) :META meta
:PAYLOAD (list :SENSOR :tool-error :message "Tool not found")))))
#+end_src
* The Final Pipeline Stage
** Act Gate (act-gate)
The exit point of the metabolic pipeline. It applies a last-mile safety check via the Deterministic Engine and dispatches the signal to the physical world.
#+begin_src lisp :tangle ../library/act.lisp
(defun act-gate (signal)
"Final Stage: Actuation and feedback generation."
(let* ((approved (getf signal :approved-action))
(type (getf signal :type))
(meta (getf signal :meta))
(source (getf meta :source))
(feedback nil)
;; context must keep internal objects for actuators to function
(context signal))
;; 1. Last-Mile Safety Check (The Bouncer & Deterministic Gates)
(when approved
(let* ((original-type (getf approved :type))
(verified (deterministic-verify approved signal)))
(if (and (listp verified)
(member (getf verified :type) '(:LOG :EVENT :log :event))
(not (member original-type '(:LOG :EVENT :log :event))))
(progn
(harness-log "ACT BLOCKED: Action failed last-mile deterministic check.")
(setf (getf signal :approved-action) nil)
(setf approved nil)
(setf feedback verified))
(progn
(setf (getf signal :approved-action) verified)
(setf approved verified)))))
;; 2. Actuation Logic
(case type
(:REQUEST (dispatch-action signal context))
(:LOG (dispatch-action signal context))
(:EVENT
(if approved
(let* ((target (getf approved :target))
(result (dispatch-action approved context)))
(cond ((and (listp result) (member (getf result :type) '(:EVENT :LOG)))
(setf feedback result))
((and result (not (member target *silent-actuators*)))
(setf feedback (list :type :EVENT :depth (1+ (getf signal :depth 0)) :meta meta
:payload (list :sensor :tool-output :result result :tool approved))))))
;; Fallback: route generic stimuli back to their origin
(when source
(dispatch-action signal context)))))
(setf (getf signal :status) :acted)
feedback))
#+end_src

91
harness/communication.org Normal file
View File

@@ -0,0 +1,91 @@
#+TITLE: Communication Protocol (communication.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:protocol:
#+STARTUP: content
* Communication Protocol (communication.lisp)
** Architectural Intent: Secure Inter-Process Communication
The Communication Protocol is the bridge between the OpenCortex microharness and the outside world. To maintain the "Zero-Bloat" mandate, the protocol must be:
1. **Lightweight:** Minimal overhead for low-latency terminal interaction.
2. **Deterministic:** Strict S-expression framing to prevent injection attacks.
3. **Transport-Agnostic:** Capable of running over TCP, Unix Sockets, or Standard I/O.
By utilizing a length-prefixed S-expression format (the "Unified Envelope"), we ensure that both human-readable text and complex Lisp data structures can be transmitted securely without the fragility of JSON or the overhead of Protobuf.
** Pipeline Initialization
#+begin_src lisp :tangle ../library/communication.lisp
(in-package :opencortex)
#+end_src
* Message Framing
** Frame Serialization (frame-message)
Every message leaving the harness must be "framed." This involves two steps:
1. *Sanitization:* Stripping raw Lisp objects (like streams or sockets) that cannot be serialized.
2. *Prefixed Framing:* Calculating the length of the S-expression and prepending it as a 6-character hexadecimal string.
Example Frame: ~00001c(:TYPE :STATUS :SCRIBE :IDLE)~
#+begin_src lisp :tangle ../library/communication.lisp
(defun sanitize-protocol-message (msg)
"Recursively strips non-serializable objects (streams, sockets) from a protocol plist."
(if (and msg (listp msg))
(let ((clean nil))
(loop for (k v) on msg by #'cddr
do (unless (member k '(:reply-stream :socket :stream))
(push k clean)
(push (if (listp v) (sanitize-protocol-message v) v) clean)))
(nreverse clean))
msg))
#+end_src
#+begin_src lisp :tangle ../library/communication.lisp
(defun frame-message (msg)
"Serializes a message plist and prefixes it with a 6-character hex length."
(let* ((sanitized (sanitize-protocol-message msg))
(payload (let ((*print-pretty* nil) (*read-eval* nil)) (format nil "~s" sanitized)))
(len (length payload)))
(format nil "~6,'0x~a" len payload)))
#+end_src
* Message Ingestion
** Framed Message Reader (read-framed-message)
The inverse of framing. This function reads exactly the number of bytes specified by the hex-length prefix. This "byte-counted" reading is a critical security measure—it prevents buffer overflow attacks and "slowloris" type hung connections.
#+begin_src lisp :tangle ../library/communication.lisp
(defun read-framed-message (stream)
"Reads a hex-prefixed message from a stream. Returns the parsed Lisp plist or :EOF."
(handler-case
(let ((len-buf (make-string 6)))
;; 1. Read the length prefix
(let ((count (read-sequence len-buf stream)))
(if (< count 6)
:eof
(let ((len (ignore-errors (parse-integer len-buf :radix 16))))
(if (and len (> len 0))
;; 2. Read exactly 'len' bytes
(let ((payload-buf (make-string len)))
(read-sequence payload-buf stream)
(let ((*read-eval* nil))
(read-from-string payload-buf)))
:error)))))
(error (c)
(harness-log "PROTOCOL ERROR: ~a" c)
:error)))
#+end_src
* Semantic Handshakes
** Hello Message (make-hello-message)
The first message sent by the daemon upon client connection. It advertises the protocol version and the agent's current capabilities.
#+begin_src lisp :tangle ../library/communication.lisp
(defun make-hello-message (version)
"Constructs the standard HELLO handshake message."
(list :TYPE :EVENT
:PAYLOAD (list :ACTION :handshake
:VERSION version
:CAPABILITIES '(:AUTH :SWANK :ORG-AST))))
#+end_src

95
harness/context.org Normal file
View File

@@ -0,0 +1,95 @@
#+TITLE: Peripheral Vision (context.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:context:
#+STARTUP: content
* Peripheral Vision (context.lisp)
** Architectural Intent: Contextual Awareness
The Context stage (often referred to as "Peripheral Vision") is responsible for assembling the situational awareness that the Probabilistic Engine needs to make informed decisions.
In most agent frameworks, context is provided as a massive, unstructured text dump of recent chat history. OpenCortex takes a more sophisticated approach:
1. **Foveal Focus:** The data immediately relevant to the current task (e.g., the specific Org headline being edited).
2. **Peripheral Awareness:** Low-resolution metadata about the rest of the Memex (e.g., list of active projects, recent system logs, current time/location).
3. **Semantic Retrieval:** Utilizing vector embeddings to pull in semantically related nodes from the long-term memory.
By balancing these three layers, we provide the agent with a "Wide Angle" view of the user's life without overflowing the LLM's context window.
** Pipeline Initialization
#+begin_src lisp :tangle ../library/context.lisp
(in-package :opencortex)
#+end_src
* Awareness Assembly
** Project Awareness (context-get-active-projects)
Identifies current active work by querying the Org Memory for nodes with the ~:PROJECT:~ tag or ~NEXT~ status.
#+begin_src lisp :tangle ../library/context.lisp
(defun context-get-active-projects ()
"Retrieves a list of project headlines currently marked as NEXT or in progress."
(let ((all-projects (list-objects-with-attribute :CATEGORY "Project")))
(loop for p in all-projects
collect (list :id (org-object-id p)
:title (getf (org-object-attributes p) :TITLE)))))
#+end_src
** Historical Awareness (context-get-recent-completed-tasks)
Provides short-term memory of what was recently achieved, allowing the agent to maintain continuity.
#+begin_src lisp :tangle ../library/context.lisp
(defun context-get-recent-completed-tasks (&optional (limit 5))
"Retrieves the last N tasks marked as DONE from the memory history."
(let ((all-completed (list-objects-with-attribute :TODO "DONE")))
(subseq (sort all-completed #'> :key #'org-object-version)
0 (min limit (length all-completed)))))
#+end_src
** Skill Awareness (context-list-all-skills)
Allows the agent to understand its own capabilities by listing the human-readable descriptions of all loaded Literate Skills.
#+begin_src lisp :tangle ../library/context.lisp
(defun context-list-all-skills ()
"Returns a list of registered skills and their documentation."
(let ((results nil))
(maphash (lambda (id skill)
(push (list :id id :name (skill-name skill)) results))
*skills-registry*)
results))
#+end_src
** System Awareness (context-get-system-logs)
Crucial for self-debugging. Provides the agent with the internal logs so it can explain why a previous action failed or was blocked by a Bouncer.
#+begin_src lisp :tangle ../library/context.lisp
(defun context-get-system-logs ()
"Retrieves the in-memory circular log buffer."
(bt:with-lock-held (*logs-lock*)
(format nil "~{~a~%~}" (reverse *system-logs*))))
#+end_src
* Global Context Generation
** Awareness Assembly (context-assemble-global-awareness)
This function acts as the "Contextual Conductor." It synthesizes the various awareness layers into a single, high-signal string suitable for the LLM system prompt.
#+begin_src lisp :tangle ../library/context.lisp
(defun context-assemble-global-awareness ()
"Assembles the full context block for a neural request."
(let ((projects (context-get-active-projects))
(time (multiple-value-bind (s m h d mo y) (get-decoded-time) (format nil "~a-~a-~a ~a:~a:~a" y mo d h m s))))
(format nil "CURRENT_TIME: ~a. ACTIVE_PROJECTS: ~s. FOVEAL_FOCUS: ~a"
time
projects
(or *foveal-focus-id* "None"))))
#+end_src
** Semantic Context Query (context-query-store)
A hook for future vector-based retrieval. In the MVP, it performs a simple keyword search over the Memory graph.
#+begin_src lisp :tangle ../library/context.lisp
(defun context-query-store (query &key (limit 5))
"Placeholder for semantic/vector search over the Memex."
(declare (ignore query limit))
nil)
#+end_src

140
harness/loop.org Normal file
View File

@@ -0,0 +1,140 @@
#+TITLE: The Metabolic Loop (loop.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:loop:
#+STARTUP: content
* The Metabolic Loop (loop.lisp)
** Architectural Intent: The Heartbeat
The Metabolic Loop is the high-level coordinator of the OpenCortex. It orchestrates the flow of energy (information) through the system by recursively calling the metabolic stages: Perceive, Reason, and Act.
Inspired by biological metabolism, the loop ensures that every stimulus is processed until it reaches "stasis" (no further actions required) or an error occurs. This recursive design allows the agent to chain multiple thoughts and tool calls together into a single cohesive cognitive session.
** Pipeline Initialization
#+begin_src lisp :tangle ../library/loop.lisp
(in-package :opencortex)
#+end_src
* Concurrency and Interrupts
** Metabolic Interrupt Flag
The harness must be able to stop gracefully. We use a thread-safe flag to signal the daemon to exit its primary loop.
#+begin_src lisp :tangle ../library/loop.lisp
(defvar *interrupt-flag* nil
"Thread-safe signal to halt the metabolic pipeline and daemon.")
#+end_src
#+begin_src lisp :tangle ../library/loop.lisp
(defvar *interrupt-lock* (bt:make-lock "harness-interrupt-lock")
"Protects the interrupt flag from concurrent access.")
#+end_src
** Heartbeat Thread Reference
#+begin_src lisp :tangle ../library/loop.lisp
(defvar *heartbeat-thread* nil
"Reference to the background thread driving autonomous reflection.")
#+end_src
* The Metabolic Pipeline
** Signal Processor (process-signal)
The primary cognitive processor. It takes a normalized signal and pushes it through the gates. If a gate generates "Feedback" (e.g., a tool result), the function recursively processes that feedback as a new stimulus.
#+begin_src lisp :tangle ../library/loop.lisp
(defun process-signal (signal)
"The entry point to the Metabolic Pipeline: Perceive -> Reason -> Act."
(let ((current-signal signal))
(loop while current-signal do
(let ((depth (getf current-signal :depth 0))
(meta (getf current-signal :meta)))
;; Safety: Prevent infinite cognitive recursion.
(when (> depth 10) (harness-log "METABOLISM ERROR: Max depth reached.") (return nil))
;; Check for graceful shutdown.
(when (bt:with-lock-held (*interrupt-lock*) *interrupt-flag*)
(harness-log "METABOLISM: Interrupted.")
(bt:with-lock-held (*interrupt-lock*) (setf *interrupt-flag* nil))
(return nil))
(handler-case
(progn
;; Stage 1: Ingest and Normalize
(setf current-signal (perceive-gate current-signal))
;; Stage 2: Cogitate and Verify
(setf current-signal (reason-gate current-signal))
;; Stage 3: Actuate and Generate Feedback
(let ((feedback (act-gate current-signal)))
(if feedback
(progn
;; Inheritance: Metadata must persist across recursive cycles.
(unless (getf feedback :meta) (setf (getf feedback :meta) meta))
(setf current-signal feedback))
(setf current-signal nil))))
(error (c)
(let ((sensor (ignore-errors (getf (getf current-signal :payload) :sensor))))
(harness-log "METABOLISM CRASH [~a]: ~a" (or sensor :unknown) c)
;; Resilience: Only rollback on critical system errors.
(unless (member sensor '(:loop-error :tool-error :syntax-error))
(harness-log "CRITICAL ERROR: Initiating Micro-Rollback.")
(rollback-memory 0))
;; If recursion is shallow, attempt to notify the user of the error.
(if (or (> depth 2) (member sensor '(:loop-error :tool-error)))
(setf current-signal nil)
(setf current-signal (list :type :EVENT :depth (1+ depth) :meta meta
:payload (list :sensor :loop-error :message (format nil "~a" c) :depth depth)))))))))))
#+end_src
* Autonomous Reflection
** Heartbeat Mechanism (start-heartbeat)
The heartbeat ensures the agent remains "alive" even in the absence of external stimuli. It allows background workers like the Scribe and Gardener to trigger periodically.
#+begin_src lisp :tangle ../library/loop.lisp
(defun start-heartbeat ()
"Starts the background heartbeat thread. Interval is loaded from HEARTBEAT_INTERVAL (default: 60s)."
(let ((interval (or (ignore-errors (parse-integer (uiop:getenv "HEARTBEAT_INTERVAL"))) 60)))
(setf *heartbeat-thread*
(bt:make-thread
(lambda ()
(loop
(sleep interval)
;; Note: inject-stimulus is synchronous for heartbeats to prevent task accumulation.
(inject-stimulus (list :type :EVENT :payload (list :sensor :heartbeat :unix-time (get-universal-time))))))
:name "opencortex-heartbeat"))))
#+end_src
* Lifecycle Management
** Main Daemon Entry Point (main)
Initializes the image, boots the gateways, and enters the primary idle loop.
#+begin_src lisp :tangle ../library/loop.lisp
(defun main ()
"Primary entry point for the OpenCortex daemon."
;; 1. Environment Hydration
(let* ((home (uiop:getenv "HOME"))
(env-file (uiop:merge-pathnames* ".local/share/opencortex/.env" (uiop:ensure-directory-pathname home))))
(when (uiop:file-exists-p env-file) (cl-dotenv:load-env env-file)))
;; 2. System Bootstrap
(initialize-actuators)
(initialize-all-skills)
;; 3. Wake up the heart.
(start-heartbeat)
;; 4. OS Signal Handling (SBCL specific)
#+sbcl
(sb-sys:enable-interrupt sb-unix:sigint
(lambda (sig code scp)
(declare (ignore sig code scp))
(harness-log "SHUTDOWN: SIGINT received. Exiting...")
(uiop:quit 0)))
;; 5. Primary Idle Loop
(let ((sleep-interval (or (ignore-errors (parse-integer (uiop:getenv "DAEMON_SLEEP_INTERVAL"))) 3600)))
(loop
(when (bt:with-lock-held (*interrupt-lock*) *interrupt-flag*) (return))
(sleep sleep-interval))))
#+end_src

86
harness/manifest.org Normal file
View File

@@ -0,0 +1,86 @@
#+TITLE: Manifest (opencortex.asd)
#+AUTHOR: Amr
#+FILETAGS: :harness:system:
#+STARTUP: content
* Manifest (opencortex.asd)
** Architectural Intent: The ASDF Skeleton
The ~opencortex.asd~ file is the physical blueprint of the Lisp Machine. It uses **Another System Definition Facility (ASDF)** to orchestrate the compilation, dependency resolution, and loading of all harness modules.
In standard Common Lisp projects, dependency graphs can be complex and non-linear. However, the OpenCortex harness mandates a strict, linear bootstrap sequence.
*** Strict Serial Loading (:serial t)
The harness uses the ~:serial t~ flag. This is a critical design choice that ensures every file is compiled and loaded in the exact order it appears in the ~:components~ list.
- *Why?* This eliminates "macro-not-found" errors by guaranteeing that the ~package.lisp~ (where the core namespace is defined) and ~skills.lisp~ (where core macros are defined) are always established before any behavioral logic or dynamic skills are loaded.
*** Separation of Concerns
The manifest defines three distinct systems to minimize runtime bloat and maximize portability.
#+begin_src mermaid
flowchart TD
Org[Literate Org Files] -- Tangle --> Lisp[Source .lisp Files]
Lisp --> ASDF[ASDF Manifest: .asd]
ASDF --> Loader[SBCL Compiler / Loader]
Loader --> Image[Live Harness Image]
Image -- Build --> Binary[Standalone Binary]
#+end_src
** Core Harness System
This system defines the "Thin Harness"—the minimalist microkernel responsible for the protocol and the metabolic loop.
#+begin_src lisp :tangle ../opencortex.asd
(defsystem :opencortex
:name "opencortex"
:author "Amr"
:version "0.1.0"
:license "AGPLv3"
:description "The Probabilistic-Deterministic Lisp Machine Harness"
:depends-on (:usocket :bordeaux-threads :dexador :uiop :cl-dotenv :cl-ppcre :hunchentoot :ironclad :str :cl-json :uuid)
:serial t
:components ((:file "src/package")
(:file "src/skills")
(:file "src/policy")
(:file "src/communication-validator")
(:file "src/communication")
(:file "src/memory")
(:file "src/context")
(:file "src/probabilistic")
(:file "src/perceive")
(:file "src/reason")
(:file "src/act")
(:file "src/loop"))
:build-operation "program-op"
:build-pathname "opencortex-server"
:entry-point "opencortex:main")
#+end_src
** Verification Suite
The Verification Suite contains the empirical tests required by the Engineering Standards. It is isolated from the core system to ensure that production environments do not load the FiveAM framework or test data.
#+begin_src lisp :tangle ../opencortex.asd
(defsystem :opencortex/tests
:depends-on (:opencortex :fiveam)
:components ((:file "tests/communication-tests")
(:file "tests/pipeline-tests")
(:file "tests/act-tests")
(:file "tests/boot-sequence-tests")
(:file "tests/memory-tests")
(:file "tests/immune-system-tests"))
:perform (test-op (o s)
(uiop:symbol-call :fiveam :run! (uiop:find-symbol* :communication-protocol-suite :opencortex-tests))
(uiop:symbol-call :fiveam :run! (uiop:find-symbol* :pipeline-suite :opencortex-pipeline-tests))
(uiop:symbol-call :fiveam :run! (uiop:find-symbol* :safety-suite :opencortex-safety-tests))
(uiop:symbol-call :fiveam :run! (uiop:find-symbol* :boot-suite :opencortex-boot-tests))
(uiop:symbol-call :fiveam :run! (uiop:find-symbol* :memory-suite :opencortex-memory-tests))
(uiop:symbol-call :fiveam :run! (uiop:find-symbol* :immune-suite :opencortex-immune-system-tests))))
#+end_src
** TUI Client
The TUI Client is a standalone consumer of the OpenCortex protocol. It uses the ~croatoan~ library for native terminal rendering.
#+begin_src lisp :tangle ../opencortex.asd
(defsystem :opencortex/tui
:depends-on (:opencortex :croatoan :usocket :bordeaux-threads)
:components ((:file "src/tui-client")))
#+end_src

150
harness/memory.org Normal file
View File

@@ -0,0 +1,150 @@
#+TITLE: Homoiconic Memory (memory.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:memory:
#+STARTUP: content
* Homoiconic Memory (memory.lisp)
** Architectural Intent: The Live Graph
The Memory module is the "conscious mind" of the OpenCortex. Unlike traditional agents that rely on slow, external databases (SQL or Vector), OpenCortex maintains your entire Memex as a live, homoiconic graph of Lisp objects in RAM.
*** Why RAM-First?
1. **Zero-Latency Inference:** Traversing complex associations between notes and tasks occurs at native Lisp speeds, without the overhead of context-switching to a database driver.
2. **Unified Data Model:** Since the program (Lisp) and the data (the Memory) share the same structure, the agent can manipulate its own memory as naturally as it manipulates its own code.
3. **Graph Sovereignty:** By keeping the graph in-process, we ensure that the user's private knowledge base never leaves the host machine unless explicitly requested by a gateway.
** Pipeline Initialization
#+begin_src lisp :tangle ../library/memory.lisp
(in-package :opencortex)
#+end_src
* Core Data Structures
** The Object Registry
#+begin_src lisp :tangle ../library/memory.lisp
(defvar *memory* (make-hash-table :test 'equal)
"The primary in-memory graph of all Org-mode entities, keyed by their unique ID.")
#+end_src
** The History Store (Merkle History)
OpenCortex maintains a history of memory states to allow for "Micro-Rollbacks" if a skill or tool execution results in an inconsistent state.
#+begin_src lisp :tangle ../library/memory.lisp
(defvar *history-store* (make-array 0 :fill-pointer 0 :adjustable t)
"A versioned log of the memory state, allowing for temporal traversal and rollback.")
#+end_src
** The Org-Object Definition
Every headline, paragraph, or task in the Memex is represented as an ~org-object~.
#+begin_src lisp :tangle ../library/memory.lisp
(defstruct org-object
"The fundamental unit of knowledge in the OpenCortex."
id
type
attributes
parent-id
children
version
last-sync
vector
content
hash)
#+end_src
* Integrity and Hashing
** Merkle Hashing (compute-merkle-hash)
To ensure data integrity and detect changes during external edits, we utilize Merkle-tree hashing. A node's hash is derived from its own content plus the hashes of its children.
#+begin_src lisp :tangle ../library/memory.lisp
(defun compute-merkle-hash (id type attributes content child-hashes)
"Computes a SHA-256 Merkle hash for a node based on its core properties and children's hashes."
(let* ((alist (loop for (k v) on attributes by #'cddr collect (cons k v)))
(sorted-alist (sort alist #'string< :key (lambda (x) (format nil "~a" (car x)))))
(attr-string (format nil "~s" sorted-alist))
(children-string (format nil "~{~a~}" child-hashes))
(raw-data (format nil "~a|~a|~a|~a|~a" id type attr-string (or content "") children-string)))
(ironclad:byte-array-to-hex-string
(ironclad:digest-sequence :sha256 (ironclad:ascii-string-to-byte-array raw-data)))))
#+end_src
* Memory Ingestion
** AST Ingestion (ingest-ast)
The primary mechanism for translating raw Org-mode Abstract Syntax Trees (provided by Emacs or a parser) into the live Lisp graph.
#+begin_src lisp :tangle ../library/memory.lisp
(defun ingest-ast (ast &optional parent-id)
"Recursively parses an Org AST into the Lisp Memory registry."
(let* ((type (getf ast :type))
(properties (getf ast :properties))
(id (or (getf properties :ID) (uuid:make-v4-uuid)))
(content (getf ast :content))
(children (getf ast :contents))
(child-ids nil))
;; Recursively ingest children and collect their IDs
(dolist (child children)
(let ((child-obj (ingest-ast child id)))
(when child-obj (push (org-object-id child-obj) child-ids))))
(let ((obj (make-org-object :id id
:type type
:attributes properties
:parent-id parent-id
:children (nreverse child-ids)
:content content
:version (get-universal-time))))
(setf (gethash id *memory*) obj)
obj)))
#+end_src
* Retrieval and Search
** Object Lookup (lookup-object)
#+begin_src lisp :tangle ../library/memory.lisp
(defun lookup-object (id)
"Retrieves an object from memory by its ID."
(gethash id *memory*))
#+end_src
** Semantic Attribute Search (list-objects-with-attribute)
Allows for querying the memory based on metadata (e.g., finding all nodes tagged :PROJECT:).
#+begin_src lisp :tangle ../library/memory.lisp
(defun list-objects-with-attribute (key value)
"Returns a list of objects that possess the specified attribute pair."
(let ((results nil))
(maphash (lambda (id obj)
(declare (ignore id))
(when (equal (getf (org-object-attributes obj) key) value)
(push obj results)))
*memory*)
results))
#+end_src
* Persistence and Resilience
** Memory Snapshots (snapshot-memory)
Captures the current state of the memory graph.
#+begin_src lisp :tangle ../library/memory.lisp
(defun snapshot-memory ()
"Creates a deep copy of the memory hash table and pushes it to the history store."
(let ((new-snap (make-hash-table :test 'equal)))
(maphash (lambda (k v) (setf (gethash k new-snap) (copy-org-object v))) *memory*)
(vector-push-extend new-snap *history-store*)))
#+end_src
** Micro-Rollbacks (rollback-memory)
The primary defense against accidental memory corruption by faulty skills.
#+begin_src lisp :tangle ../library/memory.lisp
(defun rollback-memory (&optional (steps 1))
"Restores the memory to a previous snapshot state."
(let ((index (- (length *history-store*) steps 1)))
(when (>= index 0)
(setf *memory* (aref *history-store* index))
(harness-log "IMMUNE SYSTEM: Memory rolled back ~a steps." steps))))
#+end_src

273
harness/package.org Normal file
View File

@@ -0,0 +1,273 @@
#+TITLE: System Interface (package.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:interface:
#+STARTUP: content
* System Interface (package.lisp)
The ~package.lisp~ file defines the public API of the ~opencortex~ harness. It serves as the primary membrane between the deterministic core modules and the dynamic world of skills and actuators.
** Architectural Intent: The Package Membrane
By strictly defining the public interface, we ensure that skills remain decoupled from the harness implementation details. This allows for autonomous replacement of any component (e.g., swapping the Memory or the Probabilistic Engine) without breaking existing skills.
#+begin_src mermaid
flowchart TD
External[Actuators / Clients] -- communication protocol --> Package[Package Membrane: API]
Skills[Dynamic Skills] -- API Calls --> Package
Package --> Internal[Harness Internal Modules]
style Package fill:#f9f,stroke:#333,stroke-width:4px
#+end_src
** Public API Export
#+begin_src lisp :tangle ../library/package.lisp
(defpackage :opencortex
(:use :cl)
(:export
;; --- Communication Protocol ---
#:frame-message
#:read-framed-message
#:PROTO-GET
#:LIST-OBJECTS-WITH-ATTRIBUTE
#:COSINE-SIMILARITY
#:VAULT-MASK-STRING
#:*VAULT-MEMORY*
#:parse-message
#:make-hello-message
#:validate-communication-protocol-schema
;; --- Daemon Lifecycle ---
#:start-daemon
#:stop-daemon
#:harness-log
#:main
;; --- Memory (CLOSOS) ---
#:ingest-ast
#:lookup-object
#:list-objects-by-type
#:org-id-new
#:*memory*
#:*history-store*
#:org-object
#:make-org-object
#:org-object-id
#:org-object-type
#:org-object-attributes
#:org-object-parent-id
#:org-object-children
#:org-object-version
#:org-object-last-sync
#:org-object-vector
#:org-object-content
#:org-object-hash
#:snapshot-memory
#:rollback-memory
;; --- Context API (Peripheral Vision) ---
#:context-query-store
#:context-get-active-projects
#:context-get-recent-completed-tasks
#:context-list-all-skills
#:context-get-skill-source
#:context-get-system-logs
#:context-resolve-path
#:context-get-skill-telemetry
#:harness-track-telemetry
#:context-assemble-global-awareness
;; --- Reactive Signal Pipeline ---
#:process-signal
#:perceive-gate
#:probabilistic-gate
#:consensus-gate
#:act-gate
#:reason-gate
#:perceive-gate
#:dispatch-gate
#:inject-stimulus
#:initialize-actuators
#:dispatch-action
#:register-actuator
;; --- Skill Engine ---
#:load-skill-from-org
#:initialize-all-skills
#:load-skill-with-timeout
#:topological-sort-skills
#:validate-lisp-syntax
#:defskill
#:*skills-registry*
#:skill
#:skill-name
#:skill-priority
#:skill-dependencies
#:skill-trigger-fn
#:skill-probabilistic-prompt
#:skill-deterministic-fn
;; --- Tool Registry ---
#:def-cognitive-tool
#:*cognitive-tools*
#:cognitive-tool
#:cognitive-tool-name
#:cognitive-tool-description
#:cognitive-tool-parameters
#:cognitive-tool-guard
#:cognitive-tool-body
;; --- Emacs Client Registry ---
#:*emacs-clients*
#:*clients-lock*
#:register-emacs-client
#:unregister-emacs-client
;; --- Probabilistic Engine ---
#:ask-probabilistic
#:register-probabilistic-backend
#:distill-prompt
#:*provider-cascade*
;; --- Security Vault ---
#:vault-get-secret
#:vault-set-secret
;; --- Deterministic Logic ---
#:list-objects-with-attribute
#:deterministic-verify
;; --- AST Helpers ---
#:find-headline-missing-id))
#+end_src
** Package Implementation Initialization
Ensuring the compiler enters the correct namespace for all subsequent definitions.
#+begin_src lisp :tangle ../library/package.lisp
(in-package :opencortex)
#+end_src
* System State Management
The package layer manages the core data structures that represent the live state of the harness.
** Harness Logging State
OpenCortex maintains a thread-safe circular log buffer. This is critical for two reasons:
1. *Neural Introspection:* The probabilistic engine can read the recent system logs to understand why an action failed.
2. *Real-time Debugging:* Clients can subscribe to a live log stream without needing to read the physical log file.
#+begin_src lisp :tangle ../library/package.lisp
(defvar *system-logs* nil
"Thread-safe list of the most recent system messages.")
#+end_src
#+begin_src lisp :tangle ../library/package.lisp
(defvar *logs-lock* (bt:make-lock "harness-logs-lock")
"Protects the circular log buffer from race conditions during concurrent skill execution.")
#+end_src
#+begin_src lisp :tangle ../library/package.lisp
(defvar *max-log-history* 100
"The maximum number of entries to preserve in the in-memory log buffer.")
#+end_src
** Skills Registry
All Literate Skills, once compiled, are registered here. This allows for topological sorting and priority-based execution.
#+begin_src lisp :tangle ../library/package.lisp
(defvar *skills-registry* (make-hash-table :test 'equal)
"Global registry of all loaded skills, keyed by their unique identifier.")
#+end_src
** Skill Telemetry State
To ensure the system remains performant and reliable, the harness tracks execution metrics for every skill.
#+begin_src lisp :tangle ../library/package.lisp
(defvar *skill-telemetry* (make-hash-table :test 'equal)
"Stores execution duration and failure counts for every registered skill.")
#+end_src
#+begin_src lisp :tangle ../library/package.lisp
(defvar *telemetry-lock* (bt:make-lock "harness-telemetry-lock")
"Protects the telemetry store from concurrent updates.")
#+end_src
* Support Functions
** Protocol Property Access (proto-get)
Lisp keywords can be inconsistent between capitalized and lowercase versions depending on the client (e.g., Emacs vs. Python socket). ~proto-get~ provides a robust abstraction to ensure the system correctly extracts values regardless of keyword casing.
#+begin_src lisp :tangle ../library/package.lisp
(defun proto-get (plist key)
"Robustly retrieves a value from a plist, checking both uppercase and lowercase keyword versions."
(let* ((s (string key))
(up (intern (string-upcase s) :keyword))
(dn (intern (string-downcase s) :keyword)))
(or (getf plist up) (getf plist dn))))
#+end_src
** Telemetry Tracking
The ~harness-track-telemetry~ function provides the hook for the metabolic loop to report performance data.
#+begin_src lisp :tangle ../library/package.lisp
(defun harness-track-telemetry (skill-name duration status)
"Updates performance metrics for a specific skill. Status should be :success or :rejected."
(when skill-name
(bt:with-lock-held (*telemetry-lock*)
(let ((entry (or (gethash skill-name *skill-telemetry*) (list :executions 0 :total-time 0 :failures 0))))
(incf (getf entry :executions))
(incf (getf entry :total-time) duration)
(when (eq status :rejected) (incf (getf entry :failures)))
(setf (gethash skill-name *skill-telemetry*) entry)))))
#+end_src
* Cognitive Tooling System
The Tool Registry is the agent's physical interface. It separates the /proposal/ of an action from its /execution/.
** Tool Structure
#+begin_src lisp :tangle ../library/package.lisp
(defvar *cognitive-tools* (make-hash-table :test 'equal)
"The active set of physical capabilities available to the agent.")
#+end_src
#+begin_src lisp :tangle ../library/package.lisp
(defstruct cognitive-tool
"Represents a physical or virtual capability with explicit documentation and security guards."
name
description
parameters
guard
body)
#+end_src
** Tool Registration Macro (def-cognitive-tool)
We use a macro to ensure that tools are consistently registered and accessible to the LLM's "tool-belt" prompt generator.
#+begin_src lisp :tangle ../library/package.lisp
(defmacro def-cognitive-tool (name description parameters &key guard body)
"Registers a new cognitive tool.
NAME: Keyword identifier.
DESCRIPTION: Human-readable intent (used in LLM prompts).
PARAMETERS: List of property lists defining arguments.
GUARD: (context -> boolean) function to prevent unsafe calls.
BODY: The actual Lisp execution logic."
`(setf (gethash (string-downcase (string ',name)) *cognitive-tools*)
(make-cognitive-tool :name (string-downcase (string ',name))
:description ,description
:parameters ',parameters
:guard ,guard
:body ,body)))
#+end_src
* Logging Implementation
** Centralized Logging (harness-log)
The primary mechanism for system transparency. It ensures all activity is both visible to the user and recorded for neural reasoning.
#+begin_src lisp :tangle ../library/package.lisp
(defun harness-log (msg &rest args)
"Centralized logging for the harness. Writes to STDOUT and the thread-safe circular buffer."
(let ((formatted-msg (apply #'format nil msg args)))
(bt:with-lock-held (*logs-lock*)
(push formatted-msg *system-logs*)
(when (> (length *system-logs*) *max-log-history*)
(setq *system-logs* (subseq *system-logs* 0 *max-log-history*))))
(format t "~a~%" formatted-msg)
(finish-output)))
#+end_src

106
harness/perceive.org Normal file
View File

@@ -0,0 +1,106 @@
#+TITLE: Stage 1: Perceive (perceive.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:perceive:
#+STARTUP: content
* Stage 1: Perceive (perceive.lisp)
** Architectural Intent: Sensory Ingestion
The Perceive stage is the "sensory cortex" of the OpenCortex. Its primary responsibility is to take raw, unstructured stimuli from the outside world—whether from a TCP socket, a system interrupt, or a background heartbeat—and normalize them into high-fidelity internal **Signals**.
Normalization is critical because it shields the subsequent reasoning and actuation stages from the messiness of different transport protocols. Whether a message arrives via a TUI, a Signal bot, or an internal timer, the core "Brain" perceives a consistent Lisp property list.
** Pipeline Initialization
Ensuring we are in the correct namespace for sensory processing.
#+begin_src lisp :tangle ../library/perceive.lisp
(in-package :opencortex)
#+end_src
** Sensory Concurrency (Async Sensors)
To maintain the agent's responsiveness, we distinguish between "Fast" and "Slow" sensors. Sensors that require extensive processing or external API calls are routed to asynchronous threads to prevent blocking the main metabolic pipeline.
#+begin_src lisp :tangle ../library/perceive.lisp
(defvar *async-sensors* '(:chat-message :delegation :user-command)
"List of sensors that should be processed asynchronously to avoid blocking gateways.")
#+end_src
** Foveal Focus (User Context)
The system tracks the user's current point of interaction (the "foveal focus"). This provides immediate situational awareness to the reasoning engine, allowing it to prioritize the data the human is currently looking at.
#+begin_src lisp :tangle ../library/perceive.lisp
(defvar *foveal-focus-id* nil
"The Org ID of the node the user is currently interacting with.")
#+end_src
* Primary Ingress
** Stimulus Injection (inject-stimulus)
The ~inject-stimulus~ function is the universal gateway into the OpenCortex mind. It performs two critical tasks:
1. *Envelope Wrapping:* Ensures that every raw message is wrapped in a ~:META~ envelope, preserving the source and session information.
2. *Dispatching:* Determines whether to run the metabolism synchronously or in a new thread.
#+begin_src lisp :tangle ../library/perceive.lisp
(defun inject-stimulus (raw-message &key stream (depth 0))
"Enqueues a raw message into the reactive signal pipeline."
(let* ((payload (getf raw-message :payload))
(sensor (getf payload :sensor))
(meta (getf raw-message :meta))
(async-p (or (getf payload :async-p) (member sensor *async-sensors*))))
;; Ensure META exists and contains the stream if provided
(unless meta (setf meta (list :SOURCE :SYSTEM :SESSION-ID "internal")))
(when stream (setf (getf meta :reply-stream) stream))
(setf (getf raw-message :meta) meta)
(if async-p
(bt:make-thread
(lambda ()
(restart-case (handler-bind ((error (lambda (c) (harness-log "ASYNC ERROR: ~a" c) (invoke-restart 'skip-event))))
(process-signal raw-message))
(skip-event () nil)))
:name "opencortex-async-task")
(restart-case (handler-bind ((error (lambda (c) (harness-log "SYSTEM ERROR: ~a" c) (invoke-restart 'skip-event))))
(process-signal raw-message))
(skip-event () (harness-log "SYSTEM RECOVERY: Stimulus dropped.~%"))))))
#+end_src
* The Perceive Stage
** Perception Gate (perceive-gate)
The first official stage of the metabolic loop. It performs "Pre-Cognitive" work:
1. *Logging:* Recording the arrival of the signal.
2. *State Sync:* If the signal contains an AST update (e.g., from Emacs), it immediately updates the in-memory graph.
3. *Merkle Checkpointing:* Before modifying memory, it creates a snapshot to allow for emergency rollbacks.
#+begin_src lisp :tangle ../library/perceive.lisp
(defun perceive-gate (signal)
"Initial processing: Normalizes raw stimuli and updates memory."
(let* ((payload (getf signal :payload))
(type (getf signal :type))
(meta (getf signal :meta))
(sensor (getf payload :sensor)))
(harness-log "GATE [Perceive]: ~a (~a) [Source: ~s]" type (or sensor "no-sensor") (getf meta :source))
(cond ((eq type :EVENT)
(case sensor
(:buffer-update
(let ((ast (getf payload :ast)))
(when ast
(snapshot-memory)
(ingest-ast ast))))
(:point-update
(let ((element (getf payload :element)))
(when element
(snapshot-memory)
(setf *foveal-focus-id* (ignore-errors (getf element :id)))
(ingest-ast element))))
(:interrupt
(bt:with-lock-held (*interrupt-lock*) (setf *interrupt-flag* t)))))
((eq type :RESPONSE)
(harness-log "GATE [Perceive]: Act Result -> ~a" (getf payload :status))))
(setf (getf signal :status) :perceived)
(setf (getf signal :foveal-focus) *foveal-focus-id*)
signal))
#+end_src

189
harness/reason.org Normal file
View File

@@ -0,0 +1,189 @@
#+TITLE: Stage 2: Reason (reason.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:reason:
#+STARTUP: content
* Stage 2: Reason (reason.lisp)
** Architectural Intent: Unified Cognition
The Reason stage is the cognitive engine of the OpenCortex. Its primary responsibility is to bridge the gap between raw sensory data (Perceive) and physical side-effects (Act).
Cognition is split into two distinct modes:
1. **Probabilistic Reasoning:** Utilizing LLMs to generate creative proposals and understand natural language intent.
2. **Deterministic Verification:** Utilizing native Lisp logic to verify and constrain the neural proposals against security and physics invariants.
This hybrid approach ensures the agent is both intelligent and mathematically safe.
** Pipeline Initialization
#+begin_src lisp :tangle ../library/reason.lisp
(in-package :opencortex)
#+end_src
* Probabilistic Engine Infrastructure
** Neural Backend Registry
OpenCortex is provider-agnostic. All neural backends (OpenRouter, Ollama, etc.) register themselves here.
#+begin_src lisp :tangle ../library/reason.lisp
(defvar *probabilistic-backends* (make-hash-table :test 'equal)
"A global mapping of provider identifiers (keywords) to their respective execution functions.")
#+end_src
** Provider Cascade Configuration
#+begin_src lisp :tangle ../library/reason.lisp
(defvar *provider-cascade* nil
"An ordered list of providers to attempt if the primary one fails.")
#+end_src
#+begin_src lisp :tangle ../library/reason.lisp
(defvar *model-selector-fn* nil
"A hook for dynamic model selection based on context complexity.")
#+end_src
#+begin_src lisp :tangle ../library/reason.lisp
(defvar *consensus-enabled-p* nil
"Flag to enable parallel multi-model voting (not implemented in MVP).")
#+end_src
** Backend Registration Helper
#+begin_src lisp :tangle ../library/reason.lisp
(defun register-probabilistic-backend (name fn)
"Registers a neural provider with its calling function."
(setf (gethash name *probabilistic-backends*) fn))
#+end_src
* The Cognitive Cycle
** Probabilistic Call (probabilistic-call)
The primary interface for neural reasoning. It iterates through the cascade until a successful response is achieved or the cascade is exhausted.
#+begin_src lisp :tangle ../library/reason.lisp
(defun probabilistic-call (prompt &key (system-prompt "You are the Probabilistic engine.") (cascade nil) (context nil))
"Dispatches a neural request through the provider cascade. Returns a Lisp plist or a failure log."
(let ((backends (or cascade *provider-cascade*)))
(or (dolist (backend backends)
(let ((backend-fn (gethash backend *probabilistic-backends*)))
(when backend-fn
(harness-log "PROBABILISTIC: Attempting backend ~a..." backend)
(let* ((model (when *model-selector-fn* (funcall *model-selector-fn* backend context)))
(result (if model
(funcall backend-fn prompt system-prompt :model model)
(funcall backend-fn prompt system-prompt))))
(cond ((and (listp result) (eq (getf result :status) :success))
(return (getf result :content)))
((stringp result) (return result))
(t (harness-log "PROBABILISTIC: Backend ~a failed: ~a" backend (getf result :message))))))))
(list :type :LOG :payload (list :text "Neural Cascade Failure: All providers exhausted.")))))
#+end_src
** LLM Output Sanitization (strip-markdown)
Modern LLMs often wrap Lisp code in markdown backticks. This helper ensures the code is clean before the Lisp reader touches it.
#+begin_src lisp :tangle ../library/reason.lisp
(defun strip-markdown (text)
"Strips common markdown code block markers from text to ensure valid S-expression parsing."
(if (and text (stringp text))
(let ((cleaned text))
(setf cleaned (cl-ppcre:regex-replace-all "^```[a-z]*\\n" cleaned ""))
(setf cleaned (cl-ppcre:regex-replace-all "\\n```$" cleaned ""))
(setf cleaned (cl-ppcre:regex-replace-all "```" cleaned ""))
(string-trim '(#\Space #\Newline #\Tab) cleaned))
text))
#+end_src
** The Thought Process (Think)
The core logic that prepares the "mind" for reasoning. It assembles the global awareness (Memex status, recent logs, active tasks) and provides a strict protocol template for the LLM to follow.
#+begin_src lisp :tangle ../library/reason.lisp
(defun think (context)
"Generates a Lisp action proposal based on current context."
(let* ((active-skill (find-triggered-skill context))
(tool-belt (generate-tool-belt-prompt))
(global-context (context-assemble-global-awareness))
(system-logs (context-get-system-logs))
(assistant-name (or (uiop:getenv "MEMEX_ASSISTANT") "Agent")))
(let* ((prompt-generator (when active-skill (skill-probabilistic-prompt active-skill)))
(raw-prompt (if prompt-generator
(funcall prompt-generator context)
(let ((p (proto-get (proto-get context :payload) :text)))
(if (and p (stringp p)) p "Maintain metabolic stasis."))))
(system-prompt (format nil "IDENTITY: ~a. MANDATE: Respond with ONE Lisp plist. ~a ~a RECENT_LOGS: ~a
IMPORTANT: To reply to the user, you MUST use:
(:TYPE :REQUEST :PAYLOAD (:ACTION :MESSAGE :TEXT \"<Response Text>\"))
To call a tool, you MUST use:
(:TYPE :REQUEST :TARGET :TOOL :ACTION :CALL :TOOL \"<name>\" :ARGS (:arg1 \"val\"))
PROVIDER RULE: Always use the default cascade provider unless a specific model or capability is required for the task."
assistant-name global-context tool-belt system-logs)))
(let* ((thought (probabilistic-call raw-prompt :system-prompt system-prompt :context context))
(cleaned (strip-markdown thought))
(meta (proto-get context :meta))
(source (proto-get meta :source)))
(if (and cleaned (stringp cleaned))
(let ((*read-eval* nil))
(if (and (> (length cleaned) 0) (char= (char cleaned 0) #\())
(handler-case
(let ((parsed (read-from-string cleaned)))
(let ((type (proto-get parsed :TYPE))
(target (or (proto-get parsed :TARGET) (proto-get parsed :target))))
(cond ((member type '(:REQUEST :EVENT :STATUS :RESPONSE))
(unless (proto-get parsed :target) (setf (getf parsed :target) (or source :CLI)))
parsed)
;; Handle raw plists or lists of plists that look like tool calls or data
((or (eq target :TOOL) (eq target :tool) (getf parsed :TOOL) (getf parsed :tool)
(and (listp parsed) (listp (car parsed)) (keywordp (caar parsed))))
(list :TYPE :REQUEST :TARGET :TOOL :PAYLOAD parsed))
(t (list :TYPE :REQUEST :TARGET (or source :CLI) :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned))))))
(error (c) (list :TYPE :REQUEST :TARGET (or source :CLI) :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned))))
(list :TYPE :REQUEST :TARGET (or source :CLI) :PAYLOAD (list :ACTION :MESSAGE :TEXT cleaned))))
thought)))))
#+end_src
** Deterministic Verification
The final safety check. It iterates through all active skills to verify that the proposed neural action does not violate any invariants.
#+begin_src lisp :tangle ../library/reason.lisp
(defun deterministic-verify (proposed-action context)
"Iterates through all skill deterministic-gates sorted by priority. Ensures absolute safety of the neural proposal."
(let ((current-action proposed-action)
(skills nil))
(maphash (lambda (name skill) (declare (ignore name)) (when (skill-deterministic-fn skill) (push skill skills))) *skills-registry*)
(setf skills (sort skills #'> :key #'skill-priority))
(dolist (skill skills)
(let ((trigger (skill-trigger-fn skill))
(gate (skill-deterministic-fn skill)))
(when (or (null trigger) (ignore-errors (funcall trigger context)))
(let ((next-action (funcall gate current-action context)))
(let ((original-type (proto-get current-action :type)))
(when (and (listp next-action)
(member (proto-get next-action :type) '(:LOG :EVENT :log :event))
(or (not (member original-type '(:LOG :EVENT :log :event)))
(not (eq next-action current-action))))
(harness-log "DETERMINISTIC: Intercepted by skill '~a'" (skill-name skill))
(return-from deterministic-verify next-action)))
(setf current-action next-action)))))
current-action))
#+end_src
* The Reasoning Pipeline Stage
** Reasoning Gate (reason-gate)
The stage that ties it all together. It filters stimuli that don't require cognition (like internal heartbeat pulses) and executes the hybrid neural-logical loop.
#+begin_src lisp :tangle ../library/reason.lisp
(defun reason-gate (signal)
"Unified Stage: Combines Probabilistic proposals and Deterministic verification."
(let* ((type (proto-get signal :type))
(payload (proto-get signal :payload))
(sensor (proto-get payload :sensor)))
;; Optimization: Only reason about user input or chat messages.
(unless (and (eq type :EVENT) (member sensor '(:user-input :chat-message)))
(return-from reason-gate signal))
(let ((candidate (think signal)))
(if candidate
(setf (getf signal :approved-action) (deterministic-verify candidate signal))
(setf (getf signal :approved-action) nil))
(setf (getf signal :status) :reasoned)
signal)))
#+end_src

260
harness/setup.org Normal file
View File

@@ -0,0 +1,260 @@
#+TITLE: Zero-to-One Setup (setup.org)
#+AUTHOR: Amr
#+FILETAGS: :harness:setup:
#+STARTUP: content
* Zero-to-One Setup (setup.org)
The ~setup.org~ file defines the automated installation and initialization sequence for the OpenCortex.
** The Installer Script (opencortex.sh)
#+begin_src bash :tangle ../opencortex.sh
#!/bin/bash
set -e
PORT=9105
HOST="localhost"
RED='\033[0;31m'; GREEN='\033[0;32m'; BLUE='\033[0;34m'; YELLOW='\033[0;33m'; NC='\033[0m'
command_exists() { command -v "$1" >/dev/null 2>&1; }
# Resolve symlinks to find the actual repository location
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
done
export SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
# Load environment variables if they exist
if [ -f "$SCRIPT_DIR/.env" ]; then
while IFS="=" read -r key value || [ -n "$key" ]; do
if [[ $key =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
val=$(echo "$value" | sed "s/^\"//;s/\"$//")
export "$key=$val"
fi
done < "$SCRIPT_DIR/.env"
[ -n "$ORG_AGENT_DAEMON_PORT" ] && PORT=$ORG_AGENT_DAEMON_PORT
[ -n "$DAEMON_HOST" ] && HOST=$DAEMON_HOST
fi
# --- 1. BOOTSTRAP ---
# If the script is run standalone, it clones the full repo and restarts itself.
if [ ! -d "$SCRIPT_DIR/.git" ] && [ ! -d "$HOME/.opencortex" ] && [[ ! "$(pwd)" =~ "opencortex" ]]; then
echo -e "${BLUE}=== OpenCortex: Zero-to-One Bootstrapper ===${NC}"
git clone ssh://git@10.10.10.201:2222/amr/opencortex.git ~/.opencortex
cd ~/.opencortex && git submodule update --init --recursive
exec ./opencortex.sh "$@"
fi
# --- 2. SETUP ---
setup_system() {
NON_INTERACTIVE=false
for arg in "$@"; do
if [ "$arg" == "--non-interactive" ]; then NON_INTERACTIVE=true; fi
done
echo -e "${BLUE}=== OpenCortex: Initializing System ===${NC}"
echo -e "${YELLOW}--- Installing System Dependencies ---${NC}"
if command_exists apt-get; then
sudo apt-get update && sudo apt-get install -y sbcl emacs-nox rlwrap netcat-openbsd curl git socat libssl-dev libncurses5-dev libffi-dev zlib1g-dev libsqlite3-dev
fi
if [ ! -d "$HOME/quicklisp" ]; then
curl -O https://beta.quicklisp.org/quicklisp.lisp
sbcl --non-interactive --load quicklisp.lisp --eval "(quicklisp-quickstart:install)" --eval "(ql-util:without-prompting (ql:add-to-init-file))"
rm quicklisp.lisp
fi
cd "$SCRIPT_DIR"
if [ ! -f .env ]; then
if [ "$NON_INTERACTIVE" = true ]; then
echo "Non-interactive mode: Using environment variables for .env creation."
cp .env.example .env
[ -n "$MEMEX_USER" ] && sed -i "s|MEMEX_USER=.*|MEMEX_USER=\"$MEMEX_USER\"|" .env
[ -n "$MEMEX_ASSISTANT" ] && sed -i "s|MEMEX_ASSISTANT=.*|MEMEX_ASSISTANT=\"$MEMEX_ASSISTANT\"|" .env
[ -n "$OPENROUTER_API_KEY" ] && sed -i "s|OPENROUTER_API_KEY=.*|OPENROUTER_API_KEY=\"$OPENROUTER_API_KEY\"|" .env
[ -n "$MEMEX_DIR" ] && sed -i "s|MEMEX_DIR=.*|MEMEX_DIR=\"$MEMEX_DIR\"|" .env
else
cp .env.example .env
echo -e "\n${YELLOW}--- Identity Configuration ---${NC}"
read -p "Your Name [User]: " user_name < /dev/tty
user_name=${user_name:-User}
sed -i "s|MEMEX_USER=.*|MEMEX_USER=\"$user_name\"|" .env
read -p "Agent Name [OpenCortex]: " agent_name < /dev/tty
agent_name=${agent_name:-OpenCortex}
sed -i "s|MEMEX_ASSISTANT=.*|MEMEX_ASSISTANT=\"$agent_name\"|" .env
echo -e "\n${YELLOW}--- LLM Configuration ---${NC}"
read -p "OpenRouter API Key: " openrouter_key < /dev/tty
[ -n "$openrouter_key" ] && sed -i "s|OPENROUTER_API_KEY=.*|OPENROUTER_API_KEY=\"$openrouter_key\"|" .env
echo -e "\n${YELLOW}--- Memex Folder Structure ---${NC}"
read -p "Memex Root [\$HOME/memex]: " memex_dir < /dev/tty
memex_dir=${memex_dir:-\$HOME/memex}
sed -i "s|MEMEX_DIR=.*|MEMEX_DIR=\"$memex_dir\"|" .env
fi
# Hydrate default paths
M_DIR=$(grep MEMEX_DIR .env | cut -d'"' -f2 | sed "s|\$HOME|$HOME|")
sed -i "s|SKILLS_DIR=.*|SKILLS_DIR=\"$SCRIPT_DIR/skills\"|" .env
sed -i "s|ZETTELKASTEN_DIR=.*|ZETTELKASTEN_DIR=\"$M_DIR/notes\"|" .env
mkdir -p "$M_DIR" "$M_DIR/notes" "$M_DIR/areas" "$M_DIR/resources" "$M_DIR/archives" "$M_DIR/system" "$M_DIR/inbox" "$M_DIR/daily" "$M_DIR/projects"
fi
mkdir -p src
for f in literate/*.org; do
emacs --batch --eval "(require 'org)" --eval "(org-babel-tangle-file \"$f\")" >/dev/null 2>&1 || true
done
mkdir -p "$HOME/.local/bin"
ln -sf "$SCRIPT_DIR/opencortex.sh" "$HOME/.local/bin/opencortex"
for shell_config in "$HOME/.bashrc" "$HOME/.profile"; do
if [ -f "$shell_config" ]; then
if ! grep -q ".local/bin" "$shell_config"; then
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$shell_config"
fi
fi
done
export PATH="$HOME/.local/bin:$PATH"
echo -e "${YELLOW}--- Compiling and Loading OpenCortex ---${NC}"
sbcl --non-interactive --eval '(load (merge-pathnames "quicklisp/setup.lisp" (user-homedir-pathname)))' --eval '(push (truename (uiop:getenv "SCRIPT_DIR")) asdf:*central-registry*)' --eval "(ql:quickload '(:opencortex :croatoan))"
if [ $? -ne 0 ]; then
echo -e "${RED}✗ Compilation failed.${NC}"
exit 1
fi
if [ "$NON_INTERACTIVE" = true ]; then
echo "Setup complete (Non-interactive)."
exit 0
fi
echo -e "${YELLOW}--- Finalizing: Awakening the Brain ---${NC}"
"$SCRIPT_DIR/opencortex.sh" --boot > "$SCRIPT_DIR/brain.log" 2>&1 &
success=false
for i in {1..30}; do
if nc -z localhost $PORT 2>/dev/null; then success=true; break; fi
sleep 2
echo -n "."
done
if [ "$success" = true ]; then
echo -e "\n${GREEN}✓ Brain is alive on port $PORT.${NC}"
exit 0
else
echo -e "\n${RED}✗ Brain failed to wake up.${NC}"
exit 1
fi
}
# --- 3. COMMAND ROUTER ---
COMMAND=$1
[ -z "$COMMAND" ] && COMMAND="cli"
shift || true
DEFAULT_PORT=9105
DEFAULT_HOST="localhost"
TARGET_PORT=${PORT:-$DEFAULT_PORT}
TARGET_HOST=${HOST:-$DEFAULT_HOST}
# If uninitialized, force setup.
if [ ! -f "$SCRIPT_DIR/src/package.lisp" ] || [ ! -f "$SCRIPT_DIR/.env" ]; then
COMMAND="setup"
fi
case "$COMMAND" in
setup)
setup_system "$@"
;;
--boot|boot)
export SKILLS_DIR="${SCRIPT_DIR}/skills"
[ -z "$MEMEX_DIR" ] && export MEMEX_DIR="$HOME/memex"
if [ -f "$SCRIPT_DIR/.env" ]; then
export OPENROUTER_API_KEY=$(grep OPENROUTER_API_KEY "$SCRIPT_DIR/.env" | cut -d'"' -f2)
fi
exec sbcl --non-interactive --eval '(load (merge-pathnames "quicklisp/setup.lisp" (user-homedir-pathname)))' --eval '(setf *debugger-hook* (lambda (c h) (declare (ignore h)) (format *error-output* "FATAL LISP ERROR: ~a~%" c) (uiop:print-backtrace :stream *error-output*) (uiop:quit 1)))' --eval '(push (truename (uiop:getenv "SCRIPT_DIR")) asdf:*central-registry*)' --eval '(format t "--- Quickloading OpenCortex ---~%")' --eval "(ql:quickload '(:opencortex :croatoan))" --eval '(opencortex:main)'
;;
tui)
if ! nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then
echo -e "Brain is offline. Awakening..."
"$SCRIPT_DIR/opencortex.sh" --boot > "$SCRIPT_DIR/brain.log" 2>&1 &
for i in {1..15}; do
sleep 2
if nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then break; fi
echo -n "."
done
echo ""
fi
echo -e "Launching Croatoan TUI..."
export SKILLS_DIR="${SCRIPT_DIR}/skills"
[ -z "$MEMEX_DIR" ] && export MEMEX_DIR="$HOME/memex"
exec sbcl --eval '(load (merge-pathnames "quicklisp/setup.lisp" (user-homedir-pathname)))' --eval '(push (truename (uiop:getenv "SCRIPT_DIR")) asdf:*central-registry*)' --eval '(ql:quickload :opencortex/tui)' --eval '(opencortex.tui:main)'
;;
cli)
if ! nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then
echo -e "Brain is offline. Awakening..."
"$SCRIPT_DIR/opencortex.sh" --boot > "$SCRIPT_DIR/brain.log" 2>&1 &
for i in {1..15}; do
sleep 2
if nc -z $TARGET_HOST $TARGET_PORT 2>/dev/null; then break; fi
echo -n "."
done
echo ""
fi
if command_exists socat; then
exec socat - TCP:$TARGET_HOST:$TARGET_PORT
else
exec nc $TARGET_HOST $TARGET_PORT
fi
;;
*)
echo -e "Unknown command: $COMMAND"
echo "Available commands: setup, boot, tui, cli"
exit 1
;;
esac
#+end_src
** Metabolic Docker Infrastructure (Dockerfile)
#+begin_src dockerfile :tangle ../Dockerfile
FROM debian:bullseye-slim
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \
sbcl \
emacs-nox \
curl \
git \
socat \
netcat-openbsd \
libssl-dev \
libncurses5-dev \
libffi-dev \
zlib1g-dev \
libsqlite3-dev \
&& rm -rf /var/lib/apt/lists/*
# Install Quicklisp
RUN curl -O https://beta.quicklisp.org/quicklisp.lisp \
&& sbcl --non-interactive --load quicklisp.lisp --eval "(quicklisp-quickstart:install)" --eval "(ql-util:without-prompting (ql:add-to-init-file))" \
&& rm quicklisp.lisp
WORKDIR /app
COPY . .
# Initialize system in non-interactive mode
RUN mkdir -p /root/memex /app/environment/logs && ./opencortex.sh setup --non-interactive
EXPOSE 9105
CMD ["./opencortex.sh", "boot"]
#+end_src

139
harness/skills.org Normal file
View File

@@ -0,0 +1,139 @@
#+TITLE: The Skill Engine (skills.lisp)
#+AUTHOR: Amr
#+FILETAGS: :harness:skills:
#+STARTUP: content
* The Skill Engine (skills.lisp)
** Architectural Intent: Hot-Reloadable Intelligence
The Skill Engine is the modular heart of the OpenCortex. By separating cognitive and physical capabilities into discrete "Skills," we allow the system to evolve without modifying the core Lisp microharness.
*** Core Principles
1. **Isolation:** Every skill resides in its own Lisp package, preventing global namespace pollution and variable collisions.
2. **Topological Bootstrapping:** Skills can declare dependencies on other skills. The harness automatically calculates the correct loading order.
3. **Hot-Reloading:** Since Skills are defined as Literate Org files, the agent can edit, re-tangle, and re-load its own skills at runtime without a system restart.
4. **The Bouncer Pattern:** Every skill must define a deterministic gate. This is the primary security layer where native Lisp logic verifies probabilistic AI proposals.
** Pipeline Initialization
#+begin_src lisp :tangle ../library/skills.lisp
(in-package :opencortex)
#+end_src
* Skill Definition and Registration
** The Skill Structure
#+begin_src lisp :tangle ../library/skills.lisp
(defstruct skill
"Represents a hot-reloadable module of intelligence or actuation."
name
priority
dependencies
trigger-fn
probabilistic-prompt
deterministic-fn)
#+end_src
** Skill Registration Macro (defskill)
This macro provides a clean interface for skill authors to register their modules. It automatically handles the integration with the global ~*skills-registry*~.
#+begin_src lisp :tangle ../library/skills.lisp
(defmacro defskill (name &key (priority 0) dependencies trigger probabilistic deterministic)
"Registers a new skill into the global harness registry."
`(setf (gethash (string-downcase (string ',name)) *skills-registry*)
(make-skill :name (string-downcase (string ',name))
:priority ,priority
:dependencies ,dependencies
:trigger-fn ,trigger
:probabilistic-prompt ,probabilistic
:deterministic-fn ,deterministic)))
#+end_src
* Dynamic Loading System
** Lisp Syntax Validation (validate-lisp-syntax)
Before loading a new skill into the live image, the harness performs a dry-run parse to ensure the code is syntactically valid. This prevents a single hallucinated parenthesis from crashing the entire brain.
#+begin_src lisp :tangle ../library/skills.lisp
(defun validate-lisp-syntax (file-path)
"Parses a Lisp file without evaluation to verify syntactic integrity."
(handler-case
(with-open-file (stream file-path)
(loop for form = (read stream nil :eof)
until (eq form :eof))
t)
(error (c)
(harness-log "SYNTAX ERROR in ~a: ~a" file-path c)
nil)))
#+end_src
** Literate Skill Ingestion (load-skill-from-org)
The primary mechanism for hot-reloading. It handles the Org-to-Lisp translation and ensures the resulting code is jailed within its own package.
#+begin_src lisp :tangle ../library/skills.lisp
(defun load-skill-from-org (org-file-path)
"Tangles and loads a single Org-mode skill file."
(let* ((filename (file-name-nondirectory (namestring org-file-path)))
(skill-id (pathname-name org-file-path))
(lisp-file (merge-pathnames (concatenate 'string "library/gen/" skill-id ".lisp")
(asdf:system-source-directory :opencortex))))
(ensure-directories-exist lisp-file)
(harness-log "LOADER: Loading ~a..." skill-id)
;; 1. Tangle the Org file into Lisp
(uiop:run-program (list "emacs" "--batch" "--eval" "(require 'org)"
"--eval" (format nil "(org-babel-tangle-file \"~a\")" org-file-path))
:output t)
;; 2. Verify and Load
(if (validate-lisp-syntax lisp-file)
(progn
(handler-case (load lisp-file)
(error (c) (harness-log "LOADER ERROR in skill '~a': ~a" skill-id c)))
t)
nil)))
#+end_src
* Bootstrapping Logic
** Dependency Sorting (topological-sort-skills)
Ensures that foundational skills (like the Bouncer or Policy engine) are always loaded before higher-level actuators.
#+begin_src lisp :tangle ../library/skills.lisp
(defun topological-sort-skills (skills)
"Calculates the correct loading order based on #+DEPENDS_ON metadata."
;; Placeholder: Currently sorts by priority as a proxy for dependencies.
(sort skills #'> :key #'skill-priority))
#+end_src
** Registry Initialization (initialize-all-skills)
The high-level boot sequence for the skill engine.
#+begin_src lisp :tangle ../library/skills.lisp
(defun initialize-all-skills ()
"Discovers and loads all Org files in the SKILLS_DIR."
(let* ((skills-dir (uiop:getenv "SKILLS_DIR"))
(files (when (and skills-dir (uiop:directory-exists-p skills-dir))
(uiop:directory-files skills-dir "*.org"))))
(dolist (f files)
(load-skill-from-org f))
(harness-log "LOADER: Boot Complete. [Ready: ~a] [Failed: 0]" (hash-table-count *skills-registry*))))
#+end_src
* Cognitive Dispatching
** Skill Trigger Discovery (find-triggered-skill)
Identifies which skill is best suited to handle the current metabolic signal.
#+begin_src lisp :tangle ../library/skills.lisp
(defun find-triggered-skill (context)
"Iterates through the registry and returns the first skill whose trigger returns true."
(let ((skills nil))
(maphash (lambda (name skill) (declare (ignore name)) (push skill skills)) *skills-registry*)
(setf skills (sort skills #'> :key #'skill-priority))
(dolist (s skills)
(let ((trigger (skill-trigger-fn s)))
(when (and trigger (funcall trigger context))
(return-from find-triggered-skill s))))
nil))
#+end_src

212
harness/tui-client.org Normal file
View File

@@ -0,0 +1,212 @@
#+TITLE: OpenCortex TUI Client (tui-client.lisp)
#+AUTHOR: Amr
#+FILETAGS: :tui:ux:client:
#+STARTUP: content
* OpenCortex TUI Client (tui-client.lisp)
** Architectural Intent: High-Fidelity Interaction
The TUI Client is a standalone consumer of the OpenCortex protocol. It uses the ~croatoan~ (ncurses) library to provide a split-pane, interactive terminal experience.
*** Design Requirements
1. **Concurrency:** The client must listen for incoming protocol events (heartbeats, status updates, thoughts) in a background thread to prevent the UI from freezing.
2. **Buffer Safety:** User input must be captured in a thread-safe buffer and framed correctly before being sent to the daemon.
3. **Transparency:** The status bar must provide real-time feedback on the state of background workers (Scribe and Gardener).
** Package Context
#+begin_src lisp :tangle ../library/tui-client.lisp
(in-package :cl-user)
(defpackage :opencortex.tui (:use :cl :croatoan) (:export :main))
(in-package :opencortex.tui)
#+end_src
* UI State Management
** Networking and Streams
#+begin_src lisp :tangle ../library/tui-client.lisp
(defvar *daemon-host* "127.0.0.1")
(defvar *daemon-port* 9105)
(defvar *socket* nil)
(defvar *stream* nil)
#+end_src
** Terminal Buffers
#+begin_src lisp :tangle ../library/tui-client.lisp
(defvar *chat-history* nil "A list of strings representing the scrollback buffer.")
(defvar *input-buffer* (make-array 0 :element-type 'character :fill-pointer 0 :adjustable t))
(defvar *is-running* t)
(defvar *status-text* "Connecting...")
#+end_src
** Thread-Safe Message Queue
We use a simple locked queue to move messages from the background listener thread to the foreground rendering loop.
#+begin_src lisp :tangle ../library/tui-client.lisp
(defvar *msg-queue* nil)
(defvar *queue-lock* (bt:make-lock "tui-msg-lock"))
(defun enqueue-msg (msg)
(bt:with-lock-held (*queue-lock*) (push msg *msg-queue*)))
(defun dequeue-msgs ()
(bt:with-lock-held (*queue-lock*) (let ((m (reverse *msg-queue*))) (setf *msg-queue* nil) m)))
#+end_src
* Protocol Integration
** Keyword Sanitization (clean-keywords)
Clients often receive data with inconsistent keyword casing. This helper ensures all incoming keys are normalized for easier processing.
#+begin_src lisp :tangle ../library/tui-client.lisp
(defun clean-keywords (msg)
"Ensures all keys in a plist are uppercase keywords."
(if (listp msg)
(let ((clean nil))
(loop for (k v) on msg by #'cddr
do (push (intern (string k) :keyword) clean)
(push v clean))
(nreverse clean))
msg))
#+end_src
** Payload Extraction (format-payload)
The core "intelligence" of the TUI display. It recursively searches a protocol payload for the most relevant human-readable content.
#+begin_src lisp :tangle ../library/tui-client.lisp
(defun format-payload (payload)
"Extracts human-readable text from a protocol payload, handling nested tool calls."
(let* ((action (getf payload :ACTION))
(text (getf payload :TEXT))
(msg (getf payload :MESSAGE))
(tool (getf payload :TOOL))
(prompt (getf payload :PROMPT))
(args (getf payload :ARGS))
(result (getf payload :RESULT)))
(cond (text text)
(msg msg)
((eq action :MESSAGE) (getf payload :TEXT))
((and tool prompt) (format nil "THOUGHT [~a]: ~a" tool prompt))
((and tool args)
(let ((inner-prompt (or (getf args :PROMPT) (getf args :TEXT))))
(if inner-prompt
(format nil "THOUGHT [~a]: ~a" tool inner-prompt)
(format nil "CALL [~a] (ARGS: ~s)" tool args))))
(result (format nil "RESULT: ~a" result))
(t (format nil "~s" payload)))))
#+end_src
** Background Listener (listen-thread)
Runs as a separate thread. It continuously reads framed messages from the daemon and enqueues them for the UI.
#+begin_src lisp :tangle ../library/tui-client.lisp
(defun listen-thread ()
(loop while *is-running* do
(handler-case
(when (and *stream* (open-stream-p *stream*))
(let ((raw-msg (opencortex:read-framed-message *stream*)))
(unless (member raw-msg '(:eof :error))
(let* ((msg (clean-keywords raw-msg))
(type (or (getf msg :TYPE) (getf msg :type)))
(payload (or (getf msg :PAYLOAD) (getf msg :payload))))
(cond ((and (listp msg) (eq type :EVENT))
(let ((action (or (getf payload :ACTION) (getf payload :action)))
(text (or (getf payload :TEXT) (getf payload :text) (getf payload :MESSAGE) (getf payload :message))))
(cond ((eq action :handshake) (setf *status-text* "Ready"))
(text (enqueue-msg (format nil "SYSTEM: ~a" text))))))
((and (listp msg) (eq type :STATUS))
(setf *status-text* (format nil "[Scribe: ~a] [Gardener: ~a]"
(or (getf msg :SCRIBE) (getf msg :scribe))
(or (getf msg :GARDENER) (getf msg :gardener)))))
((and (listp msg) (member type '(:REQUEST :RESPONSE :LOG)))
(let ((formatted (format-payload payload)))
(when formatted (enqueue-msg formatted))))
((and (listp msg) (eq type :EVENT) (eq (getf payload :SENSOR) :TOOL-OUTPUT))
(let ((formatted (format-payload payload)))
(when formatted (enqueue-msg formatted))))
(t (harness-log "TUI: Ignored unknown type ~a" type)))))
(when (eq raw-msg :eof) (setf *is-running* nil))
(when (eq raw-msg :error) (setf *status-text* "Protocol Error"))))
(error (c) (setf *status-text* (format nil "Net Error: ~a" c)) (setf *is-running* nil)))
(sleep 0.05)))
#+end_src
* Main Interaction Loop
** TUI Entry Point (main)
Initializes the ncurses screen, sets up the window layout, and handles user keyboard input.
#+begin_src lisp :tangle ../library/tui-client.lisp
(defun main ()
"Primary entry point for the standalone TUI client."
(handler-case
(setf *socket* (usocket:socket-connect *daemon-host* *daemon-port*))
(error (e) (format t "Error connecting: ~a~%" e) (return-from main)))
(setf *stream* (usocket:socket-stream *socket*))
(bt:make-thread #'listen-thread :name "tui-listener")
(unwind-protect
(with-screen (scr :input-echoing nil :input-blocking nil :enable-colors t :cursor-visible t)
(let* ((h (height scr))
(w (width scr))
(chat-win (make-instance 'window :height (- h 2) :width w :position (list 0 0)))
(status-win (make-instance 'window :height 1 :width w :position (list (- h 2) 0)))
(input-win (make-instance 'window :height 1 :width w :position (list (- h 1) 0)))
(last-status nil))
(setf (function-keys-enabled-p input-win) t)
(setf (input-blocking input-win) nil)
(loop while *is-running* do
;; 1. Handle incoming messages from the queue
(let ((new-msgs (dequeue-msgs)))
(when new-msgs
(dolist (msg new-msgs)
(push msg *chat-history*)
;; Maintenance: Cap scrollback to prevent memory bloat
(setf *chat-history* (subseq *chat-history* 0 (min (length *chat-history*) 500))))
(clear chat-win)
(let ((line-num 0))
(dolist (m (reverse (subseq *chat-history* 0 (min (length *chat-history*) (- h 3)))))
(add-string chat-win m :y line-num :x 0)
(incf line-num)))
(refresh chat-win)))
;; 2. Render Status Bar
(unless (equal *status-text* last-status)
(clear status-win)
(add-string status-win *status-text* :attributes '(:reverse))
(refresh status-win)
(setf last-status *status-text*))
;; 3. Handle Keyboard Input
(let* ((event (get-wide-event input-win))
(ch (and event (typep event 'event) (event-key event))))
(when ch
(cond
((or (eq ch #\Newline) (eq ch #\Return))
(let ((cmd (coerce *input-buffer* 'string)))
(setf (fill-pointer *input-buffer*) 0)
(when (> (length cmd) 0)
;; Frame and dispatch the message
(let ((framed (opencortex:frame-message (list :TYPE :EVENT
:META (list :SOURCE :tui :SESSION-ID "default")
:PAYLOAD (list :SENSOR :user-input :TEXT cmd)))))
(format *stream* "~a" framed)
(finish-output *stream*)))
(when (string= cmd "/exit") (setf *is-running* nil))))
((or (eq ch :backspace) (eq ch #\Backspace) (eq ch #\Rubout) (eq ch #\Del))
(when (> (length *input-buffer*) 0)
(decf (fill-pointer *input-buffer*))))
((characterp ch)
(vector-push-extend ch *input-buffer*))))
(clear input-win)
(add-string input-win (concatenate 'string "> " (coerce *input-buffer* 'string)))
(move input-win 0 (+ 2 (length *input-buffer*)))
(refresh input-win))
(sleep 0.02))))
(setf *is-running* nil)
(when *socket* (usocket:socket-close *socket*))))
#+end_src