v0.2.1: polish, deploy, CI, and literate refactor
Some checks failed
Deploy (Gitea) / deploy (push) Failing after 11s

- Secret Exposure Gate + Privacy Filter (Bouncer)
- Shell actuator safety harness (timeout, blocked patterns)
- REPL-first enforcement (lisp validation gate, system-prompt-augment)
- Engineering Standards lifecycle (two-track Org-first + REPL-first)
- Literate Programming discipline (one function per block, reflect-back)
- AGENTS.md: thin routing layer, skills are authoritative
- SKILLS_DIR removed, ~/notes fallback eliminated
- opencortex.sh: multi-distro (Debian+Fedora), configure, install service, backup, restore, help
- infrastructure/opencortex.service (systemd user unit)
- Docker: updated to debian:trixie, fixed build context
- GitHub CI: lint + test workflows fixed, trigger on tags only
- Gitea CI: deploy workflow paths fixed
- README: one-line curl install, badges
- USER_MANUAL: Deployment section (bare metal, Docker, backup)
- .gitignore: skills/*.lisp and tests/*.lisp as generated artifacts
- Prose/block refactor across all 35 org files
- Test suite Tier 1: 43/45 pass (env-dependent failures isolated)
This commit is contained in:
2026-05-02 17:04:33 -04:00
parent 9e77958028
commit 41de20d3f1
67 changed files with 1943 additions and 1287 deletions

View File

@@ -5,7 +5,7 @@
#+PROPERTY: header-args:lisp :tangle act.lisp
* Overview
The Act stage is where cognition meets reality. After the Probabilistic engine proposes and the Deterministic engine verifies, Act executes the approved action.
The Act stage dispatches approved actions to registered actuators. After the Probabilistic engine proposes and the Deterministic engine verifies, Act executes the approved action via the appropriate actuator (:cli, :tool, :system, :telegram, :signal, etc.). The actuator registry is extensible — skills can register new actuators at runtime.
* Implementation

View File

@@ -135,6 +135,37 @@ The ~communication.lisp~ module defines the low-level transport and framing logi
t))
#+end_src
** Protocol Smoke Test (manual for REPL evaluation)
The following script connects to a running daemon, sends "hi", and reads the response. Useful for verifying the daemon is alive and the framing protocol works end-to-end.
#+begin_src lisp :tangle no
(defun test-daemon-protocol ()
(handler-case
(let* ((socket (usocket:socket-connect "127.0.0.1" 9105))
(stream (usocket:socket-stream socket)))
(format t "Connected.~%")
(let* ((len-buf (make-string 6))
(count (read-sequence len-buf stream)))
(when (= count 6)
(let* ((len (parse-integer len-buf :radix 16))
(msg-buf (make-string len)))
(read-sequence msg-buf stream)
(format t "HELLO: ~a~%" msg-buf))))
(let* ((msg '(:TYPE :EVENT :META (:SOURCE :tui) :PAYLOAD (:SENSOR :user-input :TEXT "hi")))
(framed (frame-message msg)))
(format stream "~a" framed)
(finish-output stream)
(let* ((len-buf (make-string 6))
(count (read-sequence len-buf stream)))
(when (= count 6)
(let* ((len (parse-integer len-buf :radix 16))
(msg-buf (make-string len)))
(read-sequence msg-buf stream)
(format t "Response: ~a~%" msg-buf)))))
(usocket:socket-close socket))
(error (c) (format t "Error: ~a~%" c))))
#+end_src
* Test Suite
#+begin_src lisp :tangle ../tests/communication-tests.lisp
(eval-when (:compile-toplevel :load-toplevel :execute)

View File

@@ -34,8 +34,8 @@
(defun context-get-skill-source (skill-name)
"Reads the raw literate source of a specific skill for inspection."
(let* ((filename (format nil "~a.org" skill-name))
(skills-dir-str (or (uiop:getenv "SKILLS_DIR") (namestring (merge-pathnames "notes/" (user-homedir-pathname)))))
(skills-dir (uiop:ensure-directory-pathname (context-resolve-path skills-dir-str)))
(data-dir (uiop:ensure-directory-pathname (or (uiop:getenv "OC_DATA_DIR") (namestring (merge-pathnames ".local/share/opencortex/" (user-homedir-pathname))))))
(skills-dir (merge-pathnames "skills/" data-dir))
(full-path (merge-pathnames filename skills-dir)))
(if (uiop:file-exists-p full-path) (uiop:read-file-string full-path) nil)))
@@ -98,11 +98,47 @@
result)
path)))
(defun context-object-privacy-filtered-p (obj)
"Returns T if an org-object's :TAGS attribute matches bouncer-privacy-tags."
(let* ((attrs (org-object-attributes obj))
(tags (getf attrs :TAGS))
(privacy-tags (and (find-package :opencortex.skills.org-skill-bouncer)
(symbol-value
(find-symbol "BOUNCER-PRIVACY-TAGS"
:opencortex.skills.org-skill-bouncer)))))
(when (and tags privacy-tags)
(let ((tag-list (if (listp tags) tags (list tags))))
(some (lambda (tag)
(some (lambda (private)
(string-equal (string-trim '(#\:) tag)
(string-trim '(#\:) private)))
privacy-tags))
tag-list)))))
(defun context-object-privacy-filtered-p (obj)
"Returns T if an org-object's :TAGS attribute matches bouncer-privacy-tags."
(let* ((attrs (org-object-attributes obj))
(tags (getf attrs :TAGS))
(privacy-tags (and (find-package :opencortex.skills.org-skill-bouncer)
(symbol-value
(find-symbol "BOUNCER-PRIVACY-TAGS"
:opencortex.skills.org-skill-bouncer)))))
(when (and tags privacy-tags)
(let ((tag-list (if (listp tags) tags (list tags))))
(some (lambda (tag)
(some (lambda (private)
(string-equal (string-trim '(#\:) tag)
(string-trim '(#\:) private)))
privacy-tags))
tag-list)))))
(defun context-assemble-global-awareness (&optional signal)
"Produces a high-level skeletal outline of the current Memory for the LLM."
"Produces a high-level skeletal outline of the current Memory for the LLM.
Privacy-filtered objects (matching *privacy-filter-tags*) are excluded."
(let* ((foveal-id (or (getf signal :foveal-focus)
(ignore-errors (getf (getf signal :payload) :target-id))))
(projects (context-get-active-projects))
(all-projects (context-get-active-projects))
(projects (remove-if #'context-object-privacy-filtered-p all-projects))
(output (format nil "GLOBAL MEMEX AWARENESS (Peripheral Vision):~%")))
(if projects
(dolist (project projects)

View File

@@ -62,8 +62,8 @@ The *Context API* (Peripheral Vision) provides the opencortex with the ability t
(defun context-get-skill-source (skill-name)
"Reads the raw literate source of a specific skill for inspection."
(let* ((filename (format nil "~a.org" skill-name))
(skills-dir-str (or (uiop:getenv "SKILLS_DIR") (namestring (merge-pathnames "notes/" (user-homedir-pathname)))))
(skills-dir (uiop:ensure-directory-pathname (context-resolve-path skills-dir-str)))
(data-dir (uiop:ensure-directory-pathname (or (uiop:getenv "OC_DATA_DIR") (namestring (merge-pathnames ".local/share/opencortex/" (user-homedir-pathname))))))
(skills-dir (merge-pathnames "skills/" data-dir))
(full-path (merge-pathnames filename skills-dir)))
(if (uiop:file-exists-p full-path) (uiop:read-file-string full-path) nil)))
#+end_src
@@ -136,13 +136,53 @@ The *Context API* (Peripheral Vision) provides the opencortex with the ability t
path)))
#+end_src
** Privacy filter for context assembly
Checks if an org-object has tags matching ~*privacy-filter-tags*~. Objects with matching tags are excluded from the LLM context window.
#+begin_src lisp
(defun context-object-privacy-filtered-p (obj)
"Returns T if an org-object's :TAGS attribute matches bouncer-privacy-tags."
(let* ((attrs (org-object-attributes obj))
(tags (getf attrs :TAGS))
(privacy-tags (and (find-package :opencortex.skills.org-skill-bouncer)
(symbol-value
(find-symbol "BOUNCER-PRIVACY-TAGS"
:opencortex.skills.org-skill-bouncer)))))
(when (and tags privacy-tags)
(let ((tag-list (if (listp tags) tags (list tags))))
(some (lambda (tag)
(some (lambda (private)
(string-equal (string-trim '(#\:) tag)
(string-trim '(#\:) private)))
privacy-tags))
tag-list)))))
#+end_src
** Global Awareness (context-assemble-global-awareness)
#+begin_src lisp
(defun context-object-privacy-filtered-p (obj)
"Returns T if an org-object's :TAGS attribute matches bouncer-privacy-tags."
(let* ((attrs (org-object-attributes obj))
(tags (getf attrs :TAGS))
(privacy-tags (and (find-package :opencortex.skills.org-skill-bouncer)
(symbol-value
(find-symbol "BOUNCER-PRIVACY-TAGS"
:opencortex.skills.org-skill-bouncer)))))
(when (and tags privacy-tags)
(let ((tag-list (if (listp tags) tags (list tags))))
(some (lambda (tag)
(some (lambda (private)
(string-equal (string-trim '(#\:) tag)
(string-trim '(#\:) private)))
privacy-tags))
tag-list)))))
(defun context-assemble-global-awareness (&optional signal)
"Produces a high-level skeletal outline of the current Memory for the LLM."
"Produces a high-level skeletal outline of the current Memory for the LLM.
Privacy-filtered objects (matching *privacy-filter-tags*) are excluded."
(let* ((foveal-id (or (getf signal :foveal-focus)
(ignore-errors (getf (getf signal :payload) :target-id))))
(projects (context-get-active-projects))
(all-projects (context-get-active-projects))
(projects (remove-if #'context-object-privacy-filtered-p all-projects))
(output (format nil "GLOBAL MEMEX AWARENESS (Peripheral Vision):~%")))
(if projects
(dolist (project projects)

View File

@@ -48,13 +48,13 @@ Common Lisp's `getenv` is strictly typed in SBCL. The Doctor must ensure that mi
(test test-env-validation-fail
"Verify that an invalid MEMEX_DIR triggers a critical failure."
(let ((old-m (uiop:getenv "MEMEX_DIR"))
(old-s (uiop:getenv "SKILLS_DIR")))
(old-d (uiop:getenv "OC_DATA_DIR")))
(unwind-protect
(progn
(setf (uiop:getenv "MEMEX_DIR") "/non/existent/path/999")
(is (null (opencortex:doctor-check-env))))
(setf (uiop:getenv "MEMEX_DIR") (or old-m ""))
(setf (uiop:getenv "SKILLS_DIR") (or old-s "")))))
(setf (uiop:getenv "OC_DATA_DIR") (or old-d "")))))
#+end_src
* Phase C: Implementation (Build)

View File

@@ -5,7 +5,13 @@
#+PROPERTY: header-args:lisp :tangle memory.lisp
* Overview
The Memory module is the cognitive bedrock of the opencortex. It is not a database; it is the agent's live, active "brain" state.
The Memory module is the agent's live cognitive state — a set of Merkle-tree-versioned ~org-object~ instances stored in hash tables. Every perception, action, and decision is recorded here.
Key structures:
- ~*memory*~ — the primary object store, keyed by ID
- ~*history-store*~ — immutable archive of all past object versions, keyed by SHA-256 hash
- ~org-object~ — the universal data unit (id, type, attributes, content, vector embedding, parent, children, merkle hash)
- ~ingest-ast~ — converts an Org-mode AST into ~org-object~ instances, computing Merkle hashes for integrity
* Implementation
@@ -21,21 +27,55 @@ The Memory module is the cognitive bedrock of the opencortex. It is not a databa
"Immutable Merkle-Tree versioning store mapping hashes to objects.")
#+end_src
** Object Lookup
** Object Lookup (lookup-object)
Retrieve a single object by its ID from the active memory store.
#+begin_src lisp
(defun lookup-object (id)
"Retrieves an org-object by ID from *memory*."
(gethash id *memory*))
#+end_src
** Object search (list-objects-with-attribute)
Scan the entire memory store for objects whose attributes match a given key-value pair.
#+begin_src lisp
(defun list-objects-with-attribute (attr value)
"Returns all org-objects whose :ATTRIBUTES plist has ATTR = VALUE."
(let ((results nil))
(maphash (lambda (id obj)
(declare (ignore id))
(when (equal (getf (org-object-attributes obj) attr) value)
(push obj results)))
*memory*)
(nreverse results)))
#+end_src
** ID generation (org-id-new)
Generate a unique identifier string for a new Org node. Uses the universal time encoded in base-36 for compactness.
#+begin_src lisp
(defun org-id-new ()
"Generates a timestamp-based unique ID."
(format nil "id-~36r" (get-universal-time)))
#+end_src
** The Data Structure (org-object)
The universal data unit. Every stored entity is an ~org-object~ with an ID, type, attribute plist, content string, optional vector embedding, parent/child pointers, version timestamp, and Merkle hash.
#+begin_src lisp
(defstruct org-object
id type attributes content vector parent-id children version last-sync hash)
#+end_src
** Serialization support
Required by the Lisp runtime for saving/loading objects across image restarts.
#+begin_src lisp
(defmethod make-load-form ((obj org-object) &optional env)
(make-load-form-saving-slots obj :environment env))
#+end_src
** Deep copy
Creates an independent copy of an ~org-object~. Used by the snapshot system to capture consistent memory state.
#+begin_src lisp
(defun deep-copy-org-object (obj)
"Creates a full copy of an org-object, including a fresh list copy of attributes and children."
(make-org-object :id (org-object-id obj)
:type (org-object-type obj)
:attributes (copy-list (org-object-attributes obj))
@@ -93,24 +133,41 @@ The Memory module is the cognitive bedrock of the opencortex. It is not a databa
id)))
#+end_src
** Snapshots (snapshot-memory)
** Snapshot history (~*object-store-snapshots*~)
A stack of CoW (copy-on-write) memory snapshots for rollback. Up to 20 snapshots are retained.
#+begin_src lisp
(defvar *object-store-snapshots* nil)
#+end_src
** Hash table copy utility
Used by the rollback system to restore saved memory state.
#+begin_src lisp
(defun copy-hash-table (hash-table)
"Creates an independent copy of a hash table."
(let ((new-table (make-hash-table :test (hash-table-test hash-table)
:size (hash-table-size hash-table))))
:size (hash-table-size hash-table))))
(maphash (lambda (k v) (setf (gethash k new-table) v)) hash-table)
new-table))
#+end_src
** Memory snapshot (snapshot-memory)
Captures a point-in-time copy of ~*memory*~. Each object is deep-copied so the snapshot is independent of ongoing mutations.
#+begin_src lisp
(defun snapshot-memory ()
"Creates a CoW snapshot of *memory* for rollback recovery."
(let ((snapshot (make-hash-table :test 'equal :size (hash-table-size *memory*))))
(maphash (lambda (k v) (setf (gethash k snapshot) (deep-copy-org-object v))) *memory*)
(push (list :timestamp (get-universal-time) :data snapshot) *object-store-snapshots*)
(when (> (length *object-store-snapshots*) 20) (setf *object-store-snapshots* (subseq *object-store-snapshots* 0 20)))
(when (> (length *object-store-snapshots*) 20)
(setf *object-store-snapshots* (subseq *object-store-snapshots* 0 20)))
(harness-log "MEMORY - CoW Memory snapshot created.")))
#+end_src
** Memory rollback (rollback-memory)
Restores ~*memory*~ to a previous snapshot. By default restores the most recent snapshot (index 0).
#+begin_src lisp
(defun rollback-memory (&optional (index 0))
"Restores *memory* from a snapshot. INDEX 0 = most recent."
(let ((snapshot (nth index *object-store-snapshots*)))
(if snapshot
(progn (setf *memory* (copy-hash-table (getf snapshot :data)))
@@ -118,17 +175,24 @@ The Memory module is the cognitive bedrock of the opencortex. It is not a databa
(harness-log "MEMORY ERROR - Snapshot ~a not found." index))))
#+end_src
** Persistence (save-memory / load-memory)
** Persistence — snapshot path (~*memory-snapshot-path*~)
Configurable path for serialized memory state. Falls back to ~memory.snap~ in the home directory.
#+begin_src lisp
(defvar *memory-snapshot-path* nil)
(defun ensure-memory-snapshot-path ()
"Returns the path to the memory snapshot file, resolving env or default."
(or *memory-snapshot-path*
(let ((env-path (uiop:getenv "MEMORY_SNAPSHOT_PATH")))
(setf *memory-snapshot-path*
(or env-path (namestring (uiop:merge-pathnames* "memory.snap" (user-homedir-pathname))))))))
#+end_src
** Save to disk (save-memory-to-disk)
Serialises ~*memory*~ and ~*history-store*~ to a Lisp-readable file.
#+begin_src lisp
(defun save-memory-to-disk ()
"Writes the entire memory and history store to disk as a plist."
(let ((path (ensure-memory-snapshot-path)))
(with-open-file (stream path :direction :output :if-exists :supersede :if-does-not-exist :create)
(let ((memory-alist nil) (history-alist nil))
@@ -136,8 +200,13 @@ The Memory module is the cognitive bedrock of the opencortex. It is not a databa
(maphash (lambda (k v) (push (cons k v) history-alist)) *history-store*)
(prin1 (list :memory memory-alist :history-store history-alist) stream)))
(harness-log "MEMORY - Saved to ~a" path)))
#+end_src
** Load from disk (load-memory-from-disk)
Restores memory state from a previously saved snapshot file.
#+begin_src lisp
(defun load-memory-from-disk ()
"Reads memory state from disk and restores *memory* and *history-store*."
(let ((path (ensure-memory-snapshot-path)))
(when (uiop:file-exists-p path)
(handler-case

View File

@@ -5,16 +5,26 @@
#+PROPERTY: header-args:lisp :tangle package.lisp
* Overview
The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
~package.lisp~ defines two things: the public API of the ~opencortex~ package (the export list, above), and the implementation of low-level utility functions and global state that don't belong in a specific pipeline stage or skill.
The export list is the contract between the harness and all skills. Every function exported here is accessible to every skill via ~use-package~. Adding a symbol here is an API commitment; removing one is a breaking change.
The implementation section includes:
- ~proto-get~ — robust plist accessor used everywhere
- Logging state (~*system-logs*~, ~*logs-lock*~)
- Skill registry (~*skills-registry*~, ~defskill~)
- Cognitive tool registry (~*cognitive-tools*~, ~def-cognitive-tool~)
- Configuration variables (~*privacy-filter-tags*~, ~*secret-protected-paths*~, ~*secret-exposure-patterns*~)
- Debugger hook
* Implementation
** Public API Export
** Package Definition and Export List
The package definition. All public symbols are exported here.
#+begin_src lisp :tangle package.lisp
(defpackage :opencortex
(:use :cl)
(:export
;; --- communication protocol ---
#:frame-message
#:read-framed-message
#:PROTO-GET
@@ -25,30 +35,20 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
#:parse-message
#:make-hello-message
#:validate-communication-protocol-schema
;; --- Daemon Lifecycle ---
#:start-daemon
#:stop-daemon
#:harness-log
#:main
;; --- Diagnostic Doctor ---
#:doctor-run-all
#:doctor-main
#:doctor-check-dependencies
#:doctor-check-env
;; --- Setup Wizard ---
#:register-provider
#:system-ready-p
#:run-setup-wizard
;; --- Gateway Manager Skill ---
#:skill-gateway-register
#:skill-gateway-link
#:gateway-manager-main
;; --- Memory (CLOSOS) ---
#:ingest-ast
#:lookup-object
#:list-objects-by-type
@@ -69,8 +69,6 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
#:org-object-hash
#:snapshot-memory
#:rollback-memory
;; --- Context API (Peripheral Vision) ---
#:context-query-store
#:context-get-active-projects
#:context-get-recent-completed-tasks
@@ -81,22 +79,17 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
#:context-get-skill-telemetry
#:harness-track-telemetry
#:context-assemble-global-awareness
;; --- Reactive Signal Pipeline ---
#:process-signal
#:perceive-gate
#:probabilistic-gate
#:consensus-gate
#:act-gate
#:reason-gate
#:perceive-gate
#:dispatch-gate
#:inject-stimulus
#:initialize-actuators
#:dispatch-action
#:register-actuator
;; --- Skill Engine ---
#:load-skill-from-org
#:initialize-all-skills
#:load-skill-with-timeout
@@ -111,22 +104,14 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
#:skill-trigger-fn
#:skill-probabilistic-prompt
#:skill-deterministic-fn
;; --- Tool Registry ---
#:def-cognitive-tool
#:*cognitive-tools*
;; --- Engineering Standards Skill ---
#:verify-git-clean-p
#:engineering-standards-verify-lisp
#:engineering-standards-format-lisp
;; --- Literate Programming Skill ---
#:literate-check-block-balance
#:check-tangle-sync
#:*tangle-targets*
;; --- Utils Org Skill ---
#:utils-org-read-file
#:utils-org-write-file
#:utils-org-add-headline
@@ -138,8 +123,6 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
#:utils-org-id-format
#:utils-org-ast-to-org
#:utils-org-modify
;; --- Utils Lisp Skill ---
#:utils-lisp-validate
#:utils-lisp-check-structural
#:utils-lisp-check-syntactic
@@ -152,13 +135,9 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
#:utils-lisp-structural-inject
#:utils-lisp-structural-slurp
#:utils-lisp-register
;; --- Config Manager & Diagnostics Skill ---
#:get-oc-config-dir
#:prompt-for
#:save-secret
;; --- Tool Permissions Skill ---
#:get-tool-permission
#:set-tool-permission
#:check-tool-permission-gate
@@ -168,54 +147,61 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
#:cognitive-tool-parameters
#:cognitive-tool-guard
#:cognitive-tool-body
;; --- Emacs Client Registry ---
#:*emacs-clients*
#:*clients-lock*
#:register-emacs-client
#:unregister-emacs-client
;; --- Probabilistic Engine ---
#:ask-probabilistic
#:register-probabilistic-backend
#:distill-prompt
#:*probabilistic-backends*
#:*provider-cascade*
;; --- Security Vault ---
#:vault-get-secret
#:vault-set-secret
;; --- Deterministic Logic ---
#:list-objects-with-attribute
#:deterministic-verify
;; --- AST Helpers ---
#:find-headline-missing-id))
#+end_src
** Package Implementation
The package implementation section defines the low-level utilities and global state that are shared across all harness components and skills.
*** Robust plist access (proto-get)
Retrieves a value from a plist, checking both upper and lowercase keyword variants. This is needed because different components use different keyword conventions.
#+begin_src lisp :tangle package.lisp
(in-package :opencortex)
(defun proto-get (plist key)
"Robustly retrieves a value from a plist, checking both uppercase and lowercase keyword versions."
"Robust plist accessor — checks both :KEY and :key variants."
(let* ((s (string key))
(up (intern (string-upcase s) :keyword))
(dn (intern (string-downcase s) :keyword)))
(or (getf plist up) (getf plist dn))))
#+end_src
*** Logging state
The harness maintains a bounded ring buffer of log messages for inclusion in LLM context. Access is thread-safe via a lock.
#+begin_src lisp :tangle package.lisp
(defvar *system-logs* nil)
(defvar *logs-lock* (bordeaux-threads:make-lock "harness-logs-lock"))
(defvar *max-log-history* 100)
#+end_src
*** Skill registry
The global registry of all loaded skills. This is the authoritative list that the deterministic engine iterates.
#+begin_src lisp :tangle package.lisp
(defvar *skills-registry* (make-hash-table :test 'equal)
"Global registry of all loaded skills.")
#+end_src
*** Skill telemetry
Tracks execution metrics per skill (count, duration, failures) for diagnostics and performance analysis.
#+begin_src lisp :tangle package.lisp
(defvar *skill-telemetry* (make-hash-table :test 'equal))
(defvar *telemetry-lock* (bordeaux-threads:make-lock "harness-telemetry-lock"))
(defun harness-track-telemetry (skill-name duration status)
"Updates performance metrics for a specific skill. Status should be :success or :rejected."
"Updates performance metrics for a skill. STATUS is :success or :rejected."
(when skill-name
(bordeaux-threads:with-lock-held (*telemetry-lock*)
(let ((entry (or (gethash skill-name *skill-telemetry*) (list :executions 0 :total-time 0 :failures 0))))
@@ -223,27 +209,37 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
(incf (getf entry :total-time) duration)
(when (eq status :rejected) (incf (getf entry :failures)))
(setf (gethash skill-name *skill-telemetry*) entry)))))
#+end_src
*** Cognitive tool registry
Tools that the LLM can invoke are registered here. Each tool has a name, description, parameters, optional guard, and implementation body. The ~def-cognitive-tool~ macro handles registration. ~generate-tool-belt-prompt~ serialises the registry into the LLM's system prompt.
#+begin_src lisp :tangle package.lisp
(defvar *cognitive-tools* (make-hash-table :test 'equal))
#+end_src
#+begin_src lisp :tangle package.lisp
(defstruct cognitive-tool
name
description
parameters
guard
body)
#+end_src
#+begin_src lisp :tangle package.lisp
(defmacro def-cognitive-tool (name description parameters &key guard body)
"Registers a new cognitive tool into the global registry. Parameters must be a list of property lists."
"Registers a cognitive tool. PARAMETERS is a list of plists, one per parameter."
`(setf (gethash (string-downcase (string ',name)) *cognitive-tools*)
(make-cognitive-tool :name (string-downcase (string ',name))
:description ,description
:parameters ',parameters
:guard ,guard
:body ,body)))
:body ,body)))
#+end_src
#+begin_src lisp :tangle package.lisp
(defun generate-tool-belt-prompt ()
"Generates a prompt string describing all available cognitive tools."
"Serialises all registered tools into a prompt string for the LLM."
(let ((descriptions nil))
(maphash (lambda (k tool)
(declare (ignore k))
@@ -252,13 +248,17 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
(cognitive-tool-description tool)
(cognitive-tool-parameters tool))
descriptions))
*cognitive-tools*)
*cognitive-tools*)
(if descriptions
(format nil "Available tools:~%~a" (apply #'concatenate 'string (sort descriptions #'string<)))
"No tools registered.")))
#+end_src
*** Centralized logging (harness-log)
Thread-safe logging function that writes to both the ring buffer (for LLM context) and stdout (for the user). Bounded by ~*max-log-history*~.
#+begin_src lisp :tangle package.lisp
(defun harness-log (msg &rest args)
"Centralized logging for the harness."
"Centralized, thread-safe logging for the harness."
(let ((formatted-msg (apply #'format nil msg args)))
(bordeaux-threads:with-lock-held (*logs-lock*)
(push formatted-msg *system-logs*)
@@ -266,8 +266,10 @@ The ~package.lisp~ file defines the public API of the ~opencortex~ harness.
(setq *system-logs* (subseq *system-logs* 0 *max-log-history*))))
(format t "~a~%" formatted-msg)
(finish-output)))
#+end_src
;; --- Debugger Hook ---
*** Debugger hook
Friendly error handler that replaces the raw SBCL debugger with a diagnostic message. This prevents the agent from entering the debugger on unhandled conditions.
#+begin_src lisp :tangle package.lisp
(setf *debugger-hook* (lambda (condition hook)
"Friendly error handler - shows diagnostic message instead of raw debugger."

View File

@@ -14,14 +14,21 @@ The Reason stage implements the core Innovation of OpenCortex: the separation of
(in-package :opencortex)
#+end_src
** Probabilistic Engine Configuration
** Probabilistic Engine state
~*probabilistic-backends*~ is the hash table mapping provider keywords to backend functions. ~*provider-cascade*~ is the ordered list of providers to try. ~*model-selector-fn*~ is an optional function that selects a model per request. ~*consensus-enabled-p*~ enables multi-provider agreement.
#+begin_src lisp
(defvar *probabilistic-backends* (make-hash-table :test 'equal))
#+end_src
#+begin_src lisp
(defvar *provider-cascade* nil)
#+end_src
#+begin_src lisp
(defvar *model-selector-fn* nil)
#+end_src
#+begin_src lisp
(defvar *consensus-enabled-p* nil)
#+end_src
@@ -92,8 +99,19 @@ The Reason stage implements the core Innovation of OpenCortex: the separation of
(reflection-feedback (if rejection-trace
(format nil "~%~%PREVIOUS PROPOSAL REJECTED: ~a" rejection-trace)
""))
(system-prompt (format nil "IDENTITY: ~a~a~%~%TOOLS:~%~a~%~%CONTEXT:~%~a~%~%LOGS:~%~a"
assistant-name reflection-feedback tool-belt global-context system-logs)))
(skill-augments (let ((augments ""))
(maphash (lambda (name skill)
(declare (ignore name))
(let ((aug-fn (skill-system-prompt-augment skill)))
(when aug-fn
(let ((aug-text (ignore-errors (funcall aug-fn context))))
(when (and aug-text (stringp aug-text) (> (length aug-text) 0))
(setf augments (concatenate 'string augments aug-text (string #\Newline))))))))
*skills-registry*)
(when (> (length augments) 0) augments)))
(system-prompt (format nil "IDENTITY: ~a~a~%~%TOOLS:~%~a~%~%CONTEXT:~%~a~%~%LOGS:~%~a~%~a"
assistant-name reflection-feedback tool-belt global-context system-logs
(or skill-augments ""))))
(let* ((thought (probabilistic-call raw-prompt :system-prompt system-prompt :context context))
(cleaned (strip-markdown thought)))
(if (and cleaned (stringp cleaned) (> (length cleaned) 0) (or (char= (char cleaned 0) #\() (char= (char cleaned 0) #\[)))

View File

@@ -14,10 +14,12 @@
(defun VAULT-MASK-STRING (s) (declare (ignore s)) "[MASKED]")
(defvar *VAULT-MEMORY* (make-hash-table :test 'equal))
(defstruct skill name priority dependencies trigger-fn probabilistic-prompt deterministic-fn)
(defstruct skill name priority dependencies trigger-fn probabilistic-prompt deterministic-fn system-prompt-augment)
(defvar *skills-registry* (make-hash-table :test 'equal))
(defvar *skill-catalog* (make-hash-table :test 'equal)
"A stateful tracking table for all skill files discovered in the environment.")
"Tracks all discovered skill files and their loading state.")
(defstruct skill-entry filename (status :discovered) error-log (load-time 0))
@@ -29,21 +31,22 @@
(when (and (skill-probabilistic-prompt skill)
(ignore-errors (funcall (skill-trigger-fn skill) context)))
(push skill triggered)))
*skills-registry*)
*skills-registry*)
(first (sort triggered #'> :key #'skill-priority))))
(defmacro defskill (name &key priority dependencies trigger probabilistic deterministic)
"Registers a new skill into the global registry."
(defmacro defskill (name &key priority dependencies trigger probabilistic deterministic system-prompt-augment)
"Registers a new skill. NAME is a keyword. TRIGGER is a function (context) → bool."
`(setf (gethash (string-downcase (string ,name)) *skills-registry*)
(make-skill :name (string-downcase (string ,name))
:priority (or ,priority 10)
:dependencies ',dependencies
:trigger-fn ,trigger
:probabilistic-prompt ,probabilistic
:deterministic-fn ,deterministic)))
:deterministic-fn ,deterministic
:system-prompt-augment ,system-prompt-augment)))
(defun resolve-skill-dependencies (skill-name)
"Recursively resolves dependencies for a given skill name."
"Resolves transitive dependencies. Returns list of skill names in dependency order."
(let ((resolved nil) (seen nil))
(labels ((visit (name)
(unless (member name seen :test #'equal)
@@ -88,7 +91,9 @@
(if (uiop:string-suffix-p (namestring file) ".lisp")
(progn
(setf (gethash (string-downcase filename) name-to-file) file)
(setf (gethash (string-downcase filename) adj) nil))
;; Don't overwrite dependency info from .org files
(unless (gethash (string-downcase filename) adj)
(setf (gethash (string-downcase filename) adj) nil)))
(multiple-value-bind (id deps) (parse-skill-metadata file)
(setf (gethash (string-downcase filename) name-to-file) file)
(when id (setf (gethash (string-downcase id) id-to-file) file))
@@ -258,9 +263,9 @@
(setf (skill-entry-status entry) :failed) nil))))
(defun initialize-all-skills ()
"Initializes all skills from SKILLS_DIR."
(let* ((env-path (uiop:getenv "SKILLS_DIR"))
(skills-dir (uiop:ensure-directory-pathname (or env-path (namestring (merge-pathnames "notes/" (user-homedir-pathname)))))))
"Initializes all skills from the XDG skills directory."
(let* ((data-dir (uiop:ensure-directory-pathname (or (uiop:getenv "OC_DATA_DIR") (namestring (merge-pathnames ".local/share/opencortex/" (user-homedir-pathname))))))
(skills-dir (merge-pathnames "skills/" data-dir)))
(unless (uiop:directory-exists-p skills-dir) (return-from initialize-all-skills nil))
(let ((sorted-files (topological-sort-skills skills-dir)))
(harness-log "LOADER: Initializing ~a skills..." (length sorted-files))

View File

@@ -5,7 +5,16 @@
#+PROPERTY: header-args:lisp :tangle skills.lisp
* Overview
The ~opencortex~ Skill Engine enables **Late-Binding Intelligence**, allowing the system to discover and integrate new cognitive capabilities at runtime.
The Skill Engine is the dynamic loading and lifecycle manager for all OpenCortex skills. It discovers skill files in the skills directory, resolves their dependency order, loads them into jailed packages, exports their public symbols into the ~opencortex~ package, and provides the ~defskill~ macro that skills use to register themselves.
Key concepts:
- ~defskill~ — macro that registers a skill with its trigger, deterministic gate, and optional probabilistic prompt
- ~def-cognitive-tool~ — macro that registers a tool the LLM can invoke
- ~load-skill-from-org~ / ~load-skill-from-lisp~ — load a skill from a literate Org file or a pre-tangled Lisp file
- ~topological-sort-skills~ — orders skills by their ~#+DEPENDS_ON:~ declarations
- ~find-triggered-skill~ — returns the highest-priority skill whose trigger matches the current context
The engine supports **hot-reload** — skills can be replaced at runtime without restarting the daemon.
* Implementation
@@ -14,7 +23,11 @@ The ~opencortex~ Skill Engine enables **Late-Binding Intelligence**, allowing th
(in-package :opencortex)
#+end_src
** Global Skill Registry
** Utility functions
Helper functions used by the skill loader and other components.
*** Cosine similarity
Computes the cosine similarity between two numeric vectors. Used by the peripheral vision system for semantic relevance scoring.
#+begin_src lisp
(defun COSINE-SIMILARITY (v1 v2)
"Computes cosine similarity between two vectors."
@@ -26,17 +39,37 @@ The ~opencortex~ Skill Engine enables **Late-Binding Intelligence**, allowing th
(let* ((x (coerce (elt v1 i) 'double-float)) (y (coerce (elt v2 i) 'double-float)))
(incf dot (* x y)) (incf n1 (* x x)) (incf n2 (* y y))))
(if (or (zerop n1) (zerop n2)) 0.0 (/ dot (sqrt (* n1 n2))))))))
#+end_src
*** Secret masking
Simple mask function and the vault memory hash table. Used by the Bouncer skill and credentials vault.
#+begin_src lisp
(defun VAULT-MASK-STRING (s) (declare (ignore s)) "[MASKED]")
(defvar *VAULT-MEMORY* (make-hash-table :test 'equal))
#+end_src
(defstruct skill name priority dependencies trigger-fn probabilistic-prompt deterministic-fn)
** Skill data structures
The ~skill~ struct holds all metadata about a loaded skill: its name, priority, dependencies, trigger function, probabilistic prompt generator, deterministic gate, and system prompt augmentor. The ~skill-entry~ struct tracks the loading state of each discovered skill file.
#+begin_src lisp
(defstruct skill name priority dependencies trigger-fn probabilistic-prompt deterministic-fn system-prompt-augment)
#+end_src
#+begin_src lisp
(defvar *skills-registry* (make-hash-table :test 'equal))
#+end_src
#+begin_src lisp
(defvar *skill-catalog* (make-hash-table :test 'equal)
"A stateful tracking table for all skill files discovered in the environment.")
"Tracks all discovered skill files and their loading state.")
#+end_src
#+begin_src lisp
(defstruct skill-entry filename (status :discovered) error-log (load-time 0))
#+end_src
** Skill discovery (find-triggered-skill)
Iterates the registry and returns the highest-priority skill whose trigger function matches the current context. Only skills with a probabilistic prompt are considered (skills that are purely deterministic don't need LLM intervention).
#+begin_src lisp
(defun find-triggered-skill (context)
"Returns the highest priority skill whose trigger matches context."
(let ((triggered nil))
@@ -45,21 +78,30 @@ The ~opencortex~ Skill Engine enables **Late-Binding Intelligence**, allowing th
(when (and (skill-probabilistic-prompt skill)
(ignore-errors (funcall (skill-trigger-fn skill) context)))
(push skill triggered)))
*skills-registry*)
*skills-registry*)
(first (sort triggered #'> :key #'skill-priority))))
#+end_src
(defmacro defskill (name &key priority dependencies trigger probabilistic deterministic)
"Registers a new skill into the global registry."
** Skill registration macro (defskill)
The primary API for skills. Each skill file calls this once to register itself. The macro creates a ~skill~ struct and stores it in ~*skills-registry*~ keyed by the skill's name.
#+begin_src lisp
(defmacro defskill (name &key priority dependencies trigger probabilistic deterministic system-prompt-augment)
"Registers a new skill. NAME is a keyword. TRIGGER is a function (context) → bool."
`(setf (gethash (string-downcase (string ,name)) *skills-registry*)
(make-skill :name (string-downcase (string ,name))
:priority (or ,priority 10)
:dependencies ',dependencies
:trigger-fn ,trigger
:probabilistic-prompt ,probabilistic
:deterministic-fn ,deterministic)))
:deterministic-fn ,deterministic
:system-prompt-augment ,system-prompt-augment)))
#+end_src
** Dependency resolution (resolve-skill-dependencies)
Recursively resolves all transitive dependencies for a given skill, returning an ordered list. Uses a standard topological sort with cycle detection (a ~seen~ set prevents infinite recursion).
#+begin_src lisp
(defun resolve-skill-dependencies (skill-name)
"Recursively resolves dependencies for a given skill name."
"Resolves transitive dependencies. Returns list of skill names in dependency order."
(let ((resolved nil) (seen nil))
(labels ((visit (name)
(unless (member name seen :test #'equal)
@@ -110,7 +152,9 @@ The ~opencortex~ Skill Engine enables **Late-Binding Intelligence**, allowing th
(if (uiop:string-suffix-p (namestring file) ".lisp")
(progn
(setf (gethash (string-downcase filename) name-to-file) file)
(setf (gethash (string-downcase filename) adj) nil))
;; Don't overwrite dependency info from .org files
(unless (gethash (string-downcase filename) adj)
(setf (gethash (string-downcase filename) adj) nil)))
(multiple-value-bind (id deps) (parse-skill-metadata file)
(setf (gethash (string-downcase filename) name-to-file) file)
(when id (setf (gethash (string-downcase id) id-to-file) file))
@@ -286,9 +330,9 @@ The ~opencortex~ Skill Engine enables **Late-Binding Intelligence**, allowing th
** Initialize (initialize-all-skills)
#+begin_src lisp
(defun initialize-all-skills ()
"Initializes all skills from SKILLS_DIR."
(let* ((env-path (uiop:getenv "SKILLS_DIR"))
(skills-dir (uiop:ensure-directory-pathname (or env-path (namestring (merge-pathnames "notes/" (user-homedir-pathname)))))))
"Initializes all skills from the XDG skills directory."
(let* ((data-dir (uiop:ensure-directory-pathname (or (uiop:getenv "OC_DATA_DIR") (namestring (merge-pathnames ".local/share/opencortex/" (user-homedir-pathname))))))
(skills-dir (merge-pathnames "skills/" data-dir)))
(unless (uiop:directory-exists-p skills-dir) (return-from initialize-all-skills nil))
(let ((sorted-files (topological-sort-skills skills-dir)))
(harness-log "LOADER: Initializing ~a skills..." (length sorted-files))

View File

@@ -1,124 +0,0 @@
(defpackage :opencortex-tui-tests
(:use :cl :opencortex)
(:export #:tui-suite))
(in-package :opencortex-tui-tests)
(eval-when (:compile-toplevel :load-toplevel :execute)
(ql:quickload :fiveam :silent t))
(fiveam:def-suite tui-suite :description "Verification of the TUI parsing and styling logic")
(fiveam:in-suite tui-suite)
(fiveam:test test-tui-connection-drop
"Tier 2 Chaos: Verify that handle-return degrades gracefully when the daemon connection is lost."
;; Create a closed stream to simulate connection drop
(mock-stream (make-string-output-stream)))
(close mock-stream)
(opencortex.tui::handle-return mock-stream)
;; Check if the error was enqueued to history instead of crashing
(in-package :cl-user)
(defpackage :opencortex.tui
(:use :cl :croatoan :usocket :bordeaux-threads)
(:export :main))
(in-package :opencortex.tui)
(defun enqueue-msg (msg)
"Thread-safe addition to incoming message queue."
(defun dequeue-msgs ()
"Thread-safe retrieval of incoming messages."
msgs)))
(defun get-line-style (text)
(cond
((uiop:string-prefix-p "⬆" text) '(:cyan))
((uiop:string-prefix-p "🤔" text) '(:italic))
((uiop:string-prefix-p "ERROR" text) '(:bold :red))
(t nil)))
(defun render-chat (win)
(clear win)
(view-height (max 0 (- h 2)))
(end-idx (min history-len (+ start-idx view-height)))
(loop for msg in slice
for i from 1
do (add-string win (format nil "│ ~a" msg) :y i :x 1 :attributes (get-line-style msg)))
(refresh win)))
(defun handle-backspace ()
(defun handle-return (stream)
(when (> (length cmd) 0)
(enqueue-msg (format nil "⬆ ~a" cmd))
(handler-case
(progn
(when (and stream (open-stream-p stream))
:META (list :SOURCE :tui)
:PAYLOAD (list :SENSOR :user-input :TEXT cmd)))
(payload (format nil "~s" msg))
(len (length payload)))
(format stream "~6,'0x~a" len payload)
(finish-output stream)))
(enqueue-msg "✓ Sent"))
(error (c)
(format t "Send error: ~a~%" c)
(enqueue-msg "ERROR: Connection to daemon lost.")
(defun start-background-reader (stream)
"Starts a thread that reads framed messages from the daemon stream."
(bt:make-thread
(lambda ()
(handler-case
(count (read-sequence len-buf stream)))
(when (= count 6)
(msg-buf (make-string msg-len)))
(read-sequence msg-buf stream)
(let ((msg (read-from-string msg-buf)))
(let ((payload (getf msg :payload)))
(cond
((eq (getf payload :action) :handshake)
((and (eq (getf payload :sensor) :loop-error)
(not (string= (or (getf payload :message) "") "Neural Cascade Failure: All providers exhausted.")))
(enqueue-msg (format nil "ERROR: Daemon loop error (~a)"
(getf payload :message))))
(t
(let ((text (or (getf payload :text) (format nil "~a" payload))))
(enqueue-msg (format nil "⬇ ~a" text)))))))))
(error (c)
(enqueue-msg (format nil "ERROR: Connection lost (~a)" c))
:name "opencortex-tui-reader")))
(defun main ()
(handler-case
(error (e) (format t "Offline: ~a~%" e) (return-from main)))
;; Guard: Croatoan needs a real terminal (TERM env var, real TTY)
(unless (uiop:getenv "TERM")
(format t "TUI requires a terminal. Set TERM environment variable.~%")
(format t "Or use: echo 'your message' | nc localhost 9105~%")
(return-from main))
(unwind-protect
(handler-case
(with-screen (scr :input-echoing nil :input-blocking nil :enable-colors t)
(let ((chat-win (make-instance 'window :height (- h 5) :width (- w 2) :position '(1 1) :border t))
(input-win (make-instance 'window :height 1 :width (- w 2) :position (list (- h 2) 1) :border t)))
(setf (input-blocking input-win) nil)
(let ((msgs (dequeue-msgs)))
(when msgs
(render-chat chat-win)))
(ch (when (and ev (typep ev 'event)) (event-key ev))))
(when ch
(cond
((or (eq ch :backspace) (eq ch (code-char 127))) (handle-backspace))
(clear input-win)
(refresh input-win))
(sleep 0.02)))))
(error (c)
(format t "TUI Error: ~a~%" c)))

View File

@@ -4,7 +4,7 @@
#+PROPERTY: header-args:lisp :tangle tui-client.lisp
* Overview
The OpenCortex TUI Client is a standalone Common Lisp application built on **Croatoan**.
The TUI Client is a standalone ncurses application (built on Croatoan) that connects to the daemon via TCP. It provides a split-pane interface: a scrollable chat history window and a fixed input line at the bottom. Connected to the daemon at ~localhost:9105~, it sends user input as framed protocol messages and displays responses as they arrive from the daemon's background reader thread.
* Implementation
@@ -17,16 +17,42 @@ The OpenCortex TUI Client is a standalone Common Lisp application built on **Cro
(in-package :opencortex.tui)
#+end_src
** Global State
** Connection state
#+begin_src lisp
(defvar *daemon-host* "localhost")
#+end_src
#+begin_src lisp
(defvar *daemon-host* "127.0.0.1")
(defvar *daemon-port* 9105)
#+end_src
#+begin_src lisp
(defvar *socket* nil)
#+end_src
#+begin_src lisp
(defvar *stream* nil)
#+end_src
** UI state
#+begin_src lisp
(defvar *chat-history* nil)
(defvar *input-list* nil) ; List of characters (stored in reverse)
#+end_src
#+begin_src lisp
(defvar *input-list* nil)
#+end_src
#+begin_src lisp
(defvar *is-running* t)
(defvar *queue-lock* (bt:make-lock))
#+end_src
** Thread-safe message queue
#+begin_src lisp
(defvar *queue-lock* (bt:make-lock "incoming-queue-lock"))
#+end_src
#+begin_src lisp
(defvar *incoming-msgs* nil)
#+end_src
@@ -176,3 +202,28 @@ The OpenCortex TUI Client is a standalone Common Lisp application built on **Cro
(setf *is-running* nil)
(when *socket* (ignore-errors (usocket:socket-close *socket*)))))
#+end_src
** REPL test script (tmux)
Use this script to test the TUI non-interactively in a tmux session. It launches the TUI in a headless tmux window, sends text, and captures the output.
#+begin_src bash :tangle no
#!/bin/bash
SESSION="oct-tui-test"
tmux new-session -d -s "$SESSION" \
-e OC_CONFIG_DIR="$HOME/.config/opencortex" \
-e OC_DATA_DIR="$HOME/.local/share/opencortex" \
-e TERM="screen-256color" \
"sbcl --non-interactive \
--eval '(load (merge-pathnames \"quicklisp/setup.lisp\" (user-homedir-pathname)))' \
--eval '(push (truename \"$HOME/.local/share/opencortex/\") asdf:*central-registry*)' \
--eval '(ql:quickload :opencortex/tui)' \
--eval '(opencortex.tui:main)'"
sleep 5
tmux capture-pane -t "$SESSION" -p -S -20
tmux send-keys -t "$SESSION" 'hello' Enter
sleep 8
tmux capture-pane -t "$SESSION" -p -S -20
tmux send-keys -t "$SESSION" '/exit' Enter
sleep 1
tmux kill-session -t "$SESSION" 2>/dev/null || true
#+end_src