passepartout/harness/package.org

#+TITLE: System Interface (package.lisp)
#+AUTHOR: Agent
#+FILETAGS: :harness:interface:
#+STARTUP: content
#+PROPERTY: header-args:lisp :tangle package.lisp

* Overview
~package.lisp~ defines two things: the public API of the ~opencortex~ package (the export list, above), and the implementation of low-level utility functions and global state that don't belong in a specific pipeline stage or skill.

The export list is the contract between the harness and all skills. Every function exported here is accessible to every skill via ~use-package~. Adding a symbol here is an API commitment; removing one is a breaking change.

The implementation section includes:
- ~proto-get~ — robust plist accessor used everywhere
- Logging state (~*system-logs*~, ~*logs-lock*~)
- Skill registry (~*skills-registry*~, ~defskill~)
- Cognitive tool registry (~*cognitive-tools*~, ~def-cognitive-tool~)
- Configuration variables (~*privacy-filter-tags*~, ~*secret-protected-paths*~, ~*secret-exposure-patterns*~)
- Debugger hook

* Implementation

** Package Definition and Export List
The package definition. All public symbols are exported here.
#+begin_src lisp :tangle package.lisp
(defpackage :opencortex
  (:use :cl)
  (:export
   #:frame-message
   #:read-framed-message
   #:PROTO-GET
   #:LIST-OBJECTS-WITH-ATTRIBUTE
   #:COSINE-SIMILARITY
   #:VAULT-MASK-STRING
   #:*VAULT-MEMORY*
   #:parse-message
   #:make-hello-message
   #:validate-communication-protocol-schema
   #:start-daemon
   #:stop-daemon
   #:harness-log
   #:main
   #:doctor-run-all
   #:doctor-main
   #:doctor-check-dependencies
   #:doctor-check-env
   #:register-provider
   #:system-ready-p
   #:run-setup-wizard
   #:skill-gateway-register
   #:skill-gateway-link
   #:gateway-manager-main
   #:ingest-ast
   #:lookup-object
   #:list-objects-by-type
   #:org-id-new
   #:*memory*
   #:*history-store*
   #:org-object
   #:make-org-object
   #:org-object-id
   #:org-object-type
   #:org-object-attributes
   #:org-object-parent-id
   #:org-object-children
   #:org-object-version
   #:org-object-last-sync
   #:org-object-vector
   #:org-object-content
   #:org-object-hash
   #:snapshot-memory
   #:rollback-memory
   #:context-query-store
   #:context-get-active-projects
   #:context-get-recent-completed-tasks
   #:context-list-all-skills
   #:context-get-skill-source
   #:context-get-system-logs
   #:context-resolve-path
   #:context-get-skill-telemetry
   #:harness-track-telemetry
   #:context-assemble-global-awareness
   #:process-signal
   #:perceive-gate
   #:probabilistic-gate
   #:consensus-gate
   #:act-gate
   #:reason-gate
   #:dispatch-gate
   #:inject-stimulus
   #:initialize-actuators
   #:dispatch-action
   #:register-actuator
   #:load-skill-from-org
   #:initialize-all-skills
   #:load-skill-with-timeout
   #:topological-sort-skills
   #:validate-lisp-syntax
   #:defskill
   #:*skills-registry*
   #:skill
   #:skill-name
   #:skill-priority
   #:skill-dependencies
   #:skill-trigger-fn
   #:skill-probabilistic-prompt
   #:skill-deterministic-fn
   #:def-cognitive-tool
   #:*cognitive-tools*
   #:verify-git-clean-p
   #:engineering-standards-verify-lisp
   #:engineering-standards-format-lisp
   #:literate-check-block-balance
   #:check-tangle-sync
   #:*tangle-targets*
   #:utils-org-read-file
   #:utils-org-write-file
   #:utils-org-add-headline
   #:utils-org-set-property
   #:utils-org-set-todo
   #:utils-org-find-headline-by-id
   #:utils-org-find-headline-by-title
   #:utils-org-generate-id
   #:utils-org-id-format
   #:utils-org-ast-to-org
   #:utils-org-modify
   #:utils-lisp-validate
   #:utils-lisp-check-structural
   #:utils-lisp-check-syntactic
   #:utils-lisp-check-semantic
   #:utils-lisp-eval
   #:utils-lisp-format
   #:utils-lisp-list-definitions
   #:utils-lisp-structural-extract
   #:utils-lisp-structural-wrap
   #:utils-lisp-structural-inject
   #:utils-lisp-structural-slurp
   #:utils-lisp-register
   #:get-oc-config-dir
   #:prompt-for
   #:save-secret
   #:get-tool-permission
   #:set-tool-permission
   #:check-tool-permission-gate
   #:cognitive-tool
   #:cognitive-tool-name
   #:cognitive-tool-description
   #:cognitive-tool-parameters
   #:cognitive-tool-guard
   #:cognitive-tool-body
   #:*emacs-clients*
   #:*clients-lock*
   #:register-emacs-client
   #:unregister-emacs-client
   #:ask-probabilistic
   #:register-probabilistic-backend
   #:distill-prompt
   #:*probabilistic-backends*
   #:*provider-cascade*
   #:vault-get-secret
   #:vault-set-secret
   #:list-objects-with-attribute
   #:deterministic-verify
   #:find-headline-missing-id))
#+end_src

** Package Implementation
The package implementation section defines the low-level utilities and global state that are shared across all harness components and skills.

*** Robust plist access (proto-get)
Retrieves a value from a plist, checking both upper and lowercase keyword variants. This is needed because different components use different keyword conventions.
#+begin_src lisp :tangle package.lisp
(in-package :opencortex)

(defun proto-get (plist key)
  "Robust plist accessor — checks both :KEY and :key variants."
  (let* ((s (string key))
         (up (intern (string-upcase s) :keyword))
         (dn (intern (string-downcase s) :keyword)))
    (or (getf plist up) (getf plist dn))))
#+end_src

*** Logging state
The harness maintains a bounded ring buffer of log messages for inclusion in LLM context. Access is thread-safe via a lock.
#+begin_src lisp :tangle package.lisp
(defvar *system-logs* nil)
(defvar *logs-lock* (bordeaux-threads:make-lock "harness-logs-lock"))
(defvar *max-log-history* 100)
#+end_src

*** Skill registry
The global registry of all loaded skills. This is the authoritative list that the deterministic engine iterates.
#+begin_src lisp :tangle package.lisp
(defvar *skills-registry* (make-hash-table :test 'equal)
  "Global registry of all loaded skills.")
#+end_src

*** Skill telemetry
Tracks execution metrics per skill (count, duration, failures) for diagnostics and performance analysis.
#+begin_src lisp :tangle package.lisp
(defvar *skill-telemetry* (make-hash-table :test 'equal))
(defvar *telemetry-lock* (bordeaux-threads:make-lock "harness-telemetry-lock"))

(defun harness-track-telemetry (skill-name duration status)
  "Updates performance metrics for a skill. STATUS is :success or :rejected."
  (when skill-name
    (bordeaux-threads:with-lock-held (*telemetry-lock*)
      (let ((entry (or (gethash skill-name *skill-telemetry*) (list :executions 0 :total-time 0 :failures 0))))
        (incf (getf entry :executions))
        (incf (getf entry :total-time) duration)
        (when (eq status :rejected) (incf (getf entry :failures)))
        (setf (gethash skill-name *skill-telemetry*) entry)))))
#+end_src

*** Cognitive tool registry
Tools that the LLM can invoke are registered here. Each tool has a name, description, parameters, optional guard, and implementation body. The ~def-cognitive-tool~ macro handles registration. ~generate-tool-belt-prompt~ serialises the registry into the LLM's system prompt.
#+begin_src lisp :tangle package.lisp
(defvar *cognitive-tools* (make-hash-table :test 'equal))
#+end_src

#+begin_src lisp :tangle package.lisp
(defstruct cognitive-tool
  name
  description
  parameters
  guard
  body)
#+end_src

#+begin_src lisp :tangle package.lisp
(defmacro def-cognitive-tool (name description parameters &key guard body)
  "Registers a cognitive tool. PARAMETERS is a list of plists, one per parameter."
  `(setf (gethash (string-downcase (string ',name)) *cognitive-tools*)
         (make-cognitive-tool :name (string-downcase (string ',name))
                              :description ,description
                              :parameters ',parameters
                              :guard ,guard
                              :body ,body)))
#+end_src

#+begin_src lisp :tangle package.lisp
(defun generate-tool-belt-prompt ()
  "Serialises all registered tools into a prompt string for the LLM."
  (let ((descriptions nil))
    (maphash (lambda (k tool)
               (declare (ignore k))
               (push (format nil "- ~a: ~a~%  Parameters: ~a~%"
                             (cognitive-tool-name tool)
                             (cognitive-tool-description tool)
                             (cognitive-tool-parameters tool))
                     descriptions))
              *cognitive-tools*)
    (if descriptions
        (format nil "Available tools:~%~a" (apply #'concatenate 'string (sort descriptions #'string<)))
        "No tools registered.")))
#+end_src

*** Centralized logging (harness-log)
Thread-safe logging function that writes to both the ring buffer (for LLM context) and stdout (for the user). Bounded by ~*max-log-history*~.
#+begin_src lisp :tangle package.lisp
(defun harness-log (msg &rest args)
  "Centralized, thread-safe logging for the harness."
  (let ((formatted-msg (apply #'format nil msg args)))
    (bordeaux-threads:with-lock-held (*logs-lock*)
      (push formatted-msg *system-logs*)
      (when (> (length *system-logs*) *max-log-history*)
        (setq *system-logs* (subseq *system-logs* 0 *max-log-history*))))
    (format t "~a~%" formatted-msg)
    (finish-output)))
#+end_src

*** Debugger hook
Friendly error handler that replaces the raw SBCL debugger with a diagnostic message. This prevents the agent from entering the debugger on unhandled conditions.
#+begin_src lisp :tangle package.lisp
(setf *debugger-hook* (lambda (condition hook)
  "Friendly error handler - shows diagnostic message instead of raw debugger."
  (declare (ignore hook))
  (format t "~%")
  (format t "┌─────────────────────────────────────────────┐~%")
  (format t "│  ERROR: ~A~%" (type-of condition))
  (format t "│~%")
  (format t "│  Run: opencortex doctor~%")
  (format t "│  For system diagnostics~%")
  (format t "└─────────────────────────────────────────────┘~%")
  (format t "~%")
  (format t "Details: ~A~%" condition)
  (finish-output)
  (uiop:quit 1)))
#+end_src