DOCS: Systematic overhaul of Literate source (Granularity & Technical Reasoning)

literate/package.org

@@ -22,7 +22,7 @@ flowchart TD
 (defpackage :opencortex
   (:use :cl)
   (:export 
-   ;; --- communication protocol ---
+   ;; --- Communication Protocol ---
    #:frame-message
    #:read-framed-message
    #:PROTO-GET
@@ -138,60 +138,72 @@ flowchart TD
    #:find-headline-missing-id))
 #+end_src
 
-#+begin_src lisp :tangle ../src/package.lisp
-(in-package :opencortex)
-
-(defun proto-get (plist key)
-  "Robustly retrieves a value from a plist, checking both uppercase and lowercase keyword versions."
-  (let* ((s (string key))
-         (up (intern (string-upcase s) :keyword))
-         (dn (intern (string-downcase s) :keyword)))
-    (or (getf plist up) (getf plist dn))))
-#+end_src
-
-#+end_src
-
-#+begin_src lisp :tangle ../src/package.lisp
-(in-package :opencortex)
-
-(defun proto-get (plist key)
-  "Robustly retrieves a value from a plist, checking both uppercase and lowercase keyword versions."
-  (let* ((s (string key))
-         (up (intern (string-upcase s) :keyword))
-         (dn (intern (string-downcase s) :keyword)))
-    (or (getf plist up) (getf plist dn))))
-#+end_src
-
-#+end_src
-
-** Package Implementation
+** Package Implementation Initialization
+Ensuring the compiler enters the correct namespace for all subsequent definitions.
 #+begin_src lisp :tangle ../src/package.lisp
 (in-package :opencortex)
 #+end_src
 
+* System State Management
+The package layer manages the core data structures that represent the live state of the harness.
+
 ** Harness Logging State
-The harness maintains a thread-safe circular log buffer to provide context for debugging and neural reasoning.
+OpenCortex maintains a thread-safe circular log buffer. This is critical for two reasons:
+1. *Neural Introspection:* The probabilistic engine can read the recent system logs to understand why an action failed.
+2. *Real-time Debugging:* Clients can subscribe to a live log stream without needing to read the physical log file.
 
 #+begin_src lisp :tangle ../src/package.lisp
-(defvar *system-logs* nil)
-(defvar *logs-lock* (bt:make-lock "harness-logs-lock"))
-(defvar *max-log-history* 100)
+(defvar *system-logs* nil
+  "Thread-safe list of the most recent system messages.")
+#+end_src
+
+#+begin_src lisp :tangle ../src/package.lisp
+(defvar *logs-lock* (bt:make-lock "harness-logs-lock")
+  "Protects the circular log buffer from race conditions during concurrent skill execution.")
+#+end_src
+
+#+begin_src lisp :tangle ../src/package.lisp
+(defvar *max-log-history* 100
+  "The maximum number of entries to preserve in the in-memory log buffer.")
 #+end_src
 
 ** Skills Registry
+All Literate Skills, once compiled, are registered here. This allows for topological sorting and priority-based execution.
+
 #+begin_src lisp :tangle ../src/package.lisp
 (defvar *skills-registry* (make-hash-table :test 'equal)
-  "Global registry of all loaded skills.")
+  "Global registry of all loaded skills, keyed by their unique identifier.")
 #+end_src
 
 ** Skill Telemetry State
+To ensure the system remains performant and reliable, the harness tracks execution metrics for every skill.
+
 #+begin_src lisp :tangle ../src/package.lisp
-(defvar *skill-telemetry* (make-hash-table :test 'equal))
-(defvar *telemetry-lock* (bt:make-lock "harness-telemetry-lock"))
+(defvar *skill-telemetry* (make-hash-table :test 'equal)
+  "Stores execution duration and failure counts for every registered skill.")
 #+end_src
 
-** Telemetry Implementation
-The system tracks the performance and reliability of individual skills. This logic is currently preserved in the package layer for future expansion into a dedicated telemetry skill.
+#+begin_src lisp :tangle ../src/package.lisp
+(defvar *telemetry-lock* (bt:make-lock "harness-telemetry-lock")
+  "Protects the telemetry store from concurrent updates.")
+#+end_src
+
+* Support Functions
+
+** Protocol Property Access (proto-get)
+Lisp keywords can be inconsistent between capitalized and lowercase versions depending on the client (e.g., Emacs vs. Python socket). ~proto-get~ provides a robust abstraction to ensure the system correctly extracts values regardless of keyword casing.
+
+#+begin_src lisp :tangle ../src/package.lisp
+(defun proto-get (plist key)
+  "Robustly retrieves a value from a plist, checking both uppercase and lowercase keyword versions."
+  (let* ((s (string key))
+         (up (intern (string-upcase s) :keyword))
+         (dn (intern (string-downcase s) :keyword)))
+    (or (getf plist up) (getf plist dn))))
+#+end_src
+
+** Telemetry Tracking
+The ~harness-track-telemetry~ function provides the hook for the metabolic loop to report performance data.
 
 #+begin_src lisp :tangle ../src/package.lisp
 (defun harness-track-telemetry (skill-name duration status)
@@ -205,21 +217,36 @@ The system tracks the performance and reliability of individual skills. This log
         (setf (gethash skill-name *skill-telemetry*) entry)))))
 #+end_src
 
-** Cognitive Tool Registry
-The Tool Registry allows the agent to interact with the physical world. Every tool must define a guard (for security) and a body (for execution).
+* Cognitive Tooling System
+The Tool Registry is the agent's physical interface. It separates the /proposal/ of an action from its /execution/.
+
+** Tool Structure
+#+begin_src lisp :tangle ../src/package.lisp
+(defvar *cognitive-tools* (make-hash-table :test 'equal)
+  "The active set of physical capabilities available to the agent.")
+#+end_src
 
 #+begin_src lisp :tangle ../src/package.lisp
-(defvar *cognitive-tools* (make-hash-table :test 'equal))
-
 (defstruct cognitive-tool
+  "Represents a physical or virtual capability with explicit documentation and security guards."
   name
   description
   parameters
   guard
   body)
+#+end_src
 
+** Tool Registration Macro (def-cognitive-tool)
+We use a macro to ensure that tools are consistently registered and accessible to the LLM's "tool-belt" prompt generator.
+
+#+begin_src lisp :tangle ../src/package.lisp
 (defmacro def-cognitive-tool (name description parameters &key guard body)
-  "Registers a new cognitive tool into the global registry. Parameters must be a list of property lists."
+  "Registers a new cognitive tool. 
+   NAME: Keyword identifier.
+   DESCRIPTION: Human-readable intent (used in LLM prompts).
+   PARAMETERS: List of property lists defining arguments.
+   GUARD: (context -> boolean) function to prevent unsafe calls.
+   BODY: The actual Lisp execution logic."
   `(setf (gethash (string-downcase (string ',name)) *cognitive-tools*)
          (make-cognitive-tool :name (string-downcase (string ',name))
                               :description ,description
@@ -228,12 +255,14 @@ The Tool Registry allows the agent to interact with the physical world. Every to
                               :body ,body)))
 #+end_src
 
-** Harness Logging Implementation
-Centralized logging function. It simultaneously writes to standard output and the in-memory circular buffer.
+* Logging Implementation
+
+** Centralized Logging (harness-log)
+The primary mechanism for system transparency. It ensures all activity is both visible to the user and recorded for neural reasoning.
 
 #+begin_src lisp :tangle ../src/package.lisp
 (defun harness-log (msg &rest args)
-  "Centralized logging for the harness."
+  "Centralized logging for the harness. Writes to STDOUT and the thread-safe circular buffer."
   (let ((formatted-msg (apply #'format nil msg args)))
     (bt:with-lock-held (*logs-lock*)
       (push formatted-msg *system-logs*)
@@ -242,5 +271,3 @@ Centralized logging function. It simultaneously writes to standard output and th
     (format t "~a~%" formatted-msg)
     (finish-output)))
 #+end_src
-
-