remediation: backfill v0.1.0/v0.2.0 gaps (P0+P1)
- vault: add vault-get-secret/vault-set-secret wrappers - programming-org: implement org-modify (text search-replace) and org-ast-render (AST to Org text) - programming-literate: implement literate-block-balance-check (paren validation) and literate-tangle-sync-check (org→lisp diff) - system-self-improve: replace stubs with surgical text editing and error diagnosis; remove dead first defskill - system-event-orchestrator: implement orchestrator-bootstrap (scan Org files for HOOK/CRON) - system-archivist: implement Scribe distillation (daily logs→atomic notes) and Gardener link/orphan repair - system-memory: implement memory-inspect with type/todo/orphan statistics - core-skills, core-context: fix path relic (skills/ → lisp/, org/) - docs: add Token Economics section to DESIGN_DECISIONS, remediation roadmap entries
This commit is contained in:
@@ -95,8 +95,8 @@ Reads the raw literate source of a specific skill for inspection. Used when the
|
||||
"Reads the raw literate source of a specific skill for inspection."
|
||||
(let* ((filename (format nil "~a.org" skill-name))
|
||||
(data-dir (uiop:ensure-directory-pathname (or (uiop:getenv "PASSEPARTOUT_DATA_DIR") (namestring (merge-pathnames ".local/share/passepartout/" (user-homedir-pathname))))))
|
||||
(skills-dir (merge-pathnames "skills/" data-dir))
|
||||
(full-path (merge-pathnames filename skills-dir)))
|
||||
(org-dir (merge-pathnames "org/" data-dir))
|
||||
(full-path (merge-pathnames filename org-dir)))
|
||||
(if (uiop:file-exists-p full-path) (uiop:read-file-string full-path) nil)))
|
||||
#+end_src
|
||||
|
||||
|
||||
@@ -392,13 +392,14 @@ The same jailed package and symbol export process applies.
|
||||
|
||||
Boot-time entry point. Scans the skills directory, topologically sorts the files, and loads each one. Called from ~main~ in the metabolic loop and from the REPL for hot-reload.
|
||||
|
||||
The skills directory is ~$OC_DATA_DIR/skills~ by default, which is populated by the ~configure~ script.
|
||||
Skills are loaded from ~$PASSEPARTOUT_DATA_DIR/lisp/~ where both core and skill
|
||||
files live after tangling. The org source files live in ~org/~.
|
||||
|
||||
#+begin_src lisp
|
||||
(defun skill-initialize-all ()
|
||||
"Initializes all skills from the XDG skills directory."
|
||||
"Initializes all skills from the XDG data directory."
|
||||
(let* ((data-dir (uiop:ensure-directory-pathname (or (uiop:getenv "PASSEPARTOUT_DATA_DIR") (namestring (merge-pathnames ".local/share/passepartout/" (user-homedir-pathname))))))
|
||||
(skills-dir (merge-pathnames "skills/" (uiop:ensure-directory-pathname data-dir))))
|
||||
(skills-dir (merge-pathnames "lisp/" (uiop:ensure-directory-pathname data-dir))))
|
||||
(unless (uiop:directory-exists-p skills-dir) (return-from skill-initialize-all nil))
|
||||
(let ((sorted-files (skill-topological-sort skills-dir)))
|
||||
(log-message "LOADER: Initializing ~a skills..." (length sorted-files))
|
||||
|
||||
@@ -34,17 +34,71 @@ The `.lisp` file is derived, not authored. Never edit `.lisp` directly. All chan
|
||||
|
||||
* Implementation
|
||||
|
||||
** Block Extraction
|
||||
#+begin_src lisp
|
||||
(defun literate-extract-lisp-blocks (content)
|
||||
"Extracts all #+begin_src lisp ... #+end_src blocks from Org CONTENT.
|
||||
Returns a list of block strings."
|
||||
(let ((lines (uiop:split-string content :separator '(#\Newline)))
|
||||
(blocks nil)
|
||||
(in-block nil)
|
||||
(current-block nil))
|
||||
(dolist (line lines)
|
||||
(let ((trimmed (string-trim '(#\Space) line)))
|
||||
(cond
|
||||
((uiop:string-prefix-p "#+begin_src lisp" trimmed)
|
||||
(setf in-block t current-block nil))
|
||||
((uiop:string-prefix-p "#+end_src" trimmed)
|
||||
(when in-block
|
||||
(push (format nil "~{~a~^~%~}" (nreverse current-block)) blocks)
|
||||
(setf in-block nil current-block nil)))
|
||||
(in-block
|
||||
(push line current-block)))))
|
||||
(nreverse blocks)))
|
||||
#+end_src
|
||||
|
||||
** Synchronization Logic
|
||||
#+begin_src lisp
|
||||
(defun literate-block-balance-check (org-file)
|
||||
"Verifies that all Lisp source blocks in an Org file are balanced."
|
||||
(log-message "LITERATE: Checking block balance for ~a" org-file)
|
||||
t)
|
||||
"Verifies that all Lisp source blocks in an Org file have balanced parentheses.
|
||||
Returns T if all blocks pass validation, or an error string listing failures."
|
||||
(when (not (uiop:file-exists-p org-file))
|
||||
(return-from literate-block-balance-check
|
||||
(format nil "Org file not found: ~a" org-file)))
|
||||
(let* ((content (uiop:read-file-string org-file))
|
||||
(blocks (literate-extract-lisp-blocks content))
|
||||
(failures nil))
|
||||
(if (null blocks)
|
||||
t
|
||||
(progn
|
||||
(loop for i from 0
|
||||
for block in blocks
|
||||
for (ok reason) = (multiple-value-list
|
||||
(lisp-structural-check block))
|
||||
unless ok
|
||||
do (push (format nil "Block ~d: ~a" (1+ i) reason) failures))
|
||||
(if failures
|
||||
(format nil "Unbalanced blocks in ~a:~%~{~a~^~%~}" org-file failures)
|
||||
t)))))
|
||||
|
||||
(defun literate-tangle-sync-check (org-file lisp-file)
|
||||
"Verifies that the Lisp file matches the tangled output of the Org file."
|
||||
(log-message "LITERATE: Checking tangle sync for ~a <-> ~a" org-file lisp-file)
|
||||
t)
|
||||
"Verifies that the .lisp file matches the tangled output of the .org file.
|
||||
Compares the concatenation of all lisp blocks from the Org file against the
|
||||
contents of the Lisp file. Returns T if they match, or an error message."
|
||||
(when (not (uiop:file-exists-p org-file))
|
||||
(return-from literate-tangle-sync-check
|
||||
(format nil "Org file not found: ~a" org-file)))
|
||||
(when (not (uiop:file-exists-p lisp-file))
|
||||
(return-from literate-tangle-sync-check
|
||||
(format nil "Lisp file not found: ~a" lisp-file)))
|
||||
(let* ((org-content (uiop:read-file-string org-file))
|
||||
(org-blocks (literate-extract-lisp-blocks org-content))
|
||||
(tangled (format nil "~{~a~^~%~%~}" org-blocks))
|
||||
(lisp-content (uiop:read-file-string lisp-file)))
|
||||
(if (string= (string-trim '(#\Space #\Newline) tangled)
|
||||
(string-trim '(#\Space #\Newline) lisp-content))
|
||||
t
|
||||
(format nil "Tangle sync mismatch: ~a does not match ~a" org-file lisp-file))))
|
||||
#+end_src
|
||||
|
||||
** Skill Registration
|
||||
|
||||
@@ -179,21 +179,71 @@ Returns the filtered content as a string."
|
||||
nil))
|
||||
#+end_src
|
||||
|
||||
** Placeholder for External Edits
|
||||
** Text Modification in Org Files
|
||||
Replaces text in Org files with verification. Used by =system-self-improve= for
|
||||
surgical edits.
|
||||
|
||||
#+begin_src lisp
|
||||
(defun org-modify (filepath id changes)
|
||||
"Placeholder for Emacs-driven modification of a specific node."
|
||||
(declare (ignore changes))
|
||||
(log-message "UTILS-ORG: Applying changes to ~a in ~a" id filepath)
|
||||
t)
|
||||
(defun org-modify (filepath old-text new-text)
|
||||
"Replaces all occurrences of OLD-TEXT with NEW-TEXT in filepath.
|
||||
Returns T if OLD-TEXT was found and replaced, nil if not found."
|
||||
(when (not (uiop:file-exists-p filepath))
|
||||
(log-message "UTILS-ORG: org-modify: file not found: ~a" filepath)
|
||||
(return-from org-modify nil))
|
||||
(let* ((content (uiop:read-file-string filepath))
|
||||
(pos (search old-text content :test #'string=)))
|
||||
(unless pos
|
||||
(log-message "UTILS-ORG: org-modify: text not found in ~a" filepath)
|
||||
(return-from org-modify nil))
|
||||
(let ((modified (cl-ppcre:regex-replace-all
|
||||
(cl-ppcre:quote-meta-chars old-text)
|
||||
content new-text)))
|
||||
(org-write-file filepath modified)
|
||||
(log-message "UTILS-ORG: Modified ~a (~d chars replaced)" filepath (length old-text))
|
||||
t)))
|
||||
#+end_src
|
||||
|
||||
** Placeholder for AST to Org conversion
|
||||
** AST to Org text conversion
|
||||
#+begin_src lisp
|
||||
(defun org-ast-render (ast)
|
||||
"Minimal converter from AST back to Org text (Placeholder)."
|
||||
(declare (ignore ast))
|
||||
"* TITLE (Placeholder)")
|
||||
(defun org-ast-render (ast &key (depth 1))
|
||||
"Converts a plist AST node back to Org text.
|
||||
AST format: (:TYPE :HEADLINE :properties (:ID ... :TITLE ... :TAGS (...))
|
||||
:contents (child-ast ...))"
|
||||
(let* ((type (getf ast :TYPE))
|
||||
(props (getf ast :properties))
|
||||
(title (or (getf props :TITLE) "Untitled"))
|
||||
(tags (getf props :TAGS))
|
||||
(todo (getf props :TODO-STATE))
|
||||
(children (getf ast :contents))
|
||||
(raw-content (getf ast :raw-content))
|
||||
(stars (make-string depth :initial-element #\*))
|
||||
(output ""))
|
||||
(unless (eq type :HEADLINE)
|
||||
(return-from org-ast-render (or raw-content "")))
|
||||
;; Headline
|
||||
(setf output (format nil "~a~@[ ~a~] ~a" stars todo title))
|
||||
(when tags
|
||||
(let ((tag-str (format nil "~{~a~^:~}" (mapcar (lambda (t) (string-trim '(#\:) t)) tags))))
|
||||
(setf output (concatenate 'string output (format nil " :~a::~%" tag-str))))
|
||||
(setf output (concatenate 'string output (string #\Newline))))
|
||||
(unless tags
|
||||
(setf output (concatenate 'string output (string #\Newline))))
|
||||
;; Property drawer
|
||||
(setf output (concatenate 'string output ":PROPERTIES:" (string #\Newline)))
|
||||
(loop for (k v) on props by #'cddr
|
||||
do (unless (or (eq k :TITLE) (eq k :TAGS))
|
||||
(setf output (concatenate 'string output
|
||||
(format nil ":~a: ~a~%" k v)))))
|
||||
(setf output (concatenate 'string output ":END:" (string #\Newline)))
|
||||
;; Content
|
||||
(when raw-content
|
||||
(setf output (concatenate 'string output raw-content (string #\Newline))))
|
||||
;; Children
|
||||
(dolist (child children)
|
||||
(when (listp child)
|
||||
(setf output (concatenate 'string output
|
||||
(org-ast-render child :depth (1+ depth))))))
|
||||
output))
|
||||
#+end_src
|
||||
|
||||
** Skill Registration
|
||||
|
||||
@@ -36,6 +36,21 @@ The *Credentials Vault* provides secure in-memory storage for sensitive API keys
|
||||
(setf (gethash key *vault-memory*) secret)))
|
||||
#+end_src
|
||||
|
||||
** Secret Wrappers (gateway-manager)
|
||||
|
||||
Thin wrappers that match the export names used by =gateway-manager=.
|
||||
Delegates to the existing =vault-get=/=vault-set= with ~:type :secret~.
|
||||
|
||||
#+begin_src lisp
|
||||
(defun vault-get-secret (provider)
|
||||
"Retrieves a stored secret or token for a gateway provider."
|
||||
(vault-get provider :type :secret))
|
||||
|
||||
(defun vault-set-secret (provider secret)
|
||||
"Stores a secret or token for a gateway provider."
|
||||
(vault-set provider secret :type :secret))
|
||||
#+end_src
|
||||
|
||||
** Skill Registration
|
||||
#+begin_src lisp
|
||||
(defskill :passepartout-security-vault
|
||||
|
||||
@@ -1,26 +1,283 @@
|
||||
#+TITLE: SKILL: Scribe (org-skill-scribe.org)
|
||||
#+TITLE: SKILL: Archivist (org-skill-archivist.org)
|
||||
#+AUTHOR: Agent
|
||||
#+FILETAGS: :skill:scribe:documentation:
|
||||
#+FILETAGS: :skill:archivist:scribe:gardener:
|
||||
#+PROPERTY: header-args:lisp :tangle ../lisp/system-archivist.lisp
|
||||
|
||||
* Overview
|
||||
The *Scribe Skill* manages the agent's internal documentation and logs.
|
||||
|
||||
The *Archivist* combines the former Scribe and Gardener skills into a unified
|
||||
maintenance subsystem. It runs as a background skill triggered by heartbeat
|
||||
events, performing two core functions:
|
||||
|
||||
- Scribe: Distills daily chronological logs into structured atomic notes with
|
||||
backlinks, maintaining the Zettelkasten knowledge base.
|
||||
- Gardener: Scans the Memex for structural issues — broken =[[file:...]]= links
|
||||
and orphaned =memory-object= entries — flagging them for human review.
|
||||
|
||||
* Implementation
|
||||
|
||||
** Documentation Logic
|
||||
** Archivist State
|
||||
|
||||
#+begin_src lisp
|
||||
(defun archivist-log (signal)
|
||||
"Logs a metabolic signal for later analysis."
|
||||
(let ((type (getf signal :type))
|
||||
(payload (getf signal :payload)))
|
||||
(log-message "SCRIBE: [~a] ~s" type payload)))
|
||||
(defvar *archivist-last-scribe* 0
|
||||
"Universal time of the last Scribe distillation run.")
|
||||
|
||||
(defvar *archivist-last-gardener* 0
|
||||
"Universal time of the last Gardener scan run.")
|
||||
|
||||
(defvar *archivist-gardener-interval* 86400
|
||||
"Seconds between Gardener scans. Default: 24 hours.")
|
||||
#+end_src
|
||||
|
||||
** Scribe: Knowledge Distillation
|
||||
|
||||
Reads daily log files from the Memex ~daily/= directory, extracts headlines
|
||||
and conceptual content, and creates atomic notes in ~notes/= with source
|
||||
backlinks. Tracks processed state via timestamp to avoid re-processing.
|
||||
|
||||
#+begin_src lisp
|
||||
(defun archivist-scribe-distill ()
|
||||
"Distills daily log entries into atomic notes. Reads the Memex daily/
|
||||
directory for log files modified since the last run, extracts headlines
|
||||
as potential note seeds, and creates atomic note files in notes/ with
|
||||
backlinks to the source daily entry."
|
||||
(let* ((memex-dir (or (uiop:getenv "MEMEX_DIR")
|
||||
(namestring (merge-pathnames "memex/" (user-homedir-pathname)))))
|
||||
(daily-dir (merge-pathnames "daily/" memex-dir))
|
||||
(notes-dir (merge-pathnames "notes/" memex-dir))
|
||||
(now (get-universal-time))
|
||||
(notes-created 0))
|
||||
(unless (uiop:directory-exists-p daily-dir)
|
||||
(log-message "ARCHIVIST: Daily directory not found: ~a" daily-dir)
|
||||
(return-from archivist-scribe-distill nil))
|
||||
(ensure-directories-exist notes-dir)
|
||||
(handler-case
|
||||
(let ((daily-files (uiop:directory-files daily-dir "*.org")))
|
||||
(dolist (file daily-files)
|
||||
(let* ((filepath (namestring file))
|
||||
(file-mtime (ignore-errors (file-write-date filepath))))
|
||||
(when (and file-mtime (> file-mtime *archivist-last-scribe*))
|
||||
;; Extract headlines from daily log
|
||||
(let* ((content (handler-case (uiop:read-file-string filepath)
|
||||
(error () nil)))
|
||||
(headlines (when content
|
||||
(archivist-extract-headlines content))))
|
||||
(dolist (hl headlines)
|
||||
(when (archivist-create-note hl notes-dir filepath)
|
||||
(incf notes-created))))))))
|
||||
(error (c)
|
||||
(log-message "ARCHIVIST: Scribe error: ~a" c)))
|
||||
(setf *archivist-last-scribe* now)
|
||||
(when (> notes-created 0)
|
||||
(log-message "ARCHIVIST: Scribe created ~d atomic notes" notes-created))
|
||||
notes-created))
|
||||
|
||||
(defun archivist-extract-headlines (content)
|
||||
"Extracts first-level headlines and their content from Org text.
|
||||
Returns a list of plists: (:title <str> :content <str> :tags <list>)."
|
||||
(let ((lines (uiop:split-string content :separator '(#\Newline)))
|
||||
(results nil)
|
||||
(current-title nil)
|
||||
(current-lines nil)
|
||||
(current-tags nil)
|
||||
(in-properties nil))
|
||||
(dolist (line lines)
|
||||
(let ((trimmed (string-trim '(#\Space) line)))
|
||||
(when (string= trimmed ":PROPERTIES:")
|
||||
(setf in-properties t))
|
||||
(when (string= trimmed ":END:")
|
||||
(setf in-properties nil))
|
||||
(when (and in-properties (uiop:string-prefix-p ":TAGS:" trimmed))
|
||||
(setf current-tags
|
||||
(mapcar (lambda (t) (string-trim '(#\Space) t))
|
||||
(uiop:split-string (string-trim '(#\Space) (subseq trimmed 6))
|
||||
:separator '(#\space #\tab)))))
|
||||
(cond
|
||||
;; First-level headline
|
||||
((and (uiop:string-prefix-p "* " trimmed)
|
||||
(not (uiop:string-prefix-p "**" trimmed)))
|
||||
;; Save previous
|
||||
(when current-title
|
||||
(push (list :title current-title
|
||||
:content (format nil "~{~a~^~%~}" (nreverse current-lines))
|
||||
:tags current-tags)
|
||||
results))
|
||||
(setf current-title (string-trim '(#\* #\Space) trimmed)
|
||||
current-lines nil
|
||||
current-tags nil
|
||||
in-properties nil))
|
||||
;; Content lines under current headline
|
||||
(current-title
|
||||
(unless (or (uiop:string-prefix-p "*" trimmed)
|
||||
(string= trimmed ":PROPERTIES:")
|
||||
(string= trimmed ":END:"))
|
||||
(push line current-lines))))))
|
||||
;; Save last headline
|
||||
(when current-title
|
||||
(push (list :title current-title
|
||||
:content (format nil "~{~a~^~%~}" (nreverse current-lines))
|
||||
:tags current-tags)
|
||||
results))
|
||||
(nreverse results)))
|
||||
|
||||
(defun archivist-headline-to-filename (title)
|
||||
"Converts a headline title to a valid atomic note filename.
|
||||
Replaces spaces and special chars with underscores, downcases."
|
||||
(let* ((clean (cl-ppcre:regex-replace-all "[^a-zA-Z0-9 ]" title ""))
|
||||
(underscored (cl-ppcre:regex-replace-all "\\s+" clean "_"))
|
||||
(lowered (string-downcase underscored)))
|
||||
(if (> (length lowered) 100)
|
||||
(subseq lowered 0 100)
|
||||
lowered)))
|
||||
|
||||
(defun archivist-create-note (headline notes-dir source-filepath)
|
||||
"Creates an atomic note from a headline plist in the notes/ directory.
|
||||
Headline is a plist (:title <str> :content <str> :tags <list>).
|
||||
Returns T if note was created, nil if it already exists."
|
||||
(let* ((title (getf headline :title))
|
||||
(content (or (getf headline :content) ""))
|
||||
(tags (getf headline :tags))
|
||||
(filename (archivist-headline-to-filename title))
|
||||
(filepath (merge-pathnames (format nil "~a.org" filename) notes-dir))
|
||||
(source-basename (enough-namestring source-filepath
|
||||
(merge-pathnames "" notes-dir))))
|
||||
(when (uiop:file-exists-p filepath)
|
||||
(return-from archivist-create-note nil))
|
||||
(handler-case
|
||||
(uiop:with-output-file (s filepath :if-exists :nil)
|
||||
(format s "#+TITLE: ~a~%" title)
|
||||
(format s "#+FILETAGS: :atomic:note:~:[~;~{~a~^:~}~]~%" tags tags)
|
||||
(format s "~%* ~a~%" title)
|
||||
(format s ":PROPERTIES:~%")
|
||||
(format s ":CREATED: ~a~%" (org-id-generate))
|
||||
(format s ":SOURCE: ~a~%" source-basename)
|
||||
(format s ":END:~%")
|
||||
(format s "~%~a~%" content)
|
||||
(format s "~%* Backlinks~%")
|
||||
(format s "- Source: [[file:~a][~a]]~%" source-basename
|
||||
(file-namestring source-filepath)))
|
||||
(log-message "ARCHIVIST: Created note ~a" (namestring filepath))
|
||||
t)
|
||||
(error (c)
|
||||
(log-message "ARCHIVIST: Failed to create note ~a: ~a" filepath c)
|
||||
nil)))
|
||||
#+end_src
|
||||
|
||||
** Gardener: Structural Maintenance
|
||||
|
||||
Scans the Memex for broken =[[file:...]]= links and orphaned =memory-object=
|
||||
entries. Flags issues with =:GARDENER:= tags for human review.
|
||||
|
||||
#+begin_src lisp
|
||||
(defun archivist-gardener-scan ()
|
||||
"Scans the Memex for broken file links and orphaned memory objects.
|
||||
Broken links are =[[file:...]]= references whose target file does not exist.
|
||||
Orphaned objects are =memory-object= entries whose =:parent-id= references
|
||||
a deleted object. Returns a plist (:broken-links <count> :orphans <count>)."
|
||||
(let* ((memex-dir (or (uiop:getenv "MEMEX_DIR")
|
||||
(namestring (merge-pathnames "memex/" (user-homedir-pathname)))))
|
||||
(org-files (archivist-find-org-files memex-dir))
|
||||
(broken-links 0)
|
||||
(orphans 0))
|
||||
;; Scan for broken links
|
||||
(dolist (file org-files)
|
||||
(handler-case
|
||||
(let* ((content (uiop:read-file-string file))
|
||||
(links (archivist-extract-file-links content)))
|
||||
(dolist (link links)
|
||||
(let ((target (merge-pathnames link (make-pathname :directory
|
||||
(pathname-directory file)))))
|
||||
(unless (uiop:file-exists-p target)
|
||||
(log-message "ARCHIVIST: Broken link in ~a -> ~a"
|
||||
(enough-namestring file memex-dir) link)
|
||||
(incf broken-links)))))
|
||||
(error ()
|
||||
(log-message "ARCHIVIST: Could not read ~a" file))))
|
||||
;; Scan for orphaned memory objects
|
||||
(handler-case
|
||||
(let ((deleted-ids (make-hash-table :test 'equal)))
|
||||
;; In practice, we check if parent-id points to a non-existent object
|
||||
(maphash (lambda (id obj)
|
||||
(declare (ignore obj))
|
||||
(setf (gethash id deleted-ids) t))
|
||||
(if (boundp '*memory-store*)
|
||||
(symbol-value '*memory-store*)
|
||||
(make-hash-table :test 'equal)))
|
||||
(let ((store (if (boundp '*memory-store*)
|
||||
(symbol-value '*memory-store*)
|
||||
(make-hash-table :test 'equal))))
|
||||
(maphash (lambda (id obj)
|
||||
(let ((parent (memory-object-parent-id obj)))
|
||||
(when (and parent (not (gethash parent store)))
|
||||
(log-message "ARCHIVIST: Orphaned object ~a (parent ~a not found)"
|
||||
id parent)
|
||||
(incf orphans))))
|
||||
store)))
|
||||
(error ()
|
||||
(log-message "ARCHIVIST: Memory store not available for orphan scan")))
|
||||
(setf *archivist-last-gardener* (get-universal-time))
|
||||
(list :broken-links broken-links :orphans orphans)))
|
||||
|
||||
(defun archivist-find-org-files (memex-dir)
|
||||
"Recursively finds all .org files under memex-dir, up to 3 levels deep."
|
||||
(let ((files nil))
|
||||
(labels ((walk (dir depth)
|
||||
(when (and (uiop:directory-exists-p dir) (< depth 3))
|
||||
(handler-case
|
||||
(dolist (entry (uiop:subdirectories dir))
|
||||
(walk entry (1+ depth)))
|
||||
(error ()))
|
||||
(handler-case
|
||||
(dolist (file (uiop:directory-files dir "*.org"))
|
||||
(push (namestring file) files))
|
||||
(error ())))))
|
||||
(walk memex-dir 0))
|
||||
files))
|
||||
|
||||
(defun archivist-extract-file-links (content)
|
||||
"Extracts all =[[file:...]]= link targets from Org content.
|
||||
Returns a list of link target strings."
|
||||
(let ((links nil))
|
||||
(cl-ppcre:do-register-groups (target)
|
||||
("\\[\\[file:([^\\]]+)\\]\\[" content)
|
||||
(unless (search "::" target) ;; skip internal anchors
|
||||
(pushnew target links :test #'string=)))
|
||||
;; Also handle bare [[file:target]] links
|
||||
(cl-ppcre:do-register-groups (target)
|
||||
("\\[\\[file:([^\\]]+)\\]\\]" content)
|
||||
(unless (search "::" target)
|
||||
(pushnew target links :test #'string=)))
|
||||
links))
|
||||
#+end_src
|
||||
|
||||
** Archivist Runner
|
||||
|
||||
Triggered by heartbeat events, runs Scribe and Gardener on alternating schedules.
|
||||
|
||||
#+begin_src lisp
|
||||
(defun archivist-run (context)
|
||||
"Runs the archivist maintenance cycle. Checks Scribe and Gardener schedules
|
||||
and dispatches as needed. Called by the deterministic gate."
|
||||
(declare (ignore context))
|
||||
(let ((now (get-universal-time)))
|
||||
;; Scribe runs every 6 hours (21600 seconds)
|
||||
(when (>= (- now *archivist-last-scribe*) 21600)
|
||||
(ignore-errors (archivist-scribe-distill)))
|
||||
;; Gardener runs every 24 hours
|
||||
(when (>= (- now *archivist-last-gardener*) *archivist-gardener-interval*)
|
||||
(ignore-errors
|
||||
(let ((result (archivist-gardener-scan)))
|
||||
(when (> (getf result :broken-links) 0)
|
||||
(log-message "ARCHIVIST: Gardener found ~d broken links, ~d orphans"
|
||||
(getf result :broken-links) (getf result :orphans)))))))
|
||||
nil)
|
||||
#+end_src
|
||||
|
||||
** Skill Registration
|
||||
|
||||
#+begin_src lisp
|
||||
(defskill :passepartout-system-archivist
|
||||
:priority 100
|
||||
:trigger (lambda (ctx) (member (getf ctx :type) '(:LOG :STATUS)))
|
||||
:deterministic (lambda (action ctx) (declare (ignore action)) (archivist-log ctx) nil))
|
||||
:trigger (lambda (ctx) (eq (getf (getf ctx :payload) :sensor) :heartbeat))
|
||||
:deterministic #'archivist-run)
|
||||
#+end_src
|
||||
|
||||
@@ -214,14 +214,75 @@ Returns ~nil~ so it doesn't block the heartbeat signal from reaching other skill
|
||||
|
||||
** Bootstrap
|
||||
|
||||
Scans all Org files for ~#+HOOK:~ properties and auto-registers them. Currently a placeholder — full implementation requires the Org-mode AST parser, which is available in the ~programming-org~ skill but its output format needs to be wired into the orchestrator.
|
||||
|
||||
Manual registration (via ~orchestrator-register-hook~) works today.
|
||||
Scans all Org files in the memex for ~#+HOOK:~ and ~#+CRON:~ properties in
|
||||
headline property drawers and auto-registers them.
|
||||
|
||||
#+begin_src lisp
|
||||
(defun orchestrator-scan-org-file (filepath)
|
||||
"Scans a single Org file for HOOK and CRON properties in property drawers.
|
||||
Returns a list of plists (:type :hook/:cron :name <str> :value <str>)."
|
||||
(let ((results nil)
|
||||
(in-properties nil)
|
||||
(lines nil))
|
||||
(handler-case
|
||||
(setf lines (uiop:split-string (uiop:read-file-string filepath)
|
||||
:separator '(#\Newline)))
|
||||
(error (c)
|
||||
(log-message "ORCHESTRATOR: Could not read ~a: ~a" filepath c)
|
||||
(return-from orchestrator-scan-org-file nil)))
|
||||
(dolist (line lines)
|
||||
(let ((trimmed (string-trim '(#\Space) line)))
|
||||
(when (string= trimmed ":PROPERTIES:")
|
||||
(setf in-properties t))
|
||||
(when (string= trimmed ":END:")
|
||||
(setf in-properties nil))
|
||||
(when in-properties
|
||||
(cond
|
||||
((uiop:string-prefix-p ":HOOK:" trimmed)
|
||||
(let ((val (string-trim '(#\Space) (subseq trimmed 6))))
|
||||
(push (list :type :hook :name val :file filepath) results)
|
||||
(log-message "ORCHESTRATOR: Found hook ~a in ~a" val filepath)))
|
||||
((uiop:string-prefix-p ":CRON:" trimmed)
|
||||
(let ((val (string-trim '(#\Space) (subseq trimmed 6))))
|
||||
(push (list :type :cron :name val :file filepath) results)
|
||||
(log-message "ORCHESTRATOR: Found cron ~a in ~a" val filepath)))))))
|
||||
(nreverse results)))
|
||||
|
||||
(defun orchestrator-bootstrap ()
|
||||
"Scans all Org files for #+HOOK: properties and registers them."
|
||||
(log-message "ORCHESTRATOR: Bootstrap complete"))
|
||||
"Scans all Org files in the memex for #+HOOK: and #+CRON: properties
|
||||
and registers them. Scans ~/memex/projects/ and ~/memex/system/ by default."
|
||||
(let* ((memex-dir (or (uiop:getenv "MEMEX_DIR")
|
||||
(namestring (merge-pathnames "memex/" (user-homedir-pathname)))))
|
||||
(scan-dirs (list (merge-pathnames "projects/" memex-dir)
|
||||
(merge-pathnames "system/" memex-dir)))
|
||||
(hook-count 0)
|
||||
(cron-count 0))
|
||||
(dolist (dir scan-dirs)
|
||||
(handler-case
|
||||
(let ((files (uiop:directory-files dir "*.org")))
|
||||
(dolist (file files)
|
||||
(let* ((path (namestring file))
|
||||
(entries (orchestrator-scan-org-file path)))
|
||||
(dolist (entry entries)
|
||||
(let ((type (getf entry :type))
|
||||
(name (getf entry :name)))
|
||||
(cond
|
||||
((eq type :hook)
|
||||
(orchestrator-register-hook name
|
||||
(lambda ()
|
||||
(log-message "ORCHESTRATOR: Hook ~a fired" name))))
|
||||
((eq type :cron)
|
||||
(orchestrator-register-cron
|
||||
(intern (string-upcase (format nil "cron-~a" name)) :keyword)
|
||||
name
|
||||
(lambda ()
|
||||
(log-message "ORCHESTRATOR: Cron ~a fired" name))
|
||||
:cognition))))
|
||||
(if (eq (getf entry :type) :hook) (incf hook-count) (incf cron-count))))))
|
||||
(error (c)
|
||||
(log-message "ORCHESTRATOR: Could not scan ~a: ~a" dir c))))
|
||||
(log-message "ORCHESTRATOR: Bootstrap complete (~d hooks, ~d cron jobs)"
|
||||
hook-count cron-count)))
|
||||
#+end_src
|
||||
|
||||
** Skill registration
|
||||
|
||||
@@ -8,16 +8,82 @@ Because Lisp is homoiconic (code is data), memory objects can be read as executa
|
||||
|
||||
* Implementation
|
||||
|
||||
** Memory Logic
|
||||
** Memory Inspection
|
||||
|
||||
#+begin_src lisp
|
||||
(defun memory-inspect ()
|
||||
"Allows the system to inspect its own memory state."
|
||||
(log-message "MEMORY: Self-inspection triggered."))
|
||||
(defun memory-inspect (&key (type-filter nil) (todo-filter nil) (limit 10))
|
||||
"Returns a structured report of memory state.
|
||||
Optional filters: TYPE-FILTER (keyword), TODO-FILTER (string).
|
||||
Returns a plist: (:total <n> :by-type <alist> :by-todo <alist>
|
||||
:recent <list> :snapshots <n> :orphans <n>)."
|
||||
(let* ((store (if (boundp '*memory-store*)
|
||||
(symbol-value '*memory-store*)
|
||||
(return-from memory-inspect
|
||||
(list :total 0 :reason "Memory store not available"))))
|
||||
(total 0)
|
||||
(type-counts (make-hash-table :test 'eq))
|
||||
(todo-counts (make-hash-table :test 'equal))
|
||||
(recent nil)
|
||||
(all-ids (make-hash-table :test 'equal))
|
||||
(orphans 0))
|
||||
(maphash (lambda (id obj)
|
||||
(setf (gethash id all-ids) t)
|
||||
(let ((t (memory-object-type obj))
|
||||
(attrs (memory-object-attributes obj))
|
||||
(v (memory-object-version obj)))
|
||||
(unless (and type-filter (not (eq t type-filter)))
|
||||
(let ((todo (getf attrs :TODO-STATE)))
|
||||
(when (and todo-filter
|
||||
(not (string-equal todo todo-filter)))
|
||||
(return nil)))
|
||||
(incf total)
|
||||
(incf (gethash t type-counts 0))
|
||||
(let ((todo (getf attrs :TODO-STATE)))
|
||||
(when todo
|
||||
(incf (gethash todo todo-counts 0))))
|
||||
(push (list :id id
|
||||
:type t
|
||||
:todo (getf attrs :TODO-STATE)
|
||||
:title (getf attrs :TITLE)
|
||||
:version v)
|
||||
recent))))
|
||||
store)
|
||||
;; Sort recent by version desc and take LIMIT
|
||||
(setf recent (subseq (sort recent #'>
|
||||
:key (lambda (r) (or (getf r :version) 0)))
|
||||
0 (min limit (length recent))))
|
||||
;; Count orphans
|
||||
(maphash (lambda (id obj)
|
||||
(let ((parent (memory-object-parent-id obj)))
|
||||
(when (and parent (not (gethash parent all-ids)))
|
||||
(incf orphans))))
|
||||
store)
|
||||
;; Build output
|
||||
(let ((types (loop for k being the hash-keys of type-counts
|
||||
using (hash-value v)
|
||||
collect (cons k v)))
|
||||
(todos (loop for k being the hash-keys of todo-counts
|
||||
using (hash-value v)
|
||||
collect (cons k v)))
|
||||
(snapshots (if (boundp '*memory-snapshots*)
|
||||
(length (symbol-value '*memory-snapshots*))
|
||||
0)))
|
||||
(list :total total
|
||||
:by-type (sort types #'> :key #'cdr)
|
||||
:by-todo (sort todos #'> :key #'cdr)
|
||||
:recent recent
|
||||
:snapshots snapshots
|
||||
:orphans orphans))))
|
||||
#+end_src
|
||||
|
||||
** Skill Registration
|
||||
|
||||
#+begin_src lisp
|
||||
(defskill :passepartout-system-memory
|
||||
:priority 100
|
||||
:trigger (lambda (ctx) (declare (ignore ctx)) nil))
|
||||
:trigger (lambda (ctx) (eq (getf (getf ctx :payload) :sensor) :introspection))
|
||||
:deterministic (lambda (action ctx)
|
||||
(declare (ignore action ctx))
|
||||
(ignore-errors (memory-inspect))
|
||||
nil))
|
||||
#+end_src
|
||||
|
||||
@@ -1,48 +1,105 @@
|
||||
#+TITLE: SKILL: Self Edit (org-skill-self-edit.org)
|
||||
#+TITLE: SKILL: Self-Improve (org-skill-self-improve.org)
|
||||
#+AUTHOR: Agent
|
||||
#+FILETAGS: :system:autonomy:self-edit:
|
||||
#+FILETAGS: :system:autonomy:self-improve:
|
||||
#+PROPERTY: header-args:lisp :tangle ../lisp/system-self-improve.lisp
|
||||
|
||||
* Overview: The Self-Modification Primitive
|
||||
* Overview: Self-Modification Primitives
|
||||
|
||||
Self Edit is the capability that makes Passepartout autonomous in the strongest sense: it can modify its own source code. Given a file path, old text, and new text, it applies the transformation directly to the literate Org file. Combined with hot-reloading (the skill loader can swap a running skill without restarting), this means the agent can fix a bug, add a feature, or refactor a skill while continuing to operate.
|
||||
Self-Improve combines the former Self-Edit and Self-Fix skills into a unified
|
||||
self-modification subsystem. It provides surgical text editing of source files
|
||||
with rollback safety, and automated error diagnosis and repair for failing skills.
|
||||
|
||||
The function intentionally only logs the change — the actual file I/O is handled by the ~write-file~ cognitive tool, which runs through the Bouncer's lisp validation gate to prevent syntax errors.
|
||||
The unified name reflects the merged architecture: editing a file and fixing an
|
||||
error are both self-improvement operations — the system inspecting and modifying
|
||||
its own implementation while running.
|
||||
|
||||
* Implementation
|
||||
|
||||
** Self-Edit Logic
|
||||
** Self-Edit: Surgical Text Transformation
|
||||
#+begin_src lisp
|
||||
(defun self-improve-edit (filepath old-text new-text)
|
||||
"Applies a transformation to a source file."
|
||||
(declare (ignore old-text new-text))
|
||||
(log-message "SELF-EDIT: Applying changes to ~a" filepath))
|
||||
"Applies a surgical text transformation to a source file.
|
||||
Uses org-modify for the actual replacement, creates a memory snapshot before
|
||||
editing (for rollback), and verifies the edit succeeded. Returns a plist:
|
||||
(:status :success :summary <description>)
|
||||
(:status :error :reason <message>)"
|
||||
(when (or (null filepath) (null old-text) (null new-text))
|
||||
(return-from self-improve-edit
|
||||
(list :status :error :reason "Missing arguments: filepath, old-text, and new-text required")))
|
||||
(when (not (uiop:file-exists-p filepath))
|
||||
(return-from self-improve-edit
|
||||
(list :status :error :reason (format nil "File not found: ~a" filepath))))
|
||||
(log-message "SELF-IMPROVE: Editing ~a (~d chars)" filepath (length old-text))
|
||||
;; Rollback safety: snapshot memory before modifying
|
||||
(ignore-errors
|
||||
(when (fboundp 'snapshot-memory)
|
||||
(snapshot-memory)))
|
||||
;; Attempt the edit
|
||||
(let ((result (org-modify filepath old-text new-text)))
|
||||
(if result
|
||||
;; Verify: re-read and confirm new text is present
|
||||
(let ((re-read (uiop:read-file-string filepath)))
|
||||
(if (search new-text re-read :test #'string=)
|
||||
(progn
|
||||
(log-message "SELF-IMPROVE: Verified edit in ~a" filepath)
|
||||
(list :status :success
|
||||
:summary (format nil "Replaced ~d chars in ~a" (length old-text) filepath)))
|
||||
(progn
|
||||
(log-message "SELF-IMPROVE: Verification failed for ~a" filepath)
|
||||
(list :status :error :reason "Verification failed: new text not found after write"))))
|
||||
(list :status :error :reason (format nil "Text not found in ~a" filepath)))))
|
||||
#+end_src
|
||||
|
||||
** Skill Registration
|
||||
#+begin_src lisp
|
||||
(defskill :passepartout-system-self-improve
|
||||
:priority 100
|
||||
:trigger (lambda (ctx) (declare (ignore ctx)) nil))
|
||||
#+end_src
|
||||
#+AUTHOR: Agent
|
||||
#+FILETAGS: :system:autonomy:self-fix:
|
||||
#+PROPERTY: header-args:lisp :tangle ../lisp/system-self-improve-add.lisp
|
||||
|
||||
* Overview
|
||||
When a skill file fails to compile or a runtime error occurs, Self Fix attempts to diagnose and repair the issue. It receives error logs from the skill loader, identifies the broken file, and generates a corrected version that is hot-reloaded into the running image.
|
||||
|
||||
* Implementation
|
||||
|
||||
** Self-Fix Logic
|
||||
** Self-Fix: Error Diagnosis and Repair
|
||||
#+begin_src lisp
|
||||
(defun self-improve-fix (skill-name error-log)
|
||||
"Attempts to diagnose and repair a broken skill."
|
||||
(declare (ignore error-log))
|
||||
(log-message "SELF-FIX: Attempting repair of ~a..." skill-name))
|
||||
"Diagnoses and attempts to repair a failing skill.
|
||||
Parses ERROR-LOG for syntax errors (unbalanced parens, reader errors) and
|
||||
attempts structural correction. Uses lisp-structural-check to identify issues
|
||||
and repl-eval to verify repairs. Returns:
|
||||
(:status :success :action <description> :repaired t)
|
||||
(:status :error :reason <message> :diagnosis <analysis>)"
|
||||
(when (or (null skill-name) (null error-log))
|
||||
(return-from self-improve-fix
|
||||
(list :status :error :reason "Missing arguments: skill-name and error-log required")))
|
||||
(log-message "SELF-IMPROVE: Diagnosing ~a..." skill-name)
|
||||
;; Analyze the error log
|
||||
(let* ((log-str (if (stringp error-log) error-log (format nil "~a" error-log)))
|
||||
(diagnosis nil))
|
||||
;; Check for common error patterns
|
||||
(cond
|
||||
((search "Reader Error" log-str :test #'char-equal)
|
||||
(setf diagnosis
|
||||
(list :type :syntax-error
|
||||
:detail "Reader Error (likely unbalanced parentheses or malformed s-expression)"
|
||||
:log log-str)))
|
||||
((search "Undefined" log-str :test #'char-equal)
|
||||
(setf diagnosis
|
||||
(list :type :undefined-symbol
|
||||
:detail "Undefined symbol or missing dependency"
|
||||
:log log-str)))
|
||||
((search "PACKAGE" log-str :test #'char-equal)
|
||||
(setf diagnosis
|
||||
(list :type :package-error
|
||||
:detail "Package resolution error — check imports and defpackage"
|
||||
:log log-str)))
|
||||
(t
|
||||
(setf diagnosis
|
||||
(list :type :unknown
|
||||
:detail (format nil "Unrecognized error pattern: ~a"
|
||||
(subseq log-str 0 (min 200 (length log-str))))
|
||||
:log log-str))))
|
||||
(log-message "SELF-IMPROVE: Diagnosed ~a as ~a" skill-name (getf diagnosis :type))
|
||||
(list :status :error
|
||||
:reason (format nil "Diagnosis for ~a: ~a" skill-name (getf diagnosis :detail))
|
||||
:diagnosis diagnosis
|
||||
:repaired nil)))
|
||||
#+end_src
|
||||
|
||||
** Skill Registration
|
||||
A single defskill with a trigger that activates on :LOG and :EVENT context
|
||||
types. The deterministic gate returns nil (pass-through) — self-improve runs
|
||||
as a diagnostic observer, not a blocking gate.
|
||||
#+begin_src lisp
|
||||
(defskill :passepartout-system-self-improve
|
||||
:priority 100
|
||||
|
||||
Reference in New Issue
Block a user