From d3873bf1ffde99be83e504fff392e7f698bb43f2 Mon Sep 17 00:00:00 2001 From: Amr Gharbeia Date: Thu, 9 Apr 2026 21:07:39 -0400 Subject: [PATCH] REFAC: Complete State Persistence consolidation --- skills/org-skill-latent-reflection.org | 2 +- skills/org-skill-memory-archivist.org | 154 ----------- skills/org-skill-object-store-persistence.org | 111 -------- skills/org-skill-state-persistence.org | 255 ++++++++++++++++++ src/state-persistence.lisp | 125 +++++++++ tests/persistence-tests.lisp | 15 ++ 6 files changed, 396 insertions(+), 266 deletions(-) delete mode 100644 skills/org-skill-memory-archivist.org delete mode 100644 skills/org-skill-object-store-persistence.org create mode 100644 skills/org-skill-state-persistence.org create mode 100644 src/state-persistence.lisp create mode 100644 tests/persistence-tests.lisp diff --git a/skills/org-skill-latent-reflection.org b/skills/org-skill-latent-reflection.org index 723d338..1d2996e 100644 --- a/skills/org-skill-latent-reflection.org +++ b/skills/org-skill-latent-reflection.org @@ -5,7 +5,7 @@ #+TITLE: SKILL: Latent Reflection (Proactive Gardening) #+STARTUP: content #+FILETAGS: :memory:gardening:reflection:psf: -#+DEPENDS_ON: id:e8b500e2-3f26-4c8e-8558-528061e178ca +#+DEPENDS_ON: id:state-persistence-skill * Overview The *Latent Reflection* skill utilizes the idle cycles of the heartbeat to proactively garden the Memex. By randomly sampling the knowledge graph, it surfaces forgotten connections and synthesizes new insights without direct user prompting. This mimics the human default mode network, fostering creativity and serendipity. diff --git a/skills/org-skill-memory-archivist.org b/skills/org-skill-memory-archivist.org deleted file mode 100644 index 17167e9..0000000 --- a/skills/org-skill-memory-archivist.org +++ /dev/null @@ -1,154 +0,0 @@ -:PROPERTIES: -:ID: 98923a43-2be0-423c-8509-22592cfe9c9e -:CREATED: [2026-04-08 Wed 18:30] -:EDITED: [2026-04-09 Thu 15:30] -:END: -#+TITLE: SKILL: Memory Archivist (Universal Literate Note) -#+STARTUP: content -#+FILETAGS: :memory:persistence:ipfs:sovereignty: -#+DEPENDS_ON: id:e8b500e2-3f26-4c8e-8558-528061e178ca - -* Overview -The *Memory Archivist* provides long-term, decentralized, and immutable history for the agent's knowledge graph. It leverages IPFS to achieve "Sovereignty Above All," ensuring that memory state is preserved across space and time. - -* Implementation - -** Package Context -#+begin_src lisp -(in-package :org-agent) -#+end_src - -** Serialization (archivist-serialize-store) -Converts the live `*object-store*` into a JSON-compatible list of alists. - -#+begin_src lisp -(defun archivist-serialize-store () - "Serializes the entire object-store for archival." - (let ((objects nil)) - (maphash (lambda (id obj) - (declare (ignore id)) - (push `((:id . ,(org-object-id obj)) - (:type . ,(format nil "~s" (org-object-type obj))) - (:attributes . ,(loop for (k v) on (org-object-attributes obj) by #'cddr - collect (cons (format nil "~a" k) (format nil "~a" v)))) - (:content . ,(org-object-content obj)) - (:parent-id . ,(org-object-parent-id obj)) - (:children . ,(org-object-children obj)) - (:version . ,(org-object-version obj)) - (:last-sync . ,(org-object-last-sync obj)) - (:hash . ,(org-object-hash obj))) - objects)) - *object-store*) - objects)) -#+end_src - -** IPFS Integration (archivist-push-to-ipfs) -Pushes the serialized knowledge graph to the local IPFS daemon. - -#+begin_src lisp -(defun archivist-push-to-ipfs () - "Serializes the store and pushes it to IPFS, returning the CID." - (let* ((data (archivist-serialize-store)) - (json-payload (cl-json:encode-json-to-string data)) - (ipfs-url "http://127.0.0.1:5001/api/v0/add")) - (handler-case - (let* ((response (dex:post ipfs-url - :content `(("file" . ,json-payload)) - :headers '(("Content-Type" . "multipart/form-data")))) - (result (cl-json:decode-json-from-string response)) - (cid (cdr (assoc :hash result)))) - (kernel-log "ARCHIVIST - Memory checkpointed to IPFS. CID: ~a" cid) - cid) - (error (c) - (kernel-log "ARCHIVIST ERROR - IPFS push failed: ~a" c) - nil)))) -#+end_src - -** Restoration (archivist-pull-from-ipfs) -Fetches a knowledge graph image from IPFS and hydrates the `*object-store*`. - -#+begin_src lisp -(defun archivist-pull-from-ipfs (cid) - "Fetches data from IPFS by CID and restores the object-store." - (let ((ipfs-url (format nil "http://127.0.0.1:5001/api/v0/cat?arg=~a" cid))) - (handler-case - (let* ((response (dex:post ipfs-url)) - (data (cl-json:decode-json-from-string response))) - (clrhash *object-store*) - (dolist (item data) - (let* ((id (cdr (assoc :id item))) - (obj (make-org-object - :id id - :type (read-from-string (cdr (assoc :type item))) - :attributes (loop for attr in (cdr (assoc :attributes item)) - append (list (intern (string-upcase (car attr)) :keyword) (cdr attr))) - :content (cdr (assoc :content item)) - :parent-id (cdr (assoc :parent-id item)) - :children (cdr (assoc :children item)) - :version (cdr (assoc :version item)) - :last-sync (cdr (assoc :last-sync item)) - :hash (cdr (assoc :hash item))))) - (setf (gethash id *object-store*) obj))) - (kernel-log "ARCHIVIST - Knowledge graph restored from IPFS CID: ~a" cid) - t) - (error (c) - (kernel-log "ARCHIVIST ERROR - Restoration failed: ~a" c) - nil)))) -#+end_src - -** Cognitive Tools -Expose archival capabilities to System 1. - -#+begin_src lisp -(def-cognitive-tool :ipfs-checkpoint "Creates an immutable snapshot of the current knowledge graph on IPFS." - :parameters nil - :body (lambda (args) - (declare (ignore args)) - (let ((cid (archivist-push-to-ipfs))) - (if cid - (format nil "Checkpoint success. CID: ~a" cid) - "Checkpoint failed. Ensure IPFS daemon is running.")))) - -(def-cognitive-tool :ipfs-restore "Restores the entire knowledge graph from a specific IPFS CID." - :parameters ((:cid :type :string :description "The IPFS CID to restore from")) - :body (lambda (args) - (let ((cid (getf args :cid))) - (if (archivist-pull-from-ipfs cid) - (format nil "Restoration successful from ~a" cid) - "Restoration failed.")))) -#+end_src - -** Skill Definition -#+begin_src lisp -(defskill :skill-memory-archivist - :priority 80 - :trigger (lambda (ctx) (eq (getf (getf ctx :payload) :command) :checkpoint-ipfs)) - :neuro (lambda (ctx) "Propose an IPFS checkpoint if the user wants decentralized persistence.") - :symbolic (lambda (action ctx) - (let ((cid (archivist-push-to-ipfs))) - (if cid - `(:target :system :payload (:action :message :text ,(format nil "IPFS Checkpoint: ~a" cid))) - `(:target :system :payload (:action :message :text "IPFS Checkpoint failed.")))))) -#+end_src - -* Phase E: Chaos (Verification) -The Memory Archivist must be verified for serialization integrity. - -#+begin_src lisp -(defpackage :org-agent-archivist-tests - (:use :cl :fiveam :org-agent)) -(in-package :org-agent-archivist-tests) - -(def-suite archivist-suite :description "Tests for IPFS Archival.") -(in-suite archivist-suite) - -(test test-serialization-integrity - "Verify that the object-store can be serialized and partially reconstructed." - (clrhash org-agent::*object-store*) - (ingest-ast '(:type :HEADLINE :properties (:ID "test-id" :TITLE "Test Node") :raw-content "Body Text" :contents nil)) - (let* ((data (org-agent::archivist-serialize-store)) - (first-item (first data))) - (is (= 1 (length data))) - (is (equal "test-id" (cdr (assoc :id first-item)))) - (is (equal "Body Text" (cdr (assoc :content first-item)))))) -#+end_src diff --git a/skills/org-skill-object-store-persistence.org b/skills/org-skill-object-store-persistence.org deleted file mode 100644 index 22db738..0000000 --- a/skills/org-skill-object-store-persistence.org +++ /dev/null @@ -1,111 +0,0 @@ -:PROPERTIES: -:ID: e8b500e2-3f26-4c8e-8558-528061e178ca -:CREATED: [2026-03-31 Tue 18:28] -:EDITED: [2026-04-07 Tue 13:42] -:END: -#+TITLE: SKILL: Object Store Persistence (Universal Literate Note) -#+STARTUP: content -#+FILETAGS: :memory:persistence:closos:psf: - -* Overview -The *Object Store Persistence* skill ensures that the agent's perceptual memory (the `*object-store*`) is durable. It provides the mechanism to "dump" the in-RAM knowledge graph to a Lisp-native image file and "reload" it upon boot, eliminating the need to re-parse the entire Memex on every restart. - -* Phase A: Demand (PRD) -:PROPERTIES: -:STATUS: FROZEN -:END: - -** 1. Purpose -Define automated behaviors for knowledge graph serialization and restoration. - -** 2. User Needs -- *Instant Recall:* Rapid loading of the Object Store from a persistent image. -- *High-Fidelity Serialization:* Recursive dumping of `org-object` structs and their relations. -- *Atomic Persistence:* Save the entire graph state to a single `.el` or `.lisp` file. -- *Background Synchronization:* Periodically dump the image during heartbeats. - -** 3. Success Criteria -*** TODO Image Dump logic verification (File exists and is readable) -*** TODO Image Load logic verification (Object count matches RAM state) -*** TODO Performance audit (Loading image must be >10x faster than parsing) - -* Phase B: Blueprint (PROTOCOL) -:PROPERTIES: -:STATUS: SIGNED -:END: - -** 1. Architectural Intent -Interfaces for state dumping and restoration. Source of truth is the RAM-resident `*object-store*` and the `system/state/memory-image.lisp` file. - -** 2. Semantic Interfaces -#+begin_src lisp -(defun memory-dump-image () - "Serializes the current *object-store* to disk.") - -(defun memory-load-image () - "Restores the *object-store* from the persistent image file.") -#+end_src - -* Phase D: Build (Implementation) - -** Image Serialization -We serialize the `*history-store*` (immutable objects) and the current `*object-store*` root pointers. - -#+begin_src lisp -(defun memory-dump-image () - "Serializes the entire history store and current pointers to a Lisp image." - (let* ((state-dir (or (uiop:getenv "SYSTEM_DIR") "system/")) - (image-file (merge-pathnames "state/memory-image.lisp" state-dir))) - (ensure-directories-exist image-file) - (org-agent:kernel-log "MEMORY - Dumping Merkle-Tree history to ~a..." (uiop:native-namestring image-file)) - (with-open-file (out image-file :direction :output :if-exists :supersede) - (format out "(in-package :org-agent)~%") - ;; 1. Dump all immutable objects in the history store - (maphash (lambda (hash obj) - (print `(setf (gethash ,hash *history-store*) ,obj) out)) - org-agent:*history-store*) - ;; 2. Dump the current active pointers (the object store) - (maphash (lambda (id obj) - (print `(setf (gethash ,id *object-store*) (gethash ,(org-agent:org-object-hash obj) *history-store*)) out)) - org-agent:*object-store*)) - '(:target :system :payload (:action :message :text "Merkle-Tree image dumped.")))) - -(defun memory-load-image () - "Loads the memory image from disk." - (let* ((state-dir (or (uiop:getenv "SYSTEM_DIR") "system/")) - (image-file (merge-pathnames "state/memory-image.lisp" state-dir))) - (if (uiop:file-exists-p image-file) - (progn - (org-agent:kernel-log "MEMORY - Loading knowledge graph image...") - (load image-file) - t) - (progn - (org-agent:kernel-log "MEMORY ERROR - Image file not found.") - nil)))) -#+end_src - -** Cognitive Tools -#+begin_src lisp -(org-agent:def-cognitive-tool :persist-memory "Saves the current state of the Memex (Object Store + History) to disk." - :parameters nil - :body (lambda (args) - (declare (ignore args)) - (memory-dump-image) - "MEMORY PERSISTED SUCCESSFUL.")) - -(org-agent:def-cognitive-tool :rollback-memory "Reverts the in-memory state to a previous snapshot." - :parameters ((:index :type :integer :description "The snapshot index to roll back to (0 is most recent)")) - :body (lambda (args) - (org-agent:rollback-object-store (getf args :index)) - "ROLLBACK COMPLETE.")) -#+end_src - -** Registration -#+begin_src lisp -(org-agent:defskill :skill-object-store-persistence - :priority 100 - :trigger (lambda (context) (eq (getf (getf context :payload) :sensor) :heartbeat)) - :neuro (lambda (context) nil) - :symbolic (lambda (action context) (memory-dump-image))) -#+end_src - diff --git a/skills/org-skill-state-persistence.org b/skills/org-skill-state-persistence.org new file mode 100644 index 0000000..6f91f05 --- /dev/null +++ b/skills/org-skill-state-persistence.org @@ -0,0 +1,255 @@ +:PROPERTIES: +:ID: state-persistence-skill +:CREATED: [2026-04-09 Thu] +:END: +#+TITLE: SKILL: State Persistence Layer (Universal Literate Note) +#+STARTUP: content +#+FILETAGS: :memory:persistence:closos:ipfs:psf: +#+DEPENDS_ON: id:e8b500e2-3f26-4c8e-8558-528061e178ca + +* Overview +The *State Persistence Layer* ensures the durability and sovereignty of the agent's memory. It unifies local, high-performance Lisp image dumps with decentralized, immutable IPFS checkpointing. This dual-path approach provides both rapid operational recovery and long-term historical integrity. + +* Phase A: Demand (PRD) +:PROPERTIES: +:STATUS: SIGNED +:END: + +** 1. Purpose +Define automated behaviors for knowledge graph serialization, local persistence, and decentralized archival. + +** 2. User Needs +- *Instant Recall:* Rapid local loading of the Object Store from a persistent image. +- *Decentralized Archival:* Pushing immutable snapshots to IPFS for cross-node sync and sovereignty. +- *Merkle Integrity:* Every save operation must respect and record the Merkle-Tree history. +- *Safety:* Sanitize and validate data during restoration to prevent code injection. + +* Phase B: Blueprint (PROTOCOL) +:PROPERTIES: +:STATUS: SIGNED +:END: + +** 1. Architectural Intent +The persistence layer acts as a bridge between the volatile RAM-resident Object Store and permanent storage backends. It provides two adapters: `LOCAL` (fast, SBCL-native) and `IPFS` (sovereign, content-addressed). + +** 2. Semantic Interfaces +#+begin_src lisp +(defun persistence-dump-local () + "Serializes RAM state to a local Lisp image file.") + +(defun persistence-push-ipfs () + "Pushes an immutable snapshot of the graph to IPFS.") + +(defun persistence-restore-ipfs (cid) + "Hydrates the RAM state from an IPFS content identifier.") +#+end_src + +* Phase C: Success (QUALITY) +:PROPERTIES: +:STATUS: SIGNED +:END: + +** 1. Success Criteria +- [ ] *Speed:* Local image load must be <500ms for a 10k node graph. +- [ ] *Fidelity:* IPFS round-trip must result in a bit-identical Object Store. +- [ ] *Validation:* Restoration must block any `read-eval` reader macros in content. + +** 2. TDD Plan +Tests in `tests/persistence-tests.lisp` will verify the local dump/load cycle and the JSON serialization format for IPFS. + +* Phase D: Build (Implementation) + +** Package Context +#+begin_src lisp :tangle ../src/state-persistence.lisp +(in-package :org-agent) +#+end_src + +** Helper: Local State Path +Ensures we have a standardized location for local memory images. + +#+begin_src lisp :tangle ../src/state-persistence.lisp +(defun persistence-get-local-path () + "Returns the path to the local memory image file." + (let ((state-dir (or (uiop:getenv "SYSTEM_DIR") "system/"))) + (merge-pathnames "state/memory-image.lisp" state-dir))) +#+end_src + +** Local Image Dump (persistence-dump-local) +Serializes the Merkle history and current pointers to a Lisp file. + +#+begin_src lisp :tangle ../src/state-persistence.lisp +(defun persistence-dump-local () + "Serializes the entire history store and current pointers to a local Lisp image." + (let ((image-file (persistence-get-local-path))) + (ensure-directories-exist image-file) + (kernel-log "PERSISTENCE - Dumping local image to ~a..." (uiop:native-namestring image-file)) + (with-open-file (out image-file :direction :output :if-exists :supersede) + (format out "(in-package :org-agent)~%") + ;; 1. Dump all immutable objects in the history store + (maphash (lambda (hash obj) + (print `(setf (gethash ,hash *history-store*) ,obj) out)) + *history-store*) + ;; 2. Dump the current active pointers + (maphash (lambda (id obj) + (print `(setf (gethash ,id *object-store*) (gethash ,(org-object-hash obj) *history-store*)) out)) + *object-store*)) + t)) +#+end_src + +** Local Image Load (persistence-load-local) +Restores the state from the local disk. + +#+begin_src lisp :tangle ../src/state-persistence.lisp +(defun persistence-load-local () + "Loads the memory image from local disk." + (let ((image-file (persistence-get-local-path))) + (if (uiop:file-exists-p image-file) + (progn + (kernel-log "PERSISTENCE - Loading local image...") + (load image-file) + t) + (progn + (kernel-log "PERSISTENCE ERROR - Local image not found.") + nil)))) +#+end_src + +** IPFS Serialization (persistence-serialize-for-archival) +Converts the live `*object-store*` into a JSON-compatible list of alists. + +#+begin_src lisp :tangle ../src/state-persistence.lisp +(defun persistence-serialize-for-archival () + "Serializes the entire object-store for IPFS/JSON transport." + (let ((objects nil)) + (maphash (lambda (id obj) + (declare (ignore id)) + (push `((:id . ,(org-object-id obj)) + (:type . ,(format nil "~s" (org-object-type obj))) + (:attributes . ,(loop for (k v) on (org-object-attributes obj) by #'cddr + collect (cons (format nil "~a" k) (format nil "~a" v)))) + (:content . ,(org-object-content obj)) + (:parent-id . ,(org-object-parent-id obj)) + (:children . ,(org-object-children obj)) + (:version . ,(org-object-version obj)) + (:last-sync . ,(org-object-last-sync obj)) + (:hash . ,(org-object-hash obj))) + objects)) + *object-store*) + objects)) +#+end_src + +** IPFS Push (persistence-push-ipfs) +Pushes the serialized knowledge graph to the decentralized network. + +#+begin_src lisp :tangle ../src/state-persistence.lisp +(defun persistence-push-ipfs () + "Serializes the store and pushes it to IPFS, returning the CID." + (let* ((data (persistence-serialize-for-archival)) + (json-payload (cl-json:encode-json-to-string data)) + (ipfs-url "http://127.0.0.1:5001/api/v0/add")) + (handler-case + (let* ((response (dex:post ipfs-url + :content `(("file" . ,json-payload)) + :headers '(("Content-Type" . "multipart/form-data")))) + (result (cl-json:decode-json-from-string response)) + (cid (cdr (assoc :hash result)))) + (kernel-log "PERSISTENCE - Checkpoint to IPFS successful. CID: ~a" cid) + cid) + (error (c) + (kernel-log "PERSISTENCE ERROR - IPFS push failed: ~a" c) + nil)))) +#+end_src + +** IPFS Restore (persistence-restore-ipfs) +Restores the graph from IPFS, using a safe parser to prevent injection. + +#+begin_src lisp :tangle ../src/state-persistence.lisp +(defun persistence-restore-ipfs (cid) + "Fetches data from IPFS and safely hydrates the object-store." + (let ((ipfs-url (format nil "http://127.0.0.1:5001/api/v0/cat?arg=~a" cid))) + (handler-case + (let* ((response (dex:post ipfs-url)) + (data (cl-json:decode-json-from-string response))) + (clrhash *object-store*) + (dolist (item data) + (let* ((id (cdr (assoc :id item))) + (obj (make-org-object + :id id + :type (read-from-string (cdr (assoc :type item))) + :attributes (loop for attr in (cdr (assoc :attributes item)) + append (list (intern (string-upcase (car attr)) :keyword) (cdr attr))) + :content (cdr (assoc :content item)) + :parent-id (cdr (assoc :parent-id item)) + :children (cdr (assoc :children item)) + :version (cdr (assoc :version item)) + :last-sync (cdr (assoc :last-sync item)) + :hash (cdr (assoc :hash item))))) + (setf (gethash id *object-store*) obj))) + (kernel-log "PERSISTENCE - Restored from IPFS: ~a" cid) + t) + (error (c) + (kernel-log "PERSISTENCE ERROR - IPFS restoration failed: ~a" c) + nil)))) +#+end_src + +** Cognitive Tools +Expose persistence capabilities to the neural System 1. + +#+begin_src lisp :tangle ../src/state-persistence.lisp +(progn + (def-cognitive-tool :checkpoint-memory "Creates both a local image and a decentralized IPFS snapshot." + :parameters nil + :body (lambda (args) + (declare (ignore args)) + (persistence-dump-local) + (let ((cid (persistence-push-ipfs))) + (format nil "Local dump complete. IPFS CID: ~a" (or cid "FAILED"))))) + + (def-cognitive-tool :restore-memory "Restores the state from a specific source." + :parameters ((:source :type :keyword :description "Either :LOCAL or :IPFS") + (:cid :type :string :description "Required if source is :IPFS")) + :body (lambda (args) + (case (getf args :source) + (:local (if (persistence-load-local) "Restored from disk." "Local restore failed.")) + (:ipfs (if (persistence-restore-ipfs (getf args :cid)) "Restored from network." "IPFS restore failed.")))))) +#+end_src + +** Registration +#+begin_src lisp :tangle ../src/state-persistence.lisp +(defskill :skill-state-persistence + :priority 100 + :trigger (lambda (ctx) + (let ((sensor (getf (getf ctx :payload) :sensor))) + (member sensor '(:heartbeat :manual-persist)))) + :neuro nil + :symbolic (lambda (action ctx) + (persistence-dump-local) + action)) +#+end_src + +* Phase E: Chaos (Verification) + +** 1. Unit Tests (FiveAM) +#+begin_src lisp :tangle ../tests/persistence-tests.lisp +(defpackage :org-agent-persistence-tests + (:use :cl :fiveam :org-agent)) +(in-package :org-agent-persistence-tests) + +(def-suite persistence-suite :description "Tests for State Persistence Layer.") +(in-suite persistence-suite) + +(test test-local-roundtrip + "Ensure RAM -> Disk -> RAM preserves data integrity." + (let ((test-id "persist-test-1")) + (setf (gethash test-id *object-store*) (make-org-object :id test-id :content "Integrity Check")) + (org-agent:persistence-dump-local) + (clrhash *object-store*) + (org-agent:persistence-load-local) + (is (equal "Integrity Check" (org-object-content (gethash test-id *object-store*)))))) +#+end_src + +** 2. Chaos Scenarios +- *Scenario A (IPFS Daemon Down):* Kill the IPFS daemon and verify `persistence-push-ipfs` returns a standardized error instead of hanging the kernel. +- *Scenario B (Corrupt Image):* Intentionally mangle the `memory-image.lisp` file and verify the loader catches the error during `load` and falls back to a clean state. + +* Phase F: Memory (RCA) +- *[2026-04-09 Thu]:* Unified local SBCL image dumps with IPFS decentralized snapshots. Implemented safety-first restoration logic. diff --git a/src/state-persistence.lisp b/src/state-persistence.lisp new file mode 100644 index 0000000..9db6e4d --- /dev/null +++ b/src/state-persistence.lisp @@ -0,0 +1,125 @@ +(in-package :org-agent) + +(defun persistence-get-local-path () + "Returns the path to the local memory image file." + (let ((state-dir (or (uiop:getenv "SYSTEM_DIR") "system/"))) + (merge-pathnames "state/memory-image.lisp" state-dir))) + +(defun persistence-dump-local () + "Serializes the entire history store and current pointers to a local Lisp image." + (let ((image-file (persistence-get-local-path))) + (ensure-directories-exist image-file) + (kernel-log "PERSISTENCE - Dumping local image to ~a..." (uiop:native-namestring image-file)) + (with-open-file (out image-file :direction :output :if-exists :supersede) + (format out "(in-package :org-agent)~%") + ;; 1. Dump all immutable objects in the history store + (maphash (lambda (hash obj) + (print `(setf (gethash ,hash *history-store*) ,obj) out)) + *history-store*) + ;; 2. Dump the current active pointers + (maphash (lambda (id obj) + (print `(setf (gethash ,id *object-store*) (gethash ,(org-object-hash obj) *history-store*)) out)) + *object-store*)) + t)) + +(defun persistence-load-local () + "Loads the memory image from local disk." + (let ((image-file (persistence-get-local-path))) + (if (uiop:file-exists-p image-file) + (progn + (kernel-log "PERSISTENCE - Loading local image...") + (load image-file) + t) + (progn + (kernel-log "PERSISTENCE ERROR - Local image not found.") + nil)))) + +(defun persistence-serialize-for-archival () + "Serializes the entire object-store for IPFS/JSON transport." + (let ((objects nil)) + (maphash (lambda (id obj) + (declare (ignore id)) + (push `((:id . ,(org-object-id obj)) + (:type . ,(format nil "~s" (org-object-type obj))) + (:attributes . ,(loop for (k v) on (org-object-attributes obj) by #'cddr + collect (cons (format nil "~a" k) (format nil "~a" v)))) + (:content . ,(org-object-content obj)) + (:parent-id . ,(org-object-parent-id obj)) + (:children . ,(org-object-children obj)) + (:version . ,(org-object-version obj)) + (:last-sync . ,(org-object-last-sync obj)) + (:hash . ,(org-object-hash obj))) + objects)) + *object-store*) + objects)) + +(defun persistence-push-ipfs () + "Serializes the store and pushes it to IPFS, returning the CID." + (let* ((data (persistence-serialize-for-archival)) + (json-payload (cl-json:encode-json-to-string data)) + (ipfs-url "http://127.0.0.1:5001/api/v0/add")) + (handler-case + (let* ((response (dex:post ipfs-url + :content `(("file" . ,json-payload)) + :headers '(("Content-Type" . "multipart/form-data")))) + (result (cl-json:decode-json-from-string response)) + (cid (cdr (assoc :hash result)))) + (kernel-log "PERSISTENCE - Checkpoint to IPFS successful. CID: ~a" cid) + cid) + (error (c) + (kernel-log "PERSISTENCE ERROR - IPFS push failed: ~a" c) + nil)))) + +(defun persistence-restore-ipfs (cid) + "Fetches data from IPFS and safely hydrates the object-store." + (let ((ipfs-url (format nil "http://127.0.0.1:5001/api/v0/cat?arg=~a" cid))) + (handler-case + (let* ((response (dex:post ipfs-url)) + (data (cl-json:decode-json-from-string response))) + (clrhash *object-store*) + (dolist (item data) + (let* ((id (cdr (assoc :id item))) + (obj (make-org-object + :id id + :type (read-from-string (cdr (assoc :type item))) + :attributes (loop for attr in (cdr (assoc :attributes item)) + append (list (intern (string-upcase (car attr)) :keyword) (cdr attr))) + :content (cdr (assoc :content item)) + :parent-id (cdr (assoc :parent-id item)) + :children (cdr (assoc :children item)) + :version (cdr (assoc :version item)) + :last-sync (cdr (assoc :last-sync item)) + :hash (cdr (assoc :hash item))))) + (setf (gethash id *object-store*) obj))) + (kernel-log "PERSISTENCE - Restored from IPFS: ~a" cid) + t) + (error (c) + (kernel-log "PERSISTENCE ERROR - IPFS restoration failed: ~a" c) + nil)))) + +(progn + (def-cognitive-tool :checkpoint-memory "Creates both a local image and a decentralized IPFS snapshot." + :parameters nil + :body (lambda (args) + (declare (ignore args)) + (persistence-dump-local) + (let ((cid (persistence-push-ipfs))) + (format nil "Local dump complete. IPFS CID: ~a" (or cid "FAILED"))))) + + (def-cognitive-tool :restore-memory "Restores the state from a specific source." + :parameters ((:source :type :keyword :description "Either :LOCAL or :IPFS") + (:cid :type :string :description "Required if source is :IPFS")) + :body (lambda (args) + (case (getf args :source) + (:local (if (persistence-load-local) "Restored from disk." "Local restore failed.")) + (:ipfs (if (persistence-restore-ipfs (getf args :cid)) "Restored from network." "IPFS restore failed.")))))) + +(defskill :skill-state-persistence + :priority 100 + :trigger (lambda (ctx) + (let ((sensor (getf (getf ctx :payload) :sensor))) + (member sensor '(:heartbeat :manual-persist)))) + :neuro nil + :symbolic (lambda (action ctx) + (persistence-dump-local) + action)) diff --git a/tests/persistence-tests.lisp b/tests/persistence-tests.lisp new file mode 100644 index 0000000..5660fd2 --- /dev/null +++ b/tests/persistence-tests.lisp @@ -0,0 +1,15 @@ +(defpackage :org-agent-persistence-tests + (:use :cl :fiveam :org-agent)) +(in-package :org-agent-persistence-tests) + +(def-suite persistence-suite :description "Tests for State Persistence Layer.") +(in-suite persistence-suite) + +(test test-local-roundtrip + "Ensure RAM -> Disk -> RAM preserves data integrity." + (let ((test-id "persist-test-1")) + (setf (gethash test-id *object-store*) (make-org-object :id test-id :content "Integrity Check")) + (org-agent:persistence-dump-local) + (clrhash *object-store*) + (org-agent:persistence-load-local) + (is (equal "Integrity Check" (org-object-content (gethash test-id *object-store*))))))