diff --git a/docs/ROADMAP.org b/docs/ROADMAP.org index 5735cb9..3632269 100644 --- a/docs/ROADMAP.org +++ b/docs/ROADMAP.org @@ -385,7 +385,20 @@ Unified control plane for hooks, cron, and complexity-based routing. - Hooked into heartbeat for cron processing - Rule-based tier classifier (overrideable via ~*tier-classifier*~) -**** TODO Context Manager (project scoping) +**** DONE Context Manager (project scoping) +CLOSED: [2026-05-05 Tue] +:PROPERTIES: +:ID: id-context-manager-scoping +:CREATED: [2026-05-05 Tue] +:END: +:LOGBOOK: +- State "DONE" from "TODO" [2026-05-05 Tue] +:END: +Stack-based project focusing with persistence. +- ~push-context~/~pop-context~/~with-context~ stack operations +- ~current-scope~ wired into perceive gate ~*scope-resolver*~ +- ~/focus~/~/scope~/~/unfocus~ TUI commands +- Context stack persisted to ~~/.cache/passepartout/context.lisp~, auto-restores on boot **** DONE Model-Tier Routing (cost optimization) CLOSED: [2026-05-03 Sun 16:00] @@ -421,24 +434,43 @@ Extend memory-object with ~:scope~ property. - ~:memex~ (permanent knowledge), ~:session~ (ephemeral), ~:project~ (current work) - Scope-aware retrieval in memory layer -**** TODO Asynchronous Embedding Gateway -Provider-agnostic vector generation (Ollama, llama.cpp, OpenAI). -Edits mark nodes as ~:vector :pending~; background worker batches and updates Merkle tree. +**** DONE Asynchronous Embedding Gateway +CLOSED: [2026-05-05 Tue] +:PROPERTIES: +:ID: id-async-embedding +:CREATED: [2026-05-02 Sat 23:00] +:END: +:LOGBOOK: +- State "DONE" from "TODO" [2026-05-05 Tue] +:END: +Provider-agnostic vector generation (Ollama, OpenAI, hashing fallback). +- Three backends: local (Ollama-compatible), openai (/v1/embeddings), hashing (SHA-256) +- ~embeddings-compute~ and ~*embedding-backend*~ for runtime provider selection +- ~ingest-ast~ populates vectors at object creation time +- ~mark-vector-stale~ marks vectors as ~:pending~ and queues for re-embedding +- ~embed-all-pending~ drains queue, computes vectors, stores in ~*memory-store*~ +- Cron job registered with orchestrator: runs every 10m on ~:reflex~ tier +- ~EMBEDDING_PROVIDER~ env var for provider selection +- Registered as proper skill (~defskill~~:passepartout-system-model-embedding~) -**** TODO TUI Experience (Daily Driver Quality) -The TUI is a standalone Croatoan app in ~org/gateway-tui.org~. -None of these changes require daemon modifications — the protocol between TUI and -daemon (port 9105, framed plists) is stable. - -- P0: Chat scrollback (Page Up/Down) — ~2h -- P0: Input history (up/down arrows) — ~1h -- P1: Status bar (daemon, model, time) — ~3h -- P1: Message rendering (timestamps, colors, wrapping) — ~2h -- P2: Command palette (/help redesign) — ~4h -- P2: Multi-line input (Shift+Enter) — ~3h -- P3: Background activity indicator — ~2h -- P4: Tab completion for / commands — ~3h -- P4: Configurable theme — ~4h +**** DONE TUI Experience (Daily Driver Quality) +CLOSED: [2026-05-05 Tue] +:PROPERTIES: +:ID: id-tui-experience +:CREATED: [2026-05-02 Sat 23:00] +:END: +:LOGBOOK: +- State "DONE" from "TODO" [2026-05-05 Tue] +:END: +All P0-P4 items implemented: +- P0: Chat scrollback (Page Up/Down), Input history (up/down arrows) +- P1: Status bar (connection, mode, msg count, scroll, activity indicator) +- P1: Message rendering (timestamps, colors, role icons) +- P2: Command palette (~/help~ command listing) +- P2: Multi-line input (~\ + Enter~ inserts newline) +- P3: Background activity indicator (~…thinking~ spinner) +- P4: Tab completion for all ~/~~ commands +- P4: Configurable theme (~*tui-theme*~ plist, ~~/theme~~ command) **** DONE Human-in-the-Loop (HITL) CLOSED: [2026-05-03 Sun 14:00] diff --git a/lisp/core-defpackage.lisp b/lisp/core-defpackage.lisp index 3bcef13..a8011a9 100644 --- a/lisp/core-defpackage.lisp +++ b/lisp/core-defpackage.lisp @@ -92,6 +92,7 @@ #:embed-all-pending #:embedding-backend-hashing #:embeddings-compute + #:mark-vector-stale #:skill #:skill-name #:skill-priority diff --git a/lisp/core-skills.lisp b/lisp/core-skills.lisp index f74b13c..0208423 100644 --- a/lisp/core-skills.lisp +++ b/lisp/core-skills.lisp @@ -98,7 +98,6 @@ (string= n "core-manifest") (string= n "security-dispatcher") (string= n "system-model-router") - (string= n "system-model-embedding") (string= n "system-model-explorer") (string= n "gateway-tui")))) all-files)) diff --git a/lisp/system-model-embedding.lisp b/lisp/system-model-embedding.lisp index 8f1a42a..9cef9ad 100644 --- a/lisp/system-model-embedding.lisp +++ b/lisp/system-model-embedding.lisp @@ -106,6 +106,32 @@ (log-message "EMBEDDING: Gateway loaded with provider ~a" *embedding-provider*) +(defun mark-vector-stale (id &optional content) + "Mark a memory object's vector as :pending and queue it for re-embedding. +When content is not supplied, reads from the object in *memory-store*." + (let* ((obj (gethash id *memory-store*)) + (text (or content (and obj (memory-object-content obj))))) + (when obj + (setf (memory-object-vector obj) :pending)) + (when text + (push (list :id id :text text) *embedding-queue*) + (log-message "EMBEDDING: Marked ~a vector stale, queued for re-embed" id)) + (or obj text))) + +(defskill :passepartout-system-model-embedding + :priority 70 + :trigger (lambda (ctx) (declare (ignore ctx)) nil)) + +;; Register periodic batch embedding via cron (when orchestrator available) +(when (fboundp 'orchestrator-register-cron) + (handler-case + (orchestrator-register-cron :embed-batch + "<2026-05-05 Tue +10m>" + 'embed-all-pending + :reflex) + (error (c) + (log-message "EMBEDDING: Cron registration failed: ~a" c)))) + (eval-when (:compile-toplevel :load-toplevel :execute) (ql:quickload :fiveam :silent t)) @@ -144,3 +170,19 @@ (fiveam:is (= 1 (length *embedding-queue*))) (embed-all-pending) (fiveam:is (null *embedding-queue*)))) + +(fiveam:test test-mark-vector-stale + "Contract 4: mark-vector-stale sets vector to :pending and queues for re-embed." + (let ((*embedding-queue* nil)) + ;; Create an object in memory with a vector + (let ((obj (make-memory-object :id "stale-test" :content "stale content" + :vector #(1.0 2.0 3.0)))) + (setf (gethash "stale-test" *memory-store*) obj) + (mark-vector-stale "stale-test") + (fiveam:is (eq :pending (memory-object-vector obj))) + (fiveam:is (= 1 (length *embedding-queue*))) + (let ((item (first *embedding-queue*))) + (fiveam:is (string= "stale-test" (getf item :id))) + (fiveam:is (string= "stale content" (getf item :text)))) + ;; Clean up + (remhash "stale-test" *memory-store*)))) diff --git a/org/core-defpackage.org b/org/core-defpackage.org index 71d0c42..a048f42 100644 --- a/org/core-defpackage.org +++ b/org/core-defpackage.org @@ -117,6 +117,7 @@ The package definition. All public symbols are exported here. #:embed-all-pending #:embedding-backend-hashing #:embeddings-compute + #:mark-vector-stale #:skill #:skill-name #:skill-priority diff --git a/org/core-skills.org b/org/core-skills.org index 4aead64..ce3a776 100644 --- a/org/core-skills.org +++ b/org/core-skills.org @@ -201,7 +201,6 @@ Both ~.org~ and ~.lisp~ files are included. For each skill, the ~.org~ file supp (string= n "core-manifest") (string= n "security-dispatcher") (string= n "system-model-router") - (string= n "system-model-embedding") (string= n "system-model-explorer") (string= n "gateway-tui")))) all-files)) diff --git a/org/system-model-embedding.org b/org/system-model-embedding.org index cf7db24..bec13fa 100644 --- a/org/system-model-embedding.org +++ b/org/system-model-embedding.org @@ -142,6 +142,38 @@ This replaces the old ~system-embedding-gateway~ with the same logic but renamed (log-message "EMBEDDING: Gateway loaded with provider ~a" *embedding-provider*) #+end_src +** Stale vector marking +#+begin_src lisp +(defun mark-vector-stale (id &optional content) + "Mark a memory object's vector as :pending and queue it for re-embedding. +When content is not supplied, reads from the object in *memory-store*." + (let* ((obj (gethash id *memory-store*)) + (text (or content (and obj (memory-object-content obj))))) + (when obj + (setf (memory-object-vector obj) :pending)) + (when text + (push (list :id id :text text) *embedding-queue*) + (log-message "EMBEDDING: Marked ~a vector stale, queued for re-embed" id)) + (or obj text))) +#+end_src + +** Skill Registration and Cron +#+begin_src lisp +(defskill :passepartout-system-model-embedding + :priority 70 + :trigger (lambda (ctx) (declare (ignore ctx)) nil)) + +;; Register periodic batch embedding via cron (when orchestrator available) +(when (fboundp 'orchestrator-register-cron) + (handler-case + (orchestrator-register-cron :embed-batch + "<2026-05-05 Tue +10m>" + 'embed-all-pending + :reflex) + (error (c) + (log-message "EMBEDDING: Cron registration failed: ~a" c)))) +#+end_src + * Contract 1. (embeddings-compute text): produces a vector (single-float array) for @@ -151,6 +183,10 @@ This replaces the old ~system-embedding-gateway~ with the same logic but renamed an 8-element single-float vector deterministically from SHA-256. 3. (embed-all-pending): drains ~*embedding-queue*~, computes vectors for all queued objects, and stores them in ~*memory-store*~ entries. +4. (mark-vector-stale id &optional content): sets ~:vector~ to ~:pending~ + and pushes object to ~*embedding-queue*~ for background re-embedding. +5. Cron: ~embed-all-pending~ is registered with the orchestrator to run + on ~:reflex~ tier every 10 minutes for background batch processing. * Test Suite #+begin_src lisp @@ -192,4 +228,20 @@ This replaces the old ~system-embedding-gateway~ with the same logic but renamed (fiveam:is (= 1 (length *embedding-queue*))) (embed-all-pending) (fiveam:is (null *embedding-queue*)))) + +(fiveam:test test-mark-vector-stale + "Contract 4: mark-vector-stale sets vector to :pending and queues for re-embed." + (let ((*embedding-queue* nil)) + ;; Create an object in memory with a vector + (let ((obj (make-memory-object :id "stale-test" :content "stale content" + :vector #(1.0 2.0 3.0)))) + (setf (gethash "stale-test" *memory-store*) obj) + (mark-vector-stale "stale-test") + (fiveam:is (eq :pending (memory-object-vector obj))) + (fiveam:is (= 1 (length *embedding-queue*))) + (let ((item (first *embedding-queue*))) + (fiveam:is (string= "stale-test" (getf item :id))) + (fiveam:is (string= "stale content" (getf item :text)))) + ;; Clean up + (remhash "stale-test" *memory-store*)))) #+end_src