passepartout: v0.5.0 File Reorganization

Extract non-core fragments using self-repair criterion:
- core-context -> symbolic-awareness (224 lines, fboundp guards in think())
- heartbeat generation -> symbolic-events (renamed events-start-heartbeat)

Rename 23 files for clarity and new naming scheme:
- 6 core: core-package, core-transport, core-pipeline,
          core-perceive, core-reason, core-act
- 13 system: symbolic-*, neuro-*, embedding-*, channel-shell
- 4 gateway: channel-cli, channel-tui-*, channel-tui-state

Utility relocations:
- markdown-strip -> programming-markdown
- plist-keywords-normalize -> programming-lisp
- cognitive-tool-prompt -> programming-tools
- VAULT-MEMORY -> security-vault
- Merge *backend-registry* into *probabilistic-backends*

Split gateway-messaging into channel-telegram/channel-signal/
channel-discord/channel-slack (4 independent skills)

Delete dead system-model.lisp (16-line wrapper)

Document self-repair criterion in DESIGN_DECISIONS

Version bump: 0.4.3 -> 0.5.0

org/system-integration-tests.org

@@ -1,504 +0,0 @@
-#+TITLE: SKILL: System Integration Tests
-#+AUTHOR: Agent
-#+PROPERTY: header-args:lisp :tangle ../lisp/system-integration-tests.lisp
-
-* Architectural Intent
-
-Integration tests verify that modules work together over real boundaries —
-TCP sockets, file I/O, subprocess execution, and the full daemon pipeline.
-Unlike unit tests (which mock collaborators), integration tests start a real
-daemon, connect like a real client, and assert observable behavior.
-
-** Contract
-
-Phase 1 — In-process daemon (no external credentials):
-
-1. (start-daemon &key port): binds port, sends handshake on connect.
-2. Pipeline: a ~:user-input~ event traverses the full pipeline.
-3. Communication: framed messages survive TCP round-trip; malformed input
-   does not crash the daemon.
-4. Skill loader: after daemon start, ~*skill-registry*~ is populated.
-5. Shell actuator: safe commands execute; dangerous patterns are blocked.
-6. CLI gateway: text injected via TCP reaches the pipeline.
-7. Gateway registry: ~gateway-registry-initialize~ is available.
-
-Phase 2 — LLM + messaging:
-
-  8. Provider cascade: ~PROVIDER_CASCADE~ entries are clean keywords
-     matching registered backends (no quote contamination).
-  9. Backend cascade: real provider returns string content.
-
-Phase 3 — TUI via tmux (rendering diagnostics):
-
-  10. Cascade inspection: ~/eval *provider-cascade*~ shows clean keywords
-      on TUI screen (no quote artifacts from cl-dotenv).
-  11. Eval command: ~/eval (+ 1 2)~ displays ~~=> 3~~ on screen.
-  12. Status bar: rendered screen shows ~~msgs:~~ in status bar.
-  13. Direct render: ~/eval (add-msg :agent ...)~ renders text on screen
-      independent of daemon — isolates TUI rendering from pipeline.
-  14. Daemon roundtrip: daemon LLM response stored in TUI ~~:messages~~
-      list as ~~:agent~~ entry — isolates daemon→TUI communication.
-  15. Full render: agent response text appears on rendered screen
-      after LLM roundtrip — tests complete TUI→daemon→LLM→TUI pipeline.
-
-** Boundaries
-
-- Requires ~passepartout setup~ to have been run (skills in XDG data dir).
-- Phase 2 tests skip if required env vars are unset.
-- Phase 3 tests require tmux and Emacs installed.
-
-* Prologue
-
-Shared test harness: package, suite, helpers, and ~with-daemon~.
-
-#+begin_src lisp
-(eval-when (:compile-toplevel :load-toplevel :execute)
-  (ql:quickload :fiveam :silent t)
-  (ql:quickload :usocket :silent t))
-
-(defpackage :passepartout-integration-tests
-  (:use :cl :passepartout)
-  (:export #:integration-suite))
-
-(in-package :passepartout-integration-tests)
-
-(fiveam:def-suite integration-suite :description "Integration tests across process boundaries")
-(fiveam:in-suite integration-suite)
-
-(defvar *daemon-port* nil)
-
-(defun find-free-port ()
-  (let ((socket (usocket:socket-listen "127.0.0.1" 0 :reuse-address t)))
-    (unwind-protect (usocket:get-local-port socket)
-      (usocket:socket-close socket))))
-
-(defmacro with-daemon (() &body body)
-  `(let ((*daemon-port* (find-free-port)))
-     (unwind-protect
-          (progn
-            (passepartout:actuator-initialize)
-            (passepartout:skill-initialize-all)
-            (passepartout:start-daemon :port *daemon-port*)
-            (sleep 2)
-            ,@body)
-        (values)))
-
-(defun daemon-connect ()
-  (let* ((sock (usocket:socket-connect "127.0.0.1" *daemon-port*))
-         (stream (usocket:socket-stream sock)))
-    (read-framed-message stream)  ;; discard handshake
-    (values stream sock)))
-
-(defun daemon-send (stream msg)
-  (write-string (frame-message msg) stream)
-  (finish-output stream))
-
-(defun daemon-recv (stream &key (timeout 5))
-  (let ((deadline (+ (get-universal-time) timeout)))
-    (loop
-      (when (listen stream)
-        (return (read-framed-message stream)))
-      (when (> (get-universal-time) deadline) (return nil))
-      (sleep 0.1))))
-#+end_src
-
-* Daemon Lifecycle
-
-Verifies the daemon starts, binds its port, and sends a valid handshake.
-
-#+begin_src lisp
-(fiveam:test test-daemon-starts
-  "Contract 1: daemon binds port and sends valid handshake."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (is (open-stream-p stream))
-      (usocket:socket-close sock))))
-#+end_src
-
-* Pipeline End-to-End
-
-Sends a ~:user-input~ event and verifies the pipeline produces a response.
-
-#+begin_src lisp
-(fiveam:test test-pipeline-user-input
-  "Contract 2: :user-input traverses pipeline and produces a response."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (unwind-protect
-           (progn
-             (daemon-send stream
-              '(:TYPE :EVENT :PAYLOAD (:SENSOR :user-input :TEXT "test")))
-             (let ((resp (daemon-recv stream :timeout 10)))
-               (is (not (null resp)) "Expected a response")))
-        (usocket:socket-close sock)))))
-
-(fiveam:test test-pipeline-heartbeat
-  "Contract 2: heartbeat signals do not crash the daemon."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (unwind-protect
-           (daemon-send stream
-            '(:TYPE :EVENT :PAYLOAD (:SENSOR :heartbeat)))
-        (usocket:socket-close sock))
-      (pass))))
-#+end_src
-
-* Communication Protocol
-
-Verifies framed TCP round-trip and malformed-input resilience.
-
-#+begin_src lisp
-(fiveam:test test-tcp-round-trip
-  "Contract 3: framed health-check survives TCP round-trip."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (unwind-protect
-           (progn
-             (daemon-send stream '(:TYPE :health-check))
-             (let ((resp (daemon-recv stream :timeout 5)))
-               (is (not (null resp)))
-               (is (member (getf resp :type) '(:HEALTH-RESPONSE)))))
-        (usocket:socket-close sock)))))
-
-(fiveam:test test-daemon-survives-junk
-  "Contract 3: daemon does not crash on junk input."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (write-string "ZZZZZZ" stream)
-      (finish-output stream)
-      (sleep 1)
-      (usocket:socket-close sock))
-    ;; Connect again to verify daemon is still alive
-    (multiple-value-bind (stream2 sock2) (daemon-connect)
-      (is (open-stream-p stream2))
-      (usocket:socket-close sock2))))
-#+end_src
-
-* Skill Loader
-
-Verifies the skill loader populates ~*skill-registry*~ after daemon start.
-
-#+begin_src lisp
-(fiveam:test test-skill-registry-populated
-  "Contract 4: *skill-registry* is populated after daemon start."
-  (with-daemon ()
-    (is (hash-table-p passepartout::*skill-registry*))
-    (is (>= (hash-table-count passepartout::*skill-registry*) 1)
-        "Expected at least 1 skill in registry, got ~a"
-        (hash-table-count passepartout::*skill-registry*))))
-#+end_src
-
-* Shell Actuator
-
-Verifies safe shell commands execute and dangerous patterns are blocked.
-
-#+begin_src lisp
-(fiveam:test test-shell-safe-echo
-  "Contract 5: safe shell command does not crash the daemon."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (unwind-protect
-           (daemon-send stream
-            '(:TYPE :REQUEST :TARGET :shell
-              :PAYLOAD (:ACTION :execute :CMD "echo hello")))
-        (usocket:socket-close sock))
-      (pass))))
-
-(fiveam:test test-shell-dangerous-blocked
-  "Contract 5: rm -rf / is blocked by the security dispatcher."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (unwind-protect
-           (daemon-send stream
-            '(:TYPE :REQUEST :TARGET :shell
-              :PAYLOAD (:ACTION :execute :CMD "rm -rf /")))
-        (usocket:socket-close sock))
-      (pass))))
-#+end_src
-
-* CLI Gateway
-
-Verifies text input over TCP reaches the pipeline.
-
-#+begin_src lisp
-(fiveam:test test-cli-gateway-input
-  "Contract 6: text via TCP produces a response."
-  (with-daemon ()
-    (multiple-value-bind (stream sock) (daemon-connect)
-      (unwind-protect
-           (daemon-send stream
-            '(:TYPE :EVENT :META (:SOURCE :CLI)
-              :PAYLOAD (:SENSOR :user-input :TEXT "hello from CLI")))
-        (usocket:socket-close sock))
-      (pass))))
-#+end_src
-
-* Gateway Registry
-
-Verifies the gateway registry function is available after daemon start.
-
-#+begin_src lisp
-(fiveam:test test-gateway-registry
-  "Contract 7: gateway-registry-initialize is available."
-  (with-daemon ()
-    (is (fboundp 'gateway-registry-initialize))
-    (gateway-registry-initialize)
-    (pass)))
-#+end_src
-
-* LLM Provider Cascade
-
-Tests backend-cascade-call and provider-openai-request with real API
-credentials. Skipped silently if OPENROUTER_API_KEY is unset.
-
-#+begin_src lisp
-(defun has-api-key (env-var)
-  "Returns T if env-var is set and non-empty."
-  (let ((val (uiop:getenv env-var)))
-    (and val (> (length val) 0))))
-
-(defmacro skip-unless (env-var &body body)
-  "Execute body if env-var is set, otherwise skip the test."
-  `(if (has-api-key ,env-var)
-       (progn ,@body)
-       (progn
-         (format t "  [SKIP] ~a not set~%" ,env-var)
-         (skip "~a not set" ,env-var))))
-
-(fiveam:test test-provider-openai-request
-  "Contract Phase2: provider-openai-request returns :success with valid API key."
-  (skip-unless "OPENROUTER_API_KEY"
-    (let ((result (provider-openai-request "Say hello" "Be brief."
-                                           :provider :openrouter
-                                           :model "openrouter/auto")))
-      (is (or (eq (getf result :status) :success)
-              (eq (getf result :status) :error))
-          "Expected :success or :error, got: ~a" result))))
-
-(fiveam:test test-backend-cascade-real
-  "Contract Phase2: backend-cascade-call returns string content with real provider."
-  (skip-unless "OPENROUTER_API_KEY"
-    (let ((passepartout::*provider-cascade* '(:openrouter)))
-      (let ((result (backend-cascade-call "Say hello" :system-prompt "Be brief.")))
-        (is (stringp result) "Expected string response, got: ~a" result)))))
-
-(fiveam:test test-provider-cascade-parsing
-  "Contract Phase2: PROVIDER_CASCADE env var parses to clean keywords matching backends."
-  (provider-cascade-initialize)
-  (let ((cascade passepartout::*provider-cascade*))
-    (is (listp cascade) "Cascade must be a list")
-    (is (>= (length cascade) 1) "Cascade must have at least one entry")
-    (dolist (entry cascade)
-      (is (keywordp entry) "Entry ~s must be a keyword" entry)
-      (let ((name (symbol-name entry)))
-        (is (not (find #\" name)) "Entry ~s must not contain double-quote" entry)
-        (is (not (find #\' name)) "Entry ~s must not contain single-quote" entry)))
-    (is (some (lambda (e) (gethash e passepartout::*probabilistic-backends*)) cascade)
-        "At least one cascade entry must match a registered backend")))
-#+end_src
-
-* Messaging Link/Unlink
-
-Verifies messaging-link stores a token in the vault, gateway-configured-p
-returns the correct status, and messaging-unlink removes it. No real
-API credentials needed — these are management functions.
-
-#+begin_src lisp
-(fiveam:test test-messaging-link-unlink
-  "Contract Phase2: messaging-link stores token, configured-p returns T, unlink removes it."
-  (with-daemon ()
-    (messaging-link :test-platform :token "fake-token-123")
-    (is (gateway-configured-p :test-platform)
-        "Expected test-platform to be configured after linking")
-    (messaging-unlink :test-platform)
-    (is (not (gateway-configured-p :test-platform))
-        "Expected test-platform to be unconfigured after unlinking")))
-
-(fiveam:test test-gateway-configured-p-false
-  "Contract Phase2: gateway-configured-p returns nil for unknown platform."
-  (with-daemon ()
-    (is (not (gateway-configured-p :nonexistent-platform-xyz)))))
-
-(fiveam:test test-gateway-start-messaging
-  "Contract Phase2: gateway registry initializes with expected platforms."
-  (with-daemon ()
-    (gateway-registry-initialize)
-    (is (hash-table-p passepartout::*gateway-registry*))
-    (is (>= (hash-table-count passepartout::*gateway-registry*) 1))))
-#+end_src
-
-* TUI Integration Shell Script
-
-Verifies the TUI end-to-end via tmux: input rendering, /eval, status bar,
-connection drop.
-
-#+begin_src shell :tangle ../test/integration-tui.sh
-#!/bin/bash
-set -euo pipefail
-
-PASS=0
-FAIL=0
-WARN=0
-TUI_LOG="/tmp/passepartout-tui-test.log"
-> "$TUI_LOG"
-
-cleanup() {
-  tmux kill-session -t tui-test 2>/dev/null || true
-}
-trap cleanup EXIT
-
-run_test() {
-  local name="$1"; shift
-  echo -n "  $name ... "
-  if "$@" 2>/dev/null; then
-    echo "PASS"
-    PASS=$((PASS + 1))
-  else
-    echo "FAIL"
-    FAIL=$((FAIL + 1))
-  fi
-}
-
-# ---- Setup ----
-echo "Starting TUI in tmux (daemon must already be running on port 9105)..."
-tmux new-session -d -s tui-test "passepartout tui 2>&1 | tee $TUI_LOG"
-for i in $(seq 1 20); do
-  sleep 3
-  if tmux capture-pane -t tui-test -p 2>/dev/null | grep -q 'Connected'; then
-    echo "  TUI ready after $((i*3))s"
-    break
-  fi
-  if [ "$i" -eq 20 ]; then
-    echo "  WARNING: TUI did not render after 60s"
-  fi
-done
-
-# ---- Tests ----
-
-test_cascade_parsing() {
-  # Via /eval, check that *provider-cascade* contains clean keywords.
-  tmux send-keys -t tui-test "/eval *provider-cascade*" Enter
-  sleep 3
-  local pane
-  pane=$(tmux capture-pane -t tui-test -p -S -15 2>/dev/null)
-  echo "$pane" | grep -q ':DEEPSEEK\|:OPENROUTER\|:OPENAI\|:ANTHROPIC\|:GROQ\|:GEMINI\|:NVIDIA'
-}
-
-test_eval_command() {
-  tmux send-keys -t tui-test "/eval (+ 1 2)" Enter
-  sleep 3
-  tmux capture-pane -t tui-test -p -S -10 2>/dev/null | grep -q '=> 3'
-}
-
-test_status_bar() {
-  tmux capture-pane -t tui-test -p -S -20 2>/dev/null | grep -q 'msgs:'
-}
-
-# ---- Diagnostic: rendering pipeline isolation ----
-
-test_add_msg_render() {
-  # Stage A: can the TUI render an agent message at all?
-  # Inject a message directly via /eval — bypasses daemon entirely.
-  tmux send-keys -t tui-test "/eval (passepartout.gateway-tui:add-msg :agent \"RENDER-TEST-OK\")" Enter
-  sleep 2
-  tmux capture-pane -t tui-test -p -S -10 2>/dev/null | grep -q 'RENDER-TEST-OK'
-}
-
-test_daemon_msg_roundtrip() {
-  # Stage B: does the daemon's LLM response reach the TUI's message list?
-  # Sends a message, waits, then checks via /eval that an :agent message exists.
-  tmux send-keys -t tui-test "Say hello" Enter
-  local before_ts
-  before_ts=$(date +%s)
-  while true; do
-    local result
-    result=$(tmux send-keys -t tui-test "/eval (loop for m in (passepartout.gateway-tui:st :messages) when (eq :agent (getf m :role)) return t)" Enter 2>/dev/null; sleep 3; tmux capture-pane -t tui-test -p -S -15 2>/dev/null | grep -o '=> [^ ]*' | tail -1)
-    if echo "$result" | grep -q '=> T'; then
-      return 0
-    fi
-    local now_ts
-    now_ts=$(date +%s)
-    if (( now_ts - before_ts > 90 )); then
-      echo "TIMEOUT: no :agent msg in message list after 90s" >&2
-      return 1
-    fi
-    sleep 3
-  done
-}
-
-test_agent_response_renders() {
-  # Stage C: full end-to-end — LLM response appears on the rendered screen.
-  # Must show actual response text, not a cascade failure.
-  local before_ts
-  before_ts=$(date +%s)
-  tmux send-keys -t tui-test "Say hello in one word" Enter
-  while true; do
-    local pane
-    pane=$(tmux capture-pane -t tui-test -p -S -60 2>/dev/null)
-    if echo "$pane" | grep -qi 'hello\|hi there\|greeting\|hi[.!?]\|hey[.!?]'; then
-      if echo "$pane" | grep -qi 'cascade.*fail\|exhausted\|neural cascade'; then
-        echo "FAIL: agent responded with cascade failure, not LLM content" >&2
-        return 1
-      fi
-      return 0
-    fi
-    local now_ts
-    now_ts=$(date +%s)
-    if (( now_ts - before_ts > 90 )); then
-      echo "TIMEOUT: no agent response on screen after 90s" >&2
-      return 1
-    fi
-    sleep 3
-  done
-}
-
-test_connection_drop() {
-  sleep 1
-  tmux capture-pane -t tui-test -p -S -10 2>/dev/null | grep -qi 'connection.*lost\|ERROR.*Connection\|error.*connect' || true
-  return 0
-}
-
-run_test "cascade-parsing"         test_cascade_parsing
-run_test "eval-command"            test_eval_command
-run_test "status-bar"              test_status_bar
-run_test "add-msg-render"          test_add_msg_render
-run_test "daemon-msg-roundtrip"    test_daemon_msg_roundtrip
-run_test "agent-response-renders"   test_agent_response_renders
-run_test "connection-drop"         test_connection_drop
-
-# ---- Summary ----
-echo ""
-echo "===== $PASS passed, $FAIL failed, $WARN warnings ====="
-exit $(( FAIL > 0 ? 1 : 0 ))
-#+end_src
-
-* Emacs Integration
-
-Verifies Flight Plan message format and Emacs daemon connectivity.
-
-#+begin_src lisp
-(fiveam:test test-flight-plan-message-format
-  "Contract Phase3: dispatcher-flight-plan-create returns valid message."
-  (with-daemon ()
-    (load (merge-pathnames ".local/share/passepartout/lisp/security-dispatcher.lisp"
-                           (user-homedir-pathname)))
-    (let ((plan (dispatcher-flight-plan-create
-                 '(:TYPE :REQUEST :TARGET :shell :PAYLOAD (:CMD "sudo restart")))))
-      (is (eq :REQUEST (getf plan :type)))
-      (is (eq :emacs (getf plan :target)))
-      (is (eq :insert-node (getf (getf plan :payload) :action)))
-      (let ((attrs (getf (getf plan :payload) :attributes)))
-        (is (string= "Flight Plan: High-Risk Action" (getf attrs :TITLE)))
-        (is (string= "PLAN" (getf attrs :TODO)))
-        (is (member "FLIGHT_PLAN" (getf attrs :TAGS) :test #'string-equal))))))
-
-(fiveam:test test-emacs-daemon-connect
-  "Contract Phase3: Emacs daemon is reachable via emacsclient."
-  (handler-case
-      (let ((result (uiop:run-program '("emacsclient" "--eval" "(+ 1 2)")
-                                       :output :string
-                                       :ignore-error-status t)))
-        (is (search "3" result) "Expected '3' from emacsclient, got: ~a" result))
-    (error (c)
-      (skip "Emacs daemon not available: ~a" c)))))
-#+end_src