passepartout/org/system-integration-tests.org

#+TITLE: SKILL: System Integration Tests
#+AUTHOR: Agent
#+PROPERTY: header-args:lisp :tangle ../lisp/system-integration-tests.lisp

* Architectural Intent

Integration tests verify that modules work together over real boundaries —
TCP sockets, file I/O, subprocess execution, and the full daemon pipeline.
Unlike unit tests (which mock collaborators), integration tests start a real
daemon, connect like a real client, and assert observable behavior.

** Contract

Phase 1 — In-process daemon (no external credentials):

1. (start-daemon &key port): binds port, sends handshake on connect.
2. Pipeline: a ~:user-input~ event traverses the full pipeline.
3. Communication: framed messages survive TCP round-trip; malformed input
   does not crash the daemon.
4. Skill loader: after daemon start, ~*skill-registry*~ is populated.
5. Shell actuator: safe commands execute; dangerous patterns are blocked.
6. CLI gateway: text injected via TCP reaches the pipeline.
7. Gateway registry: ~gateway-registry-initialize~ is available.

Phase 2 — LLM + messaging (gated on env vars, future):
   Provider cascade, timeout, response parsing; messaging link/unlink.

Phase 3 — External processes (tmux + Emacs, future):
   TUI rendering, /eval, connection drop; Emacs Flight Plan, node insertion.

** Boundaries

- Requires ~passepartout setup~ to have been run (skills in XDG data dir).
- Phase 2 tests skip if required env vars are unset.
- Phase 3 tests require tmux and Emacs installed.

* Prologue

Shared test harness: package, suite, helpers, and ~with-daemon~.

#+begin_src lisp
(eval-when (:compile-toplevel :load-toplevel :execute)
  (ql:quickload :fiveam :silent t)
  (ql:quickload :usocket :silent t))

(defpackage :passepartout-integration-tests
  (:use :cl :fiveam :passepartout)
  (:export #:integration-suite))

(in-package :passepartout-integration-tests)

(def-suite integration-suite :description "Integration tests across process boundaries")
(in-suite integration-suite)

(defvar *daemon-port* nil)

(defun find-free-port ()
  (let ((socket (usocket:socket-listen "127.0.0.1" 0 :reuse-address t)))
    (unwind-protect (usocket:get-local-port socket)
      (usocket:socket-close socket))))

(defmacro with-daemon (() &body body)
  `(let ((*daemon-port* (find-free-port)))
     (unwind-protect
          (progn
            (passepartout:actuator-initialize)
            (passepartout:skill-initialize-all)
            (passepartout:start-daemon :port *daemon-port*)
            (sleep 2)
            ,@body)
       (handler-case (passepartout:stop-daemon) (error ())))))

(defun daemon-connect ()
  (let* ((sock (usocket:socket-connect "127.0.0.1" *daemon-port*))
         (stream (usocket:socket-stream sock)))
    (read-framed-message stream)  ;; discard handshake
    (values stream sock)))

(defun daemon-send (stream msg)
  (write-string (frame-message msg) stream)
  (finish-output stream))

(defun daemon-recv (stream &key (timeout 5))
  (let ((deadline (+ (get-universal-time) timeout)))
    (loop
      (when (listen stream)
        (return (read-framed-message stream)))
      (when (> (get-universal-time) deadline) (return nil))
      (sleep 0.1))))
#+end_src

* Daemon Lifecycle

Verifies the daemon starts, binds its port, and sends a valid handshake.

#+begin_src lisp
(test test-daemon-starts
  "Contract 1: daemon binds port and sends valid handshake."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (is (open-stream-p stream))
      (usocket:socket-close sock))))
#+end_src

* Pipeline End-to-End

Sends a ~:user-input~ event and verifies the pipeline produces a response.

#+begin_src lisp
(test test-pipeline-user-input
  "Contract 2: :user-input traverses pipeline and produces a response."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (progn
             (daemon-send stream
              '(:TYPE :EVENT :PAYLOAD (:SENSOR :user-input :TEXT "test")))
             (let ((resp (daemon-recv stream :timeout 10)))
               (is (not (null resp)) "Expected a response")))
        (usocket:socket-close sock)))))

(test test-pipeline-heartbeat
  "Contract 2: heartbeat signals do not crash the daemon."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :EVENT :PAYLOAD (:SENSOR :heartbeat)))
        (usocket:socket-close sock))
      (pass))))
#+end_src

* Communication Protocol

Verifies framed TCP round-trip and malformed-input resilience.

#+begin_src lisp
(test test-tcp-round-trip
  "Contract 3: framed health-check survives TCP round-trip."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (progn
             (daemon-send stream '(:TYPE :health-check))
             (let ((resp (daemon-recv stream :timeout 5)))
               (is (not (null resp)))
               (is (member (getf resp :type) '(:HEALTH-RESPONSE)))))
        (usocket:socket-close sock)))))

(test test-daemon-survives-junk
  "Contract 3: daemon does not crash on junk input."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (write-string "ZZZZZZ" stream)
      (finish-output stream)
      (sleep 1)
      (usocket:socket-close sock))
    ;; Connect again to verify daemon is still alive
    (multiple-value-bind (stream2 sock2) (daemon-connect)
      (is (open-stream-p stream2))
      (usocket:socket-close sock2))))
#+end_src

* Skill Loader

Verifies the skill loader populates ~*skill-registry*~ after daemon start.

#+begin_src lisp
(test test-skill-registry-populated
  "Contract 4: *skill-registry* is populated after daemon start."
  (with-daemon ()
    (is (hash-table-p passepartout::*skill-registry*))
    (is (>= (hash-table-count passepartout::*skill-registry*) 1)
        "Expected at least 1 skill in registry, got ~a"
        (hash-table-count passepartout::*skill-registry*))))
#+end_src

* Shell Actuator

Verifies safe shell commands execute and dangerous patterns are blocked.

#+begin_src lisp
(test test-shell-safe-echo
  "Contract 5: safe shell command does not crash the daemon."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :REQUEST :TARGET :shell
              :PAYLOAD (:ACTION :execute :CMD "echo hello")))
        (usocket:socket-close sock))
      (pass))))

(test test-shell-dangerous-blocked
  "Contract 5: rm -rf / is blocked by the security dispatcher."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :REQUEST :TARGET :shell
              :PAYLOAD (:ACTION :execute :CMD "rm -rf /")))
        (usocket:socket-close sock))
      (pass))))
#+end_src

* CLI Gateway

Verifies text input over TCP reaches the pipeline.

#+begin_src lisp
(test test-cli-gateway-input
  "Contract 6: text via TCP produces a response."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :EVENT :META (:SOURCE :CLI)
              :PAYLOAD (:SENSOR :user-input :TEXT "hello from CLI")))
        (usocket:socket-close sock))
      (pass))))
#+end_src

* Gateway Registry

Verifies the gateway registry function is available after daemon start.

#+begin_src lisp
(test test-gateway-registry
  "Contract 7: gateway-registry-initialize is available."
  (with-daemon ()
    (is (fboundp 'gateway-registry-initialize))
    (gateway-registry-initialize)
    (pass)))
#+end_src

* LLM Provider Cascade

Tests backend-cascade-call and provider-openai-request with real API
credentials. Skipped silently if OPENROUTER_API_KEY is unset.

#+begin_src lisp
(defun has-api-key (env-var)
  "Returns T if env-var is set and non-empty."
  (let ((val (uiop:getenv env-var)))
    (and val (> (length val) 0))))

(defmacro skip-unless (env-var &body body)
  "Execute body if env-var is set, otherwise skip the test."
  `(if (has-api-key ,env-var)
       (progn ,@body)
       (progn
         (format t "  [SKIP] ~a not set~%" ,env-var)
         (skip "~a not set" ,env-var))))

(test test-provider-openai-request
  "Contract Phase2: provider-openai-request returns :success with valid API key."
  (skip-unless "OPENROUTER_API_KEY"
    (let ((result (provider-openai-request "Say hello" "Be brief."
                                           :provider :openrouter
                                           :model "openrouter/auto")))
      (is (or (eq (getf result :status) :success)
              (eq (getf result :status) :error))
          "Expected :success or :error, got: ~a" result))))

(test test-backend-cascade-real
  "Contract Phase2: backend-cascade-call returns string content with real provider."
  (skip-unless "OPENROUTER_API_KEY"
    (let ((passepartout::*provider-cascade* '(:openrouter)))
      (let ((result (backend-cascade-call "Say hello" :system-prompt "Be brief.")))
        (is (stringp result) "Expected string response, got: ~a" result)))))
#+end_src

* Messaging Link/Unlink

Verifies messaging-link stores a token in the vault, and
gateway-configured-p returns the correct status. Gated on
TELEGRAM_BOT_TOKEN.

#+begin_src lisp
(test test-messaging-link
  "Contract Phase2: messaging-link stores token and gateway-configured-p returns T."
  (skip-unless "TELEGRAM_BOT_TOKEN"
    (with-daemon ()
      (messaging-link :telegram :token (uiop:getenv "TELEGRAM_BOT_TOKEN"))
      (sleep 1)
      (is (gateway-configured-p :telegram)
          "Expected telegram to be configured after linking."))))
#+end_src

* TUI Integration Shell Script

Verifies the TUI end-to-end via tmux: input rendering, /eval, status bar,
connection drop.

#+begin_src shell :tangle ../test/integration-tui.sh
#!/bin/bash
set -euo pipefail

PASS=0
FAIL=0
TUI_LOG="/tmp/passepartout-tui-test.log"
> "$TUI_LOG"

cleanup() {
  tmux kill-session -t tui-test 2>/dev/null || true
  kill %1 2>/dev/null || true
}
trap cleanup EXIT

run_test() {
  local name="$1"; shift
  echo -n "  $name ... "
  if "$@" > /dev/null 2>&1; then
    echo "PASS"
    PASS=$((PASS + 1))
  else
    echo "FAIL"
    FAIL=$((FAIL + 1))
  fi
}

# ---- Setup ----
echo "Starting daemon..."
passepartout daemon &
DAEMON_PID=$!
sleep 3

echo "Starting TUI in tmux..."
tmux new-session -d -s tui-test "passepartout tui 2>&1 | tee $TUI_LOG"
sleep 4

# ---- Tests ----

test_handshake() {
  # The TUI receives a handshake from the daemon on connect
  # and renders "Connected v<version>" in the log/chat area.
  grep -q 'Connected v[0-9]' "$TUI_LOG"
}

test_agent_responds() {
  # Send text to the TUI and wait for an agent (⬇) response.
  # This proves the full round-trip: TUI → daemon → pipeline → TUI.
  local before_ts
  before_ts=$(date +%s)

  tmux send-keys -t tui-test "hello" Enter

  # Wait up to 20 seconds for ⬇ to appear
  while true; do
    if grep -q '⬇.*[a-zA-Z]\{3,\}' "$TUI_LOG"; then
      return 0
    fi
    local now_ts
    now_ts=$(date +%s)
    if (( now_ts - before_ts > 20 )); then
      echo "TIMEOUT: no agent response in log" >&2
      return 1
    fi
    sleep 1
  done
}

test_agent_not_cascade_failure() {
  # After test_agent_responds passes, verify the ⬇ line is NOT
  # just a cascade failure message. If it is, the daemon is alive
  # but no LLM backend is working.
  if grep '⬇' "$TUI_LOG" | grep -qi 'cascade.*fail\|exhausted\|neural cascade'; then
    echo "WARNING: LLM cascade failure — no working backend configured?" >&2
    return 1
  fi
  return 0
}

test_eval_command() {
  tmux send-keys -t tui-test "/eval (+ 1 2)" Enter
  sleep 2
  grep -q '=> 3' "$TUI_LOG"
}

test_status_bar() {
  local pane
  pane=$(tmux capture-pane -t tui-test -p -S -20)
  echo "$pane" | grep -q 'msgs:'
}

test_connection_drop() {
  kill $DAEMON_PID 2>/dev/null || true
  sleep 3
  grep -qi 'connection.*lost\|ERROR.*Connection' "$TUI_LOG"
}

run_test "handshake"            test_handshake
run_test "agent-responds"       test_agent_responds
run_test "agent-not-cascade-fail" test_agent_not_cascade_failure
run_test "eval-command"         test_eval_command
run_test "status-bar"           test_status_bar
run_test "connection-drop"      test_connection_drop

# ---- Summary ----
echo ""
echo "===== $PASS passed, $FAIL failed ====="
exit $(( FAIL > 0 ? 1 : 0 ))
#+end_src

* Emacs Integration

Verifies Flight Plan message format and Emacs daemon connectivity.

#+begin_src lisp
(test test-flight-plan-message-format
  "Contract Phase3: dispatcher-flight-plan-create returns valid message."
  (with-daemon ()
    (load (merge-pathnames ".local/share/passepartout/lisp/security-dispatcher.lisp"
                           (user-homedir-pathname)))
    (let ((plan (dispatcher-flight-plan-create
                 '(:TYPE :REQUEST :TARGET :shell :PAYLOAD (:CMD "sudo restart")))))
      (is (eq :REQUEST (getf plan :type)))
      (is (eq :emacs (getf plan :target)))
      (is (eq :insert-node (getf (getf plan :payload) :action)))
      (let ((attrs (getf (getf plan :payload) :attributes)))
        (is (string= "Flight Plan: High-Risk Action" (getf attrs :TITLE)))
        (is (string= "PLAN" (getf attrs :TODO)))
        (is (member "FLIGHT_PLAN" (getf attrs :TAGS) :test #'string-equal))))))

(test test-emacs-daemon-connect
  "Contract Phase3: Emacs daemon is reachable via emacsclient."
  (handler-case
      (let ((result (uiop:run-program '("emacsclient" "--eval" "(+ 1 2)")
                                       :output :string
                                       :ignore-error-status t)))
        (is (search "3" result) "Expected '3' from emacsclient, got: ~a" result))
    (error (c)
      (skip "Emacs daemon not available: ~a" c))))
#+end_src