passepartout/org/system-integration-tests.org

#+TITLE: SKILL: System Integration Tests
#+AUTHOR: Agent
#+PROPERTY: header-args:lisp :tangle ../lisp/system-integration-tests.lisp

* Architectural Intent

Integration tests verify that modules work together over real boundaries —
TCP sockets, file I/O, subprocess execution, and the full daemon pipeline.
Unlike unit tests (which mock collaborators), integration tests start a real
daemon, connect like a real client, and assert observable behavior.

** Contract

Phase 1 — In-process daemon (no external credentials):

1. (start-daemon &key port): binds port, sends handshake on connect.
2. Pipeline: a ~:user-input~ event traverses the full pipeline.
3. Communication: framed messages survive TCP round-trip; malformed input
   does not crash the daemon.
4. Skill loader: after daemon start, ~*skill-registry*~ is populated.
5. Shell actuator: safe commands execute; dangerous patterns are blocked.
6. CLI gateway: text injected via TCP reaches the pipeline.
7. Gateway registry: ~gateway-registry-initialize~ is available.

Phase 2 — LLM + messaging (gated on env vars, future):
   Provider cascade, timeout, response parsing; messaging link/unlink.

Phase 3 — External processes (tmux + Emacs, future):
   TUI rendering, /eval, connection drop; Emacs Flight Plan, node insertion.

** Boundaries

- Requires ~passepartout setup~ to have been run (skills in XDG data dir).
- Phase 2 tests skip if required env vars are unset.
- Phase 3 tests require tmux and Emacs installed.

* Prologue

Shared test harness: package, suite, helpers, and ~with-daemon~.

#+begin_src lisp
(eval-when (:compile-toplevel :load-toplevel :execute)
  (ql:quickload :fiveam :silent t)
  (ql:quickload :usocket :silent t))

(defpackage :passepartout-integration-tests
  (:use :cl :fiveam :passepartout)
  (:export #:integration-suite))

(in-package :passepartout-integration-tests)

(def-suite integration-suite :description "Integration tests across process boundaries")
(in-suite integration-suite)

(defvar *daemon-port* nil)

(defun find-free-port ()
  (let ((socket (usocket:socket-listen "127.0.0.1" 0 :reuse-address t)))
    (unwind-protect (usocket:get-local-port socket)
      (usocket:socket-close socket))))

(defmacro with-daemon (() &body body)
  `(let ((*daemon-port* (find-free-port)))
     (unwind-protect
          (progn
            (passepartout:actuator-initialize)
            (passepartout:skill-initialize-all)
            (passepartout:start-daemon :port *daemon-port*)
            (sleep 2)
            ,@body)
       (handler-case (passepartout:stop-daemon) (error ())))))

(defun daemon-connect ()
  (let* ((sock (usocket:socket-connect "127.0.0.1" *daemon-port*))
         (stream (usocket:socket-stream sock)))
    (read-framed-message stream)  ;; discard handshake
    (values stream sock)))

(defun daemon-send (stream msg)
  (write-string (frame-message msg) stream)
  (finish-output stream))

(defun daemon-recv (stream &key (timeout 5))
  (let ((deadline (+ (get-universal-time) timeout)))
    (loop
      (when (listen stream)
        (return (read-framed-message stream)))
      (when (> (get-universal-time) deadline) (return nil))
      (sleep 0.1))))
#+end_src

* Daemon Lifecycle

Verifies the daemon starts, binds its port, and sends a valid handshake.

#+begin_src lisp
(test test-daemon-starts
  "Contract 1: daemon binds port and sends valid handshake."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (is (open-stream-p stream))
      (usocket:socket-close sock))))
#+end_src

* Pipeline End-to-End

Sends a ~:user-input~ event and verifies the pipeline produces a response.

#+begin_src lisp
(test test-pipeline-user-input
  "Contract 2: :user-input traverses pipeline and produces a response."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (progn
             (daemon-send stream
              '(:TYPE :EVENT :PAYLOAD (:SENSOR :user-input :TEXT "test")))
             (let ((resp (daemon-recv stream :timeout 10)))
               (is (not (null resp)) "Expected a response")))
        (usocket:socket-close sock)))))

(test test-pipeline-heartbeat
  "Contract 2: heartbeat signals do not crash the daemon."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :EVENT :PAYLOAD (:SENSOR :heartbeat)))
        (usocket:socket-close sock))
      (pass))))
#+end_src

* Communication Protocol

Verifies framed TCP round-trip and malformed-input resilience.

#+begin_src lisp
(test test-tcp-round-trip
  "Contract 3: framed health-check survives TCP round-trip."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (progn
             (daemon-send stream '(:TYPE :health-check))
             (let ((resp (daemon-recv stream :timeout 5)))
               (is (not (null resp)))
               (is (member (getf resp :type) '(:HEALTH-RESPONSE)))))
        (usocket:socket-close sock)))))

(test test-daemon-survives-junk
  "Contract 3: daemon does not crash on junk input."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (write-string "ZZZZZZ" stream)
      (finish-output stream)
      (sleep 1)
      (usocket:socket-close sock))
    ;; Connect again to verify daemon is still alive
    (multiple-value-bind (stream2 sock2) (daemon-connect)
      (is (open-stream-p stream2))
      (usocket:socket-close sock2))))
#+end_src

* Skill Loader

Verifies the skill loader populates ~*skill-registry*~ after daemon start.

#+begin_src lisp
(test test-skill-registry-populated
  "Contract 4: *skill-registry* is populated after daemon start."
  (with-daemon ()
    (is (hash-table-p passepartout::*skill-registry*))
    (is (>= (hash-table-count passepartout::*skill-registry*) 1)
        "Expected at least 1 skill in registry, got ~a"
        (hash-table-count passepartout::*skill-registry*))))
#+end_src

* Shell Actuator

Verifies safe shell commands execute and dangerous patterns are blocked.

#+begin_src lisp
(test test-shell-safe-echo
  "Contract 5: safe shell command does not crash the daemon."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :REQUEST :TARGET :shell
              :PAYLOAD (:ACTION :execute :CMD "echo hello")))
        (usocket:socket-close sock))
      (pass))))

(test test-shell-dangerous-blocked
  "Contract 5: rm -rf / is blocked by the security dispatcher."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :REQUEST :TARGET :shell
              :PAYLOAD (:ACTION :execute :CMD "rm -rf /")))
        (usocket:socket-close sock))
      (pass))))
#+end_src

* CLI Gateway

Verifies text input over TCP reaches the pipeline.

#+begin_src lisp
(test test-cli-gateway-input
  "Contract 6: text via TCP produces a response."
  (with-daemon ()
    (multiple-value-bind (stream sock) (daemon-connect)
      (unwind-protect
           (daemon-send stream
            '(:TYPE :EVENT :META (:SOURCE :CLI)
              :PAYLOAD (:SENSOR :user-input :TEXT "hello from CLI")))
        (usocket:socket-close sock))
      (pass))))
#+end_src

* Gateway Registry

Verifies the gateway registry function is available after daemon start.

#+begin_src lisp
(test test-gateway-registry
  "Contract 7: gateway-registry-initialize is available."
  (with-daemon ()
    (is (fboundp 'gateway-registry-initialize))
    (gateway-registry-initialize)
    (pass)))
#+end_src