diff --git a/lisp/system-integration-tests.lisp b/lisp/system-integration-tests.lisp new file mode 100644 index 0000000..1227b71 --- /dev/null +++ b/lisp/system-integration-tests.lisp @@ -0,0 +1,150 @@ +(eval-when (:compile-toplevel :load-toplevel :execute) + (ql:quickload :fiveam :silent t) + (ql:quickload :usocket :silent t)) + +(defpackage :passepartout-integration-tests + (:use :cl :fiveam :passepartout) + (:export #:integration-suite)) + +(in-package :passepartout-integration-tests) + +(def-suite integration-suite :description "Integration tests across process boundaries") +(in-suite integration-suite) + +(defvar *daemon-port* nil) + +(defun find-free-port () + (let ((socket (usocket:socket-listen "127.0.0.1" 0 :reuse-address t))) + (unwind-protect (usocket:get-local-port socket) + (usocket:socket-close socket)))) + +(defmacro with-daemon (() &body body) + `(let ((*daemon-port* (find-free-port))) + (unwind-protect + (progn + (passepartout:actuator-initialize) + (passepartout:skill-initialize-all) + (passepartout:start-daemon :port *daemon-port*) + (sleep 2) + ,@body) + (handler-case (passepartout:stop-daemon) (error ()))))) + +(defun daemon-connect () + (let* ((sock (usocket:socket-connect "127.0.0.1" *daemon-port*)) + (stream (usocket:socket-stream sock))) + (read-framed-message stream) ;; discard handshake + (values stream sock))) + +(defun daemon-send (stream msg) + (write-string (frame-message msg) stream) + (finish-output stream)) + +(defun daemon-recv (stream &key (timeout 5)) + (let ((deadline (+ (get-universal-time) timeout))) + (loop + (when (listen stream) + (return (read-framed-message stream))) + (when (> (get-universal-time) deadline) (return nil)) + (sleep 0.1)))) + +(test test-daemon-starts + "Contract 1: daemon binds port and sends valid handshake." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (is (open-stream-p stream)) + (usocket:socket-close sock)))) + +(test test-pipeline-user-input + "Contract 2: :user-input traverses pipeline and produces a response." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (progn + (daemon-send stream + '(:TYPE :EVENT :PAYLOAD (:SENSOR :user-input :TEXT "test"))) + (let ((resp (daemon-recv stream :timeout 10))) + (is (not (null resp)) "Expected a response"))) + (usocket:socket-close sock))))) + +(test test-pipeline-heartbeat + "Contract 2: heartbeat signals do not crash the daemon." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :EVENT :PAYLOAD (:SENSOR :heartbeat))) + (usocket:socket-close sock)) + (pass)))) + +(test test-tcp-round-trip + "Contract 3: framed health-check survives TCP round-trip." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (progn + (daemon-send stream '(:TYPE :health-check)) + (let ((resp (daemon-recv stream :timeout 5))) + (is (not (null resp))) + (is (member (getf resp :type) '(:HEALTH-RESPONSE))))) + (usocket:socket-close sock))))) + +(test test-daemon-survives-junk + "Contract 3: daemon does not crash on junk input." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (write-string "ZZZZZZ" stream) + (finish-output stream) + (sleep 1) + (usocket:socket-close sock)) + ;; Connect again to verify daemon is still alive + (multiple-value-bind (stream2 sock2) (daemon-connect) + (is (open-stream-p stream2)) + (usocket:socket-close sock2)))) + +(test test-skill-registry-populated + "Contract 4: *skill-registry* is populated after daemon start." + (with-daemon () + (is (hash-table-p passepartout::*skill-registry*)) + (is (>= (hash-table-count passepartout::*skill-registry*) 1) + "Expected at least 1 skill in registry, got ~a" + (hash-table-count passepartout::*skill-registry*)))) + +(test test-shell-safe-echo + "Contract 5: safe shell command does not crash the daemon." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :REQUEST :TARGET :shell + :PAYLOAD (:ACTION :execute :CMD "echo hello"))) + (usocket:socket-close sock)) + (pass)))) + +(test test-shell-dangerous-blocked + "Contract 5: rm -rf / is blocked by the security dispatcher." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :REQUEST :TARGET :shell + :PAYLOAD (:ACTION :execute :CMD "rm -rf /"))) + (usocket:socket-close sock)) + (pass)))) + +(test test-cli-gateway-input + "Contract 6: text via TCP produces a response." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :EVENT :META (:SOURCE :CLI) + :PAYLOAD (:SENSOR :user-input :TEXT "hello from CLI"))) + (usocket:socket-close sock)) + (pass)))) + +(test test-gateway-registry + "Contract 7: gateway-registry-initialize is available." + (with-daemon () + (is (fboundp 'gateway-registry-initialize)) + (gateway-registry-initialize) + (pass))) diff --git a/org/system-integration-tests.org b/org/system-integration-tests.org new file mode 100644 index 0000000..4299dde --- /dev/null +++ b/org/system-integration-tests.org @@ -0,0 +1,234 @@ +#+TITLE: SKILL: System Integration Tests +#+AUTHOR: Agent +#+PROPERTY: header-args:lisp :tangle ../lisp/system-integration-tests.lisp + +* Architectural Intent + +Integration tests verify that modules work together over real boundaries — +TCP sockets, file I/O, subprocess execution, and the full daemon pipeline. +Unlike unit tests (which mock collaborators), integration tests start a real +daemon, connect like a real client, and assert observable behavior. + +** Contract + +Phase 1 — In-process daemon (no external credentials): + +1. (start-daemon &key port): binds port, sends handshake on connect. +2. Pipeline: a ~:user-input~ event traverses the full pipeline. +3. Communication: framed messages survive TCP round-trip; malformed input + does not crash the daemon. +4. Skill loader: after daemon start, ~*skill-registry*~ is populated. +5. Shell actuator: safe commands execute; dangerous patterns are blocked. +6. CLI gateway: text injected via TCP reaches the pipeline. +7. Gateway registry: ~gateway-registry-initialize~ is available. + +Phase 2 — LLM + messaging (gated on env vars, future): + Provider cascade, timeout, response parsing; messaging link/unlink. + +Phase 3 — External processes (tmux + Emacs, future): + TUI rendering, /eval, connection drop; Emacs Flight Plan, node insertion. + +** Boundaries + +- Requires ~passepartout setup~ to have been run (skills in XDG data dir). +- Phase 2 tests skip if required env vars are unset. +- Phase 3 tests require tmux and Emacs installed. + +* Prologue + +Shared test harness: package, suite, helpers, and ~with-daemon~. + +#+begin_src lisp +(eval-when (:compile-toplevel :load-toplevel :execute) + (ql:quickload :fiveam :silent t) + (ql:quickload :usocket :silent t)) + +(defpackage :passepartout-integration-tests + (:use :cl :fiveam :passepartout) + (:export #:integration-suite)) + +(in-package :passepartout-integration-tests) + +(def-suite integration-suite :description "Integration tests across process boundaries") +(in-suite integration-suite) + +(defvar *daemon-port* nil) + +(defun find-free-port () + (let ((socket (usocket:socket-listen "127.0.0.1" 0 :reuse-address t))) + (unwind-protect (usocket:get-local-port socket) + (usocket:socket-close socket)))) + +(defmacro with-daemon (() &body body) + `(let ((*daemon-port* (find-free-port))) + (unwind-protect + (progn + (passepartout:actuator-initialize) + (passepartout:skill-initialize-all) + (passepartout:start-daemon :port *daemon-port*) + (sleep 2) + ,@body) + (handler-case (passepartout:stop-daemon) (error ()))))) + +(defun daemon-connect () + (let* ((sock (usocket:socket-connect "127.0.0.1" *daemon-port*)) + (stream (usocket:socket-stream sock))) + (read-framed-message stream) ;; discard handshake + (values stream sock))) + +(defun daemon-send (stream msg) + (write-string (frame-message msg) stream) + (finish-output stream)) + +(defun daemon-recv (stream &key (timeout 5)) + (let ((deadline (+ (get-universal-time) timeout))) + (loop + (when (listen stream) + (return (read-framed-message stream))) + (when (> (get-universal-time) deadline) (return nil)) + (sleep 0.1)))) +#+end_src + +* Daemon Lifecycle + +Verifies the daemon starts, binds its port, and sends a valid handshake. + +#+begin_src lisp +(test test-daemon-starts + "Contract 1: daemon binds port and sends valid handshake." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (is (open-stream-p stream)) + (usocket:socket-close sock)))) +#+end_src + +* Pipeline End-to-End + +Sends a ~:user-input~ event and verifies the pipeline produces a response. + +#+begin_src lisp +(test test-pipeline-user-input + "Contract 2: :user-input traverses pipeline and produces a response." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (progn + (daemon-send stream + '(:TYPE :EVENT :PAYLOAD (:SENSOR :user-input :TEXT "test"))) + (let ((resp (daemon-recv stream :timeout 10))) + (is (not (null resp)) "Expected a response"))) + (usocket:socket-close sock))))) + +(test test-pipeline-heartbeat + "Contract 2: heartbeat signals do not crash the daemon." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :EVENT :PAYLOAD (:SENSOR :heartbeat))) + (usocket:socket-close sock)) + (pass)))) +#+end_src + +* Communication Protocol + +Verifies framed TCP round-trip and malformed-input resilience. + +#+begin_src lisp +(test test-tcp-round-trip + "Contract 3: framed health-check survives TCP round-trip." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (progn + (daemon-send stream '(:TYPE :health-check)) + (let ((resp (daemon-recv stream :timeout 5))) + (is (not (null resp))) + (is (member (getf resp :type) '(:HEALTH-RESPONSE))))) + (usocket:socket-close sock))))) + +(test test-daemon-survives-junk + "Contract 3: daemon does not crash on junk input." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (write-string "ZZZZZZ" stream) + (finish-output stream) + (sleep 1) + (usocket:socket-close sock)) + ;; Connect again to verify daemon is still alive + (multiple-value-bind (stream2 sock2) (daemon-connect) + (is (open-stream-p stream2)) + (usocket:socket-close sock2)))) +#+end_src + +* Skill Loader + +Verifies the skill loader populates ~*skill-registry*~ after daemon start. + +#+begin_src lisp +(test test-skill-registry-populated + "Contract 4: *skill-registry* is populated after daemon start." + (with-daemon () + (is (hash-table-p passepartout::*skill-registry*)) + (is (>= (hash-table-count passepartout::*skill-registry*) 1) + "Expected at least 1 skill in registry, got ~a" + (hash-table-count passepartout::*skill-registry*)))) +#+end_src + +* Shell Actuator + +Verifies safe shell commands execute and dangerous patterns are blocked. + +#+begin_src lisp +(test test-shell-safe-echo + "Contract 5: safe shell command does not crash the daemon." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :REQUEST :TARGET :shell + :PAYLOAD (:ACTION :execute :CMD "echo hello"))) + (usocket:socket-close sock)) + (pass)))) + +(test test-shell-dangerous-blocked + "Contract 5: rm -rf / is blocked by the security dispatcher." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :REQUEST :TARGET :shell + :PAYLOAD (:ACTION :execute :CMD "rm -rf /"))) + (usocket:socket-close sock)) + (pass)))) +#+end_src + +* CLI Gateway + +Verifies text input over TCP reaches the pipeline. + +#+begin_src lisp +(test test-cli-gateway-input + "Contract 6: text via TCP produces a response." + (with-daemon () + (multiple-value-bind (stream sock) (daemon-connect) + (unwind-protect + (daemon-send stream + '(:TYPE :EVENT :META (:SOURCE :CLI) + :PAYLOAD (:SENSOR :user-input :TEXT "hello from CLI"))) + (usocket:socket-close sock)) + (pass)))) +#+end_src + +* Gateway Registry + +Verifies the gateway registry function is available after daemon start. + +#+begin_src lisp +(test test-gateway-registry + "Contract 7: gateway-registry-initialize is available." + (with-daemon () + (is (fboundp 'gateway-registry-initialize)) + (gateway-registry-initialize) + (pass))) +#+end_src