amr/passepartout
1
0
Fork 0
Code Issues 15 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

tests: TUI integration + cascade parsing — precise LLM diagnostics
Some checks failed
Deploy (Gitea) / deploy (push) Failing after 2s
Details

Browse Source 
- TUI agent-responds: hardened to detect and FAIL on cascade/exhausted
  responses (previously a separate WARN-only test that let real
  cascade failures slip through)
- New TUI cascade-parsing test: /eval *provider-cascade* on screen,
  checks for clean keywords (no cl-dotenv quote artifacts)
- Pre-warm step: sbcl --eval '(ql:quickload :passepartout/tui)'
  before launching tmux, cuts TUI startup from ~120s to ~10s
- Removed test_agent_not_cascade_failure (absorbed into agent-responds)
- New integration test: test-provider-cascade-parsing verifies
  PROVIDER_CASCADE entries are keywords without quotes, matching
  registered backends — catches the exact cl-dotenv quote bug
- Fixed stop-daemon ghost symbol (removed export) and paren bug
- Contract section updated with numbered Phase 2/3 items
This commit is contained in:
Amr Gharbeia
2026-05-06 08:56:07 -04:00
parent 9362c56678
commit 750918527d
4 changed files with 180 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
lisp/system-integration-tests.lisp
View File

@@ -3,13 +3,13 @@
(ql:quickload :usocket :silent t))
(defpackage :passepartout-integration-tests
(:use :cl :fiveam :passepartout)
(:use :cl :passepartout)
(:export #:integration-suite))
(in-package :passepartout-integration-tests)
(def-suite integration-suite :description "Integration tests across process boundaries")
(in-suite integration-suite)
(fiveam:def-suite integration-suite :description "Integration tests across process boundaries")
(fiveam:in-suite integration-suite)
(defvar *daemon-port* nil)
@@ -27,7 +27,7 @@
(passepartout:start-daemon :port *daemon-port*)
(sleep 2)
,@body)
(handler-case (passepartout:stop-daemon) (error ())))))
(values)))
(defun daemon-connect ()
(let* ((sock (usocket:socket-connect "127.0.0.1" *daemon-port*))
@@ -47,14 +47,14 @@
(when (> (get-universal-time) deadline) (return nil))
(sleep 0.1))))
(test test-daemon-starts
(fiveam:test test-daemon-starts
"Contract 1: daemon binds port and sends valid handshake."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
(is (open-stream-p stream))
(usocket:socket-close sock))))
(test test-pipeline-user-input
(fiveam:test test-pipeline-user-input
"Contract 2: :user-input traverses pipeline and produces a response."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
@@ -66,7 +66,7 @@
(is (not (null resp)) "Expected a response")))
(usocket:socket-close sock)))))
(test test-pipeline-heartbeat
(fiveam:test test-pipeline-heartbeat
"Contract 2: heartbeat signals do not crash the daemon."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
@@ -76,7 +76,7 @@
(usocket:socket-close sock))
(pass))))
(test test-tcp-round-trip
(fiveam:test test-tcp-round-trip
"Contract 3: framed health-check survives TCP round-trip."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
@@ -88,7 +88,7 @@
(is (member (getf resp :type) '(:HEALTH-RESPONSE)))))
(usocket:socket-close sock)))))
(test test-daemon-survives-junk
(fiveam:test test-daemon-survives-junk
"Contract 3: daemon does not crash on junk input."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
@@ -101,7 +101,7 @@
(is (open-stream-p stream2))
(usocket:socket-close sock2))))
(test test-skill-registry-populated
(fiveam:test test-skill-registry-populated
"Contract 4: *skill-registry* is populated after daemon start."
(with-daemon ()
(is (hash-table-p passepartout::*skill-registry*))
@@ -109,7 +109,7 @@
"Expected at least 1 skill in registry, got ~a"
(hash-table-count passepartout::*skill-registry*))))
(test test-shell-safe-echo
(fiveam:test test-shell-safe-echo
"Contract 5: safe shell command does not crash the daemon."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
@@ -120,7 +120,7 @@
(usocket:socket-close sock))
(pass))))
(test test-shell-dangerous-blocked
(fiveam:test test-shell-dangerous-blocked
"Contract 5: rm -rf / is blocked by the security dispatcher."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
@@ -131,7 +131,7 @@
(usocket:socket-close sock))
(pass))))
(test test-cli-gateway-input
(fiveam:test test-cli-gateway-input
"Contract 6: text via TCP produces a response."
(with-daemon ()
(multiple-value-bind (stream sock) (daemon-connect)
@@ -142,7 +142,7 @@
(usocket:socket-close sock))
(pass))))
(test test-gateway-registry
(fiveam:test test-gateway-registry
"Contract 7: gateway-registry-initialize is available."
(with-daemon ()
(is (fboundp 'gateway-registry-initialize))
@@ -162,7 +162,7 @@
(format t " [SKIP] ~a not set~%" ,env-var)
(skip "~a not set" ,env-var))))
(test test-provider-openai-request
(fiveam:test test-provider-openai-request
"Contract Phase2: provider-openai-request returns :success with valid API key."
(skip-unless "OPENROUTER_API_KEY"
(let ((result (provider-openai-request "Say hello" "Be brief."
@@ -172,14 +172,28 @@
(eq (getf result :status) :error))
"Expected :success or :error, got: ~a" result))))
(test test-backend-cascade-real
(fiveam:test test-backend-cascade-real
"Contract Phase2: backend-cascade-call returns string content with real provider."
(skip-unless "OPENROUTER_API_KEY"
(let ((passepartout::*provider-cascade* '(:openrouter)))
(let ((result (backend-cascade-call "Say hello" :system-prompt "Be brief.")))
(is (stringp result) "Expected string response, got: ~a" result)))))
(test test-messaging-link-unlink
(fiveam:test test-provider-cascade-parsing
"Contract Phase2: PROVIDER_CASCADE env var parses to clean keywords matching backends."
(provider-cascade-initialize)
(let ((cascade passepartout::*provider-cascade*))
(is (listp cascade) "Cascade must be a list")
(is (>= (length cascade) 1) "Cascade must have at least one entry")
(dolist (entry cascade)
(is (keywordp entry) "Entry ~s must be a keyword" entry)
(let ((name (symbol-name entry)))
(is (not (find #\" name)) "Entry ~s must not contain double-quote" entry)
(is (not (find #\' name)) "Entry ~s must not contain single-quote" entry)))
(is (some (lambda (e) (gethash e passepartout::*probabilistic-backends*)) cascade)
"At least one cascade entry must match a registered backend")))
(fiveam:test test-messaging-link-unlink
"Contract Phase2: messaging-link stores token, configured-p returns T, unlink removes it."
(with-daemon ()
(messaging-link :test-platform :token "fake-token-123")
@@ -189,19 +203,19 @@
(is (not (gateway-configured-p :test-platform))
"Expected test-platform to be unconfigured after unlinking")))
(test test-gateway-configured-p-false
(fiveam:test test-gateway-configured-p-false
"Contract Phase2: gateway-configured-p returns nil for unknown platform."
(with-daemon ()
(is (not (gateway-configured-p :nonexistent-platform-xyz)))))
(test test-gateway-start-messaging
(fiveam:test test-gateway-start-messaging
"Contract Phase2: gateway registry initializes with expected platforms."
(with-daemon ()
(gateway-registry-initialize)
(is (hash-table-p passepartout::*gateway-registry*))
(is (>= (hash-table-count passepartout::*gateway-registry*) 1))))
(test test-flight-plan-message-format
(fiveam:test test-flight-plan-message-format
"Contract Phase3: dispatcher-flight-plan-create returns valid message."
(with-daemon ()
(load (merge-pathnames ".local/share/passepartout/lisp/security-dispatcher.lisp"
@@ -216,7 +230,7 @@
(is (string= "PLAN" (getf attrs :TODO)))
(is (member "FLIGHT_PLAN" (getf attrs :TAGS) :test #'string-equal))))))
(test test-emacs-daemon-connect
(fiveam:test test-emacs-daemon-connect
"Contract Phase3: Emacs daemon is reachable via emacsclient."
(handler-case
(let ((result (uiop:run-program '("emacsclient" "--eval" "(+ 1 2)")
@@ -224,4 +238,4 @@
:ignore-error-status t)))
(is (search "3" result) "Expected '3' from emacsclient, got: ~a" result))
(error (c)
(skip "Emacs daemon not available: ~a" c))))
(skip "Emacs daemon not available: ~a" c)))))