tests: TUI integration + cascade parsing — precise LLM diagnostics
Some checks failed
Deploy (Gitea) / deploy (push) Failing after 2s

- TUI agent-responds: hardened to detect and FAIL on cascade/exhausted
  responses (previously a separate WARN-only test that let real
  cascade failures slip through)
- New TUI cascade-parsing test: /eval *provider-cascade* on screen,
  checks for clean keywords (no cl-dotenv quote artifacts)
- Pre-warm step: sbcl --eval '(ql:quickload :passepartout/tui)'
  before launching tmux, cuts TUI startup from ~120s to ~10s
- Removed test_agent_not_cascade_failure (absorbed into agent-responds)
- New integration test: test-provider-cascade-parsing verifies
  PROVIDER_CASCADE entries are keywords without quotes, matching
  registered backends — catches the exact cl-dotenv quote bug
- Fixed stop-daemon ghost symbol (removed export) and paren bug
- Contract section updated with numbered Phase 2/3 items
This commit is contained in:
2026-05-06 08:56:07 -04:00
parent 9362c56678
commit 750918527d
4 changed files with 180 additions and 107 deletions

View File

@@ -25,29 +25,43 @@ run_test() {
}
# ---- Setup ----
echo "Pre-warming FASL cache (speeds up TUI start from ~120s to ~10s)..."
sbcl --noinform --load ~/quicklisp/setup.lisp \
--eval '(ql:quickload :passepartout/tui :silent t)' \
--eval '(uiop:quit)' 2>/dev/null &
WARM_PID=$!
wait $WARM_PID 2>/dev/null
echo " Pre-warm complete"
echo "Starting TUI in tmux (daemon must already be running on port 9105)..."
tmux new-session -d -s tui-test "passepartout tui 2>&1 | tee $TUI_LOG"
for i in $(seq 1 40); do
sleep 3
for i in $(seq 1 15); do
sleep 2
if tmux capture-pane -t tui-test -p 2>/dev/null | grep -q 'Connected v[0-9]'; then
echo " TUI ready after $((i*3))s"
echo " TUI ready after $((i*2))s"
break
fi
if [ "$i" -eq 40 ]; then
echo " WARNING: TUI did not render after 120s"
if [ "$i" -eq 15 ]; then
echo " WARNING: TUI did not render after 30s"
fi
done
# ---- Tests ----
test_agent_responds() {
# Full round-trip: TUI → daemon → pipeline → TUI.
# Uses tmux capture-pane to read the rendered screen.
# Full round-trip: TUI → daemon → LLM → daemon → TUI.
# Must contain a real agent response (⬇), NOT a cascade failure.
local before_ts
before_ts=$(date +%s)
tmux send-keys -t tui-test "hello" Enter
tmux send-keys -t tui-test "Say hello in one word" Enter
while true; do
if tmux capture-pane -t tui-test -p -S -50 2>/dev/null | grep -q '⬇.*[a-zA-Z]\{3,\}'; then
local pane
pane=$(tmux capture-pane -t tui-test -p -S -60 2>/dev/null)
if echo "$pane" | grep -q '⬇.*[a-zA-Z]\{3,\}'; then
if echo "$pane" | grep '⬇' | grep -qi 'cascade.*fail\|exhausted\|neural cascade'; then
echo "FAIL: agent responded with cascade failure, not LLM content" >&2
return 1
fi
return 0
fi
local now_ts
@@ -60,12 +74,15 @@ test_agent_responds() {
done
}
test_agent_not_cascade_failure() {
if tmux capture-pane -t tui-test -p -S -50 2>/dev/null | grep '⬇' | grep -qi 'cascade.*fail\|exhausted\|neural cascade'; then
echo "NOTE: LLM cascade failure — no API key configured (warning only)" >&2
WARN=$((WARN + 1))
fi
return 0
test_cascade_parsing() {
# Via /eval, check that *provider-cascade* contains clean keywords.
# This catches the cl-dotenv quote contamination bug.
tmux send-keys -t tui-test "/eval *provider-cascade*" Enter
sleep 3
local pane
pane=$(tmux capture-pane -t tui-test -p -S -15 2>/dev/null)
# Must contain keyword syntax :SOMETHING (not "SOMETHING with quotes)
echo "$pane" | grep -q ':DEEPSEEK\|:OPENROUTER\|:OPENAI\|:ANTHROPIC\|:GROQ\|:GEMINI\|:NVIDIA'
}
test_eval_command() {
@@ -85,7 +102,7 @@ test_connection_drop() {
}
run_test "agent-responds" test_agent_responds
run_test "agent-not-cascade-fail" test_agent_not_cascade_failure
run_test "cascade-parsing" test_cascade_parsing
run_test "eval-command" test_eval_command
run_test "status-bar" test_status_bar
run_test "connection-drop" test_connection_drop