Add CI test runner: run-all-tests.sh + verify-api.py + verify-demo-pty.py
Three-tier verification suite: - Tier 1: FiveAM unit tests (392 tests, 12 suites) - Tier 2: API feature verification (29 checks across 20 components) - Tier 3: PTY demo integration test (17 checks through real terminal) Webhook subscription 'cl-tty-ci' configured to run on push. Gitea repo webhook configured at amr/cl-tui → Hermes gateway.
This commit is contained in:
Hermes Agent
182
scripts/verify-demo-pty.py
Executable file
182
scripts/verify-demo-pty.py
Executable file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""PTY-based interactive test for cl-tty demo.
|
||||
|
||||
Spawns the demo inside a real PTY, sends keystrokes, captures output,
|
||||
and verifies expected behavior. Exits with status 0 if all checks pass,
|
||||
non-zero otherwise.
|
||||
"""
|
||||
|
||||
import pty
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import select
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
PASS = 0
|
||||
FAIL = 0
|
||||
|
||||
def check(name, condition, detail=""):
|
||||
global PASS, FAIL
|
||||
if condition:
|
||||
PASS += 1
|
||||
print(f" OK {name}")
|
||||
else:
|
||||
FAIL += 1
|
||||
print(f" FAIL {name}" + (f" ({detail})" if detail else ""))
|
||||
|
||||
def spawn_demo():
|
||||
"""Fork PTY, exec demo.sh, return (pid, fd).
|
||||
Blocks 1s for demo to start and enter its event loop."""
|
||||
pid, fd = pty.fork()
|
||||
if pid == 0:
|
||||
os.chdir("/mnt/hermes/projects/cl-tty")
|
||||
os.execve("./demo.sh", ["./demo.sh"], {"TERM": "xterm-256color"})
|
||||
os._exit(1)
|
||||
time.sleep(1.0)
|
||||
return pid, fd
|
||||
|
||||
def read_all(fd, timeout=0.5):
|
||||
"""Drain all available output from fd within timeout."""
|
||||
data = b""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
r, _, _ = select.select([fd], [], [], max(0, deadline - time.time()))
|
||||
if r:
|
||||
try:
|
||||
chunk = os.read(fd, 65536)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
except OSError:
|
||||
break
|
||||
else:
|
||||
break
|
||||
return data
|
||||
|
||||
def strip_escapes(data):
|
||||
"""Strip ANSI escape sequences, keep visible text."""
|
||||
text = data.decode("latin-1")
|
||||
text = re.sub(r'\x1b\[[0-9;]*[a-zA-Z]', '', text)
|
||||
text = re.sub(r'\x1b\][0-9;]*[a-zA-Z].*?\x07', '', text)
|
||||
text = re.sub(r'\x1b[()][0-9A-Z]', '', text)
|
||||
text = re.sub(r'\x1b', '', text)
|
||||
text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', text)
|
||||
return text
|
||||
|
||||
def has_text(data, *patterns):
|
||||
text = strip_escapes(data)
|
||||
return all(p in text for p in patterns)
|
||||
|
||||
def last_event_count(data):
|
||||
"""Extract the last event count from output like 'Tab N/3 | M events'."""
|
||||
text = strip_escapes(data)
|
||||
matches = re.findall(r'Tab \d+/\d+ \| (\d+) events?', text)
|
||||
if matches:
|
||||
return int(matches[-1])
|
||||
return None
|
||||
|
||||
def last_tab_index(data):
|
||||
"""Extract the last tab index from output like 'Tab N/3'."""
|
||||
text = strip_escapes(data)
|
||||
matches = re.findall(r'Tab (\d+)/', text)
|
||||
if matches:
|
||||
return int(matches[-1])
|
||||
return None
|
||||
|
||||
# ── Test 1: Demo renders correctly on startup ──
|
||||
print("\n[Test 1] Demo renders correctly on startup")
|
||||
pid, fd = spawn_demo()
|
||||
output = read_all(fd, 0.5)
|
||||
os.close(fd)
|
||||
os.waitpid(pid, 0)
|
||||
|
||||
size = len(output)
|
||||
check("Output is non-empty", size > 100, f"got {size} bytes")
|
||||
check("Shows title 'cl-tty'", has_text(output, "cl-tty"))
|
||||
check("Shows component list", has_text(output, "TextInput"))
|
||||
check("Shows test count", has_text(output, "392"))
|
||||
check("Shows controls help", has_text(output, "Ctrl+C"))
|
||||
check("Shows tab bar items", has_text(output, "Home"))
|
||||
check("Shows Console tab", has_text(output, "Console"))
|
||||
check("Starts with 1 event (init log)", last_event_count(output) == 1,
|
||||
f"got {last_event_count(output)}")
|
||||
|
||||
# ── Test 2: Escape key quits the demo ──
|
||||
print("\n[Test 2] Escape key quits the demo")
|
||||
pid, fd = spawn_demo()
|
||||
os.write(fd, b"\x1b")
|
||||
output = read_all(fd, 1.0)
|
||||
os.close(fd)
|
||||
os.waitpid(pid, 0)
|
||||
check("Escape produces output", len(output) > 50, f"got {len(output)} bytes")
|
||||
# After escape, the demo sets running=nil immediately after logging.
|
||||
# The last rendered frame may still show count 1.
|
||||
# Key check: no busy-spin.
|
||||
check("No busy-spin with Escape", len(output) < 50000, f"got {len(output)} bytes")
|
||||
|
||||
# ── Test 3: Tab switches to next tab ──
|
||||
print("\n[Test 3] Tab key switches tab")
|
||||
pid, fd = spawn_demo()
|
||||
os.write(fd, b"\x09") # Tab key
|
||||
time.sleep(1.0)
|
||||
os.write(fd, b"\x09") # Tab again to trigger another render
|
||||
time.sleep(1.0)
|
||||
output = read_all(fd, 0.5)
|
||||
os.close(fd)
|
||||
os.waitpid(pid, 0)
|
||||
count = last_event_count(output)
|
||||
tab = last_tab_index(output)
|
||||
check("Events were logged", count is not None and count >= 2,
|
||||
f"last count: {count}")
|
||||
check("Tab switched from 1", tab is not None and tab > 1,
|
||||
f"last tab: {tab}")
|
||||
|
||||
# ── Test 4: 'q' types into text input, does not quit ──
|
||||
print("\n[Test 4] 'q' does NOT quit, types into text input instead")
|
||||
pid, fd = spawn_demo()
|
||||
os.write(fd, b"q")
|
||||
time.sleep(0.5)
|
||||
os.write(fd, b"a")
|
||||
time.sleep(1.0)
|
||||
output = read_all(fd, 0.5)
|
||||
os.close(fd)
|
||||
os.waitpid(pid, 0)
|
||||
count = last_event_count(output)
|
||||
check("Events were logged ('q' + 'a')", count is not None and count >= 3,
|
||||
f"last count: {count}")
|
||||
check("Demo still running after 'q' (no busy-spin)", len(output) < 50000,
|
||||
f"got {len(output)} bytes")
|
||||
|
||||
# ── Test 5: Ctrl+C quits the demo ──
|
||||
print("\n[Test 5] Ctrl+C quits the demo")
|
||||
pid, fd = spawn_demo()
|
||||
os.write(fd, b"\x03") # Ctrl+C
|
||||
output = read_all(fd, 1.0)
|
||||
os.close(fd)
|
||||
os.waitpid(pid, 0)
|
||||
check("Ctrl+C produces output", len(output) > 50, f"got {len(output)} bytes")
|
||||
|
||||
# ── Test 6: EOF on stdin quits cleanly ──
|
||||
print("\n[Test 6] EOF on stdin quits cleanly (no busy-spin)")
|
||||
result = subprocess.run(
|
||||
["timeout", "5", "bash", "-c",
|
||||
"cd /mnt/hermes/projects/cl-tty && exec sbcl --noinform --script demo.lisp < /dev/null"],
|
||||
capture_output=True, timeout=10
|
||||
)
|
||||
eof_output = result.stdout + result.stderr
|
||||
check("EOF exits quickly (not killed by timeout)",
|
||||
result.returncode == 0,
|
||||
f"exit code: {result.returncode}")
|
||||
check("No busy-spin on EOF", len(eof_output) < 50000,
|
||||
f"got {len(eof_output)} bytes")
|
||||
|
||||
# ── Summary ──
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Results: {PASS} passed, {FAIL} failed, {PASS+FAIL} total")
|
||||
if FAIL == 0:
|
||||
print("ALL CHECKS PASSED")
|
||||
else:
|
||||
print("SOME CHECKS FAILED")
|
||||
sys.exit(FAIL > 0)
|
||||
Reference in New Issue
Block a user