v0.8.3: TUI stabilization — box calls, package fixes, sandbox, configure
Bug fixes: - Fix box() calls: set color-pair before box, pass ACS default chtype integers - Fix markdown functions: move to passepartout.channel-tui package where Croatoan is imported; use add-attributes/remove-attributes instead of :bold/:underline kwargs to add-string; call theme-color in gate-trace-lines to convert theme keys to Croatoan colors - Fix sandbox: remove dex:get/dex:post from restricted symbols (blocked neuro-provider from loading) - Export *log-lock* from passepartout (was unbound in jailed skill packages) - Fix configure: always deploy to XDG, skip cp when source==dest - Fix bash crash handler format string (~~ escaping) - Revert test reorder in 28 files (caused package leakage in skill loader) Design cleanup: - Extract tui-run-screen from tui-main for clean separation - Remove inject-stimulus alias - Merge *backend-registry* into *probabilistic-backends* - Fix read-framed-message whitespace DoS (4096-iteration max) - Add *read-eval* nil to dispatcher-approvals-process read-from-string
This commit is contained in:
@@ -30,81 +30,6 @@ The tokenizer feeds three subsystems:
|
||||
model and token count (combined input+output at input prices — slight
|
||||
overestimate is safer than underestimate for budgeting).
|
||||
|
||||
* Test Suite
|
||||
#+begin_src lisp
|
||||
(eval-when (:compile-toplevel :load-toplevel :execute)
|
||||
(ql:quickload :fiveam :silent t))
|
||||
|
||||
(defpackage :passepartout-tokenizer-tests
|
||||
(:use :cl :fiveam :passepartout)
|
||||
(:export #:tokenizer-suite))
|
||||
|
||||
(in-package :passepartout-tokenizer-tests)
|
||||
|
||||
(def-suite tokenizer-suite :description "Token counting and cost estimation")
|
||||
(in-suite tokenizer-suite)
|
||||
|
||||
(test test-count-tokens-default
|
||||
"Contract 1: count-tokens returns non-zero for a non-empty string."
|
||||
(let ((count (count-tokens "hello world")))
|
||||
(is (> count 0))
|
||||
(is (integerp count))))
|
||||
|
||||
(test test-count-tokens-known-model
|
||||
"Contract 1: count-tokens with a known model returns a count."
|
||||
(let ((count (count-tokens "hello world" :model :gpt-4o-mini)))
|
||||
(is (> count 0))
|
||||
(is (integerp count))))
|
||||
|
||||
(test test-count-tokens-unknown-model
|
||||
"Contract 1: count-tokens with an unknown model falls back to default."
|
||||
(let ((count (count-tokens "hello world" :model :unknown-model-xyz)))
|
||||
(is (> count 0))
|
||||
(is (integerp count))))
|
||||
|
||||
(test test-count-tokens-empty
|
||||
"Contract 1: count-tokens on empty string returns 0."
|
||||
(let ((count (count-tokens "")))
|
||||
(is (= 0 count))))
|
||||
|
||||
(test test-model-token-ratio-known
|
||||
"Contract 2: known model returns correct ratio."
|
||||
(is (= 4.0 (model-token-ratio :gpt-4o-mini)))
|
||||
(is (= 4.5 (model-token-ratio :claude-3-5-sonnet)))
|
||||
(is (= 3.5 (model-token-ratio :llama-3.1-70b))))
|
||||
|
||||
(test test-model-token-ratio-unknown
|
||||
"Contract 2: unknown model returns default ratio."
|
||||
(is (= 4.0 (model-token-ratio :unknown-model-abc))))
|
||||
|
||||
(test test-token-cost-known
|
||||
"Contract 3: token-cost returns a number for known model."
|
||||
(let ((cost (token-cost :gpt-4o-mini 1000)))
|
||||
(is (numberp cost))
|
||||
(is (> cost 0.0))))
|
||||
|
||||
(test test-token-cost-unknown
|
||||
"Contract 3: token-cost returns 0.0 for unknown model."
|
||||
(is (= 0.0 (token-cost :no-such-model 1000))))
|
||||
|
||||
(test test-provider-token-cost
|
||||
"Contract: provider-token-cost maps provider to model price."
|
||||
(let ((cost (provider-token-cost :deepseek 1000)))
|
||||
(is (numberp cost))
|
||||
(is (> cost 0.0))))
|
||||
|
||||
(test test-count-tokens-ratio-sensitivity
|
||||
"Contract 1: longer text produces proportionally more tokens."
|
||||
(let ((short (count-tokens "hi" :model :gpt-4o-mini))
|
||||
(long (count-tokens "this is a much longer piece of text with many words in it" :model :gpt-4o-mini)))
|
||||
(is (> long short))))
|
||||
|
||||
(test test-count-tokens-non-string
|
||||
"Contract 1: non-string values are coerced and counted."
|
||||
(let ((count (count-tokens 12345)))
|
||||
(is (> count 0))))
|
||||
#+end_src
|
||||
|
||||
* Implementation
|
||||
|
||||
** Package Context
|
||||
@@ -225,3 +150,77 @@ Uses the provider's default model for pricing."
|
||||
0.0)))
|
||||
#+end_src
|
||||
|
||||
* Test Suite
|
||||
#+begin_src lisp
|
||||
(eval-when (:compile-toplevel :load-toplevel :execute)
|
||||
(ql:quickload :fiveam :silent t))
|
||||
|
||||
(defpackage :passepartout-tokenizer-tests
|
||||
(:use :cl :fiveam :passepartout)
|
||||
(:export #:tokenizer-suite))
|
||||
|
||||
(in-package :passepartout-tokenizer-tests)
|
||||
|
||||
(def-suite tokenizer-suite :description "Token counting and cost estimation")
|
||||
(in-suite tokenizer-suite)
|
||||
|
||||
(test test-count-tokens-default
|
||||
"Contract 1: count-tokens returns non-zero for a non-empty string."
|
||||
(let ((count (count-tokens "hello world")))
|
||||
(is (> count 0))
|
||||
(is (integerp count))))
|
||||
|
||||
(test test-count-tokens-known-model
|
||||
"Contract 1: count-tokens with a known model returns a count."
|
||||
(let ((count (count-tokens "hello world" :model :gpt-4o-mini)))
|
||||
(is (> count 0))
|
||||
(is (integerp count))))
|
||||
|
||||
(test test-count-tokens-unknown-model
|
||||
"Contract 1: count-tokens with an unknown model falls back to default."
|
||||
(let ((count (count-tokens "hello world" :model :unknown-model-xyz)))
|
||||
(is (> count 0))
|
||||
(is (integerp count))))
|
||||
|
||||
(test test-count-tokens-empty
|
||||
"Contract 1: count-tokens on empty string returns 0."
|
||||
(let ((count (count-tokens "")))
|
||||
(is (= 0 count))))
|
||||
|
||||
(test test-model-token-ratio-known
|
||||
"Contract 2: known model returns correct ratio."
|
||||
(is (= 4.0 (model-token-ratio :gpt-4o-mini)))
|
||||
(is (= 4.5 (model-token-ratio :claude-3-5-sonnet)))
|
||||
(is (= 3.5 (model-token-ratio :llama-3.1-70b))))
|
||||
|
||||
(test test-model-token-ratio-unknown
|
||||
"Contract 2: unknown model returns default ratio."
|
||||
(is (= 4.0 (model-token-ratio :unknown-model-abc))))
|
||||
|
||||
(test test-token-cost-known
|
||||
"Contract 3: token-cost returns a number for known model."
|
||||
(let ((cost (token-cost :gpt-4o-mini 1000)))
|
||||
(is (numberp cost))
|
||||
(is (> cost 0.0))))
|
||||
|
||||
(test test-token-cost-unknown
|
||||
"Contract 3: token-cost returns 0.0 for unknown model."
|
||||
(is (= 0.0 (token-cost :no-such-model 1000))))
|
||||
|
||||
(test test-provider-token-cost
|
||||
"Contract: provider-token-cost maps provider to model price."
|
||||
(let ((cost (provider-token-cost :deepseek 1000)))
|
||||
(is (numberp cost))
|
||||
(is (> cost 0.0))))
|
||||
|
||||
(test test-count-tokens-ratio-sensitivity
|
||||
"Contract 1: longer text produces proportionally more tokens."
|
||||
(let ((short (count-tokens "hi" :model :gpt-4o-mini))
|
||||
(long (count-tokens "this is a much longer piece of text with many words in it" :model :gpt-4o-mini)))
|
||||
(is (> long short))))
|
||||
|
||||
(test test-count-tokens-non-string
|
||||
"Contract 1: non-string values are coerced and counted."
|
||||
(let ((count (count-tokens 12345)))
|
||||
(is (> count 0))))
|
||||
#+end_src
|
||||
Reference in New Issue
Block a user