amr/passepartout
1
0
Fork 0
Code Issues 15 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

provider: thread-isolated LLM requests + in-package fix
Some checks failed
Deploy (Gitea) / deploy (push) Failing after 2s
Details

Browse Source 
- Replace bt:with-timeout with thread-per-request + polling loop
  (bt:with-timeout cannot interrupt blocking SSL reads)
- Worker thread makes the HTTP call; main thread polls for result
  with configurable LLM_REQUEST_TIMEOUT (default 30s)
- Returns timeout error after deadline; worker thread finishes naturally
- Added (in-package :passepartout) for standalone compilation
This commit is contained in:
Amr Gharbeia
2026-05-04 19:21:41 -04:00
parent f9ae84ba88
commit 78705f55ec
2 changed files with 58 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
lisp/system-model-provider.lisp
View File

@@ -1,3 +1,5 @@
(in-package :passepartout)
(defparameter *provider-configs*
'((:local . (:base-url nil :key-env nil :url-env "LOCAL_BASE_URL" :default-model "llama3"))
(:openrouter . (:base-url "https://openrouter.ai/api/v1" :key-env "OPENROUTER_API_KEY" :default-model "openrouter/auto"))
@@ -50,20 +52,33 @@
(messages . (( (role . "system") (content . ,system-prompt) )
( (role . "user") (content . ,prompt) )))))))
(handler-case
(bt:with-timeout (timeout)
(let* ((response (dex:post url :headers headers :content body
:connect-timeout (min 10 timeout)
:read-timeout (max 10 (- timeout 5))))
(json (cl-json:decode-json-from-string response))
(choices (cdr (assoc :choices json)))
(first-choice (car choices))
(message (cdr (assoc :message first-choice)))
(content (cdr (assoc :content message))))
(if content
(list :status :success :content content)
(list :status :error :message (format nil "~a: No content in response (~s)" provider json)))))
(bt:timeout ()
(list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout)))
(let* ((result nil)
(lock (bt:make-lock "llm-lock"))
(done nil)
(thread (bt:make-thread
(lambda ()
(handler-case
(let* ((response (dex:post url :headers headers :content body
:connect-timeout (min 10 timeout)
:read-timeout (max 10 (- timeout 5))))
(json (cl-json:decode-json-from-string response))
(choices (cdr (assoc :choices json)))
(first-choice (car choices))
(message (cdr (assoc :message first-choice)))
(content (cdr (assoc :content message))))
(setf result (if content
(list :status :success :content content)
(list :status :error :message (format nil "~a: No content" provider)))))
(error (c)
(setf result (list :status :error :message (format nil "~a Error: ~a" provider c)))))
(bt:with-lock-held (lock) (setf done t)))
:name (format nil "llm-~a" provider))))
(loop for waited from 0 below timeout
when (bt:with-lock-held (lock) done)
do (return result)
do (sleep 0.5))
(or result
(list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout))))
(error (c)
(list :status :error :message (format nil "~a Failure: ~a" provider c))))))

43
org/system-model-provider.org
View File

@@ -17,6 +17,8 @@ Providers register themselves at boot. No API key? That provider doesn't registe
** Provider registry
#+begin_src lisp
(in-package :passepartout)
(defparameter *provider-configs*
'((:local . (:base-url nil :key-env nil :url-env "LOCAL_BASE_URL" :default-model "llama3"))
(:openrouter . (:base-url "https://openrouter.ai/api/v1" :key-env "OPENROUTER_API_KEY" :default-model "openrouter/auto"))
@@ -78,20 +80,33 @@ Providers register themselves at boot. No API key? That provider doesn't registe
(messages . (( (role . "system") (content . ,system-prompt) )
( (role . "user") (content . ,prompt) )))))))
(handler-case
(bt:with-timeout (timeout)
(let* ((response (dex:post url :headers headers :content body
:connect-timeout (min 10 timeout)
:read-timeout (max 10 (- timeout 5))))
(json (cl-json:decode-json-from-string response))
(choices (cdr (assoc :choices json)))
(first-choice (car choices))
(message (cdr (assoc :message first-choice)))
(content (cdr (assoc :content message))))
(if content
(list :status :success :content content)
(list :status :error :message (format nil "~a: No content in response (~s)" provider json)))))
(bt:timeout ()
(list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout)))
(let* ((result nil)
(lock (bt:make-lock "llm-lock"))
(done nil)
(thread (bt:make-thread
(lambda ()
(handler-case
(let* ((response (dex:post url :headers headers :content body
:connect-timeout (min 10 timeout)
:read-timeout (max 10 (- timeout 5))))
(json (cl-json:decode-json-from-string response))
(choices (cdr (assoc :choices json)))
(first-choice (car choices))
(message (cdr (assoc :message first-choice)))
(content (cdr (assoc :content message))))
(setf result (if content
(list :status :success :content content)
(list :status :error :message (format nil "~a: No content" provider)))))
(error (c)
(setf result (list :status :error :message (format nil "~a Error: ~a" provider c)))))
(bt:with-lock-held (lock) (setf done t)))
:name (format nil "llm-~a" provider))))
(loop for waited from 0 below timeout
when (bt:with-lock-held (lock) done)
do (return result)
do (sleep 0.5))
(or result
(list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout))))
(error (c)
(list :status :error :message (format nil "~a Failure: ~a" provider c))))))
#+end_src