diff --git a/lisp/system-model-provider.lisp b/lisp/system-model-provider.lisp index 9c37806..75beaa4 100644 --- a/lisp/system-model-provider.lisp +++ b/lisp/system-model-provider.lisp @@ -1,3 +1,5 @@ +(in-package :passepartout) + (defparameter *provider-configs* '((:local . (:base-url nil :key-env nil :url-env "LOCAL_BASE_URL" :default-model "llama3")) (:openrouter . (:base-url "https://openrouter.ai/api/v1" :key-env "OPENROUTER_API_KEY" :default-model "openrouter/auto")) @@ -50,20 +52,33 @@ (messages . (( (role . "system") (content . ,system-prompt) ) ( (role . "user") (content . ,prompt) ))))))) (handler-case - (bt:with-timeout (timeout) - (let* ((response (dex:post url :headers headers :content body - :connect-timeout (min 10 timeout) - :read-timeout (max 10 (- timeout 5)))) - (json (cl-json:decode-json-from-string response)) - (choices (cdr (assoc :choices json))) - (first-choice (car choices)) - (message (cdr (assoc :message first-choice))) - (content (cdr (assoc :content message)))) - (if content - (list :status :success :content content) - (list :status :error :message (format nil "~a: No content in response (~s)" provider json))))) - (bt:timeout () - (list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout))) + (let* ((result nil) + (lock (bt:make-lock "llm-lock")) + (done nil) + (thread (bt:make-thread + (lambda () + (handler-case + (let* ((response (dex:post url :headers headers :content body + :connect-timeout (min 10 timeout) + :read-timeout (max 10 (- timeout 5)))) + (json (cl-json:decode-json-from-string response)) + (choices (cdr (assoc :choices json))) + (first-choice (car choices)) + (message (cdr (assoc :message first-choice))) + (content (cdr (assoc :content message)))) + (setf result (if content + (list :status :success :content content) + (list :status :error :message (format nil "~a: No content" provider))))) + (error (c) + (setf result (list :status :error :message (format nil "~a Error: ~a" provider c))))) + (bt:with-lock-held (lock) (setf done t))) + :name (format nil "llm-~a" provider)))) + (loop for waited from 0 below timeout + when (bt:with-lock-held (lock) done) + do (return result) + do (sleep 0.5)) + (or result + (list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout)))) (error (c) (list :status :error :message (format nil "~a Failure: ~a" provider c)))))) diff --git a/org/system-model-provider.org b/org/system-model-provider.org index 1872034..36a654f 100644 --- a/org/system-model-provider.org +++ b/org/system-model-provider.org @@ -17,6 +17,8 @@ Providers register themselves at boot. No API key? That provider doesn't registe ** Provider registry #+begin_src lisp +(in-package :passepartout) + (defparameter *provider-configs* '((:local . (:base-url nil :key-env nil :url-env "LOCAL_BASE_URL" :default-model "llama3")) (:openrouter . (:base-url "https://openrouter.ai/api/v1" :key-env "OPENROUTER_API_KEY" :default-model "openrouter/auto")) @@ -78,20 +80,33 @@ Providers register themselves at boot. No API key? That provider doesn't registe (messages . (( (role . "system") (content . ,system-prompt) ) ( (role . "user") (content . ,prompt) ))))))) (handler-case - (bt:with-timeout (timeout) - (let* ((response (dex:post url :headers headers :content body - :connect-timeout (min 10 timeout) - :read-timeout (max 10 (- timeout 5)))) - (json (cl-json:decode-json-from-string response)) - (choices (cdr (assoc :choices json))) - (first-choice (car choices)) - (message (cdr (assoc :message first-choice))) - (content (cdr (assoc :content message)))) - (if content - (list :status :success :content content) - (list :status :error :message (format nil "~a: No content in response (~s)" provider json))))) - (bt:timeout () - (list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout))) + (let* ((result nil) + (lock (bt:make-lock "llm-lock")) + (done nil) + (thread (bt:make-thread + (lambda () + (handler-case + (let* ((response (dex:post url :headers headers :content body + :connect-timeout (min 10 timeout) + :read-timeout (max 10 (- timeout 5)))) + (json (cl-json:decode-json-from-string response)) + (choices (cdr (assoc :choices json))) + (first-choice (car choices)) + (message (cdr (assoc :message first-choice))) + (content (cdr (assoc :content message)))) + (setf result (if content + (list :status :success :content content) + (list :status :error :message (format nil "~a: No content" provider))))) + (error (c) + (setf result (list :status :error :message (format nil "~a Error: ~a" provider c))))) + (bt:with-lock-held (lock) (setf done t))) + :name (format nil "llm-~a" provider)))) + (loop for waited from 0 below timeout + when (bt:with-lock-held (lock) done) + do (return result) + do (sleep 0.5)) + (or result + (list :status :error :message (format nil "~a: Request timed out after ~d seconds" provider timeout)))) (error (c) (list :status :error :message (format nil "~a Failure: ~a" provider c)))))) #+end_src