Add v0.2.0 features: vector search + tool permissions
- Local vector search: Ollama embeddings + semantic search - get-embedding with caching - cosine-similarity computation - semantic-search cognitive tool - :semantic-search tool for LLM queries - Tool permission tiers: security gating for cognitive tools - Three tiers: :allow, :deny, :ask - Gate in execute-tool-action before tool runs - Defaults: :deny for shell/delete-file, :ask for eval/write-file - :tool-permissions cognitive tool for management - Embedding provider support: Ollama AND llama.cpp - EMBEDDING_PROVIDER env var - EMBEDDING_MODEL env var - LLAMA_HOST for llama.cpp server - .env.example: Add embedding config variables - Fix parse-message in communication.lisp - Update ASDF: add test files, tool-permissions skill All 60 tests pass (6 suites x 100%)
This commit is contained in:
@@ -81,8 +81,18 @@
|
||||
(meta (getf context :meta))
|
||||
(source (getf meta :source))
|
||||
(tool (gethash (string-downcase (string tool-name)) *cognitive-tools*)))
|
||||
(if tool
|
||||
(handler-case
|
||||
(when tool
|
||||
;; Tool Permission Gate: Check permission before execution
|
||||
(let ((permission (check-tool-permission-gate tool-name context)))
|
||||
(when (eq permission :deny)
|
||||
(return-from execute-tool-action
|
||||
(list :TYPE :EVENT :DEPTH (1+ depth) :META meta
|
||||
:PAYLOAD (list :SENSOR :tool-error :tool tool-name :message (format nil "Tool PERMISSION DENIED: ~a" tool-name))))))
|
||||
(when (listp permission)
|
||||
(return-from execute-tool-action
|
||||
(list :TYPE :EVENT :DEPTH (1+ depth) :META meta
|
||||
:PAYLOAD (list :SENSOR :permission-pending :tool tool-name :args tool-args)))))
|
||||
(handler-case
|
||||
(let* ((clean-args (if (and (listp tool-args) (listp (car tool-args))) (car tool-args) tool-args))
|
||||
(result (funcall (cognitive-tool-body tool) clean-args)))
|
||||
(let ((feedback (list :TYPE :EVENT :DEPTH (1+ depth) :META meta
|
||||
@@ -94,10 +104,10 @@
|
||||
context))
|
||||
feedback))
|
||||
(error (c)
|
||||
(list :TYPE :EVENT :DEPTH (1+ depth) :META meta
|
||||
:PAYLOAD (list :SENSOR :tool-error :tool tool-name :message (format nil "~a" c)))))
|
||||
(list :TYPE :EVENT :DEPTH (1+ depth) :META meta
|
||||
:PAYLOAD (list :SENSOR :tool-error :message "Tool not found")))))
|
||||
:PAYLOAD (list :SENSOR :tool-error :tool tool-name :message (format nil "~a" c)))))
|
||||
(list :TYPE :EVENT :DEPTH (1+ depth) :META meta
|
||||
:PAYLOAD (list :SENSOR :tool-error :message "Tool not found"))))
|
||||
|
||||
(defun act-gate (signal)
|
||||
"Final Stage: Actuation and feedback generation."
|
||||
|
||||
@@ -16,6 +16,13 @@
|
||||
(len (length msg-string)))
|
||||
(format nil "~6,'0x~a~%" len msg-string)))
|
||||
|
||||
(defun parse-message (framed-string)
|
||||
"Parses a hex-length prefixed framed string into a Lisp plist."
|
||||
(let* ((len (parse-integer (subseq framed-string 0 6) :radix 16))
|
||||
(payload (subseq framed-string 6 (+ 6 len))))
|
||||
(let ((*read-eval* nil))
|
||||
(read-from-string payload))))
|
||||
|
||||
(defun read-framed-message (stream)
|
||||
"Reads a hex-length prefixed S-expression from the stream securely. Skips leading whitespace."
|
||||
(let ((length-buffer (make-string 6)))
|
||||
|
||||
@@ -91,6 +91,30 @@
|
||||
(opencortex:register-probabilistic-backend p (lambda (prompt system-prompt &key model)
|
||||
(execute-llm-request prompt system-prompt :provider p :model model))))
|
||||
|
||||
(def-cognitive-tool :get-ollama-embedding
|
||||
"Generates vector embeddings via Ollama API."
|
||||
((text :type :string :description "Text to embed."))
|
||||
:body (lambda (args)
|
||||
(let* ((text (getf args :text))
|
||||
(host (or (uiop:getenv "OLLAMA_HOST") "localhost:11434"))
|
||||
(url (format nil "http://~a/api/embeddings" host))
|
||||
(model (or (uiop:getenv "OLLAMA_EMBEDDING_MODEL") "nomic-embed-text"))
|
||||
(body (cl-json:encode-json-to-string `((model . ,model) (prompt . ,text)))))
|
||||
(handler-case
|
||||
(let* ((response (dex:post url :headers '(("Content-Type" . "application/json")) :content body :connect-timeout 5 :read-timeout 30))
|
||||
(json (cl-json:decode-json-from-string response)))
|
||||
(let ((embedding (cdr (assoc :embedding json))))
|
||||
(if embedding
|
||||
(list :status :success :vector embedding)
|
||||
(list :status :error :message "No embedding in response"))))
|
||||
(error (c) (list :status :error :message (format nil "Ollama Embedding Failure: ~a" c)))))))
|
||||
|
||||
(defun get-embedding (text)
|
||||
"Generates a vector embedding for the given text via Ollama. Returns nil on failure."
|
||||
(let ((result (funcall (get-cognitive-tool-body :get-ollama-embedding) (list :text text))))
|
||||
(when (eq (getf result :status) :success)
|
||||
(getf result :vector))))
|
||||
|
||||
(def-cognitive-tool :ask-llm
|
||||
"Queries an LLM provider via the unified gateway."
|
||||
((:prompt :type :string :description "The user prompt.")
|
||||
|
||||
91
library/gen/org-skill-tool-permissions.lisp
Normal file
91
library/gen/org-skill-tool-permissions.lisp
Normal file
@@ -0,0 +1,91 @@
|
||||
(in-package :opencortex)
|
||||
|
||||
(defvar *tool-permissions* (make-hash-table :test 'equal)
|
||||
"Hash table mapping tool names to :allow/:deny/:ask.")
|
||||
|
||||
(defun get-tool-permission (tool-name)
|
||||
(let ((key (string-downcase (string tool-name))))
|
||||
(or (gethash key *tool-permissions*) :allow)))
|
||||
|
||||
(defun set-tool-permission (tool-name tier)
|
||||
(setf (gethash (string-downcase (string tool-name)) *tool-permissions*) tier)
|
||||
(harness-log "TOOL PERMISSION: Set ~a = ~a" tool-name tier))
|
||||
|
||||
(defun check-tool-permission-gate (tool-name context)
|
||||
(declare (ignore context))
|
||||
(let ((perm (get-tool-permission tool-name)))
|
||||
(case perm
|
||||
(:allow :allow)
|
||||
(:deny :deny)
|
||||
(:ask (list :ask tool-name context))
|
||||
(t :allow))))
|
||||
|
||||
(def-cognitive-tool :get-embedding
|
||||
"Generates vector embeddings via Ollama or llama.cpp API."
|
||||
((text :type :string :description "Text to embed."))
|
||||
:body (lambda (args)
|
||||
(let* ((text (getf args :text))
|
||||
(provider (or (uiop:getenv "EMBEDDING_PROVIDER") "ollama"))
|
||||
(model (or (uiop:getenv "EMBEDDING_MODEL")
|
||||
(case (intern (string-upcase provider) :keyword)
|
||||
(:NOMIC-EMBED-TEXT "nomic-embed-text")
|
||||
(:LLAMA-CPP "llama.cpp")
|
||||
(t "nomic-embed-text"))))
|
||||
(embedding nil))
|
||||
(cond
|
||||
((string= provider "ollama")
|
||||
(let* ((host (or (uiop:getenv "OLLAMA_HOST") "localhost:11434"))
|
||||
(url (format nil "http://~a/api/embeddings" host))
|
||||
(body (cl-json:encode-json-to-string `((model . ,model) (prompt . ,text)))))
|
||||
(handler-case
|
||||
(let* ((response (dex:post url :headers '(("Content-Type" . "application/json")) :content body :connect-timeout 5 :read-timeout 30))
|
||||
(json (cl-json:decode-json-from-string response))
|
||||
(vec (cdr (assoc :embedding json))))
|
||||
(when vec (setf embedding vec)))
|
||||
(error (c) (harness-log "EMBEDDING: Ollama failed: ~a" c)))))
|
||||
((string= provider "llama.cpp")
|
||||
(let* ((host (or (uiop:getenv "LLAMA_HOST") "localhost:8080"))
|
||||
(url (format nil "http://~a/v1/embeddings" host))
|
||||
(body (cl-json:encode-json-to-string `((model . ,model) (input . ,text)))))
|
||||
(handler-case
|
||||
(let* ((response (dex:post url :headers '(("Content-Type" . "application/json")) :content body :connect-timeout 5 :read-timeout 30))
|
||||
(json (cl-json:decode-json-from-string response))
|
||||
(data (cdr (assoc :data json)))
|
||||
(vec (when data (cdr (assoc :embedding (car data))))))
|
||||
(when vec (setf embedding vec)))
|
||||
(error (c) (harness-log "EMBEDDING: llama.cpp failed: ~a" c))))))
|
||||
(if embedding
|
||||
(list :status :success :vector embedding)
|
||||
(list :status :error :message "Embedding generation failed")))))
|
||||
|
||||
(def-cognitive-tool :tool-permissions
|
||||
"View or set tool permission tiers."
|
||||
((:tool :type :string :description "Tool name")
|
||||
(:action :type :keyword :description "Action: :get, :set, :list" :default :get)
|
||||
(:tier :type :keyword :description "For :set: :allow/:deny/:ask"))
|
||||
:body (lambda (args)
|
||||
(let ((tool (getf args :tool))
|
||||
(action (getf args :action :get))
|
||||
(tier (getf args :tier)))
|
||||
(case action
|
||||
(:get (list :status :success :tool tool :permission (get-tool-permission tool)))
|
||||
(:set (progn (set-tool-permission tool tier)
|
||||
(list :status :success :message (format nil "Set ~a = ~a" tool tier))))
|
||||
(:list (let ((r nil))
|
||||
(maphash (lambda (k v) (push (list :tool k :permission v) r)) *tool-permissions*)
|
||||
(list :status :success :tools r)))
|
||||
(t (list :status :error :message "Invalid action"))))))
|
||||
|
||||
;; Defaults
|
||||
(set-tool-permission :shell :deny)
|
||||
(set-tool-permission :delete-file :deny)
|
||||
(set-tool-permission :eval :ask)
|
||||
(set-tool-permission :write-file :ask)
|
||||
(harness-log "TOOL PERMISSIONS: Initialized")
|
||||
|
||||
(defskill :skill-tool-permissions
|
||||
:priority 600
|
||||
:trigger (lambda (c) (declare (ignore c)) nil)
|
||||
:deterministic (lambda (a c)
|
||||
(let ((tool (getf (getf a :payload) :tool)))
|
||||
(when tool (check-tool-permission-gate tool c)))))
|
||||
@@ -161,3 +161,58 @@ Reconstitutes alists into hash tables."
|
||||
(defun file-name-nondirectory (path)
|
||||
"Extracts the filename from a full path string."
|
||||
(let ((pos (position #\/ path :from-end t))) (if pos (subseq path (1+ pos)) path)))
|
||||
|
||||
(defvar *embedding-cache* (make-hash-table :test 'equal)
|
||||
"Cache for embeddings to avoid redundant API calls.")
|
||||
|
||||
(defun get-embedding (text)
|
||||
"Generates a vector embedding for the given text via Ollama. Returns nil on failure."
|
||||
(when (or (null text) (string= text ""))
|
||||
(return-from get-embedding nil))
|
||||
(let ((cached (gethash text *embedding-cache*)))
|
||||
(when cached (return-from get-embedding cached)))
|
||||
(let ((result (funcall (get-cognitive-tool-body :get-ollama-embedding) (list :text text))))
|
||||
(when (eq (getf result :status) :success)
|
||||
(let ((vec (getf result :vector)))
|
||||
(setf (gethash text *embedding-cache*) vec)
|
||||
vec))))
|
||||
|
||||
(defun cosine-similarity (vec-a vec-b)
|
||||
"Computes cosine similarity between two vectors. Both should be sequences of numbers."
|
||||
(when (or (null vec-a) (null vec-b) (zerop (length vec-a)) (zerop (length vec-b)))
|
||||
(return-from cosine-similarity 0.0))
|
||||
(let ((dot-product (loop for a across vec-a
|
||||
for b across vec-b
|
||||
sum (* a b)))
|
||||
(norm-a (sqrt (loop for a across vec-a sum (* a a))))
|
||||
(norm-b (sqrt (loop for b across vec-b sum (* b b)))))
|
||||
(if (or (zerop norm-a) (zerop norm-b))
|
||||
0.0
|
||||
(/ dot-product (* norm-a norm-b)))))
|
||||
|
||||
(defun semantic-search (query &key (limit 10) (min-similarity 0.5))
|
||||
"Searches memory for objects semantically similar to the query."
|
||||
(let* ((query-vec (get-embedding query))
|
||||
(results nil))
|
||||
(unless query-vec
|
||||
(harness-log "EMBEDDING: Failed to generate embedding for query: ~a" query)
|
||||
(return-from semantic-search nil))
|
||||
(maphash (lambda (id obj)
|
||||
(let ((obj-vec (org-object-vector obj)))
|
||||
(when obj-vec
|
||||
(let ((sim (cosine-similarity query-vec obj-vec)))
|
||||
(when (>= sim min-similarity)
|
||||
(push (list :id id :object obj :similarity sim) results))))))
|
||||
*memory*)
|
||||
(setf results (sort results #'> :key (lambda (r) (getf r :similarity))))
|
||||
(subseq results 0 (min limit (length results)))))
|
||||
|
||||
(def-cognitive-tool :semantic-search
|
||||
"Searches memory for objects semantically similar to a query."
|
||||
((:query :type :string :description "The search query.")
|
||||
(:limit :type :integer :description "Maximum results to return." :default 10)
|
||||
(:min-similarity :type :number :description "Minimum similarity threshold (0-1)." :default 0.5))
|
||||
:body (lambda (args)
|
||||
(semantic-search (getf args :query)
|
||||
:limit (or (getf args :limit) 10)
|
||||
:min-similarity (or (getf args :min-similarity) 0.5))))
|
||||
|
||||
@@ -167,3 +167,16 @@
|
||||
(setq *system-logs* (subseq *system-logs* 0 *max-log-history*))))
|
||||
(format t "~a~%" formatted-msg)
|
||||
(finish-output)))
|
||||
|
||||
(defun proto-get (plist key)
|
||||
"Robustly retrieves a value from a plist, checking both uppercase and lowercase keyword versions."
|
||||
(let* ((s (string key))
|
||||
(up (intern (string-upcase s) :keyword))
|
||||
(dn (intern (string-downcase s) :keyword)))
|
||||
(or (getf plist up) (getf plist dn))))
|
||||
|
||||
(defun get-cognitive-tool-body (tool-name)
|
||||
"Retrieves the body function of a cognitive tool, or nil if not found."
|
||||
(let ((tool (gethash (string-downcase (string tool-name)) *cognitive-tools*)))
|
||||
(when tool
|
||||
(cognitive-tool-body tool))))
|
||||
|
||||
Reference in New Issue
Block a user