gptel-ollama.el (10548B)
1 ;;; gptel-ollama.el --- Ollama support for gptel -*- lexical-binding: t; -*- 2 3 ;; Copyright (C) 2023 Karthik Chikmagalur 4 5 ;; Author: Karthik Chikmagalur <karthikchikmagalur@gmail.com> 6 ;; Keywords: hypermedia 7 8 ;; This program is free software; you can redistribute it and/or modify 9 ;; it under the terms of the GNU General Public License as published by 10 ;; the Free Software Foundation, either version 3 of the License, or 11 ;; (at your option) any later version. 12 13 ;; This program is distributed in the hope that it will be useful, 14 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 ;; GNU General Public License for more details. 17 18 ;; You should have received a copy of the GNU General Public License 19 ;; along with this program. If not, see <https://www.gnu.org/licenses/>. 20 21 ;;; Commentary: 22 23 ;; This file adds support for the Ollama LLM API to gptel 24 25 ;;; Code: 26 (require 'gptel) 27 (require 'cl-generic) 28 29 (declare-function json-read "json" ()) 30 (declare-function gptel-context--wrap "gptel-context") 31 (declare-function gptel-context--collect-media "gptel-context") 32 (defvar json-object-type) 33 34 ;;; Ollama 35 (cl-defstruct (gptel-ollama (:constructor gptel--make-ollama) 36 (:copier nil) 37 (:include gptel-backend))) 38 39 (defvar-local gptel--ollama-token-count 0 40 "Token count for ollama conversations. 41 42 This variable holds the total token count for conversations with 43 Ollama models. 44 45 Intended for internal use only.") 46 47 (cl-defmethod gptel-curl--parse-stream ((_backend gptel-ollama) info) 48 "Parse response stream for the Ollama API." 49 (when (and (bobp) (re-search-forward "^{" nil t)) 50 (forward-line 0)) 51 (let* ((content-strs) (content) (pt (point))) 52 (condition-case nil 53 (while (setq content (gptel--json-read)) 54 (setq pt (point)) 55 (let ((done (map-elt content :done)) 56 (response (map-nested-elt content '(:message :content)))) 57 (push response content-strs) 58 (unless (eq done :json-false) 59 (with-current-buffer (plist-get info :buffer) 60 (cl-incf gptel--ollama-token-count 61 (+ (or (map-elt content :prompt_eval_count) 0) 62 (or (map-elt content :eval_count) 0)))) 63 (goto-char (point-max))))) 64 (error (goto-char pt))) 65 (apply #'concat (nreverse content-strs)))) 66 67 (cl-defmethod gptel--parse-response ((_backend gptel-ollama) response info) 68 "Parse a one-shot RESPONSE from the Ollama API." 69 (when-let ((context 70 (+ (or (map-elt response :prompt_eval_count) 0) 71 (or (map-elt response :eval_count) 0)))) 72 (with-current-buffer (plist-get info :buffer) 73 (cl-incf gptel--ollama-token-count context))) 74 (map-nested-elt response '(:message :content))) 75 76 (cl-defmethod gptel--request-data ((_backend gptel-ollama) prompts) 77 "JSON encode PROMPTS for sending to ChatGPT." 78 (let ((prompts-plist 79 `(:model ,(gptel--model-name gptel-model) 80 :messages [,@prompts] 81 :stream ,(or (and gptel-stream gptel-use-curl 82 (gptel-backend-stream gptel-backend)) 83 :json-false))) 84 options-plist) 85 (when gptel-temperature 86 (setq options-plist 87 (plist-put options-plist :temperature 88 gptel-temperature))) 89 (when gptel-max-tokens 90 (setq options-plist 91 (plist-put options-plist :num_predict 92 gptel-max-tokens))) 93 ;; FIXME: These options will be lost if there are model/backend-specific 94 ;; :options, since `gptel--merge-plists' does not merge plist values 95 ;; recursively. 96 (when options-plist 97 (plist-put prompts-plist :options options-plist)) 98 ;; Merge request params with model and backend params. 99 (gptel--merge-plists 100 prompts-plist 101 (gptel-backend-request-params gptel-backend) 102 (gptel--model-request-params gptel-model)))) 103 104 (cl-defmethod gptel--parse-buffer ((_backend gptel-ollama) &optional max-entries) 105 (let ((prompts) (prop) 106 (include-media (and gptel-track-media (or (gptel--model-capable-p 'media) 107 (gptel--model-capable-p 'url))))) 108 (if (or gptel-mode gptel-track-response) 109 (while (and 110 (or (not max-entries) (>= max-entries 0)) 111 (setq prop (text-property-search-backward 112 'gptel 'response 113 (when (get-char-property (max (point-min) (1- (point))) 114 'gptel) 115 t)))) 116 (if (prop-match-value prop) ;assistant role 117 (push (list :role "assistant" 118 :content (buffer-substring-no-properties (prop-match-beginning prop) 119 (prop-match-end prop))) 120 prompts) 121 (if include-media 122 (push (append '(:role "user") 123 (gptel--ollama-parse-multipart 124 (gptel--parse-media-links 125 major-mode (prop-match-beginning prop) (prop-match-end prop)))) 126 prompts) 127 (push (list :role "user" 128 :content 129 (gptel--trim-prefixes 130 (buffer-substring-no-properties (prop-match-beginning prop) 131 (prop-match-end prop)))) 132 prompts))) 133 (and max-entries (cl-decf max-entries))) 134 (push (list :role "user" 135 :content 136 (string-trim (buffer-substring-no-properties (point-min) (point-max)))) 137 prompts)) 138 (if (and (not (gptel--model-capable-p 'nosystem)) 139 gptel--system-message) 140 (cons (list :role "system" 141 :content gptel--system-message) 142 prompts) 143 prompts))) 144 145 (defun gptel--ollama-parse-multipart (parts) 146 "Convert a multipart prompt PARTS to the Ollama API format. 147 148 The input is an alist of the form 149 ((:text \"some text\") 150 (:media \"/path/to/media.png\" :mime \"image/png\") 151 (:text \"More text\")). 152 153 The output is a vector of entries in a backend-appropriate 154 format." 155 (cl-loop 156 for part in parts 157 for n upfrom 1 158 with last = (length parts) 159 for text = (plist-get part :text) 160 for media = (plist-get part :media) 161 if text do 162 (and (or (= n 1) (= n last)) (setq text (gptel--trim-prefixes text))) and 163 unless (string-empty-p text) 164 collect text into text-array end 165 else if media 166 collect (gptel--base64-encode media) into media-array end 167 finally return 168 `(,@(and text-array (list :content (mapconcat #'identity text-array " "))) 169 ,@(and media-array (list :images (vconcat media-array)))))) 170 171 (cl-defmethod gptel--wrap-user-prompt ((_backend gptel-ollama) prompts 172 &optional inject-media) 173 "Wrap the last user prompt in PROMPTS with the context string. 174 175 If INJECT-MEDIA is non-nil wrap it with base64-encoded media files in the context." 176 (if inject-media 177 ;; Wrap the first user prompt with included media files/contexts 178 (when-let* ((media-list (gptel-context--collect-media)) 179 (media-processed (gptel--ollama-parse-multipart media-list))) 180 (cl-callf (lambda (images) 181 (vconcat (plist-get media-processed :images) 182 images)) 183 (plist-get (cadr prompts) :images))) 184 ;; Wrap the last user prompt with included text contexts 185 (cl-callf gptel-context--wrap (plist-get (car (last prompts)) :content)))) 186 187 ;;;###autoload 188 (cl-defun gptel-make-ollama 189 (name &key curl-args header key models stream request-params 190 (host "localhost:11434") 191 (protocol "http") 192 (endpoint "/api/chat")) 193 "Register an Ollama backend for gptel with NAME. 194 195 Keyword arguments: 196 197 CURL-ARGS (optional) is a list of additional Curl arguments. 198 199 HOST is where Ollama runs (with port), defaults to localhost:11434 200 201 MODELS is a list of available model names, as symbols. 202 Additionally, you can specify supported LLM capabilities like 203 vision or tool-use by appending a plist to the model with more 204 information, in the form 205 206 (model-name . plist) 207 208 Currently recognized plist keys are :description, :capabilities 209 and :mime-types. An example of a model specification including 210 both kinds of specs: 211 212 :models 213 \\='(mistral:latest ;Simple specs 214 openhermes:latest 215 (llava:13b ;Full spec 216 :description 217 \"Llava 1.6: Large Lanuage and Vision Assistant\" 218 :capabilities (media) 219 :mime-types (\"image/jpeg\" \"image/png\"))) 220 221 222 STREAM is a boolean to toggle streaming responses, defaults to 223 false. 224 225 PROTOCOL (optional) specifies the protocol, http by default. 226 227 ENDPOINT (optional) is the API endpoint for completions, defaults to 228 \"/api/generate\". 229 230 HEADER (optional) is for additional headers to send with each 231 request. It should be an alist or a function that retuns an 232 alist, like: 233 ((\"Content-Type\" . \"application/json\")) 234 235 KEY (optional) is a variable whose value is the API key, or 236 function that returns the key. This is typically not required 237 for local models like Ollama. 238 239 REQUEST-PARAMS (optional) is a plist of additional HTTP request 240 parameters (as plist keys) and values supported by the API. Use 241 these to set parameters that gptel does not provide user options 242 for. 243 244 Example: 245 ------- 246 247 (gptel-make-ollama 248 \"Ollama\" 249 :host \"localhost:11434\" 250 :models \\='(mistral:latest) 251 :stream t)" 252 (declare (indent 1)) 253 (let ((backend (gptel--make-ollama 254 :curl-args curl-args 255 :name name 256 :host host 257 :header header 258 :key key 259 :models (gptel--process-models models) 260 :protocol protocol 261 :endpoint endpoint 262 :stream stream 263 :request-params request-params 264 :url (if protocol 265 (concat protocol "://" host endpoint) 266 (concat host endpoint))))) 267 (prog1 backend 268 (setf (alist-get name gptel--known-backends 269 nil nil #'equal) 270 backend)))) 271 272 (provide 'gptel-ollama) 273 ;;; gptel-ollama.el ends here 274 275