gptel-ollama.el (10776B)
1 ;;; gptel-ollama.el --- Ollama support for gptel -*- lexical-binding: t; -*- 2 3 ;; Copyright (C) 2023 Karthik Chikmagalur 4 5 ;; Author: Karthik Chikmagalur <karthikchikmagalur@gmail.com> 6 ;; Keywords: hypermedia 7 8 ;; This program is free software; you can redistribute it and/or modify 9 ;; it under the terms of the GNU General Public License as published by 10 ;; the Free Software Foundation, either version 3 of the License, or 11 ;; (at your option) any later version. 12 13 ;; This program is distributed in the hope that it will be useful, 14 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 ;; GNU General Public License for more details. 17 18 ;; You should have received a copy of the GNU General Public License 19 ;; along with this program. If not, see <https://www.gnu.org/licenses/>. 20 21 ;;; Commentary: 22 23 ;; This file adds support for the Ollama LLM API to gptel 24 25 ;;; Code: 26 (require 'gptel) 27 (require 'cl-generic) 28 29 (declare-function json-read "json" ()) 30 (declare-function gptel-context--wrap "gptel-context") 31 (declare-function gptel-context--collect-media "gptel-context") 32 (defvar json-object-type) 33 34 ;;; Ollama 35 (cl-defstruct (gptel-ollama (:constructor gptel--make-ollama) 36 (:copier nil) 37 (:include gptel-backend))) 38 39 (defvar-local gptel--ollama-token-count 0 40 "Token count for ollama conversations. 41 42 This variable holds the total token count for conversations with 43 Ollama models. 44 45 Intended for internal use only.") 46 47 (cl-defmethod gptel-curl--parse-stream ((_backend gptel-ollama) info) 48 "Parse response stream for the Ollama API." 49 (when (and (bobp) (re-search-forward "^{" nil t)) 50 (forward-line 0)) 51 (let* ((content-strs) (content) (pt (point))) 52 (condition-case nil 53 (while (setq content (gptel--json-read)) 54 (setq pt (point)) 55 (let ((done (map-elt content :done)) 56 (response (map-nested-elt content '(:message :content)))) 57 (push response content-strs) 58 (unless (eq done :json-false) 59 (with-current-buffer (plist-get info :buffer) 60 (cl-incf gptel--ollama-token-count 61 (+ (or (map-elt content :prompt_eval_count) 0) 62 (or (map-elt content :eval_count) 0)))) 63 (goto-char (point-max))))) 64 (error (goto-char pt))) 65 (apply #'concat (nreverse content-strs)))) 66 67 (cl-defmethod gptel--parse-response ((_backend gptel-ollama) response info) 68 "Parse a one-shot RESPONSE from the Ollama API." 69 (when-let ((context 70 (+ (or (map-elt response :prompt_eval_count) 0) 71 (or (map-elt response :eval_count) 0)))) 72 (with-current-buffer (plist-get info :buffer) 73 (cl-incf gptel--ollama-token-count context))) 74 (map-nested-elt response '(:message :content))) 75 76 (cl-defmethod gptel--request-data ((_backend gptel-ollama) prompts) 77 "JSON encode PROMPTS for sending to ChatGPT." 78 (when (and gptel--system-message 79 (not (gptel--model-capable-p 'nosystem))) 80 (push (list :role "system" 81 :content gptel--system-message) 82 prompts)) 83 (let ((prompts-plist 84 `(:model ,(gptel--model-name gptel-model) 85 :messages [,@prompts] 86 :stream ,(or (and gptel-stream gptel-use-curl 87 (gptel-backend-stream gptel-backend)) 88 :json-false))) 89 options-plist) 90 (when gptel-temperature 91 (setq options-plist 92 (plist-put options-plist :temperature 93 gptel-temperature))) 94 (when gptel-max-tokens 95 (setq options-plist 96 (plist-put options-plist :num_predict 97 gptel-max-tokens))) 98 ;; FIXME: These options will be lost if there are model/backend-specific 99 ;; :options, since `gptel--merge-plists' does not merge plist values 100 ;; recursively. 101 (when options-plist 102 (plist-put prompts-plist :options options-plist)) 103 ;; Merge request params with model and backend params. 104 (gptel--merge-plists 105 prompts-plist 106 (gptel-backend-request-params gptel-backend) 107 (gptel--model-request-params gptel-model)))) 108 109 (cl-defmethod gptel--parse-list ((_backend gptel-ollama) prompt-list) 110 (cl-loop for text in prompt-list 111 for role = t then (not role) 112 if text collect 113 (list :role (if role "user" "assistant") :content text))) 114 115 (cl-defmethod gptel--parse-buffer ((_backend gptel-ollama) &optional max-entries) 116 (let ((prompts) (prop) 117 (include-media (and gptel-track-media (or (gptel--model-capable-p 'media) 118 (gptel--model-capable-p 'url))))) 119 (if (or gptel-mode gptel-track-response) 120 (while (and 121 (or (not max-entries) (>= max-entries 0)) 122 (setq prop (text-property-search-backward 123 'gptel 'response 124 (when (get-char-property (max (point-min) (1- (point))) 125 'gptel) 126 t)))) 127 (if (prop-match-value prop) ;assistant role 128 (push (list :role "assistant" 129 :content (buffer-substring-no-properties (prop-match-beginning prop) 130 (prop-match-end prop))) 131 prompts) 132 (if include-media 133 (push (append '(:role "user") 134 (gptel--ollama-parse-multipart 135 (gptel--parse-media-links 136 major-mode (prop-match-beginning prop) (prop-match-end prop)))) 137 prompts) 138 (push (list :role "user" 139 :content 140 (gptel--trim-prefixes 141 (buffer-substring-no-properties (prop-match-beginning prop) 142 (prop-match-end prop)))) 143 prompts))) 144 (and max-entries (cl-decf max-entries))) 145 (push (list :role "user" 146 :content 147 (string-trim (buffer-substring-no-properties (point-min) (point-max)))) 148 prompts)) 149 prompts)) 150 151 (defun gptel--ollama-parse-multipart (parts) 152 "Convert a multipart prompt PARTS to the Ollama API format. 153 154 The input is an alist of the form 155 ((:text \"some text\") 156 (:media \"/path/to/media.png\" :mime \"image/png\") 157 (:text \"More text\")). 158 159 The output is a vector of entries in a backend-appropriate 160 format." 161 (cl-loop 162 for part in parts 163 for n upfrom 1 164 with last = (length parts) 165 for text = (plist-get part :text) 166 for media = (plist-get part :media) 167 if text do 168 (and (or (= n 1) (= n last)) (setq text (gptel--trim-prefixes text))) and 169 unless (string-empty-p text) 170 collect text into text-array end 171 else if media 172 collect (gptel--base64-encode media) into media-array end 173 finally return 174 `(,@(and text-array (list :content (mapconcat #'identity text-array " "))) 175 ,@(and media-array (list :images (vconcat media-array)))))) 176 177 (cl-defmethod gptel--wrap-user-prompt ((_backend gptel-ollama) prompts 178 &optional inject-media) 179 "Wrap the last user prompt in PROMPTS with the context string. 180 181 If INJECT-MEDIA is non-nil wrap it with base64-encoded media files in the context." 182 (if inject-media 183 ;; Wrap the first user prompt with included media files/contexts 184 (when-let* ((media-list (gptel-context--collect-media)) 185 (media-processed (gptel--ollama-parse-multipart media-list))) 186 (cl-callf (lambda (images) 187 (vconcat (plist-get media-processed :images) 188 images)) 189 (plist-get (cadr prompts) :images))) 190 ;; Wrap the last user prompt with included text contexts 191 (cl-callf gptel-context--wrap (plist-get (car (last prompts)) :content)))) 192 193 ;;;###autoload 194 (cl-defun gptel-make-ollama 195 (name &key curl-args header key models stream request-params 196 (host "localhost:11434") 197 (protocol "http") 198 (endpoint "/api/chat")) 199 "Register an Ollama backend for gptel with NAME. 200 201 Keyword arguments: 202 203 CURL-ARGS (optional) is a list of additional Curl arguments. 204 205 HOST is where Ollama runs (with port), defaults to localhost:11434 206 207 MODELS is a list of available model names, as symbols. 208 Additionally, you can specify supported LLM capabilities like 209 vision or tool-use by appending a plist to the model with more 210 information, in the form 211 212 (model-name . plist) 213 214 Currently recognized plist keys are :description, :capabilities 215 and :mime-types. An example of a model specification including 216 both kinds of specs: 217 218 :models 219 \\='(mistral:latest ;Simple specs 220 openhermes:latest 221 (llava:13b ;Full spec 222 :description 223 \"Llava 1.6: Large Lanuage and Vision Assistant\" 224 :capabilities (media) 225 :mime-types (\"image/jpeg\" \"image/png\"))) 226 227 228 STREAM is a boolean to toggle streaming responses, defaults to 229 false. 230 231 PROTOCOL (optional) specifies the protocol, http by default. 232 233 ENDPOINT (optional) is the API endpoint for completions, defaults to 234 \"/api/generate\". 235 236 HEADER (optional) is for additional headers to send with each 237 request. It should be an alist or a function that retuns an 238 alist, like: 239 ((\"Content-Type\" . \"application/json\")) 240 241 KEY (optional) is a variable whose value is the API key, or 242 function that returns the key. This is typically not required 243 for local models like Ollama. 244 245 REQUEST-PARAMS (optional) is a plist of additional HTTP request 246 parameters (as plist keys) and values supported by the API. Use 247 these to set parameters that gptel does not provide user options 248 for. 249 250 Example: 251 ------- 252 253 (gptel-make-ollama 254 \"Ollama\" 255 :host \"localhost:11434\" 256 :models \\='(mistral:latest) 257 :stream t)" 258 (declare (indent 1)) 259 (let ((backend (gptel--make-ollama 260 :curl-args curl-args 261 :name name 262 :host host 263 :header header 264 :key key 265 :models (gptel--process-models models) 266 :protocol protocol 267 :endpoint endpoint 268 :stream stream 269 :request-params request-params 270 :url (if protocol 271 (concat protocol "://" host endpoint) 272 (concat host endpoint))))) 273 (prog1 backend 274 (setf (alist-get name gptel--known-backends 275 nil nil #'equal) 276 backend)))) 277 278 (provide 'gptel-ollama) 279 ;;; gptel-ollama.el ends here 280 281