ob-python.el (23291B)
1 ;;; ob-python.el --- Babel Functions for Python -*- lexical-binding: t; -*- 2 3 ;; Copyright (C) 2009-2024 Free Software Foundation, Inc. 4 5 ;; Authors: Eric Schulte 6 ;; Dan Davison 7 ;; Maintainer: Jack Kamm <jackkamm@gmail.com> 8 ;; Keywords: literate programming, reproducible research 9 ;; URL: https://orgmode.org 10 11 ;; This file is part of GNU Emacs. 12 13 ;; GNU Emacs is free software: you can redistribute it and/or modify 14 ;; it under the terms of the GNU General Public License as published by 15 ;; the Free Software Foundation, either version 3 of the License, or 16 ;; (at your option) any later version. 17 18 ;; GNU Emacs is distributed in the hope that it will be useful, 19 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 20 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 ;; GNU General Public License for more details. 22 23 ;; You should have received a copy of the GNU General Public License 24 ;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>. 25 26 ;;; Commentary: 27 28 ;; Org-Babel support for evaluating python source code. 29 30 ;;; Code: 31 32 (require 'org-macs) 33 (org-assert-version) 34 35 (require 'ob) 36 (require 'org-macs) 37 (require 'python) 38 39 (defvar org-babel-tangle-lang-exts) 40 (add-to-list 'org-babel-tangle-lang-exts '("python" . "py")) 41 42 (defvar org-babel-default-header-args:python '()) 43 44 (defconst org-babel-header-args:python 45 '((return . :any) 46 (python . :any) 47 (async . ((yes no)))) 48 "Python-specific header arguments.") 49 50 (defcustom org-babel-python-command 'auto 51 "Command (including arguments) for interactive and non-interactive Python code. 52 When not `auto', it overrides `org-babel-python-command-session' 53 and `org-babel-python-command-nonsession'." 54 :package-version '(Org . "9.7") 55 :group 'org-babel 56 :type '(choice string (const auto))) 57 58 (defcustom org-babel-python-command-session 'auto 59 "Command (including arguments) for starting interactive Python sessions. 60 If `auto' (the default), uses the values from 61 `python-shell-interpreter' and `python-shell-interpreter-args'. 62 If `org-babel-python-command' is set, then it overrides this 63 option." 64 :package-version '(Org . "9.7") 65 :group 'org-babel 66 :type '(choice string (const auto))) 67 68 (defcustom org-babel-python-command-nonsession "python" 69 "Command (including arguments) for executing non-interactive Python code. 70 If `org-babel-python-command' is set, then it overrides this option." 71 :package-version '(Org . "9.7") 72 :group 'org-babel 73 :type 'string) 74 75 (defcustom org-babel-python-hline-to "None" 76 "Replace hlines in incoming tables with this when translating to python." 77 :group 'org-babel 78 :package-version '(Org . "8.0") 79 :type 'string) 80 81 (defcustom org-babel-python-None-to 'hline 82 "Replace `None' in python tables with this before returning." 83 :group 'org-babel 84 :package-version '(Org . "8.0") 85 :type 'symbol) 86 87 (defun org-babel-python-associate-session (session) 88 "Associate Python code buffer with an Python session. 89 Make SESSION without earmuffs be the Python buffer name." 90 (setq-local python-shell-buffer-name 91 (org-babel-python-without-earmuffs session))) 92 93 (defun org-babel-execute:python (body params) 94 "Execute Python BODY according to PARAMS. 95 This function is called by `org-babel-execute-src-block'." 96 (let* ((org-babel-python-command 97 (or (cdr (assq :python params)) 98 org-babel-python-command)) 99 (session (org-babel-python-initiate-session 100 (cdr (assq :session params)))) 101 (graphics-file (and (member "graphics" (assq :result-params params)) 102 (org-babel-graphical-output-file params))) 103 (result-params (cdr (assq :result-params params))) 104 (result-type (cdr (assq :result-type params))) 105 (return-val (when (eq result-type 'value) 106 (cdr (assq :return params)))) 107 (preamble (cdr (assq :preamble params))) 108 (async (org-babel-comint-use-async params)) 109 (full-body 110 (concat 111 (org-babel-expand-body:generic 112 body params 113 (org-babel-variable-assignments:python params)) 114 (when return-val 115 (format (if session "\n%s" "\nreturn %s") return-val)))) 116 (result (org-babel-python-evaluate 117 session full-body result-type 118 result-params preamble async graphics-file))) 119 (org-babel-reassemble-table 120 result 121 (org-babel-pick-name (cdr (assq :colname-names params)) 122 (cdr (assq :colnames params))) 123 (org-babel-pick-name (cdr (assq :rowname-names params)) 124 (cdr (assq :rownames params)))))) 125 126 (defun org-babel-prep-session:python (session params) 127 "Prepare SESSION according to the header arguments in PARAMS. 128 VARS contains resolved variable references." 129 (let* ((session (org-babel-python-initiate-session session)) 130 (var-lines 131 (org-babel-variable-assignments:python params))) 132 (org-babel-comint-in-buffer session 133 (mapc (lambda (var) 134 (end-of-line 1) (insert var) (comint-send-input) 135 (org-babel-comint-wait-for-output session)) 136 var-lines)) 137 session)) 138 139 (defun org-babel-load-session:python (session body params) 140 "Load BODY into SESSION." 141 (save-window-excursion 142 (let ((buffer (org-babel-prep-session:python session params))) 143 (with-current-buffer buffer 144 (goto-char (process-mark (get-buffer-process (current-buffer)))) 145 (insert (org-babel-chomp body))) 146 buffer))) 147 148 ;; helper functions 149 150 (defconst org-babel-python--output-graphics-wrapper "\ 151 import matplotlib.pyplot 152 matplotlib.pyplot.gcf().clear() 153 %s 154 matplotlib.pyplot.savefig('%s')" 155 "Format string for saving Python graphical output. 156 Has two %s escapes, for the Python code to be evaluated, and the 157 file to save the graphics to.") 158 159 (defconst org-babel-python--def-format-value "\ 160 def __org_babel_python_format_value(result, result_file, result_params): 161 with open(result_file, 'w') as f: 162 if 'graphics' in result_params: 163 result.savefig(result_file) 164 elif 'pp' in result_params: 165 import pprint 166 f.write(pprint.pformat(result)) 167 elif 'list' in result_params and isinstance(result, dict): 168 f.write(str(['{} :: {}'.format(k, v) for k, v in result.items()])) 169 else: 170 if not set(result_params).intersection(\ 171 ['scalar', 'verbatim', 'raw']): 172 def dict2table(res): 173 if isinstance(res, dict): 174 return [(k, dict2table(v)) for k, v in res.items()] 175 elif isinstance(res, list) or isinstance(res, tuple): 176 return [dict2table(x) for x in res] 177 else: 178 return res 179 if 'table' in result_params: 180 result = dict2table(result) 181 try: 182 import pandas 183 except ImportError: 184 pass 185 else: 186 if isinstance(result, pandas.DataFrame) and 'table' in result_params: 187 result = [[result.index.name or ''] + list(result.columns)] + \ 188 [None] + [[i] + list(row) for i, row in result.iterrows()] 189 elif isinstance(result, pandas.Series) and 'table' in result_params: 190 result = list(result.items()) 191 try: 192 import numpy 193 except ImportError: 194 pass 195 else: 196 if isinstance(result, numpy.ndarray): 197 if 'table' in result_params: 198 result = result.tolist() 199 else: 200 result = repr(result) 201 f.write(str(result))" 202 "Python function to format value result and save it to file.") 203 204 (defun org-babel-variable-assignments:python (params) 205 "Return a list of Python statements assigning the block's variables. 206 The assignments are defined in PARAMS." 207 (mapcar 208 (lambda (pair) 209 (format "%s=%s" 210 (car pair) 211 (org-babel-python-var-to-python (cdr pair)))) 212 (org-babel--get-vars params))) 213 214 (defun org-babel-python-var-to-python (var) 215 "Convert an elisp value to a python variable. 216 Convert an elisp value, VAR, into a string of python source code 217 specifying a variable of the same value." 218 (if (listp var) 219 (concat "[" (mapconcat #'org-babel-python-var-to-python var ", ") "]") 220 (if (eq var 'hline) 221 org-babel-python-hline-to 222 (format 223 (if (and (stringp var) (string-match "[\n\r]" var)) "\"\"%S\"\"" "%S") 224 (if (stringp var) (substring-no-properties var) var))))) 225 226 (defun org-babel-python-table-or-string (results) 227 "Convert RESULTS into an appropriate elisp value. 228 If the results look like a list or tuple (but not a dict), then 229 convert them into an Emacs-lisp table. Otherwise return the 230 results as a string." 231 (let ((res (if (and (> (length results) 0) 232 (string-equal "{" (substring results 0 1))) 233 results ;don't convert dicts to elisp 234 (org-babel-script-escape results)))) 235 (if (listp res) 236 (mapcar (lambda (el) (if (eq el 'None) 237 org-babel-python-None-to el)) 238 res) 239 res))) 240 241 (defvar org-babel-python-buffers '((:default . "*Python*"))) 242 243 (defun org-babel-python-session-buffer (session) 244 "Return the buffer associated with SESSION." 245 (cdr (assoc session org-babel-python-buffers))) 246 247 (defun org-babel-python-with-earmuffs (session) 248 "Return SESSION name as string, ensuring *...* around." 249 (let ((name (if (stringp session) session (format "%s" session)))) 250 (if (and (string= "*" (substring name 0 1)) 251 (string= "*" (substring name (- (length name) 1)))) 252 name 253 (format "*%s*" name)))) 254 255 (defun org-babel-python-without-earmuffs (session) 256 "Return SESSION name as string, without *...* around." 257 (let ((name (if (stringp session) session (format "%s" session)))) 258 (if (and (string= "*" (substring name 0 1)) 259 (string= "*" (substring name (- (length name) 1)))) 260 (substring name 1 (- (length name) 1)) 261 name))) 262 263 (defun org-babel-session-buffer:python (session &optional _) 264 "Return session buffer name for SESSION." 265 (or (org-babel-python-session-buffer session) 266 (org-babel-python-with-earmuffs session))) 267 268 (defun org-babel-python--python-util-comint-end-of-output-p () 269 "Return non-nil if the last prompt matches input prompt. 270 Backport of `python-util-comint-end-of-output-p' to emacs28. To 271 be removed after minimum supported version reaches emacs29." 272 (when-let* ((prompt (python-util-comint-last-prompt))) 273 (python-shell-comint-end-of-output-p 274 (buffer-substring-no-properties 275 (car prompt) (cdr prompt))))) 276 277 (defun org-babel-python--command (is-session) 278 "Helper function to return the Python command. 279 This checks `org-babel-python-command', and then 280 `org-babel-python-command-session' (if IS-SESSION) or 281 `org-babel-python-command-nonsession' (if not IS-SESSION). If 282 IS-SESSION, this might return nil, which means to use 283 `python-shell-calculate-command'." 284 (or (unless (eq org-babel-python-command 'auto) 285 org-babel-python-command) 286 (if is-session 287 (unless (eq org-babel-python-command-session 'auto) 288 org-babel-python-command-session) 289 org-babel-python-command-nonsession))) 290 291 (defvar-local org-babel-python--initialized nil 292 "Flag used to mark that python session has been initialized.") 293 (defun org-babel-python--setup-session () 294 "Babel Python session setup code, to be run once per session. 295 Function should be run from within the Python session buffer. 296 This is often run as a part of `python-shell-first-prompt-hook', 297 unless the Python session was created outside Org." 298 (python-shell-send-string-no-output org-babel-python--def-format-value) 299 (setq-local org-babel-python--initialized t)) 300 (defun org-babel-python-initiate-session-by-key (&optional session) 301 "Initiate a python session. 302 If there is not a current inferior-process-buffer matching 303 SESSION then create it. If inferior process already 304 exists (e.g. if it was manually started with `run-python'), make 305 sure it's configured to work with ob-python. If session has 306 already been configured as such, do nothing. Return the 307 initialized session." 308 (save-window-excursion 309 (let* ((session (if session (intern session) :default)) 310 (py-buffer (org-babel-session-buffer:python session)) 311 (python-shell-buffer-name 312 (org-babel-python-without-earmuffs py-buffer)) 313 (existing-session-p (comint-check-proc py-buffer)) 314 (cmd (org-babel-python--command t))) 315 (if cmd 316 (let* ((cmd-split (split-string-and-unquote cmd)) 317 (python-shell-interpreter (car cmd-split)) 318 (python-shell-interpreter-args 319 (combine-and-quote-strings 320 (append (cdr cmd-split) 321 (when (member system-type 322 '(cygwin windows-nt ms-dos)) 323 (list "-i")))))) 324 (run-python)) 325 (run-python)) 326 (with-current-buffer py-buffer 327 (if existing-session-p 328 ;; Session was created outside Org. Assume first prompt 329 ;; already happened; run session setup code directly 330 (unless org-babel-python--initialized 331 ;; Ensure first prompt. Based on python-tests.el 332 ;; (`python-tests-shell-wait-for-prompt') 333 (while (not (org-babel-python--python-util-comint-end-of-output-p)) 334 (sit-for 0.1)) 335 (org-babel-python--setup-session)) 336 ;; Adding to `python-shell-first-prompt-hook' immediately 337 ;; after `run-python' should be safe from race conditions, 338 ;; because subprocess output only arrives when Emacs is 339 ;; waiting (see elisp manual, "Output from Processes") 340 (add-hook 341 'python-shell-first-prompt-hook 342 #'org-babel-python--setup-session 343 nil 'local))) 344 ;; Wait until Python initializes 345 ;; This is more reliable compared to 346 ;; `org-babel-comint-wait-for-output' as python may emit 347 ;; multiple prompts during initialization. 348 (with-current-buffer py-buffer 349 (while (not org-babel-python--initialized) 350 (sleep-for 0.010))) 351 (setq org-babel-python-buffers 352 (cons (cons session py-buffer) 353 (assq-delete-all session org-babel-python-buffers))) 354 session))) 355 356 (defun org-babel-python-initiate-session (&optional session _params) 357 "Initiate Python session named SESSION according to PARAMS. 358 If there is not a current inferior-process-buffer matching 359 SESSION then create it. If inferior process already 360 exists (e.g. if it was manually started with `run-python'), make 361 sure it's configured to work with ob-python. If session has 362 already been configured as such, do nothing." 363 (unless (string= session "none") 364 (org-babel-python-session-buffer 365 (org-babel-python-initiate-session-by-key session)))) 366 367 (defvar org-babel-python-eoe-indicator "org_babel_python_eoe" 368 "A string to indicate that evaluation has completed.") 369 370 (defun org-babel-python-format-session-value 371 (src-file result-file result-params) 372 "Return Python code to evaluate SRC-FILE and write result to RESULT-FILE. 373 RESULT-PARAMS defines the result type." 374 (format "\ 375 import ast 376 with open('%s') as __org_babel_python_tmpfile: 377 __org_babel_python_ast = ast.parse(__org_babel_python_tmpfile.read()) 378 __org_babel_python_final = __org_babel_python_ast.body[-1] 379 if isinstance(__org_babel_python_final, ast.Expr): 380 __org_babel_python_ast.body = __org_babel_python_ast.body[:-1] 381 exec(compile(__org_babel_python_ast, '<string>', 'exec')) 382 __org_babel_python_final = eval(compile(ast.Expression( 383 __org_babel_python_final.value), '<string>', 'eval')) 384 else: 385 exec(compile(__org_babel_python_ast, '<string>', 'exec')) 386 __org_babel_python_final = None 387 __org_babel_python_format_value(__org_babel_python_final, '%s', %s)" 388 (org-babel-process-file-name src-file 'noquote) 389 (org-babel-process-file-name result-file 'noquote) 390 (org-babel-python-var-to-python result-params))) 391 392 (defun org-babel-python-evaluate 393 (session body &optional result-type result-params preamble async graphics-file) 394 "Evaluate BODY as Python code." 395 (if session 396 (if async 397 (org-babel-python-async-evaluate-session 398 session body result-type result-params graphics-file) 399 (org-babel-python-evaluate-session 400 session body result-type result-params graphics-file)) 401 (org-babel-python-evaluate-external-process 402 body result-type result-params preamble graphics-file))) 403 404 (defun org-babel-python--shift-right (body &optional count) 405 (with-temp-buffer 406 (python-mode) 407 (insert body) 408 (goto-char (point-min)) 409 (while (not (eobp)) 410 (unless (python-syntax-context 'string) 411 (python-indent-shift-right (line-beginning-position) 412 (line-end-position) 413 count)) 414 (forward-line 1)) 415 (buffer-string))) 416 417 (defun org-babel-python-evaluate-external-process 418 (body &optional result-type result-params preamble graphics-file) 419 "Evaluate BODY in external python process. 420 If RESULT-TYPE equals `output' then return standard output as a 421 string. If RESULT-TYPE equals `value' then return the value of 422 the last statement in BODY, as elisp. If GRAPHICS-FILE is 423 non-nil, then save graphical results to that file instead." 424 (let ((raw 425 (pcase result-type 426 (`output (org-babel-eval (org-babel-python--command nil) 427 (concat preamble (and preamble "\n") 428 (if graphics-file 429 (format org-babel-python--output-graphics-wrapper 430 body graphics-file) 431 body)))) 432 (`value (let ((results-file (or graphics-file 433 (org-babel-temp-file "python-")))) 434 (org-babel-eval (org-babel-python--command nil) 435 (concat 436 preamble (and preamble "\n") 437 (format 438 (concat org-babel-python--def-format-value " 439 def main(): 440 %s 441 442 __org_babel_python_format_value(main(), '%s', %s)") 443 (org-babel-python--shift-right body) 444 (org-babel-process-file-name results-file 'noquote) 445 (org-babel-python-var-to-python result-params)))) 446 (org-babel-eval-read-file results-file)))))) 447 (org-babel-result-cond result-params 448 raw 449 (org-babel-python-table-or-string raw)))) 450 451 (defun org-babel-python-send-string (session body) 452 "Pass BODY to the Python process in SESSION. 453 Return output." 454 (with-current-buffer session 455 (let* ((string-buffer "") 456 (comint-output-filter-functions 457 (cons (lambda (text) (setq string-buffer 458 (concat string-buffer text))) 459 comint-output-filter-functions)) 460 (body (format "\ 461 try: 462 %s 463 except: 464 raise 465 finally: 466 print('%s')" 467 (org-babel-python--shift-right body 4) 468 org-babel-python-eoe-indicator))) 469 (let ((python-shell-buffer-name 470 (org-babel-python-without-earmuffs session))) 471 (python-shell-send-string body)) 472 ;; same as `python-shell-comint-end-of-output-p' in emacs-25.1+ 473 (while (not (and (python-shell-comint-end-of-output-p string-buffer) 474 (string-match 475 org-babel-python-eoe-indicator 476 string-buffer))) 477 (accept-process-output (get-buffer-process (current-buffer)))) 478 (org-babel-chomp (substring string-buffer 0 (match-beginning 0)))))) 479 480 (defun org-babel-python-evaluate-session 481 (session body &optional result-type result-params graphics-file) 482 "Pass BODY to the Python process in SESSION. 483 If RESULT-TYPE equals `output' then return standard output as a 484 string. If RESULT-TYPE equals `value' then return the value of 485 the last statement in BODY, as elisp. If GRAPHICS-FILE is 486 non-nil, then save graphical results to that file instead." 487 (let* ((tmp-src-file (org-babel-temp-file "python-")) 488 (results 489 (progn 490 (with-temp-file tmp-src-file 491 (insert (if (and graphics-file (eq result-type 'output)) 492 (format org-babel-python--output-graphics-wrapper 493 body graphics-file) 494 body))) 495 (pcase result-type 496 (`output 497 (let ((body (format "\ 498 with open('%s') as f: 499 exec(compile(f.read(), f.name, 'exec'))" 500 (org-babel-process-file-name 501 tmp-src-file 'noquote)))) 502 (org-babel-python-send-string session body))) 503 (`value 504 (let* ((results-file (or graphics-file 505 (org-babel-temp-file "python-"))) 506 (body (org-babel-python-format-session-value 507 tmp-src-file results-file result-params))) 508 (org-babel-python-send-string session body) 509 (sleep-for 0.010) 510 (org-babel-eval-read-file results-file))))))) 511 (org-babel-result-cond result-params 512 results 513 (org-babel-python-table-or-string results)))) 514 515 (defun org-babel-python-read-string (string) 516 "Strip \\='s from around Python STRING." 517 (if (and (string-prefix-p "'" string) 518 (string-suffix-p "'" string)) 519 (substring string 1 -1) 520 string)) 521 522 ;; Async session eval 523 524 (defconst org-babel-python-async-indicator "print ('ob_comint_async_python_%s_%s')") 525 526 (defun org-babel-python-async-value-callback (params tmp-file) 527 (let ((result-params (cdr (assq :result-params params))) 528 (results (org-babel-eval-read-file tmp-file))) 529 (org-babel-result-cond result-params 530 results 531 (org-babel-python-table-or-string results)))) 532 533 (defun org-babel-python-async-evaluate-session 534 (session body &optional result-type result-params graphics-file) 535 "Asynchronously evaluate BODY in SESSION. 536 Returns a placeholder string for insertion, to later be replaced 537 by `org-babel-comint-async-filter'." 538 (org-babel-comint-async-register 539 session (current-buffer) 540 "ob_comint_async_python_\\(start\\|end\\|file\\)_\\(.+\\)" 541 'org-babel-chomp 'org-babel-python-async-value-callback 542 'disable-prompt-filtering) 543 (pcase result-type 544 (`output 545 (let ((uuid (org-id-uuid))) 546 (with-temp-buffer 547 (insert (format org-babel-python-async-indicator "start" uuid)) 548 (insert "\n") 549 (insert (if graphics-file 550 (format org-babel-python--output-graphics-wrapper 551 body graphics-file) 552 body)) 553 (insert "\n") 554 (insert (format org-babel-python-async-indicator "end" uuid)) 555 (let ((python-shell-buffer-name 556 (org-babel-python-without-earmuffs session))) 557 (python-shell-send-buffer))) 558 uuid)) 559 (`value 560 (let ((results-file (or graphics-file 561 (org-babel-temp-file "python-"))) 562 (tmp-src-file (org-babel-temp-file "python-"))) 563 (with-temp-file tmp-src-file (insert body)) 564 (with-temp-buffer 565 (insert (org-babel-python-format-session-value 566 tmp-src-file results-file result-params)) 567 (insert "\n") 568 (unless graphics-file 569 (insert (format org-babel-python-async-indicator "file" results-file))) 570 (let ((python-shell-buffer-name 571 (org-babel-python-without-earmuffs session))) 572 (python-shell-send-buffer))) 573 results-file)))) 574 575 (provide 'ob-python) 576 577 ;;; ob-python.el ends here