bug#68559: [PATCH] Improve Python shell completion

[Liu Hui <liuhui1610@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 1675 bytes --]

On Wed, Feb 21, 2024 at 10:55 PM Basil L. Contovounesios
<basil@contovou.net> wrote:
>
> Liu Hui [2024-02-21 18:00 +0800] wrote:
>
> > The attached patch should fix the problem.
>
> Thanks!  The patch fixes the error, but that's because
> python-shell-completion-at-point-ipython is now skipped:

Only the native completion part is skipped for the reason below.

> in particular, python-shell-readline-completer-delims evaluates to
> "\s\t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?" rather than the empty string.

"\s\t\n`..." is the delimiter used by rlcompleter, which is the
default completer used by the readline. rlcompleter cannot complete
module names or parameters, so in this case the native completion part
is skipped.

The test is intended to be used with Jedi as the completion backend,
e.g. setting PYTHONSTARTUP="$(python -m jedi repl)", or with a custom
IPython completer defined in the PYTHONSTARTUP file. I have updated the
patch to make the test use Jedi when possible.

> > -           (progn
> > -             (run-python nil t)
> > -             (insert ,contents)
> > -             (goto-char (point-min))
> > -             (python-tests-shell-wait-for-prompt)
> > -             ,@body)
> > +           ;; Prevent test failures when Jedi is used as a completion
> > +           ;; backend, either directly or indirectly (e.g., via
> > +           ;; IPython).  Jedi needs to store cache, but the
> > +           ;; "/nonexistent" HOME directory is not writable.
> > +           (ert-with-temp-directory cache-dir
>                                        ^^^^^^^^^
> Should this be an uninterned symbol instead?

Fixed.

[-- Attachment #2: 0001-Fix-Python-shell-completion-test-failures.patch --]
[-- Type: text/x-patch, Size: 5782 bytes --]

From 8cd85404f5627ffe4b41c19f1d466425eafe31b7 Mon Sep 17 00:00:00 2001
From: Liu Hui <liuhui1610@gmail.com>
Date: Wed, 21 Feb 2024 12:40:06 +0800
Subject: [PATCH] Fix Python shell completion test failures

* test/lisp/progmodes/python-tests.el
(python-tests-with-temp-buffer-with-shell): Set XDG_CACHE_HOME
to a temporary directory.
(python-tests--pythonstartup-file): New function.
(python-shell-completion-at-point-jedi-completer)
(python-shell-completion-at-point-ipython): Use Jedi as the
native completion backend when possible.  (bug#68559)
---
 test/lisp/progmodes/python-tests.el | 87 ++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 34 deletions(-)

diff --git a/test/lisp/progmodes/python-tests.el b/test/lisp/progmodes/python-tests.el
index af6c199b5bd..c5b1ab93144 100644
--- a/test/lisp/progmodes/python-tests.el
+++ b/test/lisp/progmodes/python-tests.el
@@ -55,21 +55,27 @@ python-tests-with-temp-buffer-with-shell
 always located at the beginning of buffer.  Native completion is
 turned off.  Shell buffer will be killed on exit."
   (declare (indent 1) (debug t))
-  `(with-temp-buffer
-     (let ((python-indent-guess-indent-offset nil)
-           (python-shell-completion-native-enable nil))
-       (python-mode)
-       (unwind-protect
-           (progn
-             (run-python nil t)
-             (insert ,contents)
-             (goto-char (point-min))
-             (python-tests-shell-wait-for-prompt)
-             ,@body)
-         (when (python-shell-get-buffer)
-           (python-shell-with-shell-buffer
-             (let (kill-buffer-hook kill-buffer-query-functions)
-               (kill-buffer))))))))
+  (let ((dir (make-symbol "dir")))
+    `(with-temp-buffer
+       (let ((python-indent-guess-indent-offset nil)
+             (python-shell-completion-native-enable nil))
+         (python-mode)
+         (unwind-protect
+             ;; Prevent test failures when Jedi is used as a completion
+             ;; backend, either directly or indirectly (e.g., via
+             ;; IPython).  Jedi needs to store cache, but the
+             ;; "/nonexistent" HOME directory is not writable.
+             (ert-with-temp-directory ,dir
+               (with-environment-variables (("XDG_CACHE_HOME" ,dir))
+                 (run-python nil t)
+                 (insert ,contents)
+                 (goto-char (point-min))
+                 (python-tests-shell-wait-for-prompt)
+                 ,@body))
+           (when (python-shell-get-buffer)
+             (python-shell-with-shell-buffer
+               (let (kill-buffer-hook kill-buffer-query-functions)
+                 (kill-buffer)))))))))
 
 (defmacro python-tests-with-temp-file (contents &rest body)
   "Create a `python-mode' enabled file with CONTENTS.
@@ -4860,17 +4866,28 @@ python-tests--completion-extra-context
   (should (string= "IGNORECASE"
                    (buffer-substring (line-beginning-position) (point)))))
 
+(defun python-tests--pythonstartup-file ()
+  "Return Jedi readline setup file if PYTHONSTARTUP is not set."
+  (or (getenv "PYTHONSTARTUP")
+      (with-temp-buffer
+        (if (eql 0 (call-process python-tests-shell-interpreter
+                                 nil t nil "-m" "jedi" "repl"))
+            (string-trim (buffer-string))
+          ""))))
+
 (ert-deftest python-shell-completion-at-point-jedi-completer ()
   "Check if Python shell completion works when Jedi completer is used."
   (skip-unless (executable-find python-tests-shell-interpreter))
-  (python-tests-with-temp-buffer-with-shell
-   ""
-   (python-shell-with-shell-buffer
-     (python-shell-completion-native-turn-on)
-     (skip-unless (string= python-shell-readline-completer-delims ""))
-     (python-tests--completion-module)
-     (python-tests--completion-parameters)
-     (python-tests--completion-extra-context))))
+  (with-environment-variables
+      (("PYTHONSTARTUP" (python-tests--pythonstartup-file)))
+    (python-tests-with-temp-buffer-with-shell
+     ""
+     (python-shell-with-shell-buffer
+      (python-shell-completion-native-turn-on)
+      (skip-unless (string= python-shell-readline-completer-delims ""))
+      (python-tests--completion-module)
+      (python-tests--completion-parameters)
+      (python-tests--completion-extra-context)))))
 
 (ert-deftest python-shell-completion-at-point-ipython ()
   "Check if Python shell completion works for IPython."
@@ -4880,17 +4897,19 @@ python-shell-completion-at-point-ipython
      (and
       (executable-find python-shell-interpreter)
       (eql (call-process python-shell-interpreter nil nil nil "--version") 0)))
-    (python-tests-with-temp-buffer-with-shell
-     ""
-     (python-shell-with-shell-buffer
-       (python-shell-completion-native-turn-off)
-       (python-tests--completion-module)
-       (python-tests--completion-parameters)
-       (python-shell-completion-native-turn-on)
-       (skip-unless (string= python-shell-readline-completer-delims ""))
-       (python-tests--completion-module)
-       (python-tests--completion-parameters)
-       (python-tests--completion-extra-context)))))
+    (with-environment-variables
+        (("PYTHONSTARTUP" (python-tests--pythonstartup-file)))
+      (python-tests-with-temp-buffer-with-shell
+       ""
+       (python-shell-with-shell-buffer
+         (python-shell-completion-native-turn-off)
+         (python-tests--completion-module)
+         (python-tests--completion-parameters)
+         (python-shell-completion-native-turn-on)
+         (skip-unless (string= python-shell-readline-completer-delims ""))
+         (python-tests--completion-module)
+         (python-tests--completion-parameters)
+         (python-tests--completion-extra-context))))))
 
 \f
 ;;; PDB Track integration
-- 
2.25.1