bug#28808: [PATCH] Implement Python backend for Flymake

bug#28808: [PATCH] Implement Python backend for Flymake

[Lele Gaifax]

[-- Attachment #1: Type: text/plain, Size: 348 bytes --]

Hi,

here below you can find an implementation of a Python backend for the new
Flymake facility.

I'm quite satisfied by it: I tested both the default settings (targeting
`pyflakes') and the `flake8' customization suggested in the docstrings.

As always, I'm willing to apply whatever tweak/fix you may find reasonable.

Thanks a lot,
ciao, lele.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Fix-typo.patch --]
[-- Type: text/x-diff, Size: 782 bytes --]

From 16828afebe3e732e3cbf856b093dfff65d3319ff Mon Sep 17 00:00:00 2001
From: Lele Gaifax <lele@metapensiero.it>
Date: Fri, 13 Oct 2017 10:43:13 +0200
Subject: [PATCH 1/2] Fix typo

---
 lisp/progmodes/flymake.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lisp/progmodes/flymake.el b/lisp/progmodes/flymake.el
index 8c9c4b211a..8fa763a4b8 100644
--- a/lisp/progmodes/flymake.el
+++ b/lisp/progmodes/flymake.el
@@ -124,7 +124,7 @@ flymake-gui-warnings-enabled
 			"it no longer has any effect." "26.1")
 
 (defcustom flymake-start-on-flymake-mode t
-  "Start syntax check when `flymake-mode'is enabled.
+  "Start syntax check when `flymake-mode' is enabled.
 Specifically, start it when the buffer is actually displayed."
   :type 'boolean)
 
-- 
2.15.0.rc0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #3: 0002-Add-a-Flymake-backend-for-Python.patch --]
[-- Type: text/x-diff, Size: 5394 bytes --]

From 84a4dd4ef7a6ae9e49cb7442070744b5d6e3ec95 Mon Sep 17 00:00:00 2001
From: Lele Gaifax <lele@metapensiero.it>
Date: Fri, 13 Oct 2017 10:44:02 +0200
Subject: [PATCH 2/2] Add a Flymake backend for Python

* lisp/progmodes/python.el: Implement new Flymake backend with
  related customizable settings.
  (python-flymake-command, python-flymake-command-output-regexp,
   python-flymake-msg-alist): New defcustom.
  (python-flymake): New function.
  (python-flymake-activate): New function.
---
 lisp/progmodes/python.el | 101 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index f79d9a47d3..866e02ffbd 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -5141,6 +5141,107 @@ python-util-valid-regexp-p
   (ignore-errors (string-match regexp "") t))
 
 \f
+;;; Flymake integration
+
+(defgroup python-flymake nil
+  "Integration between Python and Flymake."
+  :group 'python
+  :link '(custom-group-link :tag "Flymake" flymake)
+  :version "26.1")
+
+(defcustom python-flymake-command '("pyflakes")
+  "The external tool that will be used to perform the syntax check.
+This is a non empty list of strings, the checker tool possibly followed by
+required arguments: to use `flake8' you would set this to (\"flake8\" \"-\")."
+  :group 'python-flymake
+  :type '(repeat string))
+
+;; The default regexp accomodates for older pyflakes, which did not
+;; report the column number
+(defcustom python-flymake-command-output-regexp
+  "^\\(?:<stdin>\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
+  "The regexp used to parse the output of the specified tool.
+It must contain two or three groups: group 1 is the line number, group 2 the
+optional column number and the third is the actual message."
+  :group 'python-flymake
+  :type 'regexp)
+
+(defcustom python-flymake-msg-alist
+  '(("\\(^redefinition\\|.*unused.*\\|used$\\)" . :warning))
+  "Alist used to associate messages to their types.
+Each element should be a cons-cell (REGEXP . TYPE), where TYPE must be
+one defined in the variable `flymake-diagnostic-types-alist'.
+For example, when using `flake8' a possible configuration could be:
+
+  ((\"\\(^redefinition\\|.*unused.*\\|used$\\)\" . :warning)
+   (\"^E999\" . :error)
+   (\"^[EW][0-9]+\" . :note))
+
+By default messages are considered errors."
+  :group 'python-flymake
+  :type `(alist :key-type (regexp)
+                :value-type (symbol)))
+
+(defvar-local python--flymake-proc nil)
+
+(defun python-flymake (report-fn &rest _args)
+  "Flymake backend for Python.
+This backend uses `python-flymake-command' (which see) to launch a process
+that is passed the current buffer's content via stdin.
+REPORT-FN is Flymake's callback function."
+  (unless (executable-find (car python-flymake-command))
+    (error "Cannot find a suitable checker"))
+
+  (unless (derived-mode-p 'python-mode)
+    (error "Can only work on `python-mode' buffers"))
+
+  (when (process-live-p python--flymake-proc)
+    (kill-process python--flymake-proc))
+
+  (let ((source (current-buffer)))
+    (save-restriction
+      (widen)
+      (setq python--flymake-proc
+            (make-process
+             :name "python-flymake"
+             :noquery t
+             :connection-type 'pipe
+             :buffer (generate-new-buffer " *python-flymake*")
+             :command python-flymake-command
+             :sentinel
+             (lambda (proc _event)
+               (when (eq 'exit (process-status proc))
+                 (unwind-protect
+                     (when (eq proc python--flymake-proc)
+                       (with-current-buffer (process-buffer proc)
+                         (goto-char (point-min))
+                         (cl-loop
+                          while (search-forward-regexp
+                                 python-flymake-command-output-regexp nil t)
+                          for msg = (match-string 3)
+                          for (beg . end) = (flymake-diag-region
+                                             source
+                                             (string-to-number (match-string 1))
+                                             (and (match-string 2)
+                                                  (string-to-number
+                                                   (match-string 2))))
+                          for type = (or (assoc-default msg
+                                                        python-flymake-msg-alist
+                                                        #'string-match)
+                                         :error)
+                          collect (flymake-make-diagnostic
+                                   source beg end type msg)
+                          into diags
+                          finally (funcall report-fn diags))))
+                   (kill-buffer (process-buffer proc)))))))
+      (process-send-region python--flymake-proc (point-min) (point-max))
+      (process-send-eof python--flymake-proc))))
+
+(defun python-flymake-activate ()
+  "Activate the Flymake syntax check on all python-mode buffers."
+  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t))
+
+\f
 (defun python-electric-pair-string-delimiter ()
   (when (and electric-pair-mode
              (memq last-command-event '(?\" ?\'))
-- 
2.15.0.rc0


[-- Attachment #4: Type: text/plain, Size: 211 bytes --]


-- 
nickname: Lele Gaifax | Quando vivrò di quello che ho pensato ieri
real: Emanuele Gaifas | comincerò ad aver paura di chi mi copia.
lele@metapensiero.it  |                 -- Fortunato Depero, 1929.

bug#28821: [PATCH] Implement Python backend for Flymake

[João Távora]

Hi Lele,

Here are some comments:

Lele Gaifax <lele@metapensiero.it> writes:

> +(defgroup python-flymake nil
> +  "Integration between Python and Flymake."
> +  :group 'python
> +  :link '(custom-group-link :tag "Flymake" flymake)
> +  :version "26.1")
> +
> +(defcustom python-flymake-command '("pyflakes")
> +  "The external tool that will be used to perform the syntax check.
> +This is a non empty list of strings, the checker tool possibly followed by
> +required arguments: to use `flake8' you would set this to (\"flake8\" \"-\")."
>
I wonder if you shouldn't mention here that the command produced should,
once invoked, check (a file? a chunk?) of python source code passed to
it via its standard input.

> +  :group 'python-flymake
> +  :type '(repeat string))
> +
> +;; The default regexp accomodates for older pyflakes, which did not
> +;; report the column number
> +(defcustom python-flymake-command-output-regexp
> +  "^\\(?:<stdin>\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
> +  "The regexp used to parse the output of the specified tool.
> +It must contain two or three groups: group 1 is the line number, group 2 the
> +optional column number and the third is the actual message."

A common trick here that old flymake (and also compile.el) use is to
define the variable's value as list (REGEXP LINE COLUMN TYPE
MESSAGE). REGEXP is mandatory. LINE, COLUMN, TYPE and MESSAGE are
non-negative integer numbers designating regexp groups, or nil. In the
latter case it means the regexp cannot capture that entity.

So in your case it would become

(defcustom python-flymake-command-output-regexp
  (list
    "^\\(?:<stdin>\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
    1 2 nil 3)
  "docstring" :group 'python-flymake
  :type '(list string (choice integer symbol)
                      (choice integer symbol)
                      (choice integer symbol)
                      (choice integer symbol)))

Perhaps TYPE does not make much sense currently. But it would match
slightly better with compilation-error-regexp-alist in the future (which
you should see).

> +  (unless (derived-mode-p 'python-mode)
> +    (error "Can only work on `python-mode' buffers"))

Stefan and I arrived at the conclusion that this is cruft and isn't
needed. 

> +(defun python-flymake-activate ()

Rename this to python--flymake-setup, because "activation" is actually
enabling flymake-mode. Also, I think you should add an autoload cookie
to python--flymake-setup and then call that function from the end of
python-mode. The

> +  "Activate the Flymake syntax check on all python-mode buffers."
> +  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t))

I'd use 'python-flymake instead of #'python-flymake in add-hook, but I
can't offer a sound reason why :-)

bug#28821: Acknowledgement ([PATCH] Implement Python backend for Flymake)

[João Távora]

Sorry,

This is not at all what I meant to do, I've created a new bug but I
merely meant to reply to Lele's bug report 28808. I'll try to archive or
close 28821, since it is useless.

João

bug#28808: [PATCH] Implement Python backend for Flymake

[João Távora]

[ Sorry Lele if you're getting this twice, I mistankenly created
another bug #28821, let's continue here ]

Hi Lele,

Here are some comments:

Lele Gaifax <lele@metapensiero.it> writes:

> +(defgroup python-flymake nil
> +  "Integration between Python and Flymake."
> +  :group 'python
> +  :link '(custom-group-link :tag "Flymake" flymake)
> +  :version "26.1")
> +
> +(defcustom python-flymake-command '("pyflakes")
> +  "The external tool that will be used to perform the syntax check.
> +This is a non empty list of strings, the checker tool possibly followed by
> +required arguments: to use `flake8' you would set this to (\"flake8\" \"-\")."
>
I wonder if you shouldn't mention here that the command produced should,
once invoked, check (a file? a chunk?) of python source code passed to
it via its standard input.

> +  :group 'python-flymake
> +  :type '(repeat string))
> +
> +;; The default regexp accomodates for older pyflakes, which did not
> +;; report the column number
> +(defcustom python-flymake-command-output-regexp
> +  "^\\(?:<stdin>\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
> +  "The regexp used to parse the output of the specified tool.
> +It must contain two or three groups: group 1 is the line number, group 2 the
> +optional column number and the third is the actual message."

A common trick here that old flymake (and also compile.el) use is to
define the variable's value as list (REGEXP LINE COLUMN TYPE
MESSAGE). REGEXP is mandatory. LINE, COLUMN, TYPE and MESSAGE are
non-negative integer numbers designating regexp groups, or nil. In the
latter case it means the regexp cannot capture that entity.

So in your case it would become

(defcustom python-flymake-command-output-regexp
  (list
    "^\\(?:<stdin>\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
    1 2 nil 3)
  "docstring" :group 'python-flymake
  :type '(list string (choice integer symbol)
                      (choice integer symbol)
                      (choice integer symbol)
                      (choice integer symbol)))

Perhaps TYPE does not make much sense currently. But it would match
slightly better with compilation-error-regexp-alist in the future (which
you should see).

> +  (unless (derived-mode-p 'python-mode)
> +    (error "Can only work on `python-mode' buffers"))

Stefan and I arrived at the conclusion that this is cruft and isn't
needed. 

> +(defun python-flymake-activate ()

Rename this to python--flymake-setup, because "activation" is actually
enabling flymake-mode. Also, I think you should add an autoload cookie
to python--flymake-setup and then call that function from the end of
python-mode. The

> +  "Activate the Flymake syntax check on all python-mode buffers."
> +  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t))

I'd use 'python-flymake instead of #'python-flymake in add-hook, but I
can't offer a sound reason why :-)

João

bug#28821: Closing this bug created in error

[João Távora]

This bug was created in error by cc'ing bug-gnu-emacs@gnu.org.

The relevant bug is 28808. Sorry.

bug#28808: [PATCH] Implement Python backend for Flymake

[Lele Gaifax]

Thank you João!

joaotavora@gmail.com (João Távora) writes:

>> +(defcustom python-flymake-command '("pyflakes")
>> +  "The external tool that will be used to perform the syntax check.
>> +This is a non empty list of strings, the checker tool possibly followed by
>> +required arguments: to use `flake8' you would set this to (\"flake8\" \"-\")."
>>
> I wonder if you shouldn't mention here that the command produced should,
> once invoked, check (a file? a chunk?) of python source code passed to
> it via its standard input.

Ok, done.

>> +(defcustom python-flymake-command-output-regexp
>> +  "^\\(?:<stdin>\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
>> +  "The regexp used to parse the output of the specified tool.
>> +It must contain two or three groups: group 1 is the line number, group 2 the
>> +optional column number and the third is the actual message."
>
> A common trick here that old flymake (and also compile.el) use is to
> define the variable's value as list (REGEXP LINE COLUMN TYPE
> MESSAGE).

I will try to better understand this, as I failed to see the benefit of adding
that indirection... maybe the compile.el functionality is older than the
ability to use explicit group numbers in the regexp?

>> +  (unless (derived-mode-p 'python-mode)
>> +    (error "Can only work on `python-mode' buffers"))
>
> Stefan and I arrived at the conclusion that this is cruft and isn't
> needed. 

Ok, removed.

>> +(defun python-flymake-activate ()
>
> Rename this to python--flymake-setup, because "activation" is actually
> enabling flymake-mode.

Ok, done.

> Also, I think you should add an autoload cookie
> to python--flymake-setup and then call that function from the end of
> python-mode. The

I'm curious here: why the need of autoload cookie, if the (only?) caller site
lives in the very same source file? And if that's not only caller, why marking
it private with the double dash?

More importantly: if we unconditionally activate the Flymake feature, instead
of being an user's choice, then the python--flymake-setup function may go
away, and the add-hook moved inside the python-mode function, it already
contains lot of those...

>> +  "Activate the Flymake syntax check on all python-mode buffers."
>> +  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t))
>
> I'd use 'python-flymake instead of #'python-flymake in add-hook, but I
> can't offer a sound reason why :-)

Well, my habit is different here, and it's also the convention used by the
rest of python.el, so I will leave that as is.

See http://endlessparentheses.com/get-in-the-habit-of-using-sharp-quote.html
for what I considered a "sound reason" :)

ciao, lele.
-- 
nickname: Lele Gaifax | Quando vivrò di quello che ho pensato ieri
real: Emanuele Gaifas | comincerò ad aver paura di chi mi copia.
lele@metapensiero.it  |                 -- Fortunato Depero, 1929.

bug#28808: [PATCH] Implement Python backend for Flymake

[João Távora]

Hi Lele,

Lele Gaifax <lele@metapensiero.it> writes:

>> A common trick here that old flymake (and also compile.el) use is to
>> define the variable's value as list (REGEXP LINE COLUMN TYPE
>> MESSAGE).
>
> I will try to better understand this, as I failed to see the benefit of adding
> that indirection... maybe the compile.el functionality is older than the
> ability to use explicit group numbers in the regexp?

The benefit is that if I switch to a checker that doesn't provide
COLUMN, for example, I won't be locked out with your
expression. Likewise if I do switch to a checker that emits errors where
I can clearly identify a TYPE.

> I'm curious here: why the need of autoload cookie, if the (only?) caller site
> lives in the very same source file? And if that's not only caller, why marking
> it private with the double dash?

Sorry for the noise. There is no need for the autoload cookie. I thought
you were changing a different file than flymake.

Don't use the cookie, use the double dash.
>
> More importantly: if we unconditionally activate the Flymake feature, instead
> of being an user's choice, then the python--flymake-setup function may go
> away, and the add-hook moved inside the python-mode function, it already
> contains lot of those...

That makes sense. But we don't (yet) "unconditionally activate" Flymake, we
simply set up the buffer so that a later activation of Flymake will be
met with agreeable conditions.

> See http://endlessparentheses.com/get-in-the-habit-of-using-sharp-quote.html
> for what I considered a "sound reason" :)

Obviously, I use sharp quote myself, just not in add-hook, as that is
the style I am familiar with. But if that is the style of python.el,
that's a good reason. It's not in the rest of emacs.

But I can explain what it bothered me: As you know, in the specific
Flymake-case, backends are functions. You might be tempted to put a
closure, like a (lambda (report-fn) ...) in
flymake-diagnostic-functions.

If you do that everything will work, except for the interactive messages
that mention names of backends, where intead of the name of the symbol
you see something very ugly like

#f(compiled-function (report-fn) #<bytecode 0x1d84599>)

So, for now, as a good practice, I thought it better to use just ' so as
not to encourage others to put closures there. 

Eventually, once function-put starts working for non-symbol functions
and I can set a name property there, closures will probably not only be
accepted but encouraged.

bug#28808: [PATCH] Implement Python backend for Flymake

[Lele Gaifax]

[-- Attachment #1: Type: text/plain, Size: 231 bytes --]

Hi João,

I think I fulfilled all your suggestions/requests, and thus I'm re-attaching
the current version of the backend.

Please let me know if there's something more I can do, I will happily keep
hacking!

ciao, lele.

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Add-a-Flymake-backend-for-Python.patch --]
[-- Type: text/x-diff, Size: 6627 bytes --]

From 4e61fa030704a5e7bfcd537c503cef1da4409b6b Mon Sep 17 00:00:00 2001
From: Lele Gaifax <lele@metapensiero.it>
Date: Fri, 13 Oct 2017 10:44:02 +0200
Subject: [PATCH] Add a Flymake backend for Python

* lisp/progmodes/python.el: Implement new Flymake backend with
  related customizable settings.
  (python-flymake-command, python-flymake-command-output-regexp,
   python-flymake-msg-alist): New defcustom.
  (python-flymake): New function.
  (python-flymake-activate): New function.
---
 lisp/progmodes/python.el | 135 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 135 insertions(+)

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index 895117b9ee..831fd7e7f2 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -5142,6 +5142,141 @@ python-util-valid-regexp-p
   (ignore-errors (string-match regexp "") t))
 
 \f
+;;; Flymake integration
+
+(defgroup python-flymake nil
+  "Integration between Python and Flymake."
+  :group 'python
+  :link '(custom-group-link :tag "Flymake" flymake)
+  :version "26.1")
+
+(defcustom python-flymake-command '("pyflakes")
+  "The external tool that will be used to perform the syntax check.
+This is a non empty list of strings, the checker tool possibly followed by
+required arguments.  Once launched it will receive the Python source to be
+checked as its standard input.
+To use `flake8' you would set this to (\"flake8\" \"-\")."
+  :group 'python-flymake
+  :type '(repeat string))
+
+;; The default regexp accomodates for older pyflakes, which did not
+;; report the column number, and at the same time it's compatible with
+;; flake8 output, although it may be redefined to explicitly match the
+;; TYPE
+(defcustom python-flymake-command-output-regexp
+  (list
+   "^\\(?:<?stdin>?\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
+   1 2 nil 3)
+  "Specify how to parse the output of the specified tool.
+The value has the form (REGEXP LINE COLUMN TYPE MESSAGE): if
+REGEXP matches, the LINE'th subexpression gives the line number,
+the COLUMN'th subexpression gives the column number on that line,
+the TYPE'th subexpression gives the type of the message and
+MESSAGE'th is the message text itself.
+
+If COLUMN or TYPE are nil or that index didn't match, that
+information is not present on the matched line and a default will
+be used."
+  :group 'python-flymake
+  :type '(list regexp
+               (integer :tag "Line's index")
+               (choice
+                (const :tag "No column" nil)
+                (integer :tag "Column's index"))
+               (choice
+                (const :tag "No type" nil)
+                (integer :tag "Type's index"))
+               (integer :tag "Message's index")))
+
+(defcustom python-flymake-msg-alist
+  '(("\\(^redefinition\\|.*unused.*\\|used$\\)" . :warning))
+  "Alist used to associate messages to their types.
+Each element should be a cons-cell (REGEXP . TYPE), where TYPE must be
+one defined in the variable `flymake-diagnostic-types-alist'.
+For example, when using `flake8' a possible configuration could be:
+
+  ((\"\\(^redefinition\\|.*unused.*\\|used$\\)\" . :warning)
+   (\"^E999\" . :error)
+   (\"^[EW][0-9]+\" . :note))
+
+By default messages are considered errors."
+  :group 'python-flymake
+  :type `(alist :key-type (regexp)
+                :value-type (symbol)))
+
+(defvar-local python--flymake-proc nil)
+
+(defun python--flymake-parse-output (source proc report-fn)
+  "Collect diagnostics parsing checker tool's output line by line."
+  (let ((rx (nth 0 python-flymake-command-output-regexp))
+        (lineidx (nth 1 python-flymake-command-output-regexp))
+        (colidx (nth 2 python-flymake-command-output-regexp))
+        (typeidx (nth 3 python-flymake-command-output-regexp))
+        (msgidx (nth 4 python-flymake-command-output-regexp)))
+    (with-current-buffer (process-buffer proc)
+      (goto-char (point-min))
+      (cl-loop
+       while (search-forward-regexp rx nil t)
+       for msg = (match-string msgidx)
+       for (beg . end) = (flymake-diag-region
+                          source
+                          (string-to-number
+                           (match-string lineidx))
+                          (and colidx
+                               (match-string colidx)
+                               (string-to-number
+                                (match-string colidx))))
+       for type = (or (and typeidx
+                           (match-string typeidx)
+                           (assoc-default
+                            (match-string typeidx)
+                            python-flymake-msg-alist
+                            #'string-match))
+                      (assoc-default msg
+                                     python-flymake-msg-alist
+                                     #'string-match)
+                      :error)
+       collect (flymake-make-diagnostic
+                source beg end type msg)
+       into diags
+       finally (funcall report-fn diags)))))
+
+(defun python-flymake (report-fn &rest _args)
+  "Flymake backend for Python.
+This backend uses `python-flymake-command' (which see) to launch a process
+that is passed the current buffer's content via stdin.
+REPORT-FN is Flymake's callback function."
+  (unless (executable-find (car python-flymake-command))
+    (error "Cannot find a suitable checker"))
+
+  (when (process-live-p python--flymake-proc)
+    (kill-process python--flymake-proc))
+
+  (let ((source (current-buffer)))
+    (save-restriction
+      (widen)
+      (setq python--flymake-proc
+            (make-process
+             :name "python-flymake"
+             :noquery t
+             :connection-type 'pipe
+             :buffer (generate-new-buffer " *python-flymake*")
+             :command python-flymake-command
+             :sentinel
+             (lambda (proc _event)
+               (when (eq 'exit (process-status proc))
+                 (unwind-protect
+                     (when (eq proc python--flymake-proc)
+                       (python--flymake-parse-output source proc report-fn))
+                   (kill-buffer (process-buffer proc)))))))
+      (process-send-region python--flymake-proc (point-min) (point-max))
+      (process-send-eof python--flymake-proc))))
+
+(defun python-flymake-activate ()
+  "Activate the Flymake syntax check on all python-mode buffers."
+  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t))
+
+\f
 (defun python-electric-pair-string-delimiter ()
   (when (and electric-pair-mode
              (memq last-command-event '(?\" ?\'))
-- 
2.15.0.rc1


[-- Attachment #3: Type: text/plain, Size: 211 bytes --]


-- 
nickname: Lele Gaifax | Quando vivrò di quello che ho pensato ieri
real: Emanuele Gaifas | comincerò ad aver paura di chi mi copia.
lele@metapensiero.it  |                 -- Fortunato Depero, 1929.

bug#28808: Consider the right python--flymake-proc

[Lele Gaifax]

[-- Attachment #1: Type: text/plain, Size: 145 bytes --]

Hi,

I applied the fix about accessing the local python--flymake-proc in the right
buffer, in the inner lambda in python--flymake.

ciao, lele.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Add-a-Flymake-backend-for-Python.patch --]
[-- Type: text/x-diff, Size: 7110 bytes --]

From 739643bcf8cfcdbb09d43dd84ba12b0ab5d98d89 Mon Sep 17 00:00:00 2001
From: Lele Gaifax <lele@metapensiero.it>
Date: Fri, 13 Oct 2017 10:44:02 +0200
Subject: [PATCH] Add a Flymake backend for Python

* lisp/progmodes/python.el: Implement new Flymake backend with
  related customizable settings.
  (python-flymake-command, python-flymake-command-output-regexp,
   python-flymake-msg-alist): New defcustom.
  (python--flymake-parse-output): New function, able to parse
  python-flymake-command output accordingly to
  python-flymake-command-output-regexp.
  (python--flymake): New function implementing the backend interface
  using python--flymake-parse-output for the real work.
  (python-mode): Add python--flymake to flymake-diagnostic-functions.
---
 lisp/progmodes/python.el | 136 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 135 insertions(+), 1 deletion(-)

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index 895117b9ee..49c1ad907d 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -5142,6 +5142,138 @@ python-util-valid-regexp-p
   (ignore-errors (string-match regexp "") t))
 
 \f
+;;; Flymake integration
+
+(defgroup python-flymake nil
+  "Integration between Python and Flymake."
+  :group 'python
+  :link '(custom-group-link :tag "Flymake" flymake)
+  :version "26.1")
+
+(defcustom python-flymake-command '("pyflakes")
+  "The external tool that will be used to perform the syntax check.
+This is a non empty list of strings, the checker tool possibly followed by
+required arguments.  Once launched it will receive the Python source to be
+checked as its standard input.
+To use `flake8' you would set this to (\"flake8\" \"-\")."
+  :group 'python-flymake
+  :type '(repeat string))
+
+;; The default regexp accomodates for older pyflakes, which did not
+;; report the column number, and at the same time it's compatible with
+;; flake8 output, although it may be redefined to explicitly match the
+;; TYPE
+(defcustom python-flymake-command-output-regexp
+  (list
+   "^\\(?:<?stdin>?\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
+   1 2 nil 3)
+  "Specify how to parse the output of the specified tool.
+The value has the form (REGEXP LINE COLUMN TYPE MESSAGE): if
+REGEXP matches, the LINE'th subexpression gives the line number,
+the COLUMN'th subexpression gives the column number on that line,
+the TYPE'th subexpression gives the type of the message and
+MESSAGE'th is the message text itself.
+
+If COLUMN or TYPE are nil or that index didn't match, that
+information is not present on the matched line and a default will
+be used."
+  :group 'python-flymake
+  :type '(list regexp
+               (integer :tag "Line's index")
+               (choice
+                (const :tag "No column" nil)
+                (integer :tag "Column's index"))
+               (choice
+                (const :tag "No type" nil)
+                (integer :tag "Type's index"))
+               (integer :tag "Message's index")))
+
+(defcustom python-flymake-msg-alist
+  '(("\\(^redefinition\\|.*unused.*\\|used$\\)" . :warning))
+  "Alist used to associate messages to their types.
+Each element should be a cons-cell (REGEXP . TYPE), where TYPE must be
+one defined in the variable `flymake-diagnostic-types-alist'.
+For example, when using `flake8' a possible configuration could be:
+
+  ((\"\\(^redefinition\\|.*unused.*\\|used$\\)\" . :warning)
+   (\"^E999\" . :error)
+   (\"^[EW][0-9]+\" . :note))
+
+By default messages are considered errors."
+  :group 'python-flymake
+  :type `(alist :key-type (regexp)
+                :value-type (symbol)))
+
+(defvar-local python--flymake-proc nil)
+
+(defun python--flymake-parse-output (source proc report-fn)
+  "Collect diagnostics parsing checker tool's output line by line."
+  (let ((rx (nth 0 python-flymake-command-output-regexp))
+        (lineidx (nth 1 python-flymake-command-output-regexp))
+        (colidx (nth 2 python-flymake-command-output-regexp))
+        (typeidx (nth 3 python-flymake-command-output-regexp))
+        (msgidx (nth 4 python-flymake-command-output-regexp)))
+    (with-current-buffer (process-buffer proc)
+      (goto-char (point-min))
+      (cl-loop
+       while (search-forward-regexp rx nil t)
+       for msg = (match-string msgidx)
+       for (beg . end) = (flymake-diag-region
+                          source
+                          (string-to-number
+                           (match-string lineidx))
+                          (and colidx
+                               (match-string colidx)
+                               (string-to-number
+                                (match-string colidx))))
+       for type = (or (and typeidx
+                           (match-string typeidx)
+                           (assoc-default
+                            (match-string typeidx)
+                            python-flymake-msg-alist
+                            #'string-match))
+                      (assoc-default msg
+                                     python-flymake-msg-alist
+                                     #'string-match)
+                      :error)
+       collect (flymake-make-diagnostic
+                source beg end type msg)
+       into diags
+       finally (funcall report-fn diags)))))
+
+(defun python--flymake (report-fn &rest _args)
+  "Flymake backend for Python.
+This backend uses `python-flymake-command' (which see) to launch a process
+that is passed the current buffer's content via stdin.
+REPORT-FN is Flymake's callback function."
+  (unless (executable-find (car python-flymake-command))
+    (error "Cannot find a suitable checker"))
+
+  (when (process-live-p python--flymake-proc)
+    (kill-process python--flymake-proc))
+
+  (let ((source (current-buffer)))
+    (save-restriction
+      (widen)
+      (setq python--flymake-proc
+            (make-process
+             :name "python-flymake"
+             :noquery t
+             :connection-type 'pipe
+             :buffer (generate-new-buffer " *python-flymake*")
+             :command python-flymake-command
+             :sentinel
+             (lambda (proc _event)
+               (when (eq 'exit (process-status proc))
+                 (unwind-protect
+                     (when (with-current-buffer source
+                             (eq proc python--flymake-proc))
+                       (python--flymake-parse-output source proc report-fn))
+                   (kill-buffer (process-buffer proc)))))))
+      (process-send-region python--flymake-proc (point-min) (point-max))
+      (process-send-eof python--flymake-proc))))
+
+\f
 (defun python-electric-pair-string-delimiter ()
   (when (and electric-pair-mode
              (memq last-command-event '(?\" ?\'))
@@ -5255,7 +5387,9 @@ python-mode
   (make-local-variable 'python-shell-internal-buffer)
 
   (when python-indent-guess-indent-offset
-    (python-indent-guess-indent-offset)))
+    (python-indent-guess-indent-offset))
+
+  (add-hook 'flymake-diagnostic-functions #'python--flymake nil t))
 
 
 (provide 'python)
-- 
2.15.0.rc1


[-- Attachment #3: Type: text/plain, Size: 211 bytes --]


-- 
nickname: Lele Gaifax | Quando vivrò di quello che ho pensato ieri
real: Emanuele Gaifas | comincerò ad aver paura di chi mi copia.
lele@metapensiero.it  |                 -- Fortunato Depero, 1929.

bug#28808: Consider the right python--flymake-proc

[João Távora]

[-- Attachment #1: Type: text/plain, Size: 609 bytes --]

Good one.

I'll have a look at all the backends, including yours of course, during the
next week.

João

On Sat, Oct 21, 2017 at 8:15 AM, Lele Gaifax <lele@metapensiero.it> wrote:

> Hi,
>
> I applied the fix about accessing the local python--flymake-proc in the
> right
> buffer, in the inner lambda in python--flymake.
>
> ciao, lele.
>
>
>
> --
> nickname: Lele Gaifax | Quando vivrò di quello che ho pensato ieri
> real: Emanuele Gaifas | comincerò ad aver paura di chi mi copia.
> lele@metapensiero.it  |                 -- Fortunato Depero, 1929.
>
>


-- 
João Távora

[-- Attachment #2: Type: text/html, Size: 1122 bytes --]

bug#28808: Consider the right python--flymake-proc

[João Távora]

[-- Attachment #1: Type: text/plain, Size: 727 bytes --]

João Távora <joaotavora@gmail.com> writes:

> Good one. 
>
> I'll have a look at all the backends, including yours of course, during the next week.
>
> João
>

Sorry for the delay.

I am ready to commit your patch to emacs-26 with very minor changes:

1. The defcustom python-flymake-command-output-regexp is renamed to
python-flymake-command-output-pattern.

2. The docstring of that variable is slightly changed.

3. The function python--flymake is renamed python-flymake, since it is
public to python.el (there was some misunderstanding caused by me
earlier).

4. The commit message is very slightly changed so that the description
sentence starts on a line by its own.

Please verify,
João


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Add-a-Flymake-backend-for-Python.patch --]
[-- Type: text/x-diff, Size: 7105 bytes --]

From fd800a9e16493872ff3c8244a2e30e2d9e61fca4 Mon Sep 17 00:00:00 2001
From: Lele Gaifax <lele@metapensiero.it>
Date: Fri, 3 Nov 2017 12:20:36 +0000
Subject: [PATCH] Add a Flymake backend for Python

Implement new Flymake backend with related customizable settings.

* lisp/progmodes/python.el (python-flymake-command)
(python-flymake-command-output-pattern)
(python-flymake-msg-alist): New defcustom.
(python--flymake-parse-output): New function, able to parse
python-flymake-command output accordingly to
python-flymake-command-output-pattern.
(python-flymake): New function implementing the backend
interface using python--flymake-parse-output for the real
work.
(python-mode): Add python-flymake to flymake-diagnostic-functions.
---
 lisp/progmodes/python.el | 136 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 135 insertions(+), 1 deletion(-)

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index 895117b9ee..b7902fb978 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -5142,6 +5142,138 @@ python-util-valid-regexp-p
   (ignore-errors (string-match regexp "") t))
 
 \f
+;;; Flymake integration
+
+(defgroup python-flymake nil
+  "Integration between Python and Flymake."
+  :group 'python
+  :link '(custom-group-link :tag "Flymake" flymake)
+  :version "26.1")
+
+(defcustom python-flymake-command '("pyflakes")
+  "The external tool that will be used to perform the syntax check.
+This is a non empty list of strings, the checker tool possibly followed by
+required arguments.  Once launched it will receive the Python source to be
+checked as its standard input.
+To use `flake8' you would set this to (\"flake8\" \"-\")."
+  :group 'python-flymake
+  :type '(repeat string))
+
+;; The default regexp accomodates for older pyflakes, which did not
+;; report the column number, and at the same time it's compatible with
+;; flake8 output, although it may be redefined to explicitly match the
+;; TYPE
+(defcustom python-flymake-command-output-pattern
+  (list
+   "^\\(?:<?stdin>?\\):\\(?1:[0-9]+\\):\\(?:\\(?2:[0-9]+\\):\\)? \\(?3:.*\\)$"
+   1 2 nil 3)
+  "Specify how to parse the output of `python-flymake-command'.
+The value has the form (REGEXP LINE COLUMN TYPE MESSAGE): if
+REGEXP matches, the LINE'th subexpression gives the line number,
+the COLUMN'th subexpression gives the column number on that line,
+the TYPE'th subexpression gives the type of the message and the
+MESSAGE'th gives the message text itself.
+
+If COLUMN or TYPE are nil or that index didn't match, that
+information is not present on the matched line and a default will
+be used."
+  :group 'python-flymake
+  :type '(list regexp
+               (integer :tag "Line's index")
+               (choice
+                (const :tag "No column" nil)
+                (integer :tag "Column's index"))
+               (choice
+                (const :tag "No type" nil)
+                (integer :tag "Type's index"))
+               (integer :tag "Message's index")))
+
+(defcustom python-flymake-msg-alist
+  '(("\\(^redefinition\\|.*unused.*\\|used$\\)" . :warning))
+  "Alist used to associate messages to their types.
+Each element should be a cons-cell (REGEXP . TYPE), where TYPE must be
+one defined in the variable `flymake-diagnostic-types-alist'.
+For example, when using `flake8' a possible configuration could be:
+
+  ((\"\\(^redefinition\\|.*unused.*\\|used$\\)\" . :warning)
+   (\"^E999\" . :error)
+   (\"^[EW][0-9]+\" . :note))
+
+By default messages are considered errors."
+  :group 'python-flymake
+  :type `(alist :key-type (regexp)
+                :value-type (symbol)))
+
+(defvar-local python--flymake-proc nil)
+
+(defun python--flymake-parse-output (source proc report-fn)
+  "Collect diagnostics parsing checker tool's output line by line."
+  (let ((rx (nth 0 python-flymake-command-output-pattern))
+        (lineidx (nth 1 python-flymake-command-output-pattern))
+        (colidx (nth 2 python-flymake-command-output-pattern))
+        (typeidx (nth 3 python-flymake-command-output-pattern))
+        (msgidx (nth 4 python-flymake-command-output-pattern)))
+    (with-current-buffer (process-buffer proc)
+      (goto-char (point-min))
+      (cl-loop
+       while (search-forward-regexp rx nil t)
+       for msg = (match-string msgidx)
+       for (beg . end) = (flymake-diag-region
+                          source
+                          (string-to-number
+                           (match-string lineidx))
+                          (and colidx
+                               (match-string colidx)
+                               (string-to-number
+                                (match-string colidx))))
+       for type = (or (and typeidx
+                           (match-string typeidx)
+                           (assoc-default
+                            (match-string typeidx)
+                            python-flymake-msg-alist
+                            #'string-match))
+                      (assoc-default msg
+                                     python-flymake-msg-alist
+                                     #'string-match)
+                      :error)
+       collect (flymake-make-diagnostic
+                source beg end type msg)
+       into diags
+       finally (funcall report-fn diags)))))
+
+(defun python-flymake (report-fn &rest _args)
+  "Flymake backend for Python.
+This backend uses `python-flymake-command' (which see) to launch a process
+that is passed the current buffer's content via stdin.
+REPORT-FN is Flymake's callback function."
+  (unless (executable-find (car python-flymake-command))
+    (error "Cannot find a suitable checker"))
+
+  (when (process-live-p python--flymake-proc)
+    (kill-process python--flymake-proc))
+
+  (let ((source (current-buffer)))
+    (save-restriction
+      (widen)
+      (setq python--flymake-proc
+            (make-process
+             :name "python-flymake"
+             :noquery t
+             :connection-type 'pipe
+             :buffer (generate-new-buffer " *python-flymake*")
+             :command python-flymake-command
+             :sentinel
+             (lambda (proc _event)
+               (when (eq 'exit (process-status proc))
+                 (unwind-protect
+                     (when (with-current-buffer source
+                             (eq proc python--flymake-proc))
+                       (python--flymake-parse-output source proc report-fn))
+                   (kill-buffer (process-buffer proc)))))))
+      (process-send-region python--flymake-proc (point-min) (point-max))
+      (process-send-eof python--flymake-proc))))
+
+\f
 (defun python-electric-pair-string-delimiter ()
   (when (and electric-pair-mode
              (memq last-command-event '(?\" ?\'))
@@ -5255,7 +5387,9 @@ python-mode
   (make-local-variable 'python-shell-internal-buffer)
 
   (when python-indent-guess-indent-offset
-    (python-indent-guess-indent-offset)))
+    (python-indent-guess-indent-offset))
+
+  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t))
 
 
 (provide 'python)
-- 
2.14.2

bug#28808: Consider the right python--flymake-proc

[Lele Gaifax]

joaotavora@gmail.com (João Távora) writes:

> Please verify,

Thank you João, everything seems good to me.

Ciao, lele.
-- 
nickname: Lele Gaifax | Quando vivrò di quello che ho pensato ieri
real: Emanuele Gaifas | comincerò ad aver paura di chi mi copia.
lele@metapensiero.it  |                 -- Fortunato Depero, 1929.