bug#46834: 28.0.50; byte-compiling the standard counter closure fails

[Pip Cet <pipcet@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 1428 bytes --]

On Mon, Mar 1, 2021 at 2:23 PM Stefan Monnier <monnier@iro.umontreal.ca> wrote:
> > diff --git a/lisp/emacs-lisp/bytecomp.el b/lisp/emacs-lisp/bytecomp.el
> > index a2fe37a1ee586..7d00b453caf1c 100644
> > --- a/lisp/emacs-lisp/bytecomp.el
> > +++ b/lisp/emacs-lisp/bytecomp.el
> > @@ -2785,16 +2785,12 @@ byte-compile--reify-function
> >      (dolist (binding env)
> >        (cond
> >         ((consp binding)
> > -        ;; We check shadowing by the args, so that the `let' can be moved
> > -        ;; within the lambda, which can then be unfolded.  FIXME: Some of those
> > -        ;; bindings might be unused in `body'.
> > -        (unless (memq (car binding) args) ;Shadowed.
> > -          (push `(,(car binding) ',(cdr binding)) renv)))
> > +        (push `(,(car binding) ',(cdr binding)) renv))
> >         ((eq binding t))
> >         (t (push `(defvar ,binding) body))))
> >      (if (null renv)
> >          `(lambda ,args ,@preamble ,@body)
> > -      `(lambda ,args ,@preamble (let ,(nreverse renv) ,@body)))))
> > +      `(let ,renv (lambda ,args ,@preamble ,@body)))))
>
> This looks good, thanks, but it changes the nature of the output of
> `byte-compile` from a function value to an expression whose evaluation
> returns a function value.  So I think we should tweak `byte-compile` so
> it calls `eval` on the result in this particular case.

Thanks! That's a good catch :-)

Is this what you meant?

Pip

[-- Attachment #2: 0001-Compile-closures-that-modify-their-bound-vars-correc.patch --]
[-- Type: text/x-patch, Size: 5074 bytes --]

From 0b11af35fb3414fa1abb72bd33f4c6f769aa8847 Mon Sep 17 00:00:00 2001
From: Pip Cet <pipcet@gmail.com>
Date: Sun, 28 Feb 2021 19:43:09 +0000
Subject: [PATCH] Compile closures that modify their bound vars correctly
 (Bug#46834)

* lisp/emacs-lisp/bytecomp.el (byte-compile--reify-function): Don't
move let bindings into the lambda. Don't reverse list of
bindings. (byte-compile): Evaluate the return value if it was
previously reified.
* test/lisp/emacs-lisp/bytecomp-tests.el (bytecomp-reify-function):
Add tests.
---
 lisp/emacs-lisp/bytecomp.el            | 46 +++++++++++++-------------
 test/lisp/emacs-lisp/bytecomp-tests.el | 23 +++++++++++++
 2 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/lisp/emacs-lisp/bytecomp.el b/lisp/emacs-lisp/bytecomp.el
index a2fe37a1ee586..4e00fe6121e82 100644
--- a/lisp/emacs-lisp/bytecomp.el
+++ b/lisp/emacs-lisp/bytecomp.el
@@ -2785,16 +2785,12 @@ byte-compile--reify-function
     (dolist (binding env)
       (cond
        ((consp binding)
-        ;; We check shadowing by the args, so that the `let' can be moved
-        ;; within the lambda, which can then be unfolded.  FIXME: Some of those
-        ;; bindings might be unused in `body'.
-        (unless (memq (car binding) args) ;Shadowed.
-          (push `(,(car binding) ',(cdr binding)) renv)))
+        (push `(,(car binding) ',(cdr binding)) renv))
        ((eq binding t))
        (t (push `(defvar ,binding) body))))
     (if (null renv)
         `(lambda ,args ,@preamble ,@body)
-      `(lambda ,args ,@preamble (let ,(nreverse renv) ,@body)))))
+      `(let ,renv (lambda ,args ,@preamble ,@body)))))
 \f
 ;;;###autoload
 (defun byte-compile (form)
@@ -2819,23 +2815,27 @@ byte-compile
                  (if (symbolp form) form "provided"))
         fun)
        (t
-        (when (or (symbolp form) (eq (car-safe fun) 'closure))
-          ;; `fun' is a function *value*, so try to recover its corresponding
-          ;; source code.
-          (setq lexical-binding (eq (car fun) 'closure))
-          (setq fun (byte-compile--reify-function fun)))
-        ;; Expand macros.
-        (setq fun (byte-compile-preprocess fun))
-        (setq fun (byte-compile-top-level fun nil 'eval))
-        (if (symbolp form)
-            ;; byte-compile-top-level returns an *expression* equivalent to the
-            ;; `fun' expression, so we need to evaluate it, tho normally
-            ;; this is not needed because the expression is just a constant
-            ;; byte-code object, which is self-evaluating.
-            (setq fun (eval fun t)))
-        (if macro (push 'macro fun))
-        (if (symbolp form) (fset form fun))
-        fun))))))
+        (let (final-eval)
+          (when (or (symbolp form) (eq (car-safe fun) 'closure))
+            ;; `fun' is a function *value*, so try to recover its corresponding
+            ;; source code.
+            (setq lexical-binding (eq (car fun) 'closure))
+            (setq fun (byte-compile--reify-function fun))
+            (setq final-eval t))
+          ;; Expand macros.
+          (setq fun (byte-compile-preprocess fun))
+          (setq fun (byte-compile-top-level fun nil 'eval))
+          (if (symbolp form)
+              ;; byte-compile-top-level returns an *expression* equivalent to the
+              ;; `fun' expression, so we need to evaluate it, tho normally
+              ;; this is not needed because the expression is just a constant
+              ;; byte-code object, which is self-evaluating.
+              (setq fun (eval fun t)))
+          (if final-eval
+              (setq fun (eval fun t)))
+          (if macro (push 'macro fun))
+          (if (symbolp form) (fset form fun))
+          fun)))))))
 
 (defun byte-compile-sexp (sexp)
   "Compile and return SEXP."
diff --git a/test/lisp/emacs-lisp/bytecomp-tests.el b/test/lisp/emacs-lisp/bytecomp-tests.el
index fb84596ad3f40..03c267ccd0fef 100644
--- a/test/lisp/emacs-lisp/bytecomp-tests.el
+++ b/test/lisp/emacs-lisp/bytecomp-tests.el
@@ -1199,6 +1199,29 @@ bytecomp-local-defvar
       (should (equal (funcall (eval fun t)) '(c d)))
       (should (equal (funcall (byte-compile fun)) '(c d))))))
 
+(ert-deftest bytecomp-reify-function ()
+  "Check that closures that modify their bound variables are
+compiled correctly."
+  (cl-letf ((lexical-binding t)
+            ((symbol-function 'counter) nil))
+    (let ((x 0))
+      (defun counter () (cl-incf x))
+      (should (equal (counter) 1))
+      (should (equal (counter) 2))
+      ;; byte compiling should not cause counter to always return the
+      ;; same value (bug#46834)
+      (byte-compile 'counter)
+      (should (equal (counter) 3))
+      (should (equal (counter) 4)))
+    (let ((x 0))
+      (let ((x 1))
+        (defun counter () x)
+        (should (equal (counter) 1))
+        ;; byte compiling should not cause the outer binding to shadow
+        ;; the inner one (bug#46834)
+        (byte-compile 'counter)
+        (should (equal (counter) 1))))))
+
 ;; Local Variables:
 ;; no-byte-compile: t
 ;; End:
-- 
2.30.1