From b56b04ac23f74dddd4648c9f86c8cf7423f70829 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20Engdeg=C3=A5rd?= <mattiase@acm.org>
Date: Mon, 22 Nov 2021 16:56:38 +0100
Subject: [PATCH 1/2] Fix closure-conversion of shadowed captured lambda-lifted
 vars

* lisp/emacs-lisp/cconv.el (cconv--lifted-arg): New.
(cconv-convert):
Lambda lifted variables (ones passed explicitly to lambda-lifted
functions) that are also captured in an outer closure and shadowed
were renamed incorrectly.  Fix that by providing the correct
definiens for the closed-over variable (bug#51982).

* test/lisp/emacs-lisp/bytecomp-tests.el (bytecomp-tests--test-cases):
* test/lisp/emacs-lisp/cconv-tests.el (cconv-tests--intern-all)
(cconv-closure-convert-remap-var): Add tests.
---
 lisp/emacs-lisp/cconv.el               |  28 ++++-
 test/lisp/emacs-lisp/bytecomp-tests.el |  41 +++++++
 test/lisp/emacs-lisp/cconv-tests.el    | 152 +++++++++++++++++++++++++
 3 files changed, 215 insertions(+), 6 deletions(-)

diff --git a/lisp/emacs-lisp/cconv.el b/lisp/emacs-lisp/cconv.el
index 03e109f250..9808547b84 100644
--- a/lisp/emacs-lisp/cconv.el
+++ b/lisp/emacs-lisp/cconv.el
@@ -304,6 +304,22 @@ cconv--convert-funcbody
             `(,@(nreverse special-forms) ,@(macroexp-unprogn body))))
       funcbody)))
 
+(defun cconv--lifted-arg (var env)
+  "The argument to use for VAR in λ-lifted calls according to ENV."
+  (let ((mapping (cdr (assq var env))))
+    (pcase-exhaustive mapping
+      (`(internal-get-closed-var . ,_)
+       ;; The variable is captured.
+       mapping)
+      (`(car-safe (internal-get-closed-var . ,_))
+       ;; The variable is mutably captured; skip
+       ;; the indirection step because the variable is
+       ;; passed "by reference" to the λ-lifted function.
+       (cadr mapping))
+      ((or '() `(car-safe ,(pred symbolp)))
+       ;; The variable is not captured; use the (shadowed) variable value.
+       var))))
+
 (defun cconv-convert (form env extend)
   ;; This function actually rewrites the tree.
   "Return FORM with all its lambdas changed so they are closed.
@@ -428,10 +444,11 @@ cconv-convert
                  ;; One of the lambda-lifted vars is shadowed, so add
                  ;; a reference to the outside binding and arrange to use
                  ;; that reference.
-                 (let ((closedsym (make-symbol (format "closed-%s" var))))
+                 (let ((var-def (cconv--lifted-arg var env))
+                       (closedsym (make-symbol (format "closed-%s" var))))
                    (setq new-env (cconv--remap-llv new-env var closedsym))
                    (setq new-extend (cons closedsym (remq var new-extend)))
-                   (push `(,closedsym ,var) binders-new)))
+                   (push `(,closedsym ,var-def) binders-new)))
 
                ;; We push the element after redefined free variables are
                ;; processed.  This is important to avoid the bug when free
@@ -449,14 +466,13 @@ cconv-convert
          ;; before we know that the var will be in `new-extend' (bug#24171).
          (dolist (binder binders-new)
            (when (memq (car-safe binder) new-extend)
-             ;; One of the lambda-lifted vars is shadowed, so add
-             ;; a reference to the outside binding and arrange to use
-             ;; that reference.
+             ;; One of the lambda-lifted vars is shadowed.
              (let* ((var (car-safe binder))
+                    (var-def (cconv--lifted-arg var env))
                     (closedsym (make-symbol (format "closed-%s" var))))
                (setq new-env (cconv--remap-llv new-env var closedsym))
                (setq new-extend (cons closedsym (remq var new-extend)))
-               (push `(,closedsym ,var) binders-new)))))
+               (push `(,closedsym ,var-def) binders-new)))))
 
        `(,letsym ,(nreverse binders-new)
                  . ,(mapcar (lambda (form)
diff --git a/test/lisp/emacs-lisp/bytecomp-tests.el b/test/lisp/emacs-lisp/bytecomp-tests.el
index 816f14a18d..a75a33b2dc 100644
--- a/test/lisp/emacs-lisp/bytecomp-tests.el
+++ b/test/lisp/emacs-lisp/bytecomp-tests.el
@@ -643,6 +643,47 @@ bytecomp-tests--test-cases
 
     (cond)
     (mapcar (lambda (x) (cond ((= x 0)))) '(0 1))
+
+    ;; These expressions give different results in lexbind and dynbind modes,
+    ;; but in each the compiler and interpreter should agree!
+    (let ((f (lambda (x)
+               (lambda ()
+                 (let ((g (lambda () x)))
+                   (let ((x 'a))
+                     (list x (funcall g))))))))
+      (funcall (funcall f 'b)))
+    (let ((f (lambda (x)
+               (lambda ()
+                 (let ((g (lambda () x)))
+                   (let* ((x 'a))
+                     (list x (funcall g))))))))
+      (funcall (funcall f 'b)))
+    (let ((f (lambda (x)
+               (lambda ()
+                 (let ((g (lambda () x)))
+                   (setq x (list x x))
+                   (let ((x 'a))
+                     (list x (funcall g))))))))
+      (funcall (funcall f 'b)))
+    (let ((f (lambda (x)
+               (lambda ()
+                 (let ((g (lambda () x)))
+                   (setq x (list x x))
+                   (let* ((x 'a))
+                     (list x (funcall g))))))))
+      (funcall (funcall f 'b)))
+    (let ((f (lambda (x)
+               (let ((g (lambda () x))
+                     (h (lambda () (setq x (list x x)))))
+                 (let ((x 'a))
+                   (list x (funcall g) (funcall h)))))))
+      (funcall (funcall f 'b)))
+    (let ((f (lambda (x)
+               (let ((g (lambda () x))
+                     (h (lambda () (setq x (list x x)))))
+                 (let* ((x 'a))
+                   (list x (funcall g) (funcall h)))))))
+      (funcall (funcall f 'b)))
     )
   "List of expressions for cross-testing interpreted and compiled code.")
 
diff --git a/test/lisp/emacs-lisp/cconv-tests.el b/test/lisp/emacs-lisp/cconv-tests.el
index 4290571735..0701892b8c 100644
--- a/test/lisp/emacs-lisp/cconv-tests.el
+++ b/test/lisp/emacs-lisp/cconv-tests.el
@@ -205,5 +205,157 @@ cconv-convert-lambda-lifted
            nil 99)
           42)))
 
+(defun cconv-tests--intern-all (x)
+  "Intern all symbols in X."
+  (cond ((symbolp x) (intern (symbol-name x)))
+        ((consp x) (cons (cconv-tests--intern-all (car x))
+                         (cconv-tests--intern-all (cdr x))))
+        ;; Assume we don't need to deal with vectors etc.
+        (t x)))
+
+(ert-deftest cconv-closure-convert-remap-var ()
+  ;; Verify that we correctly remap shadowed lambda-lifted variables.
+
+  ;; We intern all symbols for ease of comparison; this works because
+  ;; the `cconv-closure-convert' result should contain no pair of
+  ;; distinct symbols having the same name.
+
+  ;; Sanity check: captured variable, no lambda-lifting or shadowing:
+  (should (equal (cconv-tests--intern-all
+           (cconv-closure-convert
+            '#'(lambda (x)
+                 #'(lambda () x))))
+           '#'(lambda (x)
+                (internal-make-closure
+                 nil (x) nil
+                 (internal-get-closed-var 0)))))
+
+  ;; Basic case:
+  (should (equal (cconv-tests--intern-all
+                  (cconv-closure-convert
+                   '#'(lambda (x)
+                        (let ((f #'(lambda () x)))
+                          (let ((x 'b))
+                            (list x (funcall f)))))))
+                 '#'(lambda (x)
+                      (let ((f #'(lambda (x) x)))
+                        (let ((x 'b)
+                              (closed-x x))
+                          (list x (funcall f closed-x)))))))
+  (should (equal (cconv-tests--intern-all
+                  (cconv-closure-convert
+                   '#'(lambda (x)
+                        (let ((f #'(lambda () x)))
+                          (let* ((x 'b))
+                            (list x (funcall f)))))))
+                 '#'(lambda (x)
+                      (let ((f #'(lambda (x) x)))
+                        (let* ((closed-x x)
+                               (x 'b))
+                          (list x (funcall f closed-x)))))))
+
+  ;; With the lambda-lifted shadowed variable also being captured:
+  (should (equal
+           (cconv-tests--intern-all
+            (cconv-closure-convert
+             '#'(lambda (x)
+                  #'(lambda ()
+                      (let ((f #'(lambda () x)))
+                        (let ((x 'a))
+                          (list x (funcall f))))))))
+           '#'(lambda (x)
+                (internal-make-closure
+                 nil (x) nil
+                 (let ((f #'(lambda (x) x)))
+                   (let ((x 'a)
+                         (closed-x (internal-get-closed-var 0)))
+                     (list x (funcall f closed-x))))))))
+  (should (equal
+           (cconv-tests--intern-all
+            (cconv-closure-convert
+             '#'(lambda (x)
+                  #'(lambda ()
+                      (let ((f #'(lambda () x)))
+                        (let* ((x 'a))
+                          (list x (funcall f))))))))
+           '#'(lambda (x)
+                (internal-make-closure
+                 nil (x) nil
+                 (let ((f #'(lambda (x) x)))
+                   (let* ((closed-x (internal-get-closed-var 0))
+                          (x 'a))
+                     (list x (funcall f closed-x))))))))
+  ;; With lambda-lifted shadowed variable also being mutably captured:
+  (should (equal
+           (cconv-tests--intern-all
+            (cconv-closure-convert
+             '#'(lambda (x)
+                  #'(lambda ()
+                      (let ((f #'(lambda () x)))
+                        (setq x x)
+                        (let ((x 'a))
+                          (list x (funcall f))))))))
+           '#'(lambda (x)
+                (let ((x (list x)))
+                  (internal-make-closure
+                   nil (x) nil
+                   (let ((f #'(lambda (x) (car-safe x))))
+                     (setcar (internal-get-closed-var 0)
+                             (car-safe (internal-get-closed-var 0)))
+                     (let ((x 'a)
+                           (closed-x (internal-get-closed-var 0)))
+                       (list x (funcall f closed-x)))))))))
+  (should (equal
+           (cconv-tests--intern-all
+            (cconv-closure-convert
+             '#'(lambda (x)
+                  #'(lambda ()
+                      (let ((f #'(lambda () x)))
+                        (setq x x)
+                        (let* ((x 'a))
+                          (list x (funcall f))))))))
+           '#'(lambda (x)
+                (let ((x (list x)))
+                  (internal-make-closure
+                   nil (x) nil
+                   (let ((f #'(lambda (x) (car-safe x))))
+                     (setcar (internal-get-closed-var 0)
+                             (car-safe (internal-get-closed-var 0)))
+                     (let* ((closed-x (internal-get-closed-var 0))
+                            (x 'a))
+                       (list x (funcall f closed-x)))))))))
+  ;; Lambda-lifted variable that isn't actually captured where it is shadowed:
+  (should (equal
+           (cconv-tests--intern-all
+            (cconv-closure-convert
+             '#'(lambda (x)
+                  (let ((g #'(lambda () x))
+                        (h #'(lambda () (setq x x))))
+                    (let ((x 'b))
+                      (list x (funcall g) (funcall h)))))))
+           '#'(lambda (x)
+                (let ((x (list x)))
+                  (let ((g #'(lambda (x) (car-safe x)))
+                        (h #'(lambda (x) (setcar x (car-safe x)))))
+                    (let ((x 'b)
+                          (closed-x x))
+                      (list x (funcall g closed-x) (funcall h closed-x))))))))
+  (should (equal
+           (cconv-tests--intern-all
+            (cconv-closure-convert
+             '#'(lambda (x)
+                  (let ((g #'(lambda () x))
+                        (h #'(lambda () (setq x x))))
+                    (let* ((x 'b))
+                      (list x (funcall g) (funcall h)))))))
+           '#'(lambda (x)
+                (let ((x (list x)))
+                  (let ((g #'(lambda (x) (car-safe x)))
+                        (h #'(lambda (x) (setcar x (car-safe x)))))
+                    (let* ((closed-x x)
+                           (x 'b))
+                      (list x (funcall g closed-x) (funcall h closed-x))))))))
+  )
+
 (provide 'cconv-tests)
 ;;; cconv-tests.el ends here
-- 
2.21.1 (Apple Git-122.3)

