bug#6991: Please keep bytecode out of *Backtrace* buffers

[npostavs@users.sourceforge.net]

[-- Attachment #1: Type: text/plain, Size: 446 bytes --]

Stefan Monnier <monnier@iro.umontreal.ca> writes:
> FWIW, I agree.  The only issue I can see here is that depending on how
> it's done, it could affect the size of the .elc files (by using two bytes
> per bytecode 0 rather than 1).  Nothing too terrible, but it's probably
> worth checking whether it does make such a difference and if so how serious
> it is.

On average it increases Emacs' .elc by ~200 bytes each, 276kB in total,
or 100.62%.


[-- Attachment #2: table of size changes for print NUL as \0 --]
[-- Type: application/octet-stream, Size: 28222 bytes --]

[-- Attachment #3: Type: text/plain, Size: 752 bytes --]


But actually, while looking at this, I understood more about what the
print_escape_nonascii flag is used for (i.e., multibyte vs unibyte
stuff), and I no longer think it makes sense for it to affect printing
the NUL byte anyway.

I propose adding a new flag print_escape_control_characters instead (see
patch #3 in the series).  I also implemented hiding the byte code
functions with text properties in #4.  It's not quite satisfactory
though, because it doesn't cover byte code functions values that are
arguments, only byte code being called.  I think printing needs to be
made more flexible in order to cleanly catch all byte code values.
Patch #5 replaces NUL bytes with "\0" in X selections (I guess it covers
w32 as well? Haven't checked yet).


[-- Attachment #4: screenshot --]
[-- Type: image/png, Size: 69444 bytes --]

[-- Attachment #5: patch --]
[-- Type: text/plain, Size: 5288 bytes --]

From 622b9b1f328cfbf527e0dd350fd45566f4716849 Mon Sep 17 00:00:00 2001
From: Noam Postavsky <npostavs@gmail.com>
Date: Sat, 11 Feb 2017 09:19:00 -0500
Subject: [PATCH v1 1/5] Operate on frame list instead of printed backtrace

* lisp/emacs-lisp/debug.el (debugger--insert-backtrace): New function,
prints the given backtrace frames.
(debugger-setup-buffer): Use it instead of editing the backtrace
buffer text.
---
 lisp/emacs-lisp/debug.el | 84 +++++++++++++++++++++++++++---------------------
 1 file changed, 48 insertions(+), 36 deletions(-)

diff --git a/lisp/emacs-lisp/debug.el b/lisp/emacs-lisp/debug.el
index cb77148c28..f24f62fa71 100644
--- a/lisp/emacs-lisp/debug.el
+++ b/lisp/emacs-lisp/debug.el
@@ -264,6 +264,40 @@ debug
       (setq debug-on-next-call debugger-step-after-exit)
       debugger-value)))
 \f
+
+(defun debugger-insert-backtrace (frames do-xrefs)
+  "Format and insert the backtrace FRAMES at point.
+Make functions into cross-reference buttons if DO-XREFS is non-nil."
+  (let ((standard-output (current-buffer))
+        (eval-buffers eval-buffer-list))
+    (require 'help-mode)     ; Define `help-function-def' button type.
+    (pcase-dolist (`(,evald ,fun ,args ,flags) frames)
+      (insert (if (plist-get flags :debug-on-exit)
+                  "* " "  "))
+      (let ((fun-file (and do-xrefs (symbol-file fun 'defun)))
+            (fun-pt (point)))
+        (cond
+         ((and evald (not debugger-stack-frame-as-list))
+          (prin1 fun)
+          (if args (prin1 args) (princ "()")))
+         (t
+          (prin1 (cons fun args))
+          (cl-incf fun-pt)))
+        (when fun-file
+          (make-text-button fun-pt (+ fun-pt (length (symbol-name fun)))
+                            :type 'help-function-def
+                            'help-args (list fun fun-file))))
+      ;; After any frame that uses eval-buffer, insert a line that
+      ;; states the buffer position it's reading at.
+      (when (and eval-buffers (memq fun '(eval-buffer eval-region)))
+        (insert (format "  ; Reading at buffer position %d"
+                        ;; This will get the wrong result if there are
+                        ;; two nested eval-region calls for the same
+                        ;; buffer.  That's not a very useful case.
+                        (with-current-buffer (pop eval-buffers)
+                          (point)))))
+      (insert "\n"))))
+
 (defun debugger-setup-buffer (args)
   "Initialize the `*Backtrace*' buffer for entry to the debugger.
 That buffer should be current already."
@@ -271,22 +305,6 @@ debugger-setup-buffer
   (erase-buffer)
   (set-buffer-multibyte t)		;Why was it nil ?  -stef
   (setq buffer-undo-list t)
-  (let ((standard-output (current-buffer))
-	(print-escape-newlines t)
-	(print-level 8)
-        (print-length 50))
-    ;; FIXME the debugger could pass a custom callback to mapbacktrace
-    ;; instead of manipulating printed results.
-    (mapbacktrace #'backtrace--print-frame 'debug))
-  (goto-char (point-min))
-  (delete-region (point)
-		 (progn
-                   (forward-line (if (eq (car args) 'debug)
-                                     ;; Remove debug--implement-debug-on-entry
-                                     ;; and the advice's `apply' frame.
-				     3
-				   1))
-		   (point)))
   (insert "Debugger entered")
   ;; lambda is for debug-on-call when a function call is next.
   ;; debug is for debug-on-entry function called.
@@ -301,10 +319,7 @@ debugger-setup-buffer
        (setq pos (point))
        (setq debugger-value (nth 1 args))
        (prin1 debugger-value (current-buffer))
-       (insert ?\n)
-       (delete-char 1)
-       (insert ? )
-       (beginning-of-line))
+       (insert ?\n))
       ;; Watchpoint triggered.
       ((and `watchpoint (let `(,symbol ,newval . ,details) (cdr args)))
        (insert
@@ -341,23 +356,20 @@ debugger-setup-buffer
                   (cdr args) args)
               (current-buffer))
        (insert ?\n)))
+    (let ((frames (nthcdr
+                   ;; Remove debug--implement-debug-on-entry and the
+                   ;; advice's `apply' frame.
+                   (if (eq (car args) 'debug) 3 1)
+                   (backtrace-frames 'debug)))
+          (print-escape-newlines t)
+          (print-level 8)
+          (print-length 50))
+      (when (eq (car args) 'exit)
+        (setf (cl-getf (nth 3 (car frames)) :debug-on-exit) nil))
+      (debugger-insert-backtrace frames t))
     ;; Place point on "stack frame 0" (bug#15101).
-    (goto-char pos))
-  ;; After any frame that uses eval-buffer,
-  ;; insert a line that states the buffer position it's reading at.
-  (save-excursion
-    (let ((tem eval-buffer-list))
-      (while (and tem
-		  (re-search-forward "^  eval-\\(buffer\\|region\\)(" nil t))
-	(end-of-line)
-	(insert (format "  ; Reading at buffer position %d"
-			;; This will get the wrong result
-			;; if there are two nested eval-region calls
-			;; for the same buffer.  That's not a very useful case.
-			(with-current-buffer (car tem)
-			  (point))))
-	(pop tem))))
-  (debugger-make-xrefs))
+    (goto-char pos)))
+
 
 (defun debugger-make-xrefs (&optional buffer)
   "Attach cross-references to function names in the `*Backtrace*' buffer."
-- 
2.11.1


[-- Attachment #6: patch --]
[-- Type: text/plain, Size: 6872 bytes --]

From 356c3e17edb19f5daef9f83f2c4d5694290f1221 Mon Sep 17 00:00:00 2001
From: Noam Postavsky <npostavs@gmail.com>
Date: Sat, 11 Feb 2017 17:19:41 -0500
Subject: [PATCH v1 2/5] Improve ert backtrace recording

Change ert to use the new `backtrace-frames' function instead of
collecting frames one by one with `backtrace-frame'.  Additionally,
collect frames starting from `signal' instead the somewhat arbitrary
"6 from the bottom".  Skipping 6 frames would skip the expression that
actually caused the signal that triggered the debugger.  Possibly 6
was chosen because in the case of a failed test, the triggering frame
is an `ert-fail' call, which is not so interesting.  But in case of
test throwing an error, this drop the `error' call which is too much.

* lisp/emacs-lisp/ert.el (ert--print-backtrace): Remove.
(ert--print-backtrace): Add DO-XREFS parameter, delegate to
`debugger-insert-backtrace'.
(ert--run-test-debugger): Record the backtrace frames starting from
the instigating `signal' call.
(ert-run-tests-batch): Pass nil for `ert--print-backtrace's new
DO-XREFS parameter.
(ert-results-pop-to-backtrace-for-test-at-point): Pass t as DO-XREFS
to `ert--print-backtrace' and remove call to `debugger-make-xrefs'.
* test/lisp/emacs-lisp/ert-tests.el (ert-test-record-backtrace): Check
the backtrace list instead of comparing its string representation.
Expect `signal' to be the first frame.
---
 lisp/emacs-lisp/ert.el            | 62 ++++++++++++---------------------------
 test/lisp/emacs-lisp/ert-tests.el |  8 ++---
 2 files changed, 21 insertions(+), 49 deletions(-)

diff --git a/lisp/emacs-lisp/ert.el b/lisp/emacs-lisp/ert.el
index 785f4aca1c..98e4cd7c24 100644
--- a/lisp/emacs-lisp/ert.el
+++ b/lisp/emacs-lisp/ert.el
@@ -669,48 +669,12 @@ ert-debug-on-error
 (cl-defstruct (ert-test-aborted-with-non-local-exit
                (:include ert-test-result)))
 
-
-(defun ert--record-backtrace ()
-  "Record the current backtrace (as a list) and return it."
-  ;; Since the backtrace is stored in the result object, result
-  ;; objects must only be printed with appropriate limits
-  ;; (`print-level' and `print-length') in place.  For interactive
-  ;; use, the cost of ensuring this possibly outweighs the advantage
-  ;; of storing the backtrace for
-  ;; `ert-results-pop-to-backtrace-for-test-at-point' given that we
-  ;; already have `ert-results-rerun-test-debugging-errors-at-point'.
-  ;; For batch use, however, printing the backtrace may be useful.
-  (cl-loop
-   ;; 6 is the number of frames our own debugger adds (when
-   ;; compiled; more when interpreted).  FIXME: Need to describe a
-   ;; procedure for determining this constant.
-   for i from 6
-   for frame = (backtrace-frame i)
-   while frame
-   collect frame))
-
-(defun ert--print-backtrace (backtrace)
+(defun ert--print-backtrace (backtrace do-xrefs)
   "Format the backtrace BACKTRACE to the current buffer."
-  ;; This is essentially a reimplementation of Fbacktrace
-  ;; (src/eval.c), but for a saved backtrace, not the current one.
   (let ((print-escape-newlines t)
         (print-level 8)
         (print-length 50))
-    (dolist (frame backtrace)
-      (pcase-exhaustive frame
-        (`(nil ,special-operator . ,arg-forms)
-         ;; Special operator.
-         (insert
-          (format "  %S\n" (cons special-operator arg-forms))))
-        (`(t ,fn . ,args)
-         ;; Function call.
-         (insert (format "  %S(" fn))
-         (cl-loop for firstp = t then nil
-                  for arg in args do
-                  (unless firstp
-                    (insert " "))
-                  (insert (format "%S" arg)))
-         (insert ")\n"))))))
+    (debugger-insert-backtrace backtrace do-xrefs)))
 
 ;; A container for the state of the execution of a single test and
 ;; environment data needed during its execution.
@@ -749,7 +713,19 @@ ert--run-test-debugger
                       ((quit) 'quit)
 		      ((ert-test-skipped) 'skipped)
                       (otherwise 'failed)))
-              (backtrace (ert--record-backtrace))
+              ;; We store the backtrace in the result object for
+              ;; `ert-results-pop-to-backtrace-for-test-at-point'.
+              ;; This means we have to limit `print-level' and
+              ;; `print-length' when printing result objects.  That
+              ;; might not be worth while when we can also use
+              ;; `ert-results-rerun-test-debugging-errors-at-point',
+              ;; (i.e., when running interactively) but having the
+              ;; backtrace ready for printing is important for batch
+              ;; use.
+              ;;
+              ;; Grab the frames starting from `signal', frames below
+              ;; that are all from the debugger.
+              (backtrace (backtrace-frames 'signal))
               (infos (reverse ert--infos)))
          (setf (ert--test-execution-info-result info)
                (cl-ecase type
@@ -1404,8 +1380,9 @@ ert-run-tests-batch
               (ert-test-result-with-condition
                (message "Test %S backtrace:" (ert-test-name test))
                (with-temp-buffer
-                 (ert--print-backtrace (ert-test-result-with-condition-backtrace
-                                        result))
+                 (ert--print-backtrace
+                  (ert-test-result-with-condition-backtrace result)
+                  nil)
                  (goto-char (point-min))
                  (while (not (eobp))
                    (let ((start (point))
@@ -2394,8 +2371,7 @@ ert-results-pop-to-backtrace-for-test-at-point
            ;; Use unibyte because `debugger-setup-buffer' also does so.
            (set-buffer-multibyte nil)
            (setq truncate-lines t)
-           (ert--print-backtrace backtrace)
-           (debugger-make-xrefs)
+           (ert--print-backtrace backtrace t)
            (goto-char (point-min))
            (insert (substitute-command-keys "Backtrace for test `"))
            (ert-insert-test-name-button (ert-test-name test))
diff --git a/test/lisp/emacs-lisp/ert-tests.el b/test/lisp/emacs-lisp/ert-tests.el
index fc5790c365..317838b250 100644
--- a/test/lisp/emacs-lisp/ert-tests.el
+++ b/test/lisp/emacs-lisp/ert-tests.el
@@ -367,12 +367,8 @@ ert-test--which-file
          (test (make-ert-test :body test-body))
          (result (ert-run-test test)))
     (should (ert-test-failed-p result))
-    (with-temp-buffer
-      (ert--print-backtrace (ert-test-failed-backtrace result))
-      (goto-char (point-min))
-      (end-of-line)
-      (let ((first-line (buffer-substring-no-properties (point-min) (point))))
-        (should (equal first-line (format "  %S()" test-body)))))))
+    (should (eq (nth 1 (car (ert-test-failed-backtrace result)))
+                'signal))))
 
 (ert-deftest ert-test-messages ()
   :tags '(:causes-redisplay)
-- 
2.11.1


[-- Attachment #7: patch --]
[-- Type: text/plain, Size: 4592 bytes --]

From 85bcf94d7bcf55e718f80fb22a3e0c8e351b0787 Mon Sep 17 00:00:00 2001
From: Noam Postavsky <npostavs@gmail.com>
Date: Sat, 11 Feb 2017 18:13:54 -0500
Subject: [PATCH v1 3/5] Escape control characters in backtraces (Bug#6991)

* src/print.c (syms_of_print): Add new variable,
print-escape-control-characters.
(print_object): Print control characters with octal escape codes when
print-escape-control-characters is true.
* lisp/subr.el (backtrace):
* lisp/emacs-lisp/debug.el (debugger-setup-buffer): Bind
`print-escape-control-characters' to t.
---
 lisp/emacs-lisp/debug.el |  2 +-
 lisp/subr.el             |  3 ++-
 src/print.c              | 40 +++++++++++++++++++++++++++++-----------
 3 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/lisp/emacs-lisp/debug.el b/lisp/emacs-lisp/debug.el
index f24f62fa71..42e44761de 100644
--- a/lisp/emacs-lisp/debug.el
+++ b/lisp/emacs-lisp/debug.el
@@ -361,7 +361,7 @@ debugger-setup-buffer
                    ;; advice's `apply' frame.
                    (if (eq (car args) 'debug) 3 1)
                    (backtrace-frames 'debug)))
-          (print-escape-newlines t)
+          (print-escape-control-characters t)
           (print-level 8)
           (print-length 50))
       (when (eq (car args) 'exit)
diff --git a/lisp/subr.el b/lisp/subr.el
index a204577ddf..6e4779ef1f 100644
--- a/lisp/subr.el
+++ b/lisp/subr.el
@@ -4472,7 +4472,8 @@ backtrace--print-frame
 (defun backtrace ()
   "Print a trace of Lisp function calls currently active.
 Output stream used is value of `standard-output'."
-  (let ((print-level (or print-level 8)))
+  (let ((print-level (or print-level 8))
+        (print-escape-control-characters t))
     (mapbacktrace #'backtrace--print-frame 'backtrace)))
 
 (defun backtrace-frames (&optional base)
diff --git a/src/print.c b/src/print.c
index db3d00f51f..1d3958a39e 100644
--- a/src/print.c
+++ b/src/print.c
@@ -1476,17 +1476,29 @@ print_object (Lisp_Object obj, Lisp_Object printcharfun, bool escapeflag)
 		  /* If we just had a hex escape, and this character
 		     could be taken as part of it,
 		     output `\ ' to prevent that.  */
-		  if (need_nonhex && c_isxdigit (c))
-		    print_c_string ("\\ ", printcharfun);
-
-		  if (c == '\n' && print_escape_newlines
-		      ? (c = 'n', true)
-		      : c == '\f' && print_escape_newlines
-		      ? (c = 'f', true)
-		      : c == '\"' || c == '\\')
-		    printchar ('\\', printcharfun);
-
-		  printchar (c, printcharfun);
+                  if (c_isxdigit (c))
+                    {
+                      if (need_nonhex)
+                        print_c_string ("\\ ", printcharfun);
+                      printchar (c, printcharfun);
+                    }
+                  else if (c == '\n' && print_escape_newlines
+                           ? (c = 'n', true)
+                           : c == '\f' && print_escape_newlines
+                           ? (c = 'f', true)
+                           : c == '\"' || c == '\\')
+                    {
+                      printchar ('\\', printcharfun);
+                      printchar (c, printcharfun);
+                    }
+                  else if (print_escape_control_characters && c_iscntrl (c))
+                    {
+                      char outbuf[5];
+                      int len = sprintf (outbuf, "\\%03o", c + 0u);
+                      strout (outbuf, len, len, printcharfun);
+                    }
+                  else
+                    printchar (c, printcharfun);
 		  need_nonhex = false;
 		}
 	    }
@@ -2264,6 +2276,11 @@ syms_of_print (void)
 Also print formfeeds as `\\f'.  */);
   print_escape_newlines = 0;
 
+  DEFVAR_BOOL ("print-escape-control-characters", print_escape_control_characters,
+	       doc: /* Non-nil means print control characters in strings as `\\OOO'.
+\(OOO is the octal representation of the character code.)*/);
+  print_escape_control_characters = 0;
+
   DEFVAR_BOOL ("print-escape-nonascii", print_escape_nonascii,
 	       doc: /* Non-nil means print unibyte non-ASCII chars in strings as \\OOO.
 \(OOO is the octal representation of the character code.)
@@ -2352,6 +2369,7 @@ representation) and `#N#' in place of each subsequent occurrence,
   DEFSYM (Qprint_escape_newlines, "print-escape-newlines");
   DEFSYM (Qprint_escape_multibyte, "print-escape-multibyte");
   DEFSYM (Qprint_escape_nonascii, "print-escape-nonascii");
+  DEFSYM (Qprint_escape_control_characters, "print-escape-control-characters");
 
   print_prune_charset_plist = Qnil;
   staticpro (&print_prune_charset_plist);
-- 
2.11.1


[-- Attachment #8: patch --]
[-- Type: text/plain, Size: 1967 bytes --]

From bbec49e74acae529ddac91a23a9c81af53454e7e Mon Sep 17 00:00:00 2001
From: Noam Postavsky <npostavs@gmail.com>
Date: Sat, 11 Feb 2017 19:00:19 -0500
Subject: [PATCH v1 4/5] Hide byte code in backtraces (Bug#6991)

* lisp/emacs-lisp/debug.el (help-byte-code): New button type, calls
`disassemble' in its action.
(debugger-insert-backtrace): Cover byte code with help-byte-code
button.
---
 lisp/emacs-lisp/debug.el | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/lisp/emacs-lisp/debug.el b/lisp/emacs-lisp/debug.el
index 42e44761de..94b683dcb9 100644
--- a/lisp/emacs-lisp/debug.el
+++ b/lisp/emacs-lisp/debug.el
@@ -265,6 +265,12 @@ debug
       debugger-value)))
 \f
 
+(define-button-type 'help-byte-code
+  'follow-link t
+  'action (lambda (button)
+            (disassemble (button-get button 'byte-code-function)))
+  'help-echo (purecopy "mouse-2, RET: disassemble this function"))
+
 (defun debugger-insert-backtrace (frames do-xrefs)
   "Format and insert the backtrace FRAMES at point.
 Make functions into cross-reference buttons if DO-XREFS is non-nil."
@@ -286,7 +292,15 @@ debugger-insert-backtrace
         (when fun-file
           (make-text-button fun-pt (+ fun-pt (length (symbol-name fun)))
                             :type 'help-function-def
-                            'help-args (list fun fun-file))))
+                            'help-args (list fun fun-file)))
+        (when (byte-code-function-p fun)
+          (goto-char fun-pt)
+          (forward-sexp)
+          (make-text-button fun-pt (point)
+                            :type 'help-byte-code
+                            'byte-code-function fun
+                            'display "#<compiled function>")
+          (end-of-line)))
       ;; After any frame that uses eval-buffer, insert a line that
       ;; states the buffer position it's reading at.
       (when (and eval-buffers (memq fun '(eval-buffer eval-region)))
-- 
2.11.1


[-- Attachment #9: patch --]
[-- Type: text/plain, Size: 802 bytes --]

From 4f49294621e64a20462c369499d128523bf4a5de Mon Sep 17 00:00:00 2001
From: Noam Postavsky <npostavs@gmail.com>
Date: Sat, 11 Feb 2017 19:47:55 -0500
Subject: [PATCH v1 5/5] Escape NUL bytes in X selections (Bug#6991)

* lisp/select.el (xselect--encode-string): Replace NUL bytes with
"\0".
---
 lisp/select.el | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lisp/select.el b/lisp/select.el
index a4a79255a9..705d6469d3 100644
--- a/lisp/select.el
+++ b/lisp/select.el
@@ -475,6 +475,9 @@ xselect--encode-string
 	   (t
 	    (error "Unknown selection type: %S" type)))))
 
+      ;; Most programs are unable to handle NUL bytes in strings.
+      (setq str (replace-regexp-in-string "\0" "\\0" str t t))
+
       (setq next-selection-coding-system nil)
       (cons type str))))
 
-- 
2.11.1