bug#22679: 25.0.91; ibuffer-do-shell-command-pipe truncate output

[Tino Calancha <f92capac@gmail.com>]

On Fri, 19 Aug 2016, Stefan Monnier wrote:

>> -   :modifier-p nil)
>> -  (shell-command-on-region
>> -   (point-min) (point-max) command))
>> +   :modifier-p nil
>> +   :opstring "Shell command executed on"
>> +   :modifier-p nil
>> +   :before (funcall #'ibuffer--before-shell-command)
>> +   :after (funcall #'ibuffer--after-shell-command))
>> +  (let ((out-buf (get-buffer "*Shell Command Output*")))
>> +    (with-current-buffer out-buf (goto-char (point-max)))
>> +    (call-process-region (point-min) (point-max) command nil 
out-buf)))
>
> I haven't looked at the rest of your patch but this part looks wrong:
> the docstring indicates that `command' is expected to be a shell command
> whereas call-process-region expects an executable.

I have corrected the call to `call-process-region': now it uses
'shell-file-name' as the executable.

I have also added one test for this bug that the new patch pass.
This test is just for documentation: i don't want to push it
to the master branch because it assumes the machine has an 'awk'
executable in 'exec-path'.

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
From b3fe9fe795d317d1798a1ad2d116ff52131f6612 Mon Sep 17 00:00:00 2001
From: Tino Calancha <tino.calancha@gmail.com>
Date: Sat, 20 Aug 2016 15:46:27 +0900
Subject: [PATCH 1/2] Fix Bug#22679

* lisp/ibuf-macs.el (define-ibuffer-op): Added optional args
'before' and 'after'.
'before' is a form to evaluate before the operation.
'after' is a form to evaluate after the operation.
* lisp/ibuf-ext.el (ibuffer--after-shell-command-pos): New defvar;
store a buffer position where to set the point in the output
buffer after operation complete.
(ibuffer--before-shell-command): New defun; erase output buffer
if 'shell-command-not-erase-buffer' is nil and
set 'ibuffer--after-shell-command-pos'.
(ibuffer--after-shell-command): New defun; set point in the
output buffer after operation complete.
(ibuffer-do-shell-command-pipe, ibuffer-do-shell-command-file):
Bind 'shell-command-not-erase-buffer' to non-nil while processing
the buffers;  use 'ibuffer--after-shell-command' to set the point
in the output buffer.
---
  lisp/ibuf-ext.el  | 87 
+++++++++++++++++++++++++++++++++++++++++++------------
  lisp/ibuf-macs.el |  8 ++++-
  2 files changed, 76 insertions(+), 19 deletions(-)

diff --git a/lisp/ibuf-ext.el b/lisp/ibuf-ext.el
index f93957e..0d79617 100644
--- a/lisp/ibuf-ext.el
+++ b/lisp/ibuf-ext.el
@@ -346,14 +346,60 @@ ibuffer-backward-filter-group
      (ibuffer-backward-filter-group 1))
    (ibuffer-forward-line 0))

+;; The value `beg-last-out' in `shell-command-not-erase-buffer'
+;; set the point at the beginning of the output of the first
+;; buffer processed.
+(defvar ibuffer--after-shell-command-pos)
+
+(defun ibuffer--before-shell-command ()
+  (let ((obuf (get-buffer-create "*Shell Command Output*"))
+        (sym shell-command-not-erase-buffer)
+        final-pos)
+    (when (buffer-live-p obuf)
+      (with-current-buffer obuf
+        (unless sym
+          (setq buffer-read-only nil)
+          (let ((inhibit-read-only t))
+            (erase-buffer)))
+        (setq final-pos
+              (cond ((or (not sym) (eq sym 'beg-last-out))
+                     (point-max))
+                    ((eq sym 'save-point)
+                     (point))))
+        (setq ibuffer--after-shell-command-pos
+              final-pos)))))
+
+(defun ibuffer--after-shell-command ()
+  (let* ((obuf (get-buffer-create "*Shell Command Output*"))
+         (pos  ibuffer--after-shell-command-pos)
+         (win  (car (get-buffer-window-list obuf))))
+    (setq ibuffer--after-shell-command-pos nil)
+    (with-current-buffer obuf
+      (unless pos (setq pos (point-max)))
+      (goto-char pos)
+      ;; Set point in the window displaying obuf, if any; otherwise
+      ;; display buf temporary in selected frame and set the point.
+      (if win
+          (set-window-point win pos)
+        (save-window-excursion
+          (let ((win (display-buffer obuf '(nil (inhibit-switch-frame . 
t)))))
+            (set-window-point win pos)))))))
+
  ;;;###autoload (autoload 'ibuffer-do-shell-command-pipe "ibuf-ext")
  (define-ibuffer-op shell-command-pipe (command)
    "Pipe the contents of each marked buffer to shell command COMMAND."
    (:interactive "sPipe to shell command: "
     :opstring "Shell command executed on"
-   :modifier-p nil)
-  (shell-command-on-region
-   (point-min) (point-max) command))
+   :modifier-p nil
+   :opstring "Shell command executed on"
+   :modifier-p nil
+   :before (funcall #'ibuffer--before-shell-command)
+   :after (funcall #'ibuffer--after-shell-command))
+  (let ((out-buf (get-buffer "*Shell Command Output*")))
+    (with-current-buffer out-buf (goto-char (point-max)))
+    (call-process-region (point-min) (point-max)
+                         shell-file-name nil out-buf nil
+                         shell-command-switch command)))

  ;;;###autoload (autoload 'ibuffer-do-shell-command-pipe-replace 
"ibuf-ext")
  (define-ibuffer-op shell-command-pipe-replace (command)
@@ -363,26 +409,31 @@ shell-command-pipe-replace
     :active-opstring "replace buffer contents in"
     :dangerous t
     :modifier-p t)
-  (with-current-buffer buf
-    (shell-command-on-region (point-min) (point-max)
-			     command nil t)))
+  (call-process-region (point-min) (point-max)
+                       shell-file-name t buf nil
+                       shell-command-switch command))

  ;;;###autoload (autoload 'ibuffer-do-shell-command-file "ibuf-ext")
  (define-ibuffer-op shell-command-file (command)
    "Run shell command COMMAND separately on files of marked buffers."
    (:interactive "sShell command on buffer's file: "
-   :opstring "Shell command executed on"
-   :modifier-p nil)
-  (shell-command (concat command " "
-			 (shell-quote-argument
-			  (or buffer-file-name
-			      (let ((file
-				     (make-temp-file
-				      (substring
-				       (buffer-name) 0
-				       (min 10 (length (buffer-name)))))))
-				(write-region nil nil file nil 0)
-				file))))))
+                :opstring "Shell command executed on"
+                :modifier-p nil
+                :before (funcall #'ibuffer--before-shell-command)
+                :after (funcall #'ibuffer--after-shell-command))
+  (let ((file (and (not (buffer-modified-p))
+                   buffer-file-name))
+        (out-buf (get-buffer "*Shell Command Output*")))
+    (when (or (null file) (not (file-exists-p file)))
+      (setq file
+            (make-temp-file
+             (substring
+              (buffer-name) 0
+              (min 10 (length (buffer-name))))))
+      (write-region nil nil file nil 0))
+    (with-current-buffer out-buf (goto-char (point-max)))
+    (call-process-shell-command (format "%s %s" command file)
+                                nil out-buf nil)))

  ;;;###autoload (autoload 'ibuffer-do-eval "ibuf-ext")
  (define-ibuffer-op eval (form)
diff --git a/lisp/ibuf-macs.el b/lisp/ibuf-macs.el
index 27e7af9..8bb05ec 100644
--- a/lisp/ibuf-macs.el
+++ b/lisp/ibuf-macs.el
@@ -169,6 +169,8 @@ ibuffer-save-marks
  				  dangerous
  				  (opstring "operated on")
  				  (active-opstring "Operate on")
+                                  before
+                                  after
  				  complex)
  				 &rest body)
    "Generate a function which operates on a buffer.
@@ -198,6 +200,8 @@ ibuffer-save-marks
  ACTIVE-OPSTRING is a string which will be displayed to the user in a
  confirmation message, in the form:
   \"Really ACTIVE-OPSTRING x buffers?\"
+BEFORE is a form to evaluate before start the operation.
+AFTER is a form to evaluate once the operation is complete.
  COMPLEX means this function is special; if COMPLEX is nil BODY
  evaluates once for each marked buffer, MBUF, with MBUF current
  and saving the point.  If COMPLEX is non-nil, BODY evaluates
@@ -206,7 +210,7 @@ ibuffer-save-marks
  marked buffer.  BODY is evaluated with `buf' bound to the
  buffer object.

-\(fn OP ARGS DOCUMENTATION (&key INTERACTIVE MARK MODIFIER-P DANGEROUS 
OPSTRING ACTIVE-OPSTRING COMPLEX) &rest BODY)"
+\(fn OP ARGS DOCUMENTATION (&key INTERACTIVE MARK MODIFIER-P DANGEROUS 
OPSTRING ACTIVE-OPSTRING BEFORE AFTER COMPLEX) &rest BODY)"
    (declare (indent 2) (doc-string 3))
    `(progn
       (defun ,(intern (concat (if (string-match "^ibuffer-do" (symbol-name 
op))
@@ -233,11 +237,13 @@ ibuffer-save-marks
  				 'ibuffer-deletion-char)
  				(_
  				 'ibuffer-marked-char))))
+         ,before ; pre-operation form.
  	 ,(let* ((finish (append
  			  '(progn)
  			  (if (eq modifier-p t)
  			      '((setq ibuffer-did-modification t))
  			    ())
+                          (and after `(,after)) ; post-operation form.
  			  `((ibuffer-redisplay t)
  			    (message ,(concat "Operation finished; " 
opstring " %s buffers") count))))
  		 (inner-body (if complex
-- 
2.8.1

From 156c63d38a78555fbdd8e04038ad96898a904019 Mon Sep 17 00:00:00 2001
From: Tino Calancha <tino.calancha@gmail.com>
Date: Sat, 20 Aug 2016 18:47:04 +0900
Subject: [PATCH 2/2] Add test for Bug#22679

* test/lisp/ibuffer-tests.el (ibuffer-test-bug22679):
---
  test/lisp/ibuffer-tests.el | 58 
++++++++++++++++++++++++++++++++++++++++++++++
  1 file changed, 58 insertions(+)

diff --git a/test/lisp/ibuffer-tests.el b/test/lisp/ibuffer-tests.el
index de281c0..5442bd2 100644
--- a/test/lisp/ibuffer-tests.el
+++ b/test/lisp/ibuffer-tests.el
@@ -30,5 +30,63 @@
      (symbol-function
       'ibuffer-mark-unsaved-buffers))))

+(ert-deftest ibuffer-test-bug22679 ()
+  "Test for http://debbugs.gnu.org/22679 ."
+  :expected-result :failed
+  (let* ((nums (generate-new-buffer "nums"))
+         (letters (generate-new-buffer "letters"))
+         (nums-file  "/tmp/nums-file")
+         (letters-file "/tmp/letters-file")
+         (str-nums "1 2 3 4 5\n6 7 8\n")
+         (str-letters "a b c d e\nf g h\n")
+         (str1 "1 2 3\n6 7 8\n")
+         (str2 "a b c\nf g h\n")
+         (buffer (get-buffer-create "*Shell Command Output*"))
+         (command "awk '{print $1 FS $2 FS $3}'")
+         (check-input-buffers (lambda ()
+                                (with-current-buffer nums
+                                  (should (string= (buffer-string) 
str1)))
+                                (with-current-buffer letters
+                                  (should (string= (buffer-string) 
str2)))))
+         (check-output-buffer (lambda ()
+                                (with-current-buffer buffer
+                                  (should (or (string= (buffer-string) 
(concat str1 str2))
+                                              (string= (buffer-string) 
(concat str2 str1)))))))
+         ;; Erase output buffer before each test.
+         (shell-command-not-erase-buffer nil)
+         ;; Don't ask for confimation to replace buffer content.
+         (ibuffer-expert t))
+    (with-current-buffer nums    (insert str-nums))
+    (with-current-buffer letters (insert str-letters))
+    (with-temp-file nums-file    (insert str-nums))
+    (with-temp-file letters-file (insert str-letters))
+    (unwind-protect
+        (save-current-buffer
+          (mapc 'find-file (list nums-file letters-file))
+          (ibuffer)
+          ;; Test ibuffer-do-shell-command-pipe[-replace]
+          (mapc (lambda (x)
+                  (ibuffer-mark-by-name-regexp (format "\\`%s\\'" x)))
+                (mapcar 'buffer-name (list nums letters)))
+          (ibuffer-do-shell-command-pipe command)
+          (funcall check-output-buffer)
+          (ibuffer-do-shell-command-pipe-replace command)
+          (funcall check-input-buffers)
+          ;; Test ibuffer-do-shell-command-file
+          (ibuffer-unmark-all-marks)
+          (mapc (lambda (x)
+                  (ibuffer-mark-by-name-regexp (format "\\`%s\\'" x)))
+                (mapcar 'buffer-name (list (get-file-buffer nums-file)
+                                           (get-file-buffer 
letters-file))))
+          (ibuffer-do-shell-command-file command)
+          (funcall check-output-buffer))
+      ;; Clean up.
+      (switch-to-buffer (current-buffer))
+      (mapc 'kill-buffer (list nums
+                               letters
+                               (get-file-buffer nums-file)
+                               (get-file-buffer letters-file)))
+      (mapc 'delete-file (list nums-file letters-file)))))
+
  (provide 'ibuffer-tests)
  ;; ibuffer-tests.el ends here
-- 
2.8.1

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

In GNU Emacs 25.1.50.14 (x86_64-pc-linux-gnu, GTK+ Version 3.20.7)
  of 2016-08-20 built on calancha-pc
Repository revision: a4ba426d25bd6a5cbe11d81b82a789b8a2c948ed