bug#51940: 29.0.50; [PATCH] Fontification and indentation in M-x shell and ielm

[miha--- via "Bug reports for GNU Emacs, the Swiss army knife of text editors" <bug-gnu-emacs@gnu.org>]

[-- Attachment #1.1: Type: text/plain, Size: 663 bytes --]

Please find the attached patches which enable fontification and
indentation in M-x shell and ielm. The main fontification and
indentation machinery is implemented in comint.el and should be general
enough for use by other comint derived modes. It works by making an
indirect buffer, putting it in a suitable major mode (sh-mode in
M-x shell buffers, for example) and fontifying and indenting user input
that buffer.

The fourth patch also implements highlighting of non-existent or
misspelled commands in M-x shell, which I find particularly useful to
immediately spot typos. This functionality is disabled by default
because it sacrifices a bit of performance.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: 0001-Implement-a-general-input-fontification-mechanism-fo.patch --]
[-- Type: text/x-patch, Size: 10703 bytes --]

From 6afffaecf1c73f43c289a749c33cce0ab8dd727f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miha=20Rihtar=C5=A1i=C4=8D?= <miha@kamnitnik.top>
Date: Mon, 15 Nov 2021 22:51:51 +0100
Subject: [PATCH 1/5] Implement a general input fontification mechanism for
 comint modes

* lisp/comint.el
(comint-indirect-setup-function): New variable for comint derived
major modes to customize.
(comint-fl-mode): New minor mode that fontifies input text through an
indirect buffer.
(comint-indirect-setup-hook):
(comint--indirect-buffer):
(comint--fl-saved-jit-lock-contextually):
(comint--fl-on):
(comint--fl-off):
(comint--fl-ppss-flush-indirect):
(comint--fl-fontify-region):
(comint--intersect-regions):
(comint-indirect-buffer):
(comint--indirect-cleanup): New functions and buffer-local variables.
---
 lisp/comint.el | 223 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 223 insertions(+)

diff --git a/lisp/comint.el b/lisp/comint.el
index 544f0b8b82..54f5bcd7f7 100644
--- a/lisp/comint.el
+++ b/lisp/comint.el
@@ -1931,6 +1931,7 @@ comint-send-input
               (when comint-highlight-input
                 (add-text-properties beg end
                                      '( font-lock-face comint-highlight-input
+                                        comint--fl-inhibit-fontification t
                                         front-sticky t )))
               (unless comint-use-prompt-regexp
                 ;; Give old user input a field property of `input', to
@@ -4042,6 +4043,228 @@ comint-osc-hyperlink-handler
              (cons (point-marker) (match-string-no-properties 1 text)))))
 
 \f
+;;; Input fontification through an indirect buffer
+;;============================================================================
+;;
+;; Modes derived from `comint-mode' can set up fontification input
+;; text with the help of an indirect buffer whose major mode and
+;; font-lock settings are set accordingly.
+
+(defvar-local comint-indirect-setup-function nil
+  "Function to set up an indirect buffer.
+This function is called by `comint-indirect-buffer' with zero
+arguments after making a comint indirect buffer.  It is usually
+set to a major-mode command whose font-locking is desired for
+input text.  In order to prevent possible mode hooks from
+running, the variable `delay-mode-hooks' is set to t prior to
+calling this function and `change-major-mode-hook' along with
+`after-change-major-mode-hook' are bound to nil.")
+
+(defcustom comint-indirect-setup-hook nil
+  "Hook run after setting up the indirect fontification buffer.
+It is run after the indirect buffer is set up for fontification
+of input regions."
+  :group 'comint
+  :type 'hook
+  :version "29.1")
+
+(defvar-local comint--indirect-buffer nil
+  "Indirect buffer used for input fontification.")
+
+(defvar-local comint--fl-saved-jit-lock-contextually nil)
+
+(define-minor-mode comint-fl-mode
+  "Enable input fontification.
+Input fontification happens through an indirect buffer created
+with `comint-indirect-buffer', which see.
+
+This function signals an error if `comint-use-prompt-regexp' is
+non-nil because input fontification isn't compatible with this
+setting."
+  :lighter nil
+  (if comint-fl-mode
+      (let ((success nil))
+        (unwind-protect
+            (progn
+              (comint--fl-on)
+              (setq success t))
+          (unless success
+            (setq comint-fl-mode nil)
+            (comint--fl-off))))
+    (comint--fl-off)))
+
+(defun comint--fl-on ()
+  "Enable input fontification for the current buffer."
+  (comint--fl-off)
+
+  (when comint-use-prompt-regexp
+    (error
+     "Input fontification is incompatible with `comint-use-prompt-regexp'"))
+
+  (add-function :around (local 'font-lock-fontify-region-function)
+                #'comint--fl-fontify-region)
+  ;; `before-change-functions' are only run in the current buffer and
+  ;; not in its indirect buffers, which means that we must manually
+  ;; flush ppss cache
+  (add-hook 'before-change-functions
+            #'comint--fl-ppss-flush-indirect 99 t)
+
+  ;; Set up contextual fontification
+  (unless (booleanp jit-lock-contextually)
+    (setq comint--fl-saved-jit-lock-contextually
+          jit-lock-contextually)
+    (setq-local jit-lock-contextually t)
+    (when jit-lock-mode
+      (jit-lock-mode t))))
+
+(defun comint--fl-off ()
+  "Disable input fontification for the current buffer."
+  (remove-function (local 'font-lock-fontify-region-function)
+                   #'comint--fl-fontify-region)
+  (remove-hook 'before-change-functions
+               #'comint--fl-ppss-flush-indirect t)
+
+  ;; Reset contextual fontification
+  (when comint--fl-saved-jit-lock-contextually
+    (setq-local jit-lock-contextually
+                comint--fl-saved-jit-lock-contextually)
+    (setq comint--fl-saved-jit-lock-contextually nil)
+    (when jit-lock-mode
+      (jit-lock-mode t)))
+
+  (font-lock-flush))
+
+(defun comint--fl-ppss-flush-indirect (beg &rest rest)
+  (when-let ((buf (comint-indirect-buffer t)))
+    (with-current-buffer buf
+      (when (memq #'syntax-ppss-flush-cache before-change-functions)
+        (apply #'syntax-ppss-flush-cache beg rest)))))
+
+(defun comint--fl-fontify-region (fun beg end verbose)
+  "Around advice for `font-lock-fontify-region-function'.
+Fontify region between BEG and END.  First, highlight it using
+FUN.  Then highlight only the input text in the region with the
+help of an indirect buffer.  VERBOSE is passed to the
+fontify-region functions.  Skip fontification of input regions
+with non-nil `comint--fl-inhibit-fontification' text property."
+  (pcase (funcall fun beg end verbose)
+    (`(jit-lock-bounds ,beg1 . ,end1)
+     (setq beg beg1 end end1)))
+  (pcase
+      (let ((min (point-min))
+            (max (point-max)))
+        (with-current-buffer (comint-indirect-buffer)
+          (narrow-to-region min max)
+          (comint--intersect-regions
+           nil (lambda (beg end)
+                 (unless (get-text-property
+                          beg 'comint--fl-inhibit-fontification)
+                   (font-lock-fontify-region beg end verbose)))
+           beg end)))
+    (`((jit-lock-bounds ,beg1 . ,_) . (jit-lock-bounds ,_ . ,end1))
+     (setq beg (min beg beg1))
+     (setq end (max end end1))))
+
+  `(jit-lock-bounds ,beg . ,end))
+
+(defun comint--intersect-regions (fun-output fun-input beg end)
+  "Run functions for suitable regions.
+Divide the region specified by BEG and END into smaller regions
+that cover either process output (its 'field property is 'output)
+or input (all remaining text).  Interchangeably call FUN-OUTPUT on
+each output region, and FUN-INPUT on each input region.
+
+FUN-OUTPUT and FUN-INPUT are passed two arguments, the beginning
+and end of the smaller region.  Before calling each function,
+narrow the buffer to the surrounding process output or input.  You
+can also pass nil as either function to skip its corresponding
+regions.
+
+Return a cons cell of return values of the first and last
+function called, or nil, if no function was called (if BEG = END)."
+  (let ((beg1 beg)
+        (end1 (copy-marker nil t))
+        (return-beg nil) (return-end nil)
+        (is-output (eq (get-text-property beg 'field) 'output)))
+    (setq end (copy-marker end t))
+
+    (while (< beg1 end)
+      (set-marker
+       end1 (or (if is-output
+                    (text-property-not-all beg1 end 'field 'output)
+                  (text-property-any beg1 end 'field 'output))
+                end))
+      (when-let ((fun (if is-output fun-output fun-input)))
+        (save-restriction
+          (let ((beg2 beg1)
+                (end2 end1))
+            (when (= beg2 beg)
+              (setq beg2 (field-beginning beg2)))
+            (when (= end2 end)
+              (setq end2 (field-end end2)))
+            ;; Narrow to the whole field surrounding the region
+            (narrow-to-region beg2 end2))
+          (setq return-end (list (funcall fun beg1
+                                          (marker-position end1)))))
+        (unless return-beg
+          (setq return-beg return-end)))
+      (setq beg1 (marker-position end1))
+      (setq is-output (not is-output)))
+
+    (set-marker end nil)
+    (set-marker end1 nil)
+    (when return-beg
+      (cons (car return-beg) (car return-end)))))
+
+(defun comint-indirect-buffer (&optional no-create)
+  "Return an indirect buffer.
+If an indirect buffer for the current buffer already exists,
+return it, otherwise create it first and set it up by calling
+`comint-indirect-setup-function' with zero arguments, turning on
+font-lock, and running `comint-indirect-setup-hook'.  This setup
+happens with `delay-mode-hooks' set to t in order to prevent
+possible SETUP-FUN's mode hooks from running.
+
+If an indirect buffer doesn't exist and NO-CREATE is non-nil,
+return nil."
+  (or
+   comint--indirect-buffer
+   (unless no-create
+     (let ((setup-hook
+            (if (local-variable-p 'comint-indirect-setup-hook)
+                (list comint-indirect-setup-hook)))
+           (setup-fun comint-indirect-setup-function))
+
+       (add-hook 'change-major-mode-hook #'comint--indirect-cleanup
+                 nil t)
+
+       (with-current-buffer
+           (setq comint--indirect-buffer
+                 (make-indirect-buffer
+                  (current-buffer)
+                  (generate-new-buffer-name
+                   (concat " " (buffer-name) "-comint-indirect"))))
+         (setq-local delay-mode-hooks t)
+         (when setup-fun
+           (let ((change-major-mode-hook nil)
+                 (after-change-major-mode-hook nil))
+             (funcall setup-fun)))
+         (setq-local font-lock-dont-widen t)
+         (setq-local font-lock-support-mode nil)
+         (font-lock-mode)
+         (when setup-hook
+           (setq-local comint-indirect-setup-hook
+                       (car setup-hook)))
+         (run-hooks 'comint-indirect-setup-hook))
+       comint--indirect-buffer))))
+
+(defun comint--indirect-cleanup ()
+  (when comint--indirect-buffer
+    (kill-buffer comint--indirect-buffer)
+    (setq comint--indirect-buffer nil)))
+
+\f
+
 ;;; Converting process modes to use comint mode
 ;;============================================================================
 ;; The code in the Emacs 19 distribution has all been modified to use comint
-- 
2.34.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.3: 0002-Input-fontification-for-M-x-shell.patch --]
[-- Type: text/x-patch, Size: 3300 bytes --]

From ca566e8813cba89236c97f9f35e3fe78a947fd11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miha=20Rihtar=C5=A1i=C4=8D?= <miha@kamnitnik.top>
Date: Mon, 15 Nov 2021 23:13:03 +0100
Subject: [PATCH 2/5] Input fontification for M-x shell

* lisp/shell.el (shell-comint-fl-enable): New user option to control
input fontification.
(shell-comint-fl-mode-setup-hook): New hook.
(shell-mode): Set up and enable input fontification.
---
 lisp/shell.el | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/lisp/shell.el b/lisp/shell.el
index 370532ea46..347b3e04b9 100644
--- a/lisp/shell.el
+++ b/lisp/shell.el
@@ -306,6 +306,22 @@ shell-input-autoexpand
 		 (const :tag "on" t))
   :group 'shell)
 
+(defcustom shell-comint-fl-enable t
+  "Enable highlighting of input in shell buffers.
+This variable only has effect when the shell is started.  Use the
+command `comint-fl-mode' to toggle highlighting of input."
+  :type 'boolean
+  :group 'shell
+  :safe 'booleanp
+  :version "29.1")
+
+(defcustom shell-comint-fl-mode-setup-hook nil
+  "Hook run after setting up an indirect shell fontification buffer."
+  :type 'boolean
+  :group 'shell
+  :safe 'booleanp
+  :version "29.1")
+
 (defvar shell-dirstack nil
   "List of directories saved by pushd in this buffer's shell.
 Thus, this does not include the shell's current directory.")
@@ -516,6 +532,8 @@ shell-completion-vars
 
 (put 'shell-mode 'mode-class 'special)
 
+(defvar sh-shell-file)
+
 (define-derived-mode shell-mode comint-mode "Shell"
   "Major mode for interacting with an inferior shell.
 \\<shell-mode-map>
@@ -572,6 +590,11 @@ shell-mode
 control whether input and output cause the window to scroll to the end of the
 buffer."
   :interactive nil
+  :after-hook
+  (and (null comint-use-prompt-regexp)
+       shell-comint-fl-enable
+       (comint-fl-mode))
+
   (setq comint-prompt-regexp shell-prompt-pattern)
   (shell-completion-vars)
   (setq-local paragraph-separate "\\'")
@@ -591,6 +614,19 @@ shell-mode
   (setq-local ansi-color-apply-face-function #'shell-apply-ansi-color)
   (shell-reapply-ansi-color)
 
+  (add-hook 'comint-indirect-setup-hook
+            #'shell-comint-fl-mode-setup-hook 'append t)
+  (setq comint-indirect-setup-function
+        (let ((shell shell--start-prog))
+          (lambda ()
+            (require 'sh-script)
+            (cl-letf
+                (((default-value 'sh-shell-file)
+                  (or shell sh-shell-file))
+                 (inhibit-message t)
+                 (message-log-max nil))
+              (sh-mode)))))
+
   ;; This is not really correct, since the shell buffer does not really
   ;; edit this directory.  But it is useful in the buffer list and menus.
   (setq list-buffers-directory (expand-file-name default-directory))
@@ -645,6 +681,10 @@ shell-mode
                     ": [[:digit:]]+:[[:digit:]]+;")))
     (comint-read-input-ring t)))
 
+(defun shell-comint-fl-mode-setup-hook ()
+  "Run `shell-comint-fl-mode-setup-hook'."
+  (run-hooks 'shell-comint-fl-mode-setup-hook))
+
 (defun shell-apply-ansi-color (beg end face)
   "Apply FACE as the ansi-color face for the text between BEG and END."
   (when face
-- 
2.34.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.4: 0003-Input-fontification-for-M-x-ielm.patch --]
[-- Type: text/x-patch, Size: 2453 bytes --]

From c368fd4f75ac43c38e493e86f2abb16a0c378bbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miha=20Rihtar=C5=A1i=C4=8D?= <miha@kamnitnik.top>
Date: Mon, 15 Nov 2021 23:16:23 +0100
Subject: [PATCH 3/5] Input fontification for M-x ielm

* lisp/ielm.el (ielm-comint-fl-enable): New user option to control
input fontification.
(ielm-indirect-setup-hook): New hook.
(inferior-emacs-lisp-mode): Set up and enable input fontification.
---
 lisp/ielm.el | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/lisp/ielm.el b/lisp/ielm.el
index 39820a893a..d792bd871d 100644
--- a/lisp/ielm.el
+++ b/lisp/ielm.el
@@ -472,6 +472,26 @@ ielm-set-pm
   ;; Set the process mark in the current buffer to POS.
   (set-marker (process-mark (get-buffer-process (current-buffer))) pos))
 
+;;; Input fontification
+
+(defcustom ielm-comint-fl-enable t
+  "Enable highlighting of input in ielm buffers.
+This variable only has effect when the shell is started.  Use the
+command `comint-fl-mode' to toggle highlighting of input."
+  :type 'boolean
+  :safe 'booleanp
+  :version "29.1")
+
+(defcustom ielm-indirect-setup-hook nil
+  "Hook run after setting up an indirect ielm fontification buffer."
+  :type 'boolean
+  :safe 'booleanp
+  :version "29.1")
+
+(defun ielm-indirect-setup-hook ()
+  "Run `ielm-indirect-setup-hook'."
+  (run-hooks 'ielm-indirect-setup-hook))
+
 ;;; Major mode
 
 (define-derived-mode inferior-emacs-lisp-mode comint-mode "IELM"
@@ -526,6 +546,10 @@ inferior-emacs-lisp-mode
 Customized bindings may be defined in `ielm-map', which currently contains:
 \\{ielm-map}"
   :syntax-table emacs-lisp-mode-syntax-table
+  :after-hook
+  (and (null comint-use-prompt-regexp)
+       ielm-comint-fl-enable
+       (comint-fl-mode))
 
   (setq comint-prompt-regexp (concat "^" (regexp-quote ielm-prompt)))
   (setq-local paragraph-separate "\\'")
@@ -564,6 +588,10 @@ inferior-emacs-lisp-mode
   (setq-local font-lock-defaults
        '(ielm-font-lock-keywords nil nil ((?: . "w") (?- . "w") (?* . "w"))))
 
+  (add-hook 'comint-indirect-setup-hook
+            #'ielm-indirect-setup-hook 'append t)
+  (setq comint-indirect-setup-function #'emacs-lisp-mode)
+
   ;; A dummy process to keep comint happy. It will never get any input
   (unless (comint-check-proc (current-buffer))
     ;; Was cat, but on non-Unix platforms that might not exist, so
-- 
2.34.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.5: 0004-Highlight-non-existent-commands-in-M-x-shell.patch --]
[-- Type: text/x-patch, Size: 11066 bytes --]

From 9f2ab28d4c9b044de707acbfb0d5f4558f378738 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miha=20Rihtar=C5=A1i=C4=8D?= <miha@kamnitnik.top>
Date: Mon, 15 Nov 2021 23:21:03 +0100
Subject: [PATCH 4/5] Highlight non-existent commands in M-x shell

* lisp/shell.el (shell-mode): Enable highlighting of non-existent
commands if requested.
(shell-highlight-undef-aliases):
(shell-highlight-undef-remote-file-name-inhibit-cache): New user
options.
(shell-highlight-undef-mode): New minor mode.
(shell-highlight-undef-defined-face):
(shell-highlight-undef-undefined-face):
(shell-highlight-undef-alias-face): New faces.
(shell-highlight-undef--exec-cache):
(shell-highlight-undef--face):
(shell-highlight-undef-keywords):
(shell-highlight-undef-regexp):
(shell-highlight-undef--executable-find):
(shell-highlight-undef-matcher):
(shell-highlight-undef--indirect):
(shell-highlight--setup):
(shell-highlight-undef-reset-mode): New functions and buffer local
variables.
---
 lisp/shell.el | 227 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 224 insertions(+), 3 deletions(-)

diff --git a/lisp/shell.el b/lisp/shell.el
index 347b3e04b9..3c008ec508 100644
--- a/lisp/shell.el
+++ b/lisp/shell.el
@@ -322,6 +322,16 @@ shell-comint-fl-mode-setup-hook
   :safe 'booleanp
   :version "29.1")
 
+(defcustom shell-highlight-undef-enable nil
+  "Enable highlighting of undefined commands in shell buffers.
+This variable only has effect when the shell is started.  Use the
+command `shell-highlight-undef-mode' to toggle highlighting of
+undefined commands."
+  :type 'boolean
+  :group 'shell
+  :safe 'booleanp
+  :version "29.1")
+
 (defvar shell-dirstack nil
   "List of directories saved by pushd in this buffer's shell.
 Thus, this does not include the shell's current directory.")
@@ -591,9 +601,11 @@ shell-mode
 buffer."
   :interactive nil
   :after-hook
-  (and (null comint-use-prompt-regexp)
-       shell-comint-fl-enable
-       (comint-fl-mode))
+  (unless comint-use-prompt-regexp
+    (if shell-comint-fl-enable
+        (comint-fl-mode))
+    (if shell-highlight-undef-enable
+        (shell-highlight-undef-mode)))
 
   (setq comint-prompt-regexp shell-prompt-pattern)
   (shell-completion-vars)
@@ -1471,6 +1483,215 @@ shell-narrow-to-prompt
            (point-max)
          (shell--prompt-begin-position))))))
 
+;;; Highlight undefined commands
+;;
+;; To highlight non-existent commands, customize
+;; `shell-highlight-undef-enable' to t.  To highlight some commands as
+;; aliases, add them to `shell-highlight-undef-aliases'.
+
+(defcustom shell-highlight-undef-aliases nil
+  "List commands to highlight as a command alias."
+  :group 'shell
+  :type '(repeat string)
+  :version "29.1")
+
+(defface shell-highlight-undef-defined-face
+  '((t :inherit 'font-lock-function-name-face))
+  "Face used for existent commands."
+  :group 'shell
+  :version "29.1")
+
+(defface shell-highlight-undef-undefined-face
+  '((t :inherit 'font-lock-warning-face))
+  "Face used for non-existent commands."
+  :group 'shell
+  :version "29.1")
+
+(defface shell-highlight-undef-alias-face
+  '((t :inherit 'font-lock-variable-name-face))
+  "Face used command aliases."
+  :group 'shell
+  :version "29.1")
+
+(defcustom shell-highlight-undef-remote-file-name-inhibit-cache nil
+  "Whether to use cache for finding remote executables.
+See `remote-file-name-inhibit-cache' for description."
+  :group 'faces
+  :type '(choice
+          (const :tag "Do not inhibit file name cache" nil)
+          (const :tag "Do not use file name cache" t)
+          (integer :tag "Do not use file name cache"
+                   :format "Do not use file name cache older than %v seconds"
+                   :value 10))
+  :version "29.1")
+
+(defvar shell--highlight-undef-exec-cache nil
+  "Cache of executable files found in `exec-path'.
+An alist, whose elements are of the form
+\(REMOTE TIME EXECUTABLES), where REMOTE is a string, returned by
+`file-remote-p', TIME is the return value of `float-time' end
+EXECUTABLES is a hash table with keys being the base-names of
+executable files.
+
+Cache expiry is controlled by the user option
+`remote-file-name-inhibit-cache'.")
+
+(defvar shell--highlight-undef-face 'shell-highlight-undef-defined-face)
+
+(defvar shell-highlight-undef-keywords
+  `((,#'shell-highlight-undef-matcher 6 shell--highlight-undef-face)))
+
+(defvar-local shell-highlight-undef-regexp regexp-unmatchable)
+
+(defun shell--highlight-undef-executable-find (command)
+  "Return non-nil if COMMAND is found in `exec-path'.
+Similar to `executable-find', but use cache stored in
+`shell--highlight-undef-exec-cache'."
+  (let ((remote (file-remote-p default-directory))
+        as ret found-in-cache delta-time)
+    (if (null remote)
+        (executable-find command)
+
+      (setq delta-time
+            shell-highlight-undef-remote-file-name-inhibit-cache)
+
+      (pcase (setq as (assoc remote shell--highlight-undef-exec-cache))
+        (`(,_ ,time ,hash)
+         (when (pcase delta-time
+                 ((pred numberp) (<= (float-time) (+ time delta-time)))
+                 ('t nil)
+                 ('nil t))
+           (setq ret (gethash command hash))
+           (setq found-in-cache t)))
+        (_ (setq as (list remote 0 (make-hash-table :test #'equal)))
+           (push as shell--highlight-undef-exec-cache)))
+
+      (if found-in-cache
+          ret
+        ;; Build cache
+        (setcar (cdr as) (float-time))
+        (let ((hash (clrhash (caddr as))))
+          (dolist (dir (exec-path))
+            (pcase-dolist (`(,f . ,attr)
+                           (condition-case nil
+                               (directory-files-and-attributes
+                                (concat remote dir) nil nil 'nosort 'integer)
+                             (file-error nil)))
+              ;; Approximation.  Assume every non-directory file in $PATH is an
+              ;; executable.  Alternatively, we could check
+              ;; `file-executable-p', but doing so for every file in $PATH is
+              ;; slow on remote machines.
+              (unless (eq t (file-attribute-type attr))
+                (puthash f t hash))))
+          (gethash command hash))))))
+
+(defun shell-highlight-undef-matcher (end)
+  "Matcher used to highlight commands up to END."
+  (when (re-search-forward shell-highlight-undef-regexp end t)
+    (save-match-data
+      (let ((cmd (match-string 6))
+            (beg (match-beginning 6)))
+        (setq shell--highlight-undef-face
+              (let* ((buf (buffer-base-buffer))
+                     (default-directory
+                      (if buf (buffer-local-value 'default-directory buf)
+                        default-directory)))
+                (cond
+                 ;; Don't highlight command output.  Mostly useful if
+                 ;; `shell-comint-fl-mode' is disabled.
+                 ((text-property-any beg (point) 'field 'output)
+                  nil)
+                 ((member cmd shell-highlight-undef-aliases)
+                  'shell-highlight-undef-alias-face)
+                 ;; Check if it contains a directory separator
+                 ((file-name-directory cmd)
+                  (when (file-name-absolute-p cmd)
+                    (setq cmd (concat
+                               (or (bound-and-true-p comint-file-name-prefix)
+                                   (file-remote-p default-directory))
+                               cmd)))
+                  (if (or (file-executable-p cmd)
+                          (file-directory-p cmd))
+                      'shell-highlight-undef-defined-face
+                    'shell-highlight-undef-undefined-face))
+                 ((shell--highlight-undef-executable-find cmd)
+                  'shell-highlight-undef-defined-face)
+                 (t 'shell-highlight-undef-undefined-face))))))
+    t))
+
+(defvar-local shell--highlight-undef-indirect nil
+  "Whether we are using `shell-comint-fl-mode' indirect buffer.")
+
+(declare-function sh-feature "sh-script" (alist &optional function))
+(defvar sh-leading-keywords)
+(defvar sh-other-keywords)
+
+(define-minor-mode shell-highlight-undef-mode
+  "Highlight undefined shell commands and aliases."
+  :init-value nil
+  (if shell--highlight-undef-indirect
+      (progn
+        (setq shell--highlight-undef-indirect nil)
+        (when-let ((buf (comint-indirect-buffer t)))
+          (with-current-buffer buf
+            (font-lock-remove-keywords nil shell-highlight-undef-keywords))))
+    (font-lock-remove-keywords nil shell-highlight-undef-keywords))
+  (remove-hook 'comint-fl-mode-hook
+               #'shell-highlight-undef-reset-mode t)
+  (remove-hook 'comint-indirect-setup-hook #'shell--highlight-undef-setup t)
+
+  (when shell-highlight-undef-mode
+    (when comint-use-prompt-regexp
+      (setq shell-highlight-undef-mode nil)
+      (error
+       "`shell-highlight-undef-mode' is incompatible with `comint-use-prompt-regexp'"))
+
+    (require 'sh-script)
+
+    (let ((regexp
+           ;; Adapted from `sh-font-lock-keywords-1'
+           (concat
+            "\\("
+            "[;(){}`|&]"
+            (if comint-fl-mode
+                ;; `comint-fl-mode' already puts point-min on end of
+                ;; prompt
+                ""
+              (concat "\\|" comint-prompt-regexp))
+            "\\|^"
+            "\\)"
+            "[ \t]*\\(\\("
+            (regexp-opt (sh-feature sh-leading-keywords) t)
+            "[ \t]+\\)?"
+            (regexp-opt (append (sh-feature sh-leading-keywords)
+                                (sh-feature sh-other-keywords))
+                        t)
+            "[ \t]+\\)?\\_<\\(\\(?:\\s_\\|\\sw\\|/\\)+\\)\\_>")))
+      (cond (comint-fl-mode
+             (setq shell--highlight-undef-indirect t)
+             (if-let ((buf (comint-indirect-buffer t)))
+                 (with-current-buffer buf
+                   (shell--highlight-undef-setup regexp))
+               (add-hook 'comint-indirect-setup-hook
+                         (lambda ()
+                           (shell--highlight-undef-setup regexp))
+                         nil t)))
+            (t (shell--highlight-undef-setup regexp))))
+
+    (add-hook 'comint-fl-mode-hook
+              #'shell-highlight-undef-reset-mode nil t))
+
+  (font-lock-flush))
+
+(defun shell--highlight-undef-setup (regexp)
+  (font-lock-add-keywords nil shell-highlight-undef-keywords t)
+  (setq shell-highlight-undef-regexp regexp))
+
+(defun shell-highlight-undef-reset-mode ()
+  "If `shell-highlight-undef-mode' is on, turn it off and on."
+  (when shell-highlight-undef-mode
+    (shell-highlight-undef-mode 1)))
+
 (provide 'shell)
 
 ;;; shell.el ends here
-- 
2.34.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.6: 0005-Input-indentation-for-M-x-shell.patch --]
[-- Type: text/x-patch, Size: 6791 bytes --]

From cfa2293b71af972432969f784b33a29ec0ac7b44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miha=20Rihtar=C5=A1i=C4=8D?= <miha@kamnitnik.top>
Date: Mon, 15 Nov 2021 23:37:36 +0100
Subject: [PATCH 5/5] Input indentation for M-x shell

* lisp/comint.el
(comint-indent-input-line):
(comint-indent-input-line-default):
(comint-indent-input-region):
(comint-indent-input-region-default): New functions the implement a
general mechanism for input indentation through an indirect buffer in
comint derived major modes.
* lisp/shell.el (shell-mode): Set up input indentation according
sh-mode.
---
 lisp/comint.el | 95 +++++++++++++++++++++++++++++++++++++++++++++-----
 lisp/shell.el  |  4 +++
 2 files changed, 90 insertions(+), 9 deletions(-)

diff --git a/lisp/comint.el b/lisp/comint.el
index 54f5bcd7f7..cc3cf2a7fa 100644
--- a/lisp/comint.el
+++ b/lisp/comint.el
@@ -4043,22 +4043,22 @@ comint-osc-hyperlink-handler
              (cons (point-marker) (match-string-no-properties 1 text)))))
 
 \f
-;;; Input fontification through an indirect buffer
+;;; Input fontification and indentation through an indirect buffer
 ;;============================================================================
 ;;
-;; Modes derived from `comint-mode' can set up fontification input
-;; text with the help of an indirect buffer whose major mode and
-;; font-lock settings are set accordingly.
+;; Modes derived from `comint-mode' can set up fontification and
+;; indentation of input text with the help of an indirect buffer whose
+;; major mode and font-lock settings are set accordingly.
 
 (defvar-local comint-indirect-setup-function nil
   "Function to set up an indirect buffer.
 This function is called by `comint-indirect-buffer' with zero
 arguments after making a comint indirect buffer.  It is usually
-set to a major-mode command whose font-locking is desired for
-input text.  In order to prevent possible mode hooks from
-running, the variable `delay-mode-hooks' is set to t prior to
-calling this function and `change-major-mode-hook' along with
-`after-change-major-mode-hook' are bound to nil.")
+set to a major-mode command whose font-locking and indentation
+are desired for input text.  In order to prevent possible mode
+hooks from running, the variable `delay-mode-hooks' is set to t
+prior to calling this function and `change-major-mode-hook' along
+with `after-change-major-mode-hook' are bound to nil.")
 
 (defcustom comint-indirect-setup-hook nil
   "Hook run after setting up the indirect fontification buffer.
@@ -4216,6 +4216,83 @@ comint--intersect-regions
     (when return-beg
       (cons (car return-beg) (car return-end)))))
 
+(defun comint-indent-input-line (fun)
+  "Indent current input or process output line.
+If point is on output, call FUN, otherwise indent the current
+line in the indirect buffer created by `comint-indirect-buffer',
+which see."
+  (if (or comint-use-prompt-regexp
+          (eq (get-text-property (point) 'field) 'output))
+      (funcall fun)
+    (let ((point (point))
+          (min (point-min))
+          (max (point-max)))
+      (unwind-protect
+          (with-current-buffer (comint-indirect-buffer)
+            (narrow-to-region min max)
+            (goto-char point)
+            (narrow-to-region (field-beginning) (field-end))
+            (unwind-protect (funcall indent-line-function)
+              (setq point (point))))
+        (goto-char point)))))
+
+(defun comint-indent-input-region (fun start end)
+  "Indent the region between START and END.
+Output text between START and END is indented with FUN and input
+text is indented in the indirect buffer created by
+`comint-indirect-buffer', which see."
+  (if comint-use-prompt-regexp
+      (funcall fun start end)
+    (let ((opoint (copy-marker (point)))
+          final-point)
+      (unwind-protect
+          (comint--intersect-regions
+           (lambda (start end)
+             (goto-char opoint)
+             (if (= opoint (point))
+                 (unwind-protect (funcall fun start end)
+                   (setq final-point (copy-marker (point))))
+               (funcall fun start end)))
+           (lambda (start end)
+             (let ((min (point-min))
+                   (max (point-max))
+                   (final-point1 nil))
+               (unwind-protect
+                   (with-current-buffer (comint-indirect-buffer)
+                     (narrow-to-region min max)
+                     (goto-char opoint)
+                     (if (= opoint (point))
+                         (unwind-protect
+                             (funcall indent-region-function start end)
+                           (setq final-point1 (point)))
+                       (funcall indent-region-function start end)))
+                 (when final-point1
+                   (setq final-point (copy-marker final-point1))))))
+           start end)
+        (if final-point
+            (progn
+              (goto-char final-point)
+              (set-marker final-point nil))
+          (goto-char opoint))
+        (set-marker opoint nil)))))
+
+(defun comint-indent-input-line-default ()
+  "Indent current input or output line.
+If point is on output, indent the current line according to the
+default value of `indent-line-function', otherwise indent the
+current line in the indirect buffer created by
+`comint-indirect-buffer', which see."
+  (comint-indent-input-line (default-value 'indent-line-function)))
+
+(defun comint-indent-input-region-default (start end)
+  "Indent the region between START and END.
+Output text between START and END is indented according to the
+default value of `indent-region-function' and input text is
+indented in the indirect buffer created by
+`comint-indirect-buffer', which see."
+  (comint-indent-input-region (default-value 'indent-line-function)
+                              start end))
+
 (defun comint-indirect-buffer (&optional no-create)
   "Return an indirect buffer.
 If an indirect buffer for the current buffer already exists,
diff --git a/lisp/shell.el b/lisp/shell.el
index 3c008ec508..3629171fbe 100644
--- a/lisp/shell.el
+++ b/lisp/shell.el
@@ -639,6 +639,10 @@ shell-mode
                  (message-log-max nil))
               (sh-mode)))))
 
+  (setq-local indent-line-function #'comint-indent-input-line-default)
+  (setq-local indent-region-function
+              #'comint-indent-input-region-default)
+
   ;; This is not really correct, since the shell buffer does not really
   ;; edit this directory.  But it is useful in the buffer list and menus.
   (setq list-buffers-directory (expand-file-name default-directory))
-- 
2.34.0


[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 861 bytes --]