bug#40808: 27.0.91; inaccuracy in isearch-lazy-count

[Juri Linkov <juri@linkov.net>]

[-- Attachment #1: Type: text/plain, Size: 957 bytes --]

> What's the status of this bug?  Apart from the request
> by Juri for changes to be made to Org mode, is the code
> Juri prescribed for this bug fix going to be applied,
> or has it been applied?
>
> Is fixing this bug (e.g. with Juri's code) somehow
> dependent on also fixing some Org-mode code?

Fortunately, the isearch fix doesn't depend on org-mode.
So here is the complete implementation that was made possible
thanks to the suggestions in
https://lists.gnu.org/archive/html/emacs-devel/2022-06/msg01208.html

For example, searching for the word "nongnu" in etc/ORG-NEWS
when all outlines are collapsed with isearch-lazy-count=t
will show such prompts:

search-invisible = t      1/4 I-search:
search-invisible = open   1/2 (invisible 2) I-search:
search-invisible = nil    0/0 (invisible 4) I-search:

where such numbers as "1/2" show the matches that can be visited by isearch,
and "(invisible 2)" are the rest of matches unreachable by isearch.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: lazy-count-invisible.patch --]
[-- Type: text/x-diff, Size: 7164 bytes --]

diff --git a/lisp/isearch.el b/lisp/isearch.el
index db7b53c014..945dce029a 100644
--- a/lisp/isearch.el
+++ b/lisp/isearch.el
@@ -466,6 +466,12 @@ lazy-count-suffix-format
   :group 'lazy-count
   :version "27.1")
 
+(defvar lazy-count-invisible-format " (invisible %s)"
+  "Format of the number of invisible matches for the prompt.
+When invisible matches exist, their number is appended
+after the total number of matches.  Display nothing when
+this variable is nil.")
+
 \f
 ;; Define isearch help map.
 
@@ -1277,6 +1283,7 @@ isearch-mode
 
 	isearch-lazy-count-current nil
 	isearch-lazy-count-total nil
+	isearch-lazy-count-invisible nil
 
 	;; Save the original value of `minibuffer-message-timeout', and
 	;; set it to nil so that isearch's messages don't get timed out.
@@ -3529,7 +3536,12 @@ isearch-lazy-count-format
                     (- isearch-lazy-count-total
                        isearch-lazy-count-current
                        -1)))
-                (or isearch-lazy-count-total "?"))
+                (if (and isearch-lazy-count-invisible
+                         lazy-count-invisible-format)
+                    (concat (format "%s" (or isearch-lazy-count-total "?"))
+                            (format lazy-count-invisible-format
+                                    isearch-lazy-count-invisible))
+                  (or isearch-lazy-count-total "?")))
       "")))
 
 \f
@@ -3780,10 +3792,11 @@ isearch-range-invisible
     (save-excursion
       (goto-char beg)
       (let (;; can-be-opened keeps track if we can open some overlays.
-	    (can-be-opened (eq search-invisible 'open))
+	    (can-be-opened (memq search-invisible '(open can-be-opened)))
 	    ;; the list of overlays that could be opened
 	    (crt-overlays nil))
-	(when (and can-be-opened isearch-hide-immediately)
+	(when (and can-be-opened isearch-hide-immediately
+                   (not (eq search-invisible 'can-be-opened)))
 	  (isearch-close-unnecessary-overlays beg end))
 	;; If the following character is currently invisible,
 	;; skip all characters with that same `invisible' property value.
@@ -3822,9 +3835,10 @@ isearch-range-invisible
 	(if (>= (point) end)
 	    (if (and can-be-opened (consp crt-overlays))
 		(progn
-		  (setq isearch-opened-overlays
-			(append isearch-opened-overlays crt-overlays))
-		  (mapc 'isearch-open-overlay-temporary crt-overlays)
+		  (unless (eq search-invisible 'can-be-opened)
+                    (setq isearch-opened-overlays
+			  (append isearch-opened-overlays crt-overlays))
+		    (mapc 'isearch-open-overlay-temporary crt-overlays))
 		  nil)
 	      (setq isearch-hidden t)))))))
 
@@ -4008,6 +4022,7 @@ isearch-lazy-highlight-forward
 (defvar isearch-lazy-highlight-error nil)
 (defvar isearch-lazy-count-current nil)
 (defvar isearch-lazy-count-total nil)
+(defvar isearch-lazy-count-invisible nil)
 (defvar isearch-lazy-count-hash (make-hash-table))
 (defvar lazy-count-update-hook nil
   "Hook run after new lazy count results are computed.")
@@ -4086,7 +4101,8 @@ isearch-lazy-highlight-new-loop
         ;; Reset old counter before going to count new numbers
         (clrhash isearch-lazy-count-hash)
         (setq isearch-lazy-count-current nil
-              isearch-lazy-count-total nil)
+              isearch-lazy-count-total nil
+              isearch-lazy-count-invisible nil)
         ;; Delay updating the message if possible, to avoid flicker
         (when (string-equal isearch-string "")
           (when (and isearch-mode (null isearch-message-function))
@@ -4166,11 +4182,13 @@ isearch-lazy-highlight-search
 	    (isearch-regexp-lax-whitespace
 	     isearch-lazy-highlight-regexp-lax-whitespace)
 	    (isearch-forward isearch-lazy-highlight-forward)
-	    ;; Don't match invisible text unless it can be opened
-	    ;; or when counting matches and user can visit hidden matches
-	    (search-invisible (or (eq search-invisible 'open)
-				  (and isearch-lazy-count search-invisible)))
-	    (retry t)
+            ;; Count all invisible matches, but highlight only
+            ;; according to search-invisible without opening overlays.
+	    (search-invisible (or isearch-lazy-count
+                                  (if (eq search-invisible 'open)
+                                      'can-be-opened
+                                    search-invisible)))
+            (retry t)
 	    (success nil))
 	;; Use a loop like in `isearch-search'.
 	(while retry
@@ -4186,15 +4204,23 @@ isearch-lazy-highlight-search
     (error nil)))
 
 (defun isearch-lazy-highlight-match (mb me)
-  (let ((ov (make-overlay mb me)))
-    (push ov isearch-lazy-highlight-overlays)
-    ;; 1000 is higher than ediff's 100+,
-    ;; but lower than isearch main overlay's 1001
-    (overlay-put ov 'priority 1000)
-    (overlay-put ov 'face 'lazy-highlight)
-    (unless (or (eq isearch-lazy-highlight 'all-windows)
-                isearch-lazy-highlight-buffer)
-      (overlay-put ov 'window (selected-window)))))
+  (unless (and isearch-lazy-count
+               ;; Recheck the match that possibly was intended
+               ;; for counting only, but not for highlighting
+               (not (let ((search-invisible
+                           (if (eq search-invisible 'open)
+                               'can-be-opened
+                             search-invisible)))
+                      (funcall isearch-filter-predicate mb me))))
+    (let ((ov (make-overlay mb me)))
+      (push ov isearch-lazy-highlight-overlays)
+      ;; 1000 is higher than ediff's 100+,
+      ;; but lower than isearch main overlay's 1001
+      (overlay-put ov 'priority 1000)
+      (overlay-put ov 'face 'lazy-highlight)
+      (unless (or (eq isearch-lazy-highlight 'all-windows)
+                  isearch-lazy-highlight-buffer)
+        (overlay-put ov 'window (selected-window))))))
 
 (defun isearch-lazy-highlight-start ()
   "Start a new lazy-highlight updating loop."
@@ -4328,11 +4354,19 @@ isearch-lazy-highlight-buffer-update
 				(setq found nil)
 			      (forward-char -1)))
 			(when isearch-lazy-count
-			  (setq isearch-lazy-count-total
-				(1+ (or isearch-lazy-count-total 0)))
-			  (puthash (if isearch-lazy-highlight-forward me mb)
-				   isearch-lazy-count-total
-				   isearch-lazy-count-hash))
+                          ;; Count as invisible when can't open overlay
+			  (if (not (let ((search-invisible
+                                          (if (eq search-invisible 'open)
+                                              'can-be-opened
+                                            search-invisible)))
+                                     (funcall isearch-filter-predicate mb me)))
+			      (setq isearch-lazy-count-invisible
+				    (1+ (or isearch-lazy-count-invisible 0)))
+			    (setq isearch-lazy-count-total
+				  (1+ (or isearch-lazy-count-total 0)))
+			    (puthash (if isearch-lazy-highlight-forward me mb)
+				     isearch-lazy-count-total
+				     isearch-lazy-count-hash)))
 			;; Don't highlight the match when this loop is used
 			;; only to count matches or when matches were already
 			;; highlighted within the current window boundaries