elisp based address completion, round 5.

elisp based address completion, round 5.

[David Bremner]

Changes since last round:

	- resend as one series to (hopefully) reduce confusion
	- update notmuch-company.el to use notmuch-message-mode

[Patch v5 1/3] Emacs: Add address completion mechanism implemented in elisp

[David Bremner]

From: Michal Sojka <sojkam1@fel.cvut.cz>

Currently, notmuch has an address completion mechanism that requires
external command to provide completion candidates. This patch adds a
completion mechanism inspired by https://github.com/tjim/nevermore,
which is implemented in Emacs lisp only.

The preexisting address completion mechanism, activated by pressing TAB
on To/Cc lines, is extended to use the new mechanism when no external
command is configured, i.e. when notmuch-address-command to nil, which
is the new default.

The core of the new mechanism is the function notmuch-address-harvest,
which collects the completion candidates from the notmuch database and
stores them in notmuch-address-completions variable. The address
harvesting can run either synchronously (same as with the previous
mechanism) or asynchronously. When the user presses TAB for the first
time, synchronous harvesting limited to user entered text is performed.
If the entered text is reasonably long, this operation is relatively
fast. Then, asynchronous harvesting over the full database is triggered.
This operation may take long time (minutes on rotating disk). After it
finishes, no harvesting is normally performed again and subsequent
completion requests use the harvested data cached in memory. Completion
cache is updated after 24 hours.

Note that the change of the notmuch-address-command default value
may *BREAK EXISTING SETUPS* when the user used external command named
"notmuch-addresses", i.e. the previous default. The result will be that
the user will use the new mechanism instead of the his command. I
believe that many users may not even recognize this because the new
mechanism works the same as
http://commonmeasure.org/~jkr/git/notmuch_addresses.git and perhaps also
as other commands suggested at
http://notmuchmail.org/emacstips/#address_completion.
---
 emacs/notmuch-address.el | 123 ++++++++++++++++++++++++++++++++++++++++++++---
 emacs/notmuch-lib.el     |   3 ++
 2 files changed, 118 insertions(+), 8 deletions(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index fde3c1b..9f6711b 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -20,14 +20,18 @@
 ;; Authors: David Edmondson <dme@dme.org>
 
 (require 'message)
+(require 'notmuch-query)
+(require 'notmuch-parser)
 
 ;;
 
-(defcustom notmuch-address-command "notmuch-addresses"
-  "The command which generates possible addresses. It must take a
-single argument and output a list of possible matches, one per
-line."
-  :type 'string
+(defcustom notmuch-address-command nil
+  "The command which generates possible addresses for completion.
+It must take a single argument and output a list of possible
+matches, one per line. If set to nil, addresses are generated by
+a built-in completion mechanism."
+  :type '(radio (const :tag "No command: Use built-in completion" nil)
+		 (string :tag "Custom command" :value "notmuch-addresses"))
   :group 'notmuch-send
   :group 'notmuch-external)
 
@@ -42,6 +46,17 @@ to know how address selection is made by default."
   :group 'notmuch-send
   :group 'notmuch-external)
 
+(defvar notmuch-address-last-harvest 0
+  "Time of last address harvest")
+
+(defvar notmuch-address-completions (make-hash-table :test 'equal)
+  "Hash of email addresses for completion during email composition.
+  This variable is set by calling `notmuch-address-harvest'.")
+
+(defvar notmuch-address-full-harvest-finished nil
+  "t indicates that full completion address harvesting has been
+finished")
+
 (defun notmuch-address-selection-function (prompt collection initial-input)
   "Call (`completing-read'
       PROMPT COLLECTION nil nil INITIAL-INPUT 'notmuch-address-history)"
@@ -59,8 +74,32 @@ to know how address selection is made by default."
     (setq message-completion-alist
 	  (push notmuch-address-message-alist-member message-completion-alist))))
 
+(defun notmuch-address-matching (substring)
+  "Returns a list of completion candidates matching SUBSTRING.
+The candidates are taked form `notmuch-address-completions'."
+  (let ((candidates)
+	(re (regexp-quote substring)))
+    (maphash (lambda (key val)
+	       (when (string-match re key)
+		 (push key candidates)))
+	     notmuch-address-completions)
+    candidates))
+
 (defun notmuch-address-options (original)
-  (process-lines notmuch-address-command original))
+  "Returns a list of completion candidates. Uses either
+elisp-based implementation or older implementation requiring
+external commands."
+  (cond
+   ((null notmuch-address-command)
+    (when (not notmuch-address-full-harvest-finished)
+      ;; First, run quick synchronous harvest based on what the user
+      ;; entered so far
+      (notmuch-address-harvest (format "to:%s*" original) t))
+    (prog1 (notmuch-address-matching original)
+      ;; Then (re)start potentially long-running full asynchronous harvesting
+      (notmuch-address-harvest-trigger)))
+   (t
+    (process-lines notmuch-address-command original))))
 
 (defun notmuch-address-expand-name ()
   (let* ((end (point))
@@ -109,11 +148,79 @@ to know how address selection is made by default."
 			   (not (file-directory-p bin))))
 	      (throw 'found-command bin))))))))
 
+(defun notmuch-address-harvest-msg (msg)
+  (let* ((headers (plist-get msg :headers))
+	 (to (ignore-errors (mail-extract-address-components (plist-get headers :To) t)))
+	 (cc (ignore-errors (mail-extract-address-components (plist-get headers :Cc) t)))
+	 (bcc (ignore-errors (mail-extract-address-components (plist-get headers :Bcc) t))))
+    (mapc (lambda (parts)
+	    (let* ((name (car parts))
+		   (email (cadr parts))
+		   (entry (if name (format "%s <%s>" name email) email)))
+	      (puthash entry t notmuch-address-completions)))
+	  (append to cc bcc))
+    nil))
+
+(defun notmuch-address-harvest-handle-result (obj)
+  (notmuch-query-map-threads 'notmuch-address-harvest-msg (list obj)))
+
+(defun notmuch-address-harvest-filter (proc string)
+  (when (buffer-live-p (process-buffer proc))
+    (with-current-buffer (process-buffer proc)
+      (save-excursion
+	(goto-char (point-max))
+	(insert string))
+      (notmuch-sexp-parse-partial-list
+       'notmuch-address-harvest-handle-result (process-buffer proc)))))
+
+(defvar notmuch-address-harvest-proc nil)   ; the process of a harvest underway
+
+(defun notmuch-address-harvest (&optional filter-query synchronous callback)
+  "Collect addresses completion candidates. It queries the
+notmuch database for all messages sent by the user optionally
+matching FILTER-QUERY (if not nil). It collects the destination
+addresses from those messages and stores them in
+`notmuch-address-completions'. Address harvesting may take some
+time so the address collection runs asynchronously unless
+SYNCHRONOUS is t. In case of asynchronous execution, CALLBACK is
+called when harvesting finishes."
+  (let* ((from-me-query (mapconcat (lambda (x) (concat "from:" x)) (notmuch-user-emails) " or "))
+	 (query (if filter-query
+		    (format "(%s) and (%s)" from-me-query filter-query)
+		  from-me-query))
+	 (args `("show" "--format=sexp" "--format-version=2"
+		 "--body=false" "--entire-thread=false" ,query)))
+    (if synchronous
+	(notmuch-query-map-threads 'notmuch-address-harvest-msg
+				   (apply 'notmuch-call-notmuch-sexp args))
+      ;; Asynchronous
+      (when notmuch-address-harvest-proc
+	(kill-buffer (process-buffer notmuch-address-harvest-proc))) ; this also kills the process
+      (setq notmuch-address-harvest-proc
+	    (apply 'notmuch-start-notmuch
+	     "notmuch-address-harvest"		; process name
+	     " *notmuch-address-harvest*"	; process buffer
+	     callback				; process sentinel
+	     args))
+      (set-process-filter notmuch-address-harvest-proc 'notmuch-address-harvest-filter)
+      (set-process-query-on-exit-flag notmuch-address-harvest-proc nil)))
+  ;; return value
+  nil)
+
 ;; If we can find the program specified by `notmuch-address-command',
-;; insinuate ourselves into `message-mode'.
-(when (notmuch-address-locate-command notmuch-address-command)
+;; or if it is nil, insinuate ourselves into `message-mode'.
+(when (or (null notmuch-address-command)
+	  (notmuch-address-locate-command notmuch-address-command))
   (notmuch-address-message-insinuate))
 
+(defun notmuch-address-harvest-trigger ()
+  (let ((now (float-time)))
+    (when (> (- now notmuch-address-last-harvest) 86400)
+      (setq notmuch-address-last-harvest now)
+      (notmuch-address-harvest nil nil
+			       (lambda (proc event)
+				 (setq notmuch-address-full-harvest-finished t))))))
+
 ;;
 
 (provide 'notmuch-address)
diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el
index 201d7ec..1c3a9fe 100644
--- a/emacs/notmuch-lib.el
+++ b/emacs/notmuch-lib.el
@@ -232,6 +232,9 @@ on the command line, and then retry your notmuch command")))
   "Return the user.other_email value (as a list) from the notmuch configuration."
   (split-string (notmuch-config-get "user.other_email") "\n" t))
 
+(defun notmuch-user-emails ()
+  (cons (notmuch-user-primary-email) (notmuch-user-other-email)))
+
 (defun notmuch-poll ()
   "Run \"notmuch new\" or an external script to import mail.
 
-- 
2.6.1

[Patch v5 2/3] emacs: convert notmuch-address-harvester to use notmuch-address

[David Bremner]

No attempt is made to optimize anything here, just drop in the new
command. In particular the use of --output=recipients is known to be
slower than --output=senders, but it fit the existing logic better.
---
 emacs/notmuch-address.el | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index 9f6711b..d54a8bb 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -20,7 +20,6 @@
 ;; Authors: David Edmondson <dme@dme.org>
 
 (require 'message)
-(require 'notmuch-query)
 (require 'notmuch-parser)
 
 ;;
@@ -148,21 +147,12 @@ external commands."
 			   (not (file-directory-p bin))))
 	      (throw 'found-command bin))))))))
 
-(defun notmuch-address-harvest-msg (msg)
-  (let* ((headers (plist-get msg :headers))
-	 (to (ignore-errors (mail-extract-address-components (plist-get headers :To) t)))
-	 (cc (ignore-errors (mail-extract-address-components (plist-get headers :Cc) t)))
-	 (bcc (ignore-errors (mail-extract-address-components (plist-get headers :Bcc) t))))
-    (mapc (lambda (parts)
-	    (let* ((name (car parts))
-		   (email (cadr parts))
-		   (entry (if name (format "%s <%s>" name email) email)))
-	      (puthash entry t notmuch-address-completions)))
-	  (append to cc bcc))
-    nil))
+(defun notmuch-address-harvest-addr (result)
+  (let ((name-addr (plist-get result :name-addr)))
+    (puthash name-addr t notmuch-address-completions)))
 
 (defun notmuch-address-harvest-handle-result (obj)
-  (notmuch-query-map-threads 'notmuch-address-harvest-msg (list obj)))
+  (notmuch-address-harvest-addr obj))
 
 (defun notmuch-address-harvest-filter (proc string)
   (when (buffer-live-p (process-buffer proc))
@@ -188,10 +178,11 @@ called when harvesting finishes."
 	 (query (if filter-query
 		    (format "(%s) and (%s)" from-me-query filter-query)
 		  from-me-query))
-	 (args `("show" "--format=sexp" "--format-version=2"
-		 "--body=false" "--entire-thread=false" ,query)))
+	 (args `("address" "--format=sexp" "--format-version=2"
+		 "--output=recipients"
+		 ,query)))
     (if synchronous
-	(notmuch-query-map-threads 'notmuch-address-harvest-msg
+	(mapc #'notmuch-address-harvest-addr
 				   (apply 'notmuch-call-notmuch-sexp args))
       ;; Asynchronous
       (when notmuch-address-harvest-proc
-- 
2.6.1

Re: [Patch v5 2/3] emacs: convert notmuch-address-harvester to use notmuch-address

[Jani Nikula]

On Sat, 24 Oct 2015, David Bremner <david@tethera.net> wrote:
> No attempt is made to optimize anything here, just drop in the new
> command. In particular the use of --output=recipients is known to be
> slower than --output=senders, but it fit the existing logic better.

Would it be useful to (optionally) use --deduplicate=address here?
Having a dozen name variants with the same address part bugs me most
about the completions.

Another thing that might be useful as a follow-up is adding an optional
date: search filter to only take into account recent emails. This would
gradually phase out obsolete addresses from the results. Also speeds up
searches if the database is huge.

BR,
Jani.


> ---
>  emacs/notmuch-address.el | 25 ++++++++-----------------
>  1 file changed, 8 insertions(+), 17 deletions(-)
>
> diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
> index 9f6711b..d54a8bb 100644
> --- a/emacs/notmuch-address.el
> +++ b/emacs/notmuch-address.el
> @@ -20,7 +20,6 @@
>  ;; Authors: David Edmondson <dme@dme.org>
>  
>  (require 'message)
> -(require 'notmuch-query)
>  (require 'notmuch-parser)
>  
>  ;;
> @@ -148,21 +147,12 @@ external commands."
>  			   (not (file-directory-p bin))))
>  	      (throw 'found-command bin))))))))
>  
> -(defun notmuch-address-harvest-msg (msg)
> -  (let* ((headers (plist-get msg :headers))
> -	 (to (ignore-errors (mail-extract-address-components (plist-get headers :To) t)))
> -	 (cc (ignore-errors (mail-extract-address-components (plist-get headers :Cc) t)))
> -	 (bcc (ignore-errors (mail-extract-address-components (plist-get headers :Bcc) t))))
> -    (mapc (lambda (parts)
> -	    (let* ((name (car parts))
> -		   (email (cadr parts))
> -		   (entry (if name (format "%s <%s>" name email) email)))
> -	      (puthash entry t notmuch-address-completions)))
> -	  (append to cc bcc))
> -    nil))
> +(defun notmuch-address-harvest-addr (result)
> +  (let ((name-addr (plist-get result :name-addr)))
> +    (puthash name-addr t notmuch-address-completions)))
>  
>  (defun notmuch-address-harvest-handle-result (obj)
> -  (notmuch-query-map-threads 'notmuch-address-harvest-msg (list obj)))
> +  (notmuch-address-harvest-addr obj))
>  
>  (defun notmuch-address-harvest-filter (proc string)
>    (when (buffer-live-p (process-buffer proc))
> @@ -188,10 +178,11 @@ called when harvesting finishes."
>  	 (query (if filter-query
>  		    (format "(%s) and (%s)" from-me-query filter-query)
>  		  from-me-query))
> -	 (args `("show" "--format=sexp" "--format-version=2"
> -		 "--body=false" "--entire-thread=false" ,query)))
> +	 (args `("address" "--format=sexp" "--format-version=2"
> +		 "--output=recipients"
> +		 ,query)))
>      (if synchronous
> -	(notmuch-query-map-threads 'notmuch-address-harvest-msg
> +	(mapc #'notmuch-address-harvest-addr
>  				   (apply 'notmuch-call-notmuch-sexp args))
>        ;; Asynchronous
>        (when notmuch-address-harvest-proc
> -- 
> 2.6.1
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> https://notmuchmail.org/mailman/listinfo/notmuch

Re: [Patch v5 2/3] emacs: convert notmuch-address-harvester to use notmuch-address

[Mark Walters]

Hi

I have looked through and tried patches 1 and 2 and it looks OK to
me. So at least a sort of +1 on them combined (I didn't really look at
them separately as they seem to make more sense as 1 lump).

I agree with Jani about both the date option, and deduplicate
possibility, and we may also want to add an option to use
--output=sender.

However, I think anything like that could come as a later patch. For the
moment the only thing I think we need to consider is the variable
notmuch-address-command. Do we want to have something other than nil to
mean use the elisp completion? In particular we could have something
which can be extended to specify the options above.

Additionally, aesthetically, I like the idea that setting this variable
to nil disables address completion.

Best wishes

Mark

[Patch v5 3/3] Emacs: Add address completion based on company-mode

[David Bremner]

From: Michal Sojka <sojkam1@fel.cvut.cz>

With this patch, address completion candidates are shown automatically
after short typing delay in a nice popup box. This requires company-mode
to be installed and it works only on Emacs >= 24. The completion is
based entirely on the asynchronous address harvesting from
notmuch-address.el so the GUI is theoretically not blocked for long
time.

The completion works similarly as the TAB-initiated completion from
notmuch-address.el, i.e. quick harvest based on user input is executed
first and only after full harvesting is finished, in-memory cached data
is used.

The notmuch-company.el is excluded from byte-compilation, because it
would require every person who want to compile notmuch to have
company-mode installed.
---
 emacs/Makefile.local     |  6 +++-
 emacs/notmuch-company.el | 81 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 emacs/notmuch-company.el

diff --git a/emacs/Makefile.local b/emacs/Makefile.local
index 1109cfa..6c93e73 100644
--- a/emacs/Makefile.local
+++ b/emacs/Makefile.local
@@ -20,6 +20,7 @@ emacs_sources := \
 	$(dir)/notmuch-print.el \
 	$(dir)/notmuch-version.el \
 	$(dir)/notmuch-jump.el \
+	$(dir)/notmuch-company.el
 
 $(dir)/notmuch-version.el: $(dir)/Makefile.local version.stamp
 $(dir)/notmuch-version.el: $(srcdir)/$(dir)/notmuch-version.el.tmpl
@@ -30,7 +31,10 @@ $(dir)/notmuch-version.el: $(srcdir)/$(dir)/notmuch-version.el.tmpl
 emacs_images := \
 	$(srcdir)/$(dir)/notmuch-logo.png
 
-emacs_bytecode = $(emacs_sources:.el=.elc)
+# Do not try to install files that are not byte-compiled.
+emacs_no_byte_compile := $(dir)/notmuch-company.el
+
+emacs_bytecode = $(patsubst %.el,%.elc,$(filter-out $(emacs_no_byte_compile),$(emacs_sources)))
 
 # Because of defmacro's and defsubst's, we have to account for load
 # dependencies between Elisp files when byte compiling.  Otherwise,
diff --git a/emacs/notmuch-company.el b/emacs/notmuch-company.el
new file mode 100644
index 0000000..f34aec4
--- /dev/null
+++ b/emacs/notmuch-company.el
@@ -0,0 +1,81 @@
+;; notmuch-company.el --- Mail address completion for notmuch via company-mode  -*- no-byte-compile: t; lexical-binding: t -*-
+
+
+;; Authors: Trevor Jim <tjim@mac.com>
+;; 	    Michal Sojka <sojkam1@fel.cvut.cz>
+;;
+;; Keywords: mail, completion
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; To enable this, install company mode (https://company-mode.github.io/)
+;; and add
+;;
+;;     (require 'notmuch-company)
+;;
+;; to your .emacs.
+;;
+;; NB company-minimum-prefix-length defaults to 3 so you don't get
+;; completion unless you type 3 characters
+
+;;; Code:
+
+(require 'company)
+(require 'message)
+(require 'notmuch-address)
+(require 'cl-lib)
+
+(defvar-local notmuch-company-last-prefix nil)
+
+;;;###autoload
+(defun notmuch-company (command &optional arg &rest ignore)
+  "`company-mode' completion back-end for `notmuch'."
+  (interactive (list 'interactive))
+  (let ((case-fold-search t)
+	(completion-ignore-case t))
+    (cl-case command
+      (interactive (company-begin-backend 'notmuch-company))
+      (prefix (and (derived-mode-p 'message-mode)
+		   (looking-back "^\\(To\\|Cc\\|Bcc\\):.*"
+				 (line-beginning-position))
+		   (setq notmuch-company-last-prefix (company-grab "[:,][ \t]*\\(.*\\)" 1 (point-at-bol)))))
+      (candidates (cond
+		   (notmuch-address-full-harvest-finished
+		    ;; Update harvested addressed from time to time
+		    (notmuch-address-harvest-trigger)
+		    (notmuch-address-matching arg))
+		   (t
+		    (cons :async
+			  (lambda (callback)
+			    ;; First run quick asynchronous harvest based on what the user entered so far
+			    (notmuch-address-harvest
+			     (format "to:%s*" arg) nil
+			     (lambda (proc event)
+			       (funcall callback (notmuch-address-matching arg))
+			       ;; Then (re)start potentially long-running full asynchronous harvesting
+			       (notmuch-address-harvest-trigger))))))))
+      (match (if (string-match notmuch-company-last-prefix arg)
+		 (match-end 0)
+	       0))
+      (no-cache t))))
+
+;;;###autoload
+(add-hook 'notmuch-message-mode-hook '(lambda ()
+					(company-mode)
+					(make-local-variable 'company-backends)
+					(setq company-backends '(notmuch-company))))
+
+(provide 'notmuch-company)
-- 
2.6.1

Re: [Patch v5 3/3] Emacs: Add address completion based on company-mode

[David Bremner]

David Bremner <david@tethera.net> writes:

> +;;;###autoload
> +(add-hook 'notmuch-message-mode-hook '(lambda ()
> +					(company-mode)
> +					(make-local-variable 'company-backends)
> +					(setq company-backends '(notmuch-company))))
> +
> +(provide 'notmuch-company)
> -- 

I know I just sent this message a few minutes ago, but I didn't write it
;).  Why is this second autoload cookie here? this means that in
principle this form will end up in the autoloads file.

[PATCH] autoload notmuch-company if we have company

[Mark Walters]

---

Does this work for people with company installed (on top of the first
3 patches in the series)? I hope it should autoload notmuch-company if
the user has company. I don't on this computer (and am still running
emacs 23) so haven't tested.

Best wishes

Mark


emacs/notmuch-address.el | 5 +++++
 emacs/notmuch-company.el | 1 -
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index d54a8bb..4a3fc23 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -22,6 +22,11 @@
 (require 'message)
 (require 'notmuch-parser)
 
+;; Only load notmuch-company if we have company (but we don't want an
+;; error if we don't have company
+(when (require 'company nil 't)
+  (require 'notmuch-company))
+
 ;;
 
 (defcustom notmuch-address-command nil
diff --git a/emacs/notmuch-company.el b/emacs/notmuch-company.el
index f34aec4..9a44cf5 100644
--- a/emacs/notmuch-company.el
+++ b/emacs/notmuch-company.el
@@ -33,7 +33,6 @@
 
 ;;; Code:
 
-(require 'company)
 (require 'message)
 (require 'notmuch-address)
 (require 'cl-lib)
-- 
2.1.4

elisp completion patches v6

[David Bremner]

Main changes since v5:

   - replace use of notmuch-message-message-insinuate. I'm not sure if
     we should leave a stub function, or just break people's .emacs,
     and tell them not to call it anymore.

   - company is autoloaded, and used, if present. This can be disabled
     by setting notmuch-message-use-company to nil

   - notmuch-message-command being nil is now an explicit disabling of
     of completion

   - deduplicate=address is hardcoded

Things I didn't have time to do

       - date timeout
       - make deduplication configurable
       - make TAB invoke company

I'm still not sure about the use of notmuch-message-command as opposed
to a seperate variable. This particular setup will be a smooth
transition for those that have customized the variable.

I'm also not sure if enabling company should turn off non-company
based completion.

A more conservative default would be to set
notmuch-message-use-company to nil.

[PATCH 1/3] emacs: replace use of notmuch-address-message-insinuate

[David Bremner]

This allows e.g. Gnus users to load this file without changing
message-mode behaviour.

This will disable completion for those that did not customize the
variable but relied on the existence of a file named "notmuch-addresses"
in their path. In the next commit the default behaviour will change to
use a "workalike" internal completion mechanism.
---
 emacs/notmuch-address.el | 20 +++++---------------
 emacs/notmuch-mua.el     |  7 +++++--
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index fde3c1b..39200ef 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -23,11 +23,13 @@
 
 ;;
 
-(defcustom notmuch-address-command "notmuch-addresses"
+(defcustom notmuch-address-command nil
   "The command which generates possible addresses. It must take a
 single argument and output a list of possible matches, one per
-line."
-  :type 'string
+line. The default value of nil disables address completion."
+  :type '(radio
+	  (const :tag "Disable address completion" nil)
+	  (string :tag "Use external completion command" "notmuch-addresses"))
   :group 'notmuch-send
   :group 'notmuch-external)
 
@@ -54,11 +56,6 @@ to know how address selection is made by default."
 
 (defvar notmuch-address-history nil)
 
-(defun notmuch-address-message-insinuate ()
-  (unless (memq notmuch-address-message-alist-member message-completion-alist)
-    (setq message-completion-alist
-	  (push notmuch-address-message-alist-member message-completion-alist))))
-
 (defun notmuch-address-options (original)
   (process-lines notmuch-address-command original))
 
@@ -109,11 +106,4 @@ to know how address selection is made by default."
 			   (not (file-directory-p bin))))
 	      (throw 'found-command bin))))))))
 
-;; If we can find the program specified by `notmuch-address-command',
-;; insinuate ourselves into `message-mode'.
-(when (notmuch-address-locate-command notmuch-address-command)
-  (notmuch-address-message-insinuate))
-
-;;
-
 (provide 'notmuch-address)
diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el
index 57465b2..6cc9656 100644
--- a/emacs/notmuch-mua.el
+++ b/emacs/notmuch-mua.el
@@ -269,7 +269,11 @@ Note that these functions use `mail-citation-hook' if that is non-nil."
   (set-buffer-modified-p nil))
 
 (define-derived-mode notmuch-message-mode message-mode "Message[Notmuch]"
-  "Notmuch message composition mode. Mostly like `message-mode'")
+  "Notmuch message composition mode. Mostly like `message-mode'"
+  (when notmuch-address-command
+    (unless (memq notmuch-address-message-alist-member message-completion-alist)
+      (setq message-completion-alist
+	    (push notmuch-address-message-alist-member message-completion-alist)))))
 
 (define-key notmuch-message-mode-map (kbd "C-c C-c") #'notmuch-mua-send-and-exit)
 (define-key notmuch-message-mode-map (kbd "C-c C-s") #'notmuch-mua-send)
@@ -296,7 +300,6 @@ OTHER-ARGS are passed through to `message-mail'."
   (message-hide-headers)
   (set-buffer-modified-p nil)
   (notmuch-mua-maybe-set-window-dedicated)
-
   (message-goto-to))
 
 (defcustom notmuch-identities nil
-- 
2.6.1

[PATCH 2/3] Emacs: Add address completion mechanism implemented in elisp

[David Bremner]

From: Michal Sojka <sojkam1@fel.cvut.cz>

Currently, notmuch has an address completion mechanism that requires
external command to provide completion candidates. This patch adds a
completion mechanism inspired by https://github.com/tjim/nevermore,
which is implemented in Emacs lisp only.

The preexisting address completion mechanism, activated by pressing TAB
on To/Cc lines, is extended to use the new mechanism when no external
command is configured, i.e. when notmuch-address-command to nil, which
is the new default.

The core of the new mechanism is the function notmuch-address-harvest,
which collects the completion candidates from the notmuch database and
stores them in notmuch-address-completions variable. The address
harvesting can run either synchronously (same as with the previous
mechanism) or asynchronously. When the user presses TAB for the first
time, synchronous harvesting limited to user entered text is performed.
If the entered text is reasonably long, this operation is relatively
fast. Then, asynchronous harvesting over the full database is triggered.
This operation may take long time (minutes on rotating disk). After it
finishes, no harvesting is normally performed again and subsequent
completion requests use the harvested data cached in memory. Completion
cache is updated after 24 hours.

Note that this commit restores (different) completion functionality for
users when the user used external command named "notmuch-addresses",
i.e. the old default.  The result will be that the user will use
the new mechanism instead of this command. I believe that many users may
not even recognize this because the new mechanism works the same as
http://commonmeasure.org/~jkr/git/notmuch_addresses.git and perhaps also
as other commands suggested at
http://notmuchmail.org/emacstips/#address_completion.
---
 emacs/notmuch-address.el | 108 +++++++++++++++++++++++++++++++++++++++++++++--
 emacs/notmuch-lib.el     |   3 ++
 2 files changed, 107 insertions(+), 4 deletions(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index 39200ef..2a748ec 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -20,14 +20,17 @@
 ;; Authors: David Edmondson <dme@dme.org>
 
 (require 'message)
-
+(require 'notmuch-parser)
+(require 'notmuch-lib)
 ;;
 
-(defcustom notmuch-address-command nil
+(defcustom notmuch-address-command 'internal
   "The command which generates possible addresses. It must take a
 single argument and output a list of possible matches, one per
-line. The default value of nil disables address completion."
+line. The default value of `internal' uses built-in address
+completion."
   :type '(radio
+	  (const :tag "Use internal address completion" internal)
 	  (const :tag "Disable address completion" nil)
 	  (string :tag "Use external completion command" "notmuch-addresses"))
   :group 'notmuch-send
@@ -44,6 +47,17 @@ to know how address selection is made by default."
   :group 'notmuch-send
   :group 'notmuch-external)
 
+(defvar notmuch-address-last-harvest 0
+  "Time of last address harvest")
+
+(defvar notmuch-address-completions (make-hash-table :test 'equal)
+  "Hash of email addresses for completion during email composition.
+  This variable is set by calling `notmuch-address-harvest'.")
+
+(defvar notmuch-address-full-harvest-finished nil
+  "t indicates that full completion address harvesting has been
+finished")
+
 (defun notmuch-address-selection-function (prompt collection initial-input)
   "Call (`completing-read'
       PROMPT COLLECTION nil nil INITIAL-INPUT 'notmuch-address-history)"
@@ -56,8 +70,32 @@ to know how address selection is made by default."
 
 (defvar notmuch-address-history nil)
 
+(defun notmuch-address-matching (substring)
+  "Returns a list of completion candidates matching SUBSTRING.
+The candidates are taked form `notmuch-address-completions'."
+  (let ((candidates)
+	(re (regexp-quote substring)))
+    (maphash (lambda (key val)
+	       (when (string-match re key)
+		 (push key candidates)))
+	     notmuch-address-completions)
+    candidates))
+
 (defun notmuch-address-options (original)
-  (process-lines notmuch-address-command original))
+  "Returns a list of completion candidates. Uses either
+elisp-based implementation or older implementation requiring
+external commands."
+  (cond
+   ((eq notmuch-address-command 'internal)
+    (when (not notmuch-address-full-harvest-finished)
+      ;; First, run quick synchronous harvest based on what the user
+      ;; entered so far
+      (notmuch-address-harvest (format "to:%s*" original) t))
+    (prog1 (notmuch-address-matching original)
+      ;; Then (re)start potentially long-running full asynchronous harvesting
+      (notmuch-address-harvest-trigger)))
+   (t
+    (process-lines notmuch-address-command original))))
 
 (defun notmuch-address-expand-name ()
   (let* ((end (point))
@@ -106,4 +144,66 @@ to know how address selection is made by default."
 			   (not (file-directory-p bin))))
 	      (throw 'found-command bin))))))))
 
+(defun notmuch-address-harvest-addr (result)
+  (let ((name-addr (plist-get result :name-addr)))
+    (puthash name-addr t notmuch-address-completions)))
+
+(defun notmuch-address-harvest-handle-result (obj)
+  (notmuch-address-harvest-addr obj))
+
+(defun notmuch-address-harvest-filter (proc string)
+  (when (buffer-live-p (process-buffer proc))
+    (with-current-buffer (process-buffer proc)
+      (save-excursion
+	(goto-char (point-max))
+	(insert string))
+      (notmuch-sexp-parse-partial-list
+       'notmuch-address-harvest-handle-result (process-buffer proc)))))
+
+(defvar notmuch-address-harvest-proc nil)   ; the process of a harvest underway
+
+(defun notmuch-address-harvest (&optional filter-query synchronous callback)
+  "Collect addresses completion candidates. It queries the
+notmuch database for all messages sent by the user optionally
+matching FILTER-QUERY (if not nil). It collects the destination
+addresses from those messages and stores them in
+`notmuch-address-completions'. Address harvesting may take some
+time so the address collection runs asynchronously unless
+SYNCHRONOUS is t. In case of asynchronous execution, CALLBACK is
+called when harvesting finishes."
+  (let* ((from-me-query (mapconcat (lambda (x) (concat "from:" x)) (notmuch-user-emails) " or "))
+	 (query (if filter-query
+		    (format "(%s) and (%s)" from-me-query filter-query)
+		  from-me-query))
+	 (args `("address" "--format=sexp" "--format-version=2"
+		 "--output=recipients"
+		 "--deduplicate=address"
+		 ,query)))
+    (if synchronous
+	(mapc #'notmuch-address-harvest-addr
+				   (apply 'notmuch-call-notmuch-sexp args))
+      ;; Asynchronous
+      (when notmuch-address-harvest-proc
+	(kill-buffer (process-buffer notmuch-address-harvest-proc))) ; this also kills the process
+      (setq notmuch-address-harvest-proc
+	    (apply 'notmuch-start-notmuch
+	     "notmuch-address-harvest"		; process name
+	     " *notmuch-address-harvest*"	; process buffer
+	     callback				; process sentinel
+	     args))
+      (set-process-filter notmuch-address-harvest-proc 'notmuch-address-harvest-filter)
+      (set-process-query-on-exit-flag notmuch-address-harvest-proc nil)))
+  ;; return value
+  nil)
+
+(defun notmuch-address-harvest-trigger ()
+  (let ((now (float-time)))
+    (when (> (- now notmuch-address-last-harvest) 86400)
+      (setq notmuch-address-last-harvest now)
+      (notmuch-address-harvest nil nil
+			       (lambda (proc event)
+				 (setq notmuch-address-full-harvest-finished t))))))
+
+;;
+
 (provide 'notmuch-address)
diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el
index 201d7ec..1c3a9fe 100644
--- a/emacs/notmuch-lib.el
+++ b/emacs/notmuch-lib.el
@@ -232,6 +232,9 @@ on the command line, and then retry your notmuch command")))
   "Return the user.other_email value (as a list) from the notmuch configuration."
   (split-string (notmuch-config-get "user.other_email") "\n" t))
 
+(defun notmuch-user-emails ()
+  (cons (notmuch-user-primary-email) (notmuch-user-other-email)))
+
 (defun notmuch-poll ()
   "Run \"notmuch new\" or an external script to import mail.
 
-- 
2.6.1

[PATCH 3/3] Emacs: Add address completion based on company-mode

[David Bremner]

From: Michal Sojka <sojkam1@fel.cvut.cz>

With this patch, address completion candidates are shown automatically
after short typing delay in a nice popup box. This requires company-mode
to be installed and it works only on Emacs >= 24. The completion is
based entirely on the asynchronous address harvesting from
notmuch-address.el so the GUI is theoretically not blocked for long
time.

The completion works similarly as the TAB-initiated completion from
notmuch-address.el, i.e. quick harvest based on user input is executed
first and only after full harvesting is finished, in-memory cached data
is used.
---
 emacs/Makefile.local     |  1 +
 emacs/notmuch-company.el | 73 ++++++++++++++++++++++++++++++++++++++++++++++++
 emacs/notmuch-mua.el     | 13 ++++++++-
 3 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 emacs/notmuch-company.el

diff --git a/emacs/Makefile.local b/emacs/Makefile.local
index 1109cfa..4c06c52 100644
--- a/emacs/Makefile.local
+++ b/emacs/Makefile.local
@@ -20,6 +20,7 @@ emacs_sources := \
 	$(dir)/notmuch-print.el \
 	$(dir)/notmuch-version.el \
 	$(dir)/notmuch-jump.el \
+	$(dir)/notmuch-company.el
 
 $(dir)/notmuch-version.el: $(dir)/Makefile.local version.stamp
 $(dir)/notmuch-version.el: $(srcdir)/$(dir)/notmuch-version.el.tmpl
diff --git a/emacs/notmuch-company.el b/emacs/notmuch-company.el
new file mode 100644
index 0000000..03c492f
--- /dev/null
+++ b/emacs/notmuch-company.el
@@ -0,0 +1,73 @@
+;; notmuch-company.el --- Mail address completion for notmuch via company-mode  -*- lexical-binding: t -*-
+
+
+;; Authors: Trevor Jim <tjim@mac.com>
+;; 	    Michal Sojka <sojkam1@fel.cvut.cz>
+;;
+;; Keywords: mail, completion
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; To enable this, install company mode (https://company-mode.github.io/)
+;; and customize notmuch-message-use-company
+;;
+;; NB company-minimum-prefix-length defaults to 3 so you don't get
+;; completion unless you type 3 characters
+
+;;; Code:
+
+(require 'notmuch-address)
+(require 'cl-lib)
+
+(defvar-local notmuch-company-last-prefix nil)
+(declare-function company-begin-backend "company")
+(declare-function company-grab "company")
+
+;;;###autoload
+(defun notmuch-company (command &optional arg &rest _ignore)
+  "`company-mode' completion back-end for `notmuch'."
+  (interactive (list 'interactive))
+  (require 'company)
+  (let ((case-fold-search t)
+	(completion-ignore-case t))
+    (cl-case command
+      (interactive (company-begin-backend 'notmuch-company))
+      (prefix (and (derived-mode-p 'message-mode)
+		   (looking-back "^\\(To\\|Cc\\|Bcc\\):.*"
+				 (line-beginning-position))
+		   (setq notmuch-company-last-prefix (company-grab "[:,][ \t]*\\(.*\\)" 1 (point-at-bol)))))
+      (candidates (cond
+		   (notmuch-address-full-harvest-finished
+		    ;; Update harvested addressed from time to time
+		    (notmuch-address-harvest-trigger)
+		    (notmuch-address-matching arg))
+		   (t
+		    (cons :async
+			  (lambda (callback)
+			    ;; First run quick asynchronous harvest based on what the user entered so far
+			    (notmuch-address-harvest
+			     (format "to:%s*" arg) nil
+			     (lambda (_proc _event)
+			       (funcall callback (notmuch-address-matching arg))
+			       ;; Then (re)start potentially long-running full asynchronous harvesting
+			       (notmuch-address-harvest-trigger))))))))
+      (match (if (string-match notmuch-company-last-prefix arg)
+		 (match-end 0)
+	       0))
+      (no-cache t))))
+
+
+(provide 'notmuch-company)
diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el
index 6cc9656..c90381d 100644
--- a/emacs/notmuch-mua.el
+++ b/emacs/notmuch-mua.el
@@ -25,6 +25,7 @@
 
 (require 'notmuch-lib)
 (require 'notmuch-address)
+(require 'notmuch-company)
 
 (eval-when-compile (require 'cl))
 
@@ -268,12 +269,22 @@ Note that these functions use `mail-citation-hook' if that is non-nil."
   (message-goto-body)
   (set-buffer-modified-p nil))
 
+(defcustom notmuch-message-use-company t
+  "If available, use company mode for completion in notmuch-message-mode"
+  :type 'boolean
+  :group 'notmuch-send)
+
 (define-derived-mode notmuch-message-mode message-mode "Message[Notmuch]"
   "Notmuch message composition mode. Mostly like `message-mode'"
   (when notmuch-address-command
     (unless (memq notmuch-address-message-alist-member message-completion-alist)
       (setq message-completion-alist
-	    (push notmuch-address-message-alist-member message-completion-alist)))))
+	    (push notmuch-address-message-alist-member message-completion-alist))))
+  (when (and notmuch-message-use-company
+	     (require 'company nil t))
+    (company-mode)
+    (make-local-variable 'company-backends)
+    (setq company-backends '(notmuch-company))))
 
 (define-key notmuch-message-mode-map (kbd "C-c C-c") #'notmuch-mua-send-and-exit)
 (define-key notmuch-message-mode-map (kbd "C-c C-s") #'notmuch-mua-send)
-- 
2.6.1

[PATCH] fix compilation on emacs23

[Mark Walters]

The series so far does not compile on emacs23. The two problems are
that cl-lib doesn't exist and that defvar-local is not defined. The
latter can trivially be replaced by defvar followed by
make-variable-buffer-local.

For the former just require 'cl as we do elsewhere, and change the one
call to a cl function, namely to cl-case to case.
---
Lightly tested and seems to compile and work on emacs 23 and emacs 24.

Best wishes

Mark


emacs/notmuch-company.el | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/emacs/notmuch-company.el b/emacs/notmuch-company.el
index 03c492f..a9e769c 100644
--- a/emacs/notmuch-company.el
+++ b/emacs/notmuch-company.el
@@ -1,6 +1,5 @@
 ;; notmuch-company.el --- Mail address completion for notmuch via company-mode  -*- lexical-binding: t -*-
 
-
 ;; Authors: Trevor Jim <tjim@mac.com>
 ;; 	    Michal Sojka <sojkam1@fel.cvut.cz>
 ;;
@@ -30,9 +29,10 @@
 ;;; Code:
 
 (require 'notmuch-address)
-(require 'cl-lib)
+(eval-when-compile (require 'cl))
 
-(defvar-local notmuch-company-last-prefix nil)
+(defvar notmuch-company-last-prefix nil)
+(make-variable-buffer-local 'notmuch-company-last-prefix)
 (declare-function company-begin-backend "company")
 (declare-function company-grab "company")
 
@@ -43,7 +43,7 @@ (defun notmuch-company (command &optional arg &rest _ignore)
   (require 'company)
   (let ((case-fold-search t)
 	(completion-ignore-case t))
-    (cl-case command
+    (case command
       (interactive (company-begin-backend 'notmuch-company))
       (prefix (and (derived-mode-p 'message-mode)
 		   (looking-back "^\\(To\\|Cc\\|Bcc\\):.*"
-- 
2.1.4

Re: [PATCH 3/3] Emacs: Add address completion based on company-mode

[Mark Walters]

On Sat, 24 Oct 2015, David Bremner <david@tethera.net> wrote:
> From: Michal Sojka <sojkam1@fel.cvut.cz>
>
> With this patch, address completion candidates are shown automatically
> after short typing delay in a nice popup box. This requires company-mode
> to be installed and it works only on Emacs >= 24. The completion is
> based entirely on the asynchronous address harvesting from
> notmuch-address.el so the GUI is theoretically not blocked for long
> time.
>
> The completion works similarly as the TAB-initiated completion from
> notmuch-address.el, i.e. quick harvest based on user input is executed
> first and only after full harvesting is finished, in-memory cached data
> is used.
> ---
>  emacs/Makefile.local     |  1 +
>  emacs/notmuch-company.el | 73 ++++++++++++++++++++++++++++++++++++++++++++++++
>  emacs/notmuch-mua.el     | 13 ++++++++-
>  3 files changed, 86 insertions(+), 1 deletion(-)
>  create mode 100644 emacs/notmuch-company.el
>
> diff --git a/emacs/Makefile.local b/emacs/Makefile.local
> index 1109cfa..4c06c52 100644
> --- a/emacs/Makefile.local
> +++ b/emacs/Makefile.local
> @@ -20,6 +20,7 @@ emacs_sources := \
>  	$(dir)/notmuch-print.el \
>  	$(dir)/notmuch-version.el \
>  	$(dir)/notmuch-jump.el \
> +	$(dir)/notmuch-company.el
>  
>  $(dir)/notmuch-version.el: $(dir)/Makefile.local version.stamp
>  $(dir)/notmuch-version.el: $(srcdir)/$(dir)/notmuch-version.el.tmpl
> diff --git a/emacs/notmuch-company.el b/emacs/notmuch-company.el
> new file mode 100644
> index 0000000..03c492f
> --- /dev/null
> +++ b/emacs/notmuch-company.el
> @@ -0,0 +1,73 @@
> +;; notmuch-company.el --- Mail address completion for notmuch via company-mode  -*- lexical-binding: t -*-
> +
> +
> +;; Authors: Trevor Jim <tjim@mac.com>
> +;; 	    Michal Sojka <sojkam1@fel.cvut.cz>
> +;;
> +;; Keywords: mail, completion
> +
> +;; This program is free software; you can redistribute it and/or modify
> +;; it under the terms of the GNU General Public License as published by
> +;; the Free Software Foundation, either version 3 of the License, or
> +;; (at your option) any later version.
> +
> +;; This program is distributed in the hope that it will be useful,
> +;; but WITHOUT ANY WARRANTY; without even the implied warranty of
> +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +;; GNU General Public License for more details.
> +
> +;; You should have received a copy of the GNU General Public License
> +;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
> +
> +;;; Commentary:
> +
> +;; To enable this, install company mode (https://company-mode.github.io/)
> +;; and customize notmuch-message-use-company
> +;;
> +;; NB company-minimum-prefix-length defaults to 3 so you don't get
> +;; completion unless you type 3 characters
> +
> +;;; Code:
> +
> +(require 'notmuch-address)
> +(require 'cl-lib)
> +
> +(defvar-local notmuch-company-last-prefix nil)
> +(declare-function company-begin-backend "company")
> +(declare-function company-grab "company")
> +
> +;;;###autoload
> +(defun notmuch-company (command &optional arg &rest _ignore)
> +  "`company-mode' completion back-end for `notmuch'."
> +  (interactive (list 'interactive))
> +  (require 'company)
> +  (let ((case-fold-search t)
> +	(completion-ignore-case t))
> +    (cl-case command
> +      (interactive (company-begin-backend 'notmuch-company))
> +      (prefix (and (derived-mode-p 'message-mode)
> +		   (looking-back "^\\(To\\|Cc\\|Bcc\\):.*"
> +				 (line-beginning-position))
> +		   (setq notmuch-company-last-prefix (company-grab "[:,][ \t]*\\(.*\\)" 1 (point-at-bol)))))
> +      (candidates (cond
> +		   (notmuch-address-full-harvest-finished
> +		    ;; Update harvested addressed from time to time
> +		    (notmuch-address-harvest-trigger)
> +		    (notmuch-address-matching arg))
> +		   (t
> +		    (cons :async
> +			  (lambda (callback)
> +			    ;; First run quick asynchronous harvest based on what the user entered so far
> +			    (notmuch-address-harvest
> +			     (format "to:%s*" arg) nil
> +			     (lambda (_proc _event)
> +			       (funcall callback (notmuch-address-matching arg))
> +			       ;; Then (re)start potentially long-running full asynchronous harvesting
> +			       (notmuch-address-harvest-trigger))))))))

I have found a bug in this but I don't know the best way to fix it.

If you start notmuch in emacs, start composing a message, type 3 letters
(say) in the to line and press tab then you get the "quick" address
completion, and it starts the full harvest. However if you don't select
one of the addresses and just type another character and pause then
notmuch-company starts and it sees that you haven't got a full harvest
so it asynchronously gets the quick completions. But this kills the main
harvest (the "; this also kills the process" line in
notmuch-address.el). The sentinel for the full harvest then marks the
full harvest complete.

Since it is marked complete the full harvest won't ever get done (or
not until the completion timeouts in 24 hours). 

The first part of the fix is probably to only mark the harvest complete
if the sentinel returns normally. But it may need more, perhaps to run
the async quick case in a separate buffer from the full harvest.

Best wishes

Mark


> +      (match (if (string-match notmuch-company-last-prefix arg)
> +		 (match-end 0)
> +	       0))
> +      (no-cache t))))
> +
> +
> +(provide 'notmuch-company)
> diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el
> index 6cc9656..c90381d 100644
> --- a/emacs/notmuch-mua.el
> +++ b/emacs/notmuch-mua.el
> @@ -25,6 +25,7 @@
>  
>  (require 'notmuch-lib)
>  (require 'notmuch-address)
> +(require 'notmuch-company)
>  
>  (eval-when-compile (require 'cl))
>  
> @@ -268,12 +269,22 @@ Note that these functions use `mail-citation-hook' if that is non-nil."
>    (message-goto-body)
>    (set-buffer-modified-p nil))
>  
> +(defcustom notmuch-message-use-company t
> +  "If available, use company mode for completion in notmuch-message-mode"
> +  :type 'boolean
> +  :group 'notmuch-send)
> +
>  (define-derived-mode notmuch-message-mode message-mode "Message[Notmuch]"
>    "Notmuch message composition mode. Mostly like `message-mode'"
>    (when notmuch-address-command
>      (unless (memq notmuch-address-message-alist-member message-completion-alist)
>        (setq message-completion-alist
> -	    (push notmuch-address-message-alist-member message-completion-alist)))))
> +	    (push notmuch-address-message-alist-member message-completion-alist))))
> +  (when (and notmuch-message-use-company
> +	     (require 'company nil t))
> +    (company-mode)
> +    (make-local-variable 'company-backends)
> +    (setq company-backends '(notmuch-company))))
>  
>  (define-key notmuch-message-mode-map (kbd "C-c C-c") #'notmuch-mua-send-and-exit)
>  (define-key notmuch-message-mode-map (kbd "C-c C-s") #'notmuch-mua-send)
> -- 
> 2.6.1

[PATCH] Fix clash of company mode and async harvest

[Mark Walters]

The current code has a bug such that an async partial harvest kills
the full harvest, and marks the full harvest complete. This means the
full harvest is not reattempted for another 24 hours (during which
time the user will only get partial results).

Fix this by using separate buffers for full and partial harvests. Also
check the return value of the full harvest and only mark the harvest
complete if it finishes successfully.
---

This seems to fix the bug mentioned in my previous email. It works
under light testing in both emacs 23 and emacs 24.

Best wishes

Mark


emacs/notmuch-address.el | 41 +++++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index d82c2aa..228135e 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -161,7 +161,10 @@ (defun notmuch-address-harvest-filter (proc string)
       (notmuch-sexp-parse-partial-list
        'notmuch-address-harvest-handle-result (process-buffer proc)))))
 
-(defvar notmuch-address-harvest-proc nil)   ; the process of a harvest underway
+(defvar notmuch-address-harvest-procs '(nil . nil)
+  "The currently running harvests.
+
+The car is a partial harvest, and the cdr is a full harvest")
 
 (defun notmuch-address-harvest (&optional filter-query synchronous callback)
   "Collect addresses completion candidates. It queries the
@@ -184,16 +187,25 @@ (defun notmuch-address-harvest (&optional filter-query synchronous callback)
 	(mapc #'notmuch-address-harvest-addr
 				   (apply 'notmuch-call-notmuch-sexp args))
       ;; Asynchronous
-      (when notmuch-address-harvest-proc
-	(kill-buffer (process-buffer notmuch-address-harvest-proc))) ; this also kills the process
-      (setq notmuch-address-harvest-proc
-	    (apply 'notmuch-start-notmuch
-	     "notmuch-address-harvest"		; process name
-	     " *notmuch-address-harvest*"	; process buffer
-	     callback				; process sentinel
-	     args))
-      (set-process-filter notmuch-address-harvest-proc 'notmuch-address-harvest-filter)
-      (set-process-query-on-exit-flag notmuch-address-harvest-proc nil)))
+      (let* ((current-proc (if filter-query
+			       (car notmuch-address-harvest-procs)
+			     (cdr notmuch-address-harvest-procs)))
+	     (proc-name (format "notmuch-address-%s-harvest"
+				(if filter-query "partial" "full")))
+	     (proc-buf (concat " *" proc-name "*")))
+	;; Kill any existing process
+	(when current-proc
+	  (kill-buffer (process-buffer current-proc))) ; this also kills the process
+
+	(setq current-proc
+	      (apply 'notmuch-start-notmuch proc-name proc-buf
+		     callback				; process sentinel
+		     args))
+	(set-process-filter current-proc 'notmuch-address-harvest-filter)
+	(set-process-query-on-exit-flag current-proc nil)
+	(if filter-query
+	    (setcar notmuch-address-harvest-procs current-proc)
+	  (setcdr notmuch-address-harvest-procs current-proc)))))
   ;; return value
   nil)
 
@@ -203,7 +215,12 @@ (defun notmuch-address-harvest-trigger ()
       (setq notmuch-address-last-harvest now)
       (notmuch-address-harvest nil nil
 			       (lambda (proc event)
-				 (setq notmuch-address-full-harvest-finished t))))))
+				 ;; If harvest fails, we want to try
+				 ;; again when the trigger is next
+				 ;; called
+				 (if (string= event "finished\n")
+				     (setq notmuch-address-full-harvest-finished t)
+				   (setq notmuch-address-last-harvest 0)))))))
 
 ;;
 
-- 
2.1.4

[PATCH] address completion tweaks

[Mark Walters]

If the user has company installed and notmuch is configured to use it,
then use it for all the completions. In particular, pressing tab now
forces company to run immediately.

In addition this patch fixes a semi-bug where address completion was
used for more headers than company mode was. With the change above
this became important. Thus make company mode work for all headers
that address completion does.
---

I think this is an improvement: without pressing tab quickly brings up
the normal mini-buffer completion mode, but waiting a second means
company has started and then tab doesn't bring up the mini-buffer.

The change in notmuch-mua is just so that company-setup runs before we
setup the message-completion-alist.

Best wishes

Mark



emacs/notmuch-address.el |  7 ++++---
 emacs/notmuch-company.el |  5 +++--
 emacs/notmuch-mua.el     | 10 +++++-----
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index 228135e..0a73ddf 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -64,9 +64,11 @@ (defun notmuch-address-selection-function (prompt collection initial-input)
   (completing-read
    prompt collection nil nil initial-input 'notmuch-address-history))
 
+(defvar notmuch-address-completion-headers-regexp
+  "^\\(Resent-\\)?\\(To\\|B?Cc\\|Reply-To\\|From\\|Mail-Followup-To\\|Mail-Copies-To\\):")
+
 (defvar notmuch-address-message-alist-member
-  '("^\\(Resent-\\)?\\(To\\|B?Cc\\|Reply-To\\|From\\|Mail-Followup-To\\|Mail-Copies-To\\):"
-	      . notmuch-address-expand-name))
+  (cons notmuch-address-completion-headers-regexp 'notmuch-address-expand-name))
 
 (defvar notmuch-address-history nil)
 
@@ -221,7 +223,6 @@ (defun notmuch-address-harvest-trigger ()
 				 (if (string= event "finished\n")
 				     (setq notmuch-address-full-harvest-finished t)
 				   (setq notmuch-address-last-harvest 0)))))))
-
 ;;
 
 (provide 'notmuch-address)
diff --git a/emacs/notmuch-company.el b/emacs/notmuch-company.el
index 9e57914..3157ed7 100644
--- a/emacs/notmuch-company.el
+++ b/emacs/notmuch-company.el
@@ -41,6 +41,8 @@ (defvar company-backends)
 ;;;###autoload
 (defun notmuch-company-setup ()
   (company-mode)
+  (setq notmuch-address-message-alist-member
+	(cons notmuch-address-completion-headers-regexp 'company-manual-begin))
   (make-local-variable 'company-backends)
   (setq company-backends '(notmuch-company)))
 
@@ -54,7 +56,7 @@ (defun notmuch-company (command &optional arg &rest _ignore)
     (case command
       (interactive (company-begin-backend 'notmuch-company))
       (prefix (and (derived-mode-p 'message-mode)
-		   (looking-back "^\\(To\\|Cc\\|Bcc\\):.*"
+		   (looking-back (concat notmuch-address-completion-headers-regexp ".*")
 				 (line-beginning-position))
 		   (setq notmuch-company-last-prefix (company-grab "[:,][ \t]*\\(.*\\)" 1 (point-at-bol)))))
       (candidates (cond
@@ -77,5 +79,4 @@ (defun notmuch-company (command &optional arg &rest _ignore)
 	       0))
       (no-cache t))))
 
-
 (provide 'notmuch-company)
diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el
index 63fc8db..8cbf38c 100644
--- a/emacs/notmuch-mua.el
+++ b/emacs/notmuch-mua.el
@@ -276,13 +276,13 @@ (defcustom notmuch-message-use-company t
 (define-derived-mode notmuch-message-mode message-mode "Message[Notmuch]"
   "Notmuch message composition mode. Mostly like `message-mode'"
   (when notmuch-address-command
+    (when (and notmuch-message-use-company
+	       (eq notmuch-address-command 'internal)
+	       (require 'company nil t))
+      (notmuch-company-setup))
     (unless (memq notmuch-address-message-alist-member message-completion-alist)
       (setq message-completion-alist
-	    (push notmuch-address-message-alist-member message-completion-alist))))
-  (when (and notmuch-message-use-company
-	     (eq notmuch-address-command 'internal)
-	     (require 'company nil t))
-    (notmuch-company-setup)))
+	    (push notmuch-address-message-alist-member message-completion-alist)))))
 
 (define-key notmuch-message-mode-map (kbd "C-c C-c") #'notmuch-mua-send-and-exit)
 (define-key notmuch-message-mode-map (kbd "C-c C-s") #'notmuch-mua-send)
-- 
2.1.4

Re: elisp completion patches v6

[David Bremner]

David Bremner <david@tethera.net> writes:

> Main changes since v5:
>
>    - replace use of notmuch-message-message-insinuate. I'm not sure if
>      we should leave a stub function, or just break people's .emacs,
>      and tell them not to call it anymore.
>
>    - company is autoloaded, and used, if present. This can be disabled
>      by setting notmuch-message-use-company to nil
>
>    - notmuch-message-command being nil is now an explicit disabling of
>      of completion
>
>    - deduplicate=address is hardcoded

One more important change is that this no longer special-cases
notmuch-company with respect to byte-compilation. It compiles fine
without company-mode (although I noticed that I missed one variable
declaration). On the other hand I didn't test this with emacs23. I
wouldn't expect it to work, but hopefully it still compiles.

d

[PATCH] Fix non internal cases for notmuch-address-command

[Mark Walters]

This makes tab completion do nothing when notmuch-address-command is
nil.

It also stops company mode from being used when an external command is
specified, which was giving rise to some strange behaviour.
---

This patch is only really a 3 line patch (if you do a word-diff): the
rest is a whitespace change.

Best wishes

Mark

emacs/notmuch-address.el | 53 ++++++++++++++++++++++++------------------------
 emacs/notmuch-mua.el     |  1 +
 2 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/emacs/notmuch-address.el b/emacs/notmuch-address.el
index 2a748ec..d82c2aa 100644
--- a/emacs/notmuch-address.el
+++ b/emacs/notmuch-address.el
@@ -98,32 +98,33 @@ (defun notmuch-address-options (original)
     (process-lines notmuch-address-command original))))
 
 (defun notmuch-address-expand-name ()
-  (let* ((end (point))
-	 (beg (save-excursion
-		(re-search-backward "\\(\\`\\|[\n:,]\\)[ \t]*")
-		(goto-char (match-end 0))
-		(point)))
-	 (orig (buffer-substring-no-properties beg end))
-	 (completion-ignore-case t)
-	 (options (with-temp-message "Looking for completion candidates..."
-		    (notmuch-address-options orig)))
-	 (num-options (length options))
-	 (chosen (cond
-		  ((eq num-options 0)
-		   nil)
-		  ((eq num-options 1)
-		   (car options))
-		  (t
-		   (funcall notmuch-address-selection-function
-			    (format "Address (%s matches): " num-options)
-			    (cdr options) (car options))))))
-    (if chosen
-	(progn
-	  (push chosen notmuch-address-history)
-	  (delete-region beg end)
-	  (insert chosen))
-      (message "No matches.")
-      (ding))))
+  (when notmuch-address-command
+    (let* ((end (point))
+	   (beg (save-excursion
+		  (re-search-backward "\\(\\`\\|[\n:,]\\)[ \t]*")
+		  (goto-char (match-end 0))
+		  (point)))
+	   (orig (buffer-substring-no-properties beg end))
+	   (completion-ignore-case t)
+	   (options (with-temp-message "Looking for completion candidates..."
+		      (notmuch-address-options orig)))
+	   (num-options (length options))
+	   (chosen (cond
+		    ((eq num-options 0)
+		     nil)
+		    ((eq num-options 1)
+		     (car options))
+		    (t
+		     (funcall notmuch-address-selection-function
+			      (format "Address (%s matches): " num-options)
+			      (cdr options) (car options))))))
+      (if chosen
+	  (progn
+	    (push chosen notmuch-address-history)
+	    (delete-region beg end)
+	    (insert chosen))
+	(message "No matches.")
+	(ding)))))
 
 ;; Copied from `w3m-which-command'.
 (defun notmuch-address-locate-command (command)
diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el
index 3feea29..63fc8db 100644
--- a/emacs/notmuch-mua.el
+++ b/emacs/notmuch-mua.el
@@ -280,6 +280,7 @@ (define-derived-mode notmuch-message-mode message-mode "Message[Notmuch]"
       (setq message-completion-alist
 	    (push notmuch-address-message-alist-member message-completion-alist))))
   (when (and notmuch-message-use-company
+	     (eq notmuch-address-command 'internal)
 	     (require 'company nil t))
     (notmuch-company-setup)))
 
-- 
2.1.4

Re: elisp completion patches v6

[Aaron Ecay]

Hi David,

2015ko urriak 24an, David Bremner-ek idatzi zuen:
> 
> Main changes since v5:
> 
>    - replace use of notmuch-message-message-insinuate. I'm not sure if
>      we should leave a stub function, or just break people's .emacs,
>      and tell them not to call it anymore.

Maybe you could use a combination of ‘make-obsolete’ (for byte-compiler
warnings) and ‘display-warning’ (for runtime)?  I understand the desire
to phase the function out eventually rather than have an eternal stub,
but intentionally breaking .emacs is a really drastic step.

-- 
Aaron Ecay