* [PATCH v3 1/6] cli: remove useless talloc_strdup
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
@ 2013-03-31 9:45 ` Jani Nikula
2013-03-31 9:45 ` [PATCH v3 2/6] cli: extract count printing to a separate function in notmuch count Jani Nikula
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jani Nikula @ 2013-03-31 9:45 UTC (permalink / raw)
To: notmuch; +Cc: Tomi Ollila
If the condition holds, query_string_from_args() has already returned
a talloc allocated empty string. There's no need to duplicate that.
---
notmuch-count.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/notmuch-count.c b/notmuch-count.c
index 390794f..c2f1b7d 100644
--- a/notmuch-count.c
+++ b/notmuch-count.c
@@ -71,10 +71,6 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
return 1;
}
- if (*query_str == '\0') {
- query_str = talloc_strdup (config, "");
- }
-
query = notmuch_query_create (notmuch, query_str);
if (query == NULL) {
fprintf (stderr, "Out of memory\n");
--
1.7.10.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v3 2/6] cli: extract count printing to a separate function in notmuch count
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
2013-03-31 9:45 ` [PATCH v3 1/6] cli: remove useless talloc_strdup Jani Nikula
@ 2013-03-31 9:45 ` Jani Nikula
2013-03-31 9:46 ` [PATCH v3 3/6] cli: add --batch option to " Jani Nikula
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jani Nikula @ 2013-03-31 9:45 UTC (permalink / raw)
To: notmuch; +Cc: Tomi Ollila
Make count printing on a query string reusable. No functional changes.
---
notmuch-count.c | 59 +++++++++++++++++++++++++++++++++----------------------
1 file changed, 36 insertions(+), 23 deletions(-)
diff --git a/notmuch-count.c b/notmuch-count.c
index c2f1b7d..630f036 100644
--- a/notmuch-count.c
+++ b/notmuch-count.c
@@ -32,16 +32,47 @@ enum {
EXCLUDE_FALSE,
};
+static int
+print_count (notmuch_database_t *notmuch, const char *query_str,
+ const char **exclude_tags, size_t exclude_tags_length, int output)
+{
+ notmuch_query_t *query;
+ size_t i;
+
+ query = notmuch_query_create (notmuch, query_str);
+ if (query == NULL) {
+ fprintf (stderr, "Out of memory\n");
+ return 1;
+ }
+
+ for (i = 0; i < exclude_tags_length; i++)
+ notmuch_query_add_tag_exclude (query, exclude_tags[i]);
+
+ switch (output) {
+ case OUTPUT_MESSAGES:
+ printf ("%u\n", notmuch_query_count_messages (query));
+ break;
+ case OUTPUT_THREADS:
+ printf ("%u\n", notmuch_query_count_threads (query));
+ break;
+ }
+
+ notmuch_query_destroy (query);
+
+ return 0;
+}
+
int
notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
{
notmuch_database_t *notmuch;
- notmuch_query_t *query;
char *query_str;
int opt_index;
int output = OUTPUT_MESSAGES;
int exclude = EXCLUDE_TRUE;
- unsigned int i;
+ const char **search_exclude_tags = NULL;
+ size_t search_exclude_tags_length = 0;
+ int ret;
notmuch_opt_desc_t options[] = {
{ NOTMUCH_OPT_KEYWORD, &output, "output", 'o',
@@ -71,33 +102,15 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
return 1;
}
- query = notmuch_query_create (notmuch, query_str);
- if (query == NULL) {
- fprintf (stderr, "Out of memory\n");
- return 1;
- }
-
if (exclude == EXCLUDE_TRUE) {
- const char **search_exclude_tags;
- size_t search_exclude_tags_length;
-
search_exclude_tags = notmuch_config_get_search_exclude_tags
(config, &search_exclude_tags_length);
- for (i = 0; i < search_exclude_tags_length; i++)
- notmuch_query_add_tag_exclude (query, search_exclude_tags[i]);
}
- switch (output) {
- case OUTPUT_MESSAGES:
- printf ("%u\n", notmuch_query_count_messages (query));
- break;
- case OUTPUT_THREADS:
- printf ("%u\n", notmuch_query_count_threads (query));
- break;
- }
+ ret = print_count (notmuch, query_str, search_exclude_tags,
+ search_exclude_tags_length, output);
- notmuch_query_destroy (query);
notmuch_database_destroy (notmuch);
- return 0;
+ return ret;
}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v3 3/6] cli: add --batch option to notmuch count
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
2013-03-31 9:45 ` [PATCH v3 1/6] cli: remove useless talloc_strdup Jani Nikula
2013-03-31 9:45 ` [PATCH v3 2/6] cli: extract count printing to a separate function in notmuch count Jani Nikula
@ 2013-03-31 9:46 ` Jani Nikula
2013-03-31 9:46 ` [PATCH v3 4/6] man: document notmuch count --batch and --input options Jani Nikula
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jani Nikula @ 2013-03-31 9:46 UTC (permalink / raw)
To: notmuch; +Cc: Tomi Ollila
Add support for reading queries from stdin, one per line, and writing
results to stdout, one per line.
This will bring considerable performance improvements when utilized in
Emacs notmuch-hello, especially so when running remote notmuch.
---
notmuch-count.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 50 insertions(+), 2 deletions(-)
diff --git a/notmuch-count.c b/notmuch-count.c
index 630f036..8772cff 100644
--- a/notmuch-count.c
+++ b/notmuch-count.c
@@ -62,6 +62,27 @@ print_count (notmuch_database_t *notmuch, const char *query_str,
return 0;
}
+static int
+count_file (notmuch_database_t *notmuch, FILE *input, const char **exclude_tags,
+ size_t exclude_tags_length, int output)
+{
+ char *line = NULL;
+ ssize_t line_len;
+ size_t line_size;
+ int ret = 0;
+
+ while (!ret && (line_len = getline (&line, &line_size, input)) != -1) {
+ chomp_newline (line);
+ ret = print_count (notmuch, line, exclude_tags, exclude_tags_length,
+ output);
+ }
+
+ if (line)
+ free (line);
+
+ return ret;
+}
+
int
notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
{
@@ -72,6 +93,9 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
int exclude = EXCLUDE_TRUE;
const char **search_exclude_tags = NULL;
size_t search_exclude_tags_length = 0;
+ notmuch_bool_t batch = FALSE;
+ FILE *input = stdin;
+ char *input_file_name = NULL;
int ret;
notmuch_opt_desc_t options[] = {
@@ -83,6 +107,8 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
(notmuch_keyword_t []){ { "true", EXCLUDE_TRUE },
{ "false", EXCLUDE_FALSE },
{ 0, 0 } } },
+ { NOTMUCH_OPT_BOOLEAN, &batch, "batch", 0, 0 },
+ { NOTMUCH_OPT_STRING, &input_file_name, "input", 'i', 0 },
{ 0, 0, 0, 0, 0 }
};
@@ -92,6 +118,21 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
return 1;
}
+ if (input_file_name) {
+ batch = TRUE;
+ input = fopen (input_file_name, "r");
+ if (input == NULL) {
+ fprintf (stderr, "Error opening %s for reading: %s\n",
+ input_file_name, strerror (errno));
+ return 1;
+ }
+ }
+
+ if (batch && opt_index != argc) {
+ fprintf (stderr, "--batch and query string are not compatible\n");
+ return 1;
+ }
+
if (notmuch_database_open (notmuch_config_get_database_path (config),
NOTMUCH_DATABASE_MODE_READ_ONLY, ¬much))
return 1;
@@ -107,10 +148,17 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
(config, &search_exclude_tags_length);
}
- ret = print_count (notmuch, query_str, search_exclude_tags,
- search_exclude_tags_length, output);
+ if (batch)
+ ret = count_file (notmuch, input, search_exclude_tags,
+ search_exclude_tags_length, output);
+ else
+ ret = print_count (notmuch, query_str, search_exclude_tags,
+ search_exclude_tags_length, output);
notmuch_database_destroy (notmuch);
+ if (input != stdin)
+ fclose (input);
+
return ret;
}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v3 4/6] man: document notmuch count --batch and --input options
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
` (2 preceding siblings ...)
2013-03-31 9:46 ` [PATCH v3 3/6] cli: add --batch option to " Jani Nikula
@ 2013-03-31 9:46 ` Jani Nikula
2013-03-31 9:46 ` [PATCH v3 5/6] test: " Jani Nikula
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jani Nikula @ 2013-03-31 9:46 UTC (permalink / raw)
To: notmuch; +Cc: Tomi Ollila
---
man/man1/notmuch-count.1 | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/man/man1/notmuch-count.1 b/man/man1/notmuch-count.1
index 86a67fe..7fc4378 100644
--- a/man/man1/notmuch-count.1
+++ b/man/man1/notmuch-count.1
@@ -46,6 +46,26 @@ Output the number of matching threads.
Specify whether to omit messages matching search.tag_exclude from the
count (the default) or not.
.RE
+
+.RS 4
+.TP 4
+.BR \-\-batch
+
+Read queries from a file (stdin by default), one per line, and output
+the number of matching messages (or threads) to stdout, one per
+line. On an empty input line the count of all messages (or threads) in
+the database will be output. This option is not compatible with
+specifying search terms on the command line.
+.RE
+
+.RS 4
+.TP 4
+.BR "\-\-input=" <filename>
+
+Read input from given file, instead of from stdin. Implies
+.BR --batch .
+.RE
+
.RE
.RE
--
1.7.10.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v3 5/6] test: notmuch count --batch and --input options
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
` (3 preceding siblings ...)
2013-03-31 9:46 ` [PATCH v3 4/6] man: document notmuch count --batch and --input options Jani Nikula
@ 2013-03-31 9:46 ` Jani Nikula
2013-03-31 9:46 ` [PATCH v3 6/6] emacs: hello: use batch count Jani Nikula
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jani Nikula @ 2013-03-31 9:46 UTC (permalink / raw)
To: notmuch; +Cc: Tomi Ollila
---
test/count | 46 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 46 insertions(+)
diff --git a/test/count b/test/count
index 879b114..05713fd 100755
--- a/test/count
+++ b/test/count
@@ -38,4 +38,50 @@ test_expect_equal \
"0" \
"`notmuch count --output=threads from:cworth and not from:cworth`"
+test_begin_subtest "message count is the default for batch count"
+notmuch count --batch >OUTPUT <<EOF
+
+from:cworth
+EOF
+notmuch count --output=messages >EXPECTED
+notmuch count --output=messages from:cworth >>EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "batch message count"
+notmuch count --batch --output=messages >OUTPUT <<EOF
+from:cworth
+
+tag:inbox
+EOF
+notmuch count --output=messages from:cworth >EXPECTED
+notmuch count --output=messages >>EXPECTED
+notmuch count --output=messages tag:inbox >>EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "batch thread count"
+notmuch count --batch --output=threads >OUTPUT <<EOF
+
+from:cworth
+from:cworth and not from:cworth
+foo
+EOF
+notmuch count --output=threads >EXPECTED
+notmuch count --output=threads from:cworth >>EXPECTED
+notmuch count --output=threads from:cworth and not from:cworth >>EXPECTED
+notmuch count --output=threads foo >>EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "batch message count with input file"
+cat >INPUT <<EOF
+from:cworth
+
+tag:inbox
+EOF
+notmuch count --input=INPUT --output=messages >OUTPUT
+notmuch count --output=messages from:cworth >EXPECTED
+notmuch count --output=messages >>EXPECTED
+notmuch count --output=messages tag:inbox >>EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+
test_done
--
1.7.10.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v3 6/6] emacs: hello: use batch count
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
` (4 preceding siblings ...)
2013-03-31 9:46 ` [PATCH v3 5/6] test: " Jani Nikula
@ 2013-03-31 9:46 ` Jani Nikula
2013-03-31 16:38 ` [PATCH v3 0/6] batch count for notmuch-hello speedup Tomi Ollila
2013-04-01 13:43 ` David Bremner
7 siblings, 0 replies; 9+ messages in thread
From: Jani Nikula @ 2013-03-31 9:46 UTC (permalink / raw)
To: notmuch; +Cc: Tomi Ollila
From: Mark Walters <markwalters1009@gmail.com>
This modifies notmuch hello to use the new count --batch
functionality. It should give exactly the same results as before but
under many conditions it should be much faster. In particular it is
much faster for remote use.
The code is a little ugly as it has to do some working out of the
query when asking the query and some when dealing with the result.
However, the code path is exactly the same in both local and remote
use.
---
emacs/notmuch-hello.el | 52 +++++++++++++++++++++++++++++-------------------
1 file changed, 32 insertions(+), 20 deletions(-)
diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index 00b78e1..cda79f1 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -381,26 +381,38 @@ The result is the list of elements of the form (NAME QUERY COUNT).
The values :show-empty-searches, :filter and :filter-count from
options will be handled as specified for
`notmuch-hello-insert-searches'."
- (notmuch-remove-if-not
- #'identity
- (mapcar
- (lambda (elem)
- (let* ((name (car elem))
- (query-and-count (if (consp (cdr elem))
- ;; do we have a different query for the message count?
- (cons (second elem) (third elem))
- (cons (cdr elem) (cdr elem))))
- (message-count
- (string-to-number
- (notmuch-saved-search-count
- (notmuch-hello-filtered-query (cdr query-and-count)
- (or (plist-get options :filter-count)
- (plist-get options :filter)))))))
- (and (or (plist-get options :show-empty-searches) (> message-count 0))
- (list name (notmuch-hello-filtered-query
- (car query-and-count) (plist-get options :filter))
- message-count))))
- query-alist)))
+ (with-temp-buffer
+ (dolist (elem query-alist nil)
+ (let ((count-query (if (consp (cdr elem))
+ ;; do we have a different query for the message count?
+ (third elem)
+ (cdr elem))))
+ (insert
+ (notmuch-hello-filtered-query count-query
+ (or (plist-get options :filter-count)
+ (plist-get options :filter)))
+ "\n")))
+
+ (call-process-region (point-min) (point-max) notmuch-command
+ t t nil "count" "--batch")
+ (goto-char (point-min))
+
+ (notmuch-remove-if-not
+ #'identity
+ (mapcar
+ (lambda (elem)
+ (let ((name (car elem))
+ (search-query (if (consp (cdr elem))
+ ;; do we have a different query for the message count?
+ (second elem)
+ (cdr elem)))
+ (message-count (prog1 (read (current-buffer))
+ (forward-line 1))))
+ (and (or (plist-get options :show-empty-searches) (> message-count 0))
+ (list name (notmuch-hello-filtered-query
+ search-query (plist-get options :filter))
+ message-count))))
+ query-alist))))
(defun notmuch-hello-insert-buttons (searches)
"Insert buttons for SEARCHES.
--
1.7.10.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* Re: [PATCH v3 0/6] batch count for notmuch-hello speedup
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
` (5 preceding siblings ...)
2013-03-31 9:46 ` [PATCH v3 6/6] emacs: hello: use batch count Jani Nikula
@ 2013-03-31 16:38 ` Tomi Ollila
2013-04-01 13:43 ` David Bremner
7 siblings, 0 replies; 9+ messages in thread
From: Tomi Ollila @ 2013-03-31 16:38 UTC (permalink / raw)
To: Jani Nikula, notmuch
On Sun, Mar 31 2013, Jani Nikula <jani@nikula.org> wrote:
> This is v3 of id:cover.1362841844.git.jani@nikula.org, with the commit
> messages of patches 1/6 and 3/6 fixed. No other changes. Thanks to
> Jameson and Tomi for review, and of course Mark for the elisp part.
+1 -- I verified there is no (accidental) changes between
id:cover.1362841844.git.jani@nikula.org and
id:cover.1364722841.git.jani@nikula.org
>
> BR,
> Jani.
Tomi
>
> Jani Nikula (5):
> cli: remove useless talloc_strdup
> cli: extract count printing to a separate function in notmuch count
> cli: add --batch option to notmuch count
> man: document notmuch count --batch and --input options
> test: notmuch count --batch and --input options
>
> Mark Walters (1):
> emacs: hello: use batch count
>
> emacs/notmuch-hello.el | 52 +++++++++++++---------
> man/man1/notmuch-count.1 | 20 +++++++++
> notmuch-count.c | 111 +++++++++++++++++++++++++++++++++++-----------
> test/count | 46 +++++++++++++++++++
> 4 files changed, 182 insertions(+), 47 deletions(-)
>
> --
> 1.7.10.4
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> http://notmuchmail.org/mailman/listinfo/notmuch
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCH v3 0/6] batch count for notmuch-hello speedup
2013-03-31 9:45 [PATCH v3 0/6] batch count for notmuch-hello speedup Jani Nikula
` (6 preceding siblings ...)
2013-03-31 16:38 ` [PATCH v3 0/6] batch count for notmuch-hello speedup Tomi Ollila
@ 2013-04-01 13:43 ` David Bremner
7 siblings, 0 replies; 9+ messages in thread
From: David Bremner @ 2013-04-01 13:43 UTC (permalink / raw)
To: Jani Nikula, notmuch; +Cc: Tomi Ollila
Jani Nikula <jani@nikula.org> writes:
> This is v3 of id:cover.1362841844.git.jani@nikula.org, with the commit
> messages of patches 1/6 and 3/6 fixed. No other changes. Thanks to
> Jameson and Tomi for review, and of course Mark for the elisp part.
Pushed,
d
^ permalink raw reply [flat|nested] 9+ messages in thread