[PATCH v5 0/6] lib/cli: limit number of messages in search results

[PATCH v5 0/6] lib/cli: limit number of messages in search results

[Jani Nikula]

Hi, output limiting patches v5, with only documentation changes since the
previous version id:"cover.1320615322.git.jani@nikula.org".

The changes are as suggested by Austin Clements:
id:"87ehxcufr0.fsf@awakening.csail.mit.edu"
id:"87bosgufcf.fsf@awakening.csail.mit.edu"


BR,
Jani.


Jani Nikula (6):
  lib: add function to get the number of threads matching a search
  cli: add options --first and --maxitems to notmuch search
  cli: drop unused code from notmuch count
  cli: add support for --output parameter in notmuch count
  test: add tests for notmuch count
  test: add tests for notmuch search --first and --maxitems

 NEWS                 |   10 +++++++
 lib/notmuch.h        |   14 ++++++++++
 lib/query.cc         |   44 +++++++++++++++++++++++++++++++
 notmuch-count.c      |   44 +++++++++----------------------
 notmuch-search.c     |   70 ++++++++++++++++++++++++++++++++++++++++--------
 notmuch.1            |   48 +++++++++++++++++++++++++++++----
 test/count           |   40 ++++++++++++++++++++++++++++
 test/notmuch-test    |    2 +
 test/search-limiting |   71 ++++++++++++++++++++++++++++++++++++++++++++++++++
 9 files changed, 294 insertions(+), 49 deletions(-)
 create mode 100755 test/count
 create mode 100755 test/search-limiting

-- 
1.7.5.4

[PATCH v5 1/6] lib: add function to get the number of threads matching a search

[Jani Nikula]

Add function notmuch_query_count_threads() to get the number of threads
matching a search. This is done by performing a search and figuring out the
number of unique thread IDs in the matching messages, a significantly
heavier operation than notmuch_query_count_messages().

Signed-off-by: Jani Nikula <jani@nikula.org>
---
 lib/notmuch.h |   14 ++++++++++++++
 lib/query.cc  |   44 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 0 deletions(-)

diff --git a/lib/notmuch.h b/lib/notmuch.h
index c4330e4..9f23a10 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -609,6 +609,20 @@ notmuch_threads_destroy (notmuch_threads_t *threads);
 unsigned
 notmuch_query_count_messages (notmuch_query_t *query);
  
+/* Return the number of threads matching a search.
+ *
+ * This function performs a search and returns the number of unique thread IDs
+ * in the matching messages. This is the same as number of threads matching a
+ * search.
+ *
+ * Note that this is a significantly heavier operation than
+ * notmuch_query_count_messages().
+ *
+ * If an error occurs, this function may return 0.
+ */
+unsigned
+notmuch_query_count_threads (notmuch_query_t *query);
+
 /* Get the thread ID of 'thread'.
  *
  * The returned string belongs to 'thread' and as such, should not be
diff --git a/lib/query.cc b/lib/query.cc
index 6f02b04..b6c0f12 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -457,3 +457,47 @@ notmuch_query_count_messages (notmuch_query_t *query)
 
     return count;
 }
+
+unsigned
+notmuch_query_count_threads (notmuch_query_t *query)
+{
+    notmuch_messages_t *messages;
+    GHashTable *hash;
+    unsigned int count;
+    notmuch_sort_t sort;
+
+    sort = query->sort;
+    query->sort = NOTMUCH_SORT_UNSORTED;
+    messages = notmuch_query_search_messages (query);
+    query->sort = sort;
+    if (messages == NULL)
+	return 0;
+
+    hash = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL);
+    if (hash == NULL) {
+	talloc_free (messages);
+	return 0;
+    }
+
+    while (notmuch_messages_valid (messages)) {
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	const char *thread_id = notmuch_message_get_thread_id (message);
+	char *thread_id_copy = talloc_strdup (messages, thread_id);
+	if (unlikely (thread_id_copy == NULL)) {
+	    notmuch_message_destroy (message);
+	    count = 0;
+	    goto DONE;
+	}
+	g_hash_table_insert (hash, thread_id_copy, NULL);
+	notmuch_message_destroy (message);
+	notmuch_messages_move_to_next (messages);
+    }
+
+    count = g_hash_table_size (hash);
+
+  DONE:
+    g_hash_table_unref (hash);
+    talloc_free (messages);
+
+    return count;
+}
-- 
1.7.5.4

[PATCH v5 2/6] cli: add options --first and --maxitems to notmuch search

[Jani Nikula]

Add options --first=[-]N and --maxitems=M to notmuch search to determine
the first result and maximum number of results to display.

Option --maxitems=M limits the maximum number of results to display to M.

Option --first=[-]N skips the first N results; with the leading '-' skip
until the Nth result from the end (showing a total of N results if within
bounds of the total number of results and not limited with --maxitems).

Note that --first with a negative N for thread or summary output requires
counting the number of matching threads in advance.

Signed-off-by: Jani Nikula <jani@nikula.org>
---
 NEWS             |    5 ++++
 notmuch-search.c |   70 ++++++++++++++++++++++++++++++++++++++++++++---------
 notmuch.1        |   19 +++++++++++++-
 3 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/NEWS b/NEWS
index 71c7c9a..abc749a 100644
--- a/NEWS
+++ b/NEWS
@@ -23,6 +23,11 @@ Add search terms to  "notmuch dump"
   search/show/tag. The output file argument of dump is deprecated in
   favour of using stdout.
 
+Add "notmuch search" --first and --maxitems options
+
+  The search command now takes options --first=[-]N and --maxitems=N to limit
+  the number of results shown.
+
 Notmuch 0.9 (2011-10-01)
 ========================
 
diff --git a/notmuch-search.c b/notmuch-search.c
index 6f04d9a..c62a594 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -194,13 +194,22 @@ static int
 do_search_threads (const search_format_t *format,
 		   notmuch_query_t *query,
 		   notmuch_sort_t sort,
-		   output_t output)
+		   output_t output,
+		   int first,
+		   int maxitems)
 {
     notmuch_thread_t *thread;
     notmuch_threads_t *threads;
     notmuch_tags_t *tags;
     time_t date;
     int first_thread = 1;
+    int i;
+
+    if (first < 0) {
+	first += notmuch_query_count_threads (query);
+	if (first < 0)
+	    first = 0;
+    }
 
     threads = notmuch_query_search_threads (query);
     if (threads == NULL)
@@ -208,17 +217,23 @@ do_search_threads (const search_format_t *format,
 
     fputs (format->results_start, stdout);
 
-    for (;
-	 notmuch_threads_valid (threads);
-	 notmuch_threads_move_to_next (threads))
+    for (i = 0;
+	 notmuch_threads_valid (threads) &&
+	     (maxitems < 0 || i < first + maxitems);
+	 notmuch_threads_move_to_next (threads), i++)
     {
 	int first_tag = 1;
 
+	thread = notmuch_threads_get (threads);
+
+	if (i < first) {
+	    notmuch_thread_destroy (thread);
+	    continue;
+	}
+
 	if (! first_thread)
 	    fputs (format->item_sep, stdout);
 
-	thread = notmuch_threads_get (threads);
-
 	if (output == OUTPUT_THREADS) {
 	    format->item_id (thread, "thread:",
 			     notmuch_thread_get_thread_id (thread));
@@ -271,12 +286,21 @@ do_search_threads (const search_format_t *format,
 static int
 do_search_messages (const search_format_t *format,
 		    notmuch_query_t *query,
-		    output_t output)
+		    output_t output,
+		    int first,
+		    int maxitems)
 {
     notmuch_message_t *message;
     notmuch_messages_t *messages;
     notmuch_filenames_t *filenames;
     int first_message = 1;
+    int i;
+
+    if (first < 0) {
+	first += notmuch_query_count_messages (query);
+	if (first < 0)
+	    first = 0;
+    }
 
     messages = notmuch_query_search_messages (query);
     if (messages == NULL)
@@ -284,10 +308,14 @@ do_search_messages (const search_format_t *format,
 
     fputs (format->results_start, stdout);
 
-    for (;
-	 notmuch_messages_valid (messages);
-	 notmuch_messages_move_to_next (messages))
+    for (i = 0;
+	 notmuch_messages_valid (messages) &&
+	     (maxitems < 0 || i < first + maxitems);
+	 notmuch_messages_move_to_next (messages), i++)
     {
+	if (i < first)
+	    continue;
+
 	message = notmuch_messages_get (messages);
 
 	if (output == OUTPUT_FILES) {
@@ -394,6 +422,8 @@ notmuch_search_command (void *ctx, int argc, char *argv[])
     const search_format_t *format = &format_text;
     int i, ret;
     output_t output = OUTPUT_SUMMARY;
+    int maxitems = -1; /* unlimited */
+    int first = 0;
 
     argc--; argv++; /* skip subcommand argument */
 
@@ -412,6 +442,22 @@ notmuch_search_command (void *ctx, int argc, char *argv[])
 		fprintf (stderr, "Invalid value for --sort: %s\n", opt);
 		return 1;
 	    }
+	} else if (STRNCMP_LITERAL (argv[i], "--first=") == 0) {
+	    char *p;
+	    opt = argv[i] + sizeof ("--first=") - 1;
+	    first = strtol (opt, &p, 10);
+	    if (*opt == '\0' || p == opt || *p != '\0') {
+		fprintf (stderr, "Invalid value for --first: %s\n", opt);
+		return 1;
+	    }
+	} else if (STRNCMP_LITERAL (argv[i], "--maxitems=") == 0) {
+	    char *p;
+	    opt = argv[i] + sizeof ("--maxitems=") - 1;
+	    maxitems = strtoul (opt, &p, 10);
+	    if (*opt == '\0' || p == opt || *p != '\0') {
+		fprintf (stderr, "Invalid value for --maxitems: %s\n", opt);
+		return 1;
+	    }
 	} else if (STRNCMP_LITERAL (argv[i], "--format=") == 0) {
 	    opt = argv[i] + sizeof ("--format=") - 1;
 	    if (strcmp (opt, "text") == 0) {
@@ -478,11 +524,11 @@ notmuch_search_command (void *ctx, int argc, char *argv[])
     default:
     case OUTPUT_SUMMARY:
     case OUTPUT_THREADS:
-	ret = do_search_threads (format, query, sort, output);
+	ret = do_search_threads (format, query, sort, output, first, maxitems);
 	break;
     case OUTPUT_MESSAGES:
     case OUTPUT_FILES:
-	ret = do_search_messages (format, query, output);
+	ret = do_search_messages (format, query, output, first, maxitems);
 	break;
     case OUTPUT_TAGS:
 	ret = do_search_tags (notmuch, format, query);
diff --git a/notmuch.1 b/notmuch.1
index bba479e..15c3147 100644
--- a/notmuch.1
+++ b/notmuch.1
@@ -214,11 +214,26 @@ when sorting by
 .B newest\-first
 the threads will be sorted by the newest message in each thread.
 
-.RE
-.RS 4
 By default, results will be displayed in reverse chronological order,
 (that is, the newest results will be displayed first).
+.RE
+
+.RS 4
+.TP 4
+.BR \-\-first=[\-]N
+
+Skip displaying the first N results. With the leading '\-', start at the Nth
+result from the end.
+.RE
+
+.RS 4
+.TP 4
+.BR \-\-maxitems=N
 
+Limit the number of displayed results to N.
+.RE
+
+.RS 4
 See the
 .B "SEARCH SYNTAX"
 section below for details of the supported syntax for <search-terms>.
-- 
1.7.5.4

[PATCH v5 3/6] cli: drop unused code from notmuch count

[Jani Nikula]

Remove unused code within #if 0 blocks from notmuch count.

Signed-off-by: Jani Nikula <jani@nikula.org>
---
 notmuch-count.c |   32 --------------------------------
 1 files changed, 0 insertions(+), 32 deletions(-)

diff --git a/notmuch-count.c b/notmuch-count.c
index 0d700a9..a35be40 100644
--- a/notmuch-count.c
+++ b/notmuch-count.c
@@ -29,11 +29,6 @@ notmuch_count_command (void *ctx, int argc, char *argv[])
     notmuch_query_t *query;
     char *query_str;
     int i;
-#if 0
-    char *opt, *end;
-    int i, first = 0, max_threads = -1;
-    notmuch_sort_t sort = NOTMUCH_SORT_NEWEST_FIRST;
-#endif
 
     argc--; argv++; /* skip subcommand argument */
 
@@ -42,33 +37,6 @@ notmuch_count_command (void *ctx, int argc, char *argv[])
 	    i++;
 	    break;
 	}
-#if 0
-	if (STRNCMP_LITERAL (argv[i], "--first=") == 0) {
-	    opt = argv[i] + sizeof ("--first=") - 1;
-	    first = strtoul (opt, &end, 10);
-	    if (*opt == '\0' || *end != '\0') {
-		fprintf (stderr, "Invalid value for --first: %s\n", opt);
-		return 1;
-	    }
-	} else if (STRNCMP_LITERAL (argv[i], "--max-threads=") == 0) {
-	    opt = argv[i] + sizeof ("--max-threads=") - 1;
-	    max_threads = strtoul (opt, &end, 10);
-	    if (*opt == '\0' || *end != '\0') {
-		fprintf (stderr, "Invalid value for --max-threads: %s\n", opt);
-		return 1;
-	    }
-	} else if (STRNCMP_LITERAL (argv[i], "--sort=") == 0) {
-	    opt = argv[i] + sizeof ("--sort=") - 1;
-	    if (strcmp (opt, "oldest-first") == 0) {
-		sort = NOTMUCH_SORT_OLDEST_FIRST;
-	    } else if (strcmp (opt, "newest-first") == 0) {
-		sort = NOTMUCH_SORT_NEWEST_FIRST;
-	    } else {
-		fprintf (stderr, "Invalid value for --sort: %s\n", opt);
-		return 1;
-	    }
-	} else
-#endif
 	{
 	    fprintf (stderr, "Unrecognized option: %s\n", argv[i]);
 	    return 1;
-- 
1.7.5.4

[PATCH v5 4/6] cli: add support for --output parameter in notmuch count

[Jani Nikula]

Add support for --output=messages (which remains the default) and
--output=threads to notmuch count.

Signed-off-by: Jani Nikula <jani@nikula.org>
---
 NEWS            |    5 +++++
 notmuch-count.c |   18 ++++++++++++++++--
 notmuch.1       |   29 +++++++++++++++++++++++++----
 3 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/NEWS b/NEWS
index abc749a..78434d9 100644
--- a/NEWS
+++ b/NEWS
@@ -28,6 +28,11 @@ Add "notmuch search" --first and --maxitems options
   The search command now takes options --first=[-]N and --maxitems=N to limit
   the number of results shown.
 
+Add "notmuch count --output" option
+
+  The count command is now capable of counting threads in addition to
+  messages. This is selected using the new --output=(threads|messages) option.
+
 Notmuch 0.9 (2011-10-01)
 ========================
 
diff --git a/notmuch-count.c b/notmuch-count.c
index a35be40..20ce334 100644
--- a/notmuch-count.c
+++ b/notmuch-count.c
@@ -29,6 +29,7 @@ notmuch_count_command (void *ctx, int argc, char *argv[])
     notmuch_query_t *query;
     char *query_str;
     int i;
+    notmuch_bool_t output_messages = TRUE;
 
     argc--; argv++; /* skip subcommand argument */
 
@@ -37,7 +38,17 @@ notmuch_count_command (void *ctx, int argc, char *argv[])
 	    i++;
 	    break;
 	}
-	{
+	if (STRNCMP_LITERAL (argv[i], "--output=") == 0) {
+	    const char *opt = argv[i] + sizeof ("--output=") - 1;
+	    if (strcmp (opt, "threads") == 0) {
+		output_messages = FALSE;
+	    } else if (strcmp (opt, "messages") == 0) {
+		output_messages = TRUE;
+	    } else {
+		fprintf (stderr, "Invalid value for --output: %s\n", opt);
+		return 1;
+	    }
+	} else {
 	    fprintf (stderr, "Unrecognized option: %s\n", argv[i]);
 	    return 1;
 	}
@@ -71,7 +82,10 @@ notmuch_count_command (void *ctx, int argc, char *argv[])
 	return 1;
     }
 
-    printf ("%u\n", notmuch_query_count_messages(query));
+    if (output_messages)
+	printf ("%u\n", notmuch_query_count_messages (query));
+    else
+	printf ("%u\n", notmuch_query_count_threads (query));
 
     notmuch_query_destroy (query);
     notmuch_database_close (notmuch);
diff --git a/notmuch.1 b/notmuch.1
index 15c3147..64114db 100644
--- a/notmuch.1
+++ b/notmuch.1
@@ -372,14 +372,35 @@ section below for details of the supported syntax for <search-terms>.
 .RE
 .RS 4
 .TP 4
-.BR count " <search-term>..."
+.BR count " [options...] <search-term>..."
 
 Count messages matching the search terms.
 
-The number of matching messages is output to stdout.
+The number of matching messages (or threads) is output to stdout.
 
-With no search terms, a count of all messages in the database will be
-displayed.
+With no search terms, a count of all messages (or threads) in the database will
+be displayed.
+
+Supported options for
+.B count
+include
+.RS 4
+.TP 4
+.B \-\-output=(messages|threads)
+
+.RS 4
+.TP 4
+.B messages
+
+Output the number of matching messages. This is the default.
+.RE
+.RS 4
+.TP 4
+.B threads
+
+Output the number of matching threads.
+.RE
+.RE
 .RE
 .RE
 
-- 
1.7.5.4

[PATCH v5 5/6] test: add tests for notmuch count

[Jani Nikula]

Signed-off-by: Jani Nikula <jani@nikula.org>
---
 test/count        |   40 ++++++++++++++++++++++++++++++++++++++++
 test/notmuch-test |    1 +
 2 files changed, 41 insertions(+), 0 deletions(-)
 create mode 100755 test/count

diff --git a/test/count b/test/count
new file mode 100755
index 0000000..300b171
--- /dev/null
+++ b/test/count
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+test_description='"notmuch count" for messages and threads'
+. ./test-lib.sh
+
+add_email_corpus
+
+SEARCH="\"*\""
+
+test_begin_subtest "message count is the default for notmuch count"
+test_expect_equal \
+    "`notmuch search --output=messages ${SEARCH} | wc -l`" \
+    "`notmuch count ${SEARCH}`"
+
+test_begin_subtest "message count with --output=messages"
+test_expect_equal \
+    "`notmuch search --output=messages ${SEARCH} | wc -l`" \
+    "`notmuch count --output=messages ${SEARCH}`"
+
+test_begin_subtest "thread count with --output=threads"
+test_expect_equal \
+    "`notmuch search --output=threads ${SEARCH} | wc -l`" \
+    "`notmuch count --output=threads ${SEARCH}`"
+
+test_begin_subtest "thread count is the default for notmuch search"
+test_expect_equal \
+    "`notmuch search ${SEARCH} | wc -l`" \
+    "`notmuch count --output=threads ${SEARCH}`"
+
+SEARCH="from:cworth and not from:cworth"
+test_begin_subtest "count with no matching messages"
+test_expect_equal \
+    "0" \
+    "`notmuch count --output=messages ${SEARCH}`"
+
+test_begin_subtest "count with no matching threads"
+test_expect_equal \
+    "0" \
+    "`notmuch count --output=threads ${SEARCH}`"
+
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index 738f8f6..587adb5 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -19,6 +19,7 @@ cd $(dirname "$0")
 TESTS="
   basic
   new
+  count
   search
   search-output
   search-by-folder
-- 
1.7.5.4

[PATCH v5 6/6] test: add tests for notmuch search --first and --maxitems

[Jani Nikula]

Signed-off-by: Jani Nikula <jani@nikula.org>
---
 test/notmuch-test    |    1 +
 test/search-limiting |   71 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 0 deletions(-)
 create mode 100755 test/search-limiting

diff --git a/test/notmuch-test b/test/notmuch-test
index 587adb5..435b469 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -25,6 +25,7 @@ TESTS="
   search-by-folder
   search-position-overlap-bug
   search-insufficient-from-quoting
+  search-limiting
   json
   multipart
   thread-naming
diff --git a/test/search-limiting b/test/search-limiting
new file mode 100755
index 0000000..45cc0a9
--- /dev/null
+++ b/test/search-limiting
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+test_description='"notmuch search" --first and --maxitems parameters'
+. ./test-lib.sh
+
+add_email_corpus
+
+for outp in messages threads; do
+    test_begin_subtest "${outp}: maxitems does the right thing"
+    notmuch search --output=${outp} "*" | head -n 20 >expected
+    notmuch search --output=${outp} --maxitems=20 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: concatenation of limited searches"
+    notmuch search --output=${outp} "*" | head -n 20 >expected
+    notmuch search --output=${outp} --maxitems=10 "*" >output
+    notmuch search --output=${outp} --maxitems=10 --first=10 "*" >>output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: maxitems larger than result set"
+    N=`notmuch count --output=${outp} "*"`
+    notmuch search --output=${outp} "*" >expected
+    notmuch search --output=${outp} --maxitems=$((1 + ${N})) "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: maxitems = 0"
+    test_expect_equal "`notmuch search --output=${outp} --maxitems=0 "*"`" ""
+
+    test_begin_subtest "${outp}: first does the right thing"
+    # note: tail -n +N is 1-based
+    notmuch search --output=${outp} "*" | tail -n +21 >expected
+    notmuch search --output=${outp} --first=20 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: first = 0"
+    notmuch search --output=${outp} "*" >expected
+    notmuch search --output=${outp} --first=0 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: negative first"
+    notmuch search --output=${outp} "*" | tail -n 20 >expected
+    notmuch search --output=${outp} --first=-20 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: negative first"
+    notmuch search --output=${outp} "*" | tail -n 1 >expected
+    notmuch search --output=${outp} --first=-1 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: negative first combined with maxitems"
+    notmuch search --output=${outp} "*" | tail -n 20 | head -n 10 >expected
+    notmuch search --output=${outp} --first=-20 --maxitems=10 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: negative first combined with equal maxitems"
+    notmuch search --output=${outp} "*" | tail -n 20 >expected
+    notmuch search --output=${outp} --first=-20 --maxitems=20 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: negative first combined with large maxitems"
+    notmuch search --output=${outp} "*" | tail -n 20 >expected
+    notmuch search --output=${outp} --first=-20 --maxitems=50 "*" >output
+    test_expect_equal_file expected output
+
+    test_begin_subtest "${outp}: negative first larger then results"
+    N=`notmuch count --output=${outp} "*"`
+    notmuch search --output=${outp} "*" >expected
+    notmuch search --output=${outp} --first=-$((1 + ${N})) "*" >output
+    test_expect_equal_file expected output
+done
+
+test_done
-- 
1.7.5.4