[PATCH 0/4] Add a --include-text-content option to notmuch-show.c

[PATCH 0/4] Add a --include-text-content option to notmuch-show.c

[Mark Walters]

Currently notmuch show includes text/* parts (except text/html) in the
JSON and text outputs. This patch adds a new option
--include-text-content=true|false which allows the caller to disable
this behaviour.

This is similar to the --headers-only option proposed in
id:"1341041595-5858-1-git-send-email-markwalters1009@gmail.com". The
key difference is that this version does not change the JSON output
schema: the schema says that "A leaf part's body content is optional,
but may be included if it can be correctly encoded as a string."

This means that the emacs show mode works correctly if switched to
specifying --include-text-content=false (it fetches each part
individually). Indeed, it may be desirable to make that the default
(as suggested by Jamie in
id:"877gu7gzy9.fsf@servo.finestructure.net"). However, this may make
show significantly slower so we do not change the default here.

There have been other suggestions of allowing the user to specify
exactly which parts/headers etc notmuch should return and even of
unifying show and search. That is obviously a much bigger change than
this (and this would not make that any more difficult).

Finally, the advantage of this change is that it can substantially
speed up some uses (when the caller only wants the headers): see
id:"87r4sei8yu.fsf@qmul.ac.uk" for some examples with a factor of 8
speed up.

Best wishes

Mark



Mark Walters (4):
  cli: show: add --include-text-content=true|false option
  cli: show: implement --include-text-content for --format=text
  test: add some tests for --include-text-content option
  man: update man page for --include-text-option

 man/man1/notmuch-show.1 |   12 ++++++++++
 notmuch-client.h        |    3 +-
 notmuch-reply.c         |    2 +-
 notmuch-show.c          |   57 ++++++++++++++++++++++++++++-------------------
 test/json               |    9 +++++++
 5 files changed, 58 insertions(+), 25 deletions(-)

-- 
1.7.9.1

[PATCH 1/4] cli: show: add --include-text-content=true|false option

[Mark Walters]

Normally notmuch show includes any text/* parts (except text/html) in
the output.  This option allows the caller to suppress this behaviour.

This is used by notmuch-pick.el (although not needed) because it gives
a speed-up of at least a factor of a two; moreover it reduces the
memory usage in emacs hugely.

This commit just implements it for --format-json. The next commit adds
it for --format=text. The option does not make sense for --format=raw
or --format=mbox.
---

This patch is a little bigger than I would like but a lot of it is code movement.

 notmuch-client.h |    3 ++-
 notmuch-reply.c  |    2 +-
 notmuch-show.c   |   44 +++++++++++++++++++++++++-------------------
 3 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/notmuch-client.h b/notmuch-client.h
index 0c17b79..edb356d 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -87,6 +87,7 @@ typedef struct notmuch_crypto {
 typedef struct notmuch_show_params {
     notmuch_bool_t entire_thread;
     notmuch_bool_t omit_excluded;
+    notmuch_bool_t include_text_content;
     notmuch_bool_t raw;
     int part;
     notmuch_crypto_t crypto;
@@ -176,7 +177,7 @@ notmuch_status_t
 show_one_part (const char *filename, int part);
 
 void
-format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first);
+format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first, notmuch_bool_t include_text_content);
 
 void
 format_headers_json (const void *ctx, GMimeMessage *message, notmuch_bool_t reply);
diff --git a/notmuch-reply.c b/notmuch-reply.c
index 3a038ed..de21f3b 100644
--- a/notmuch-reply.c
+++ b/notmuch-reply.c
@@ -620,7 +620,7 @@ notmuch_reply_format_json(void *ctx,
     /* Start the original */
     printf (", \"original\": ");
 
-    format_part_json (ctx, node, TRUE);
+    format_part_json (ctx, node, TRUE, TRUE);
 
     /* End */
     printf ("}\n");
diff --git a/notmuch-show.c b/notmuch-show.c
index 8f3c60e..35e3f22 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -559,7 +559,7 @@ format_part_text (const void *ctx, mime_node_t *node,
 }
 
 void
-format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
+format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first, notmuch_bool_t include_text_content)
 {
     /* Any changes to the JSON format should be reflected in the file
      * devel/schemata. */
@@ -572,7 +572,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	format_headers_json (ctx, GMIME_MESSAGE (node->part), FALSE);
 
 	printf (", \"body\": [");
-	format_part_json (ctx, mime_node_child (node, 0), first);
+	format_part_json (ctx, mime_node_child (node, 0), first, include_text_content);
 
 	printf ("]}");
 	return;
@@ -623,19 +623,21 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	 * makes charset decoding the responsibility on the caller, we
 	 * report the charset for text/html parts.
 	 */
-	if (g_mime_content_type_is_type (content_type, "text", "html")) {
-	    const char *content_charset = g_mime_object_get_content_type_parameter (meta, "charset");
-
-	    if (content_charset != NULL)
-		printf (", \"content-charset\": %s", json_quote_str (local, content_charset));
-	} else if (g_mime_content_type_is_type (content_type, "text", "*")) {
-	    GMimeStream *stream_memory = g_mime_stream_mem_new ();
-	    GByteArray *part_content;
-	    show_text_part_content (node->part, stream_memory, 0);
-	    part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
-
-	    printf (", \"content\": %s", json_quote_chararray (local, (char *) part_content->data, part_content->len));
-	    g_object_unref (stream_memory);
+	if (g_mime_content_type_is_type (content_type, "text", "*")) {
+	    if (g_mime_content_type_is_type (content_type, "text", "html") || !include_text_content) {
+		const char *content_charset = g_mime_object_get_content_type_parameter (meta, "charset");
+
+		if (content_charset != NULL)
+		    printf (", \"content-charset\": %s", json_quote_str (local, content_charset));
+	    } else {
+		GMimeStream *stream_memory = g_mime_stream_mem_new ();
+		GByteArray *part_content;
+		show_text_part_content (node->part, stream_memory, 0);
+		part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
+
+		printf (", \"content\": %s", json_quote_chararray (local, (char *) part_content->data, part_content->len));
+		g_object_unref (stream_memory);
+	    }
 	}
     } else if (GMIME_IS_MULTIPART (node->part)) {
 	printf (", \"content\": [");
@@ -652,17 +654,16 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
     talloc_free (local);
 
     for (i = 0; i < node->nchildren; i++)
-	format_part_json (ctx, mime_node_child (node, i), i == 0);
+	format_part_json (ctx, mime_node_child (node, i), i == 0, include_text_content);
 
     printf ("%s}", terminator);
 }
 
 static notmuch_status_t
 format_part_json_entry (const void *ctx, mime_node_t *node, unused (int indent),
-			unused (const notmuch_show_params_t *params))
+			const notmuch_show_params_t *params)
 {
-    format_part_json (ctx, node, TRUE);
-
+    format_part_json (ctx, node, TRUE, params->include_text_content);
     return NOTMUCH_STATUS_SUCCESS;
 }
 
@@ -1004,6 +1005,7 @@ notmuch_show_command (void *ctx, unused (int argc), unused (char *argv[]))
     notmuch_show_params_t params = {
 	.part = -1,
 	.omit_excluded = TRUE,
+	.include_text_content = TRUE,
 	.crypto = {
 	    .verify = FALSE,
 	    .decrypt = FALSE
@@ -1032,6 +1034,7 @@ notmuch_show_command (void *ctx, unused (int argc), unused (char *argv[]))
 	{ NOTMUCH_OPT_INT, &params.part, "part", 'p', 0 },
 	{ NOTMUCH_OPT_BOOLEAN, &params.crypto.decrypt, "decrypt", 'd', 0 },
 	{ NOTMUCH_OPT_BOOLEAN, &params.crypto.verify, "verify", 'v', 0 },
+	{ NOTMUCH_OPT_BOOLEAN, &params.include_text_content, "include-text-content", 'h', 0 },
 	{ 0, 0, 0, 0, 0 }
     };
 
@@ -1086,6 +1089,9 @@ notmuch_show_command (void *ctx, unused (int argc), unused (char *argv[]))
 	    entire_thread = ENTIRE_THREAD_FALSE;
     }
 
+    if (!params.include_text_content && (format == &format_mbox || format == &format_raw))
+	fprintf (stderr, "Warning: --include_text_content=false only makes sense for format=json and format=text\n");
+
     if (entire_thread == ENTIRE_THREAD_TRUE)
 	params.entire_thread = TRUE;
     else
-- 
1.7.9.1

[PATCH 2/4] cli: show: implement --include-text-content for --format=text

[Mark Walters]

---
 notmuch-show.c |   13 +++++++++----
 1 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 35e3f22..6261b6d 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -537,11 +537,15 @@ format_part_text (const void *ctx, mime_node_t *node,
 	if (g_mime_content_type_is_type (content_type, "text", "*") &&
 	    !g_mime_content_type_is_type (content_type, "text", "html"))
 	{
-	    GMimeStream *stream_stdout = g_mime_stream_file_new (stdout);
-	    g_mime_stream_file_set_owner (GMIME_STREAM_FILE (stream_stdout), FALSE);
-	    show_text_part_content (node->part, stream_stdout, 0);
-	    g_object_unref(stream_stdout);
+	    if (params->include_text_content) {
+		GMimeStream *stream_stdout = g_mime_stream_file_new (stdout);
+		g_mime_stream_file_set_owner (GMIME_STREAM_FILE (stream_stdout), FALSE);
+		show_text_part_content (node->part, stream_stdout, 0);
+		g_object_unref(stream_stdout);
+	    } else {
+		printf ("Text part (not included): %s\n",
+			g_mime_content_type_to_string (content_type));
+	    }
 	} else {
 	    printf ("Non-text part: %s\n",
 		    g_mime_content_type_to_string (content_type));
-- 
1.7.9.1

[PATCH 3/4] test: add some tests for --include-text-content option

[Mark Walters]

---
 test/json |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/test/json b/test/json
index 6439788..53cb039 100755
--- a/test/json
+++ b/test/json
@@ -7,6 +7,15 @@ add_message "[subject]=\"json-show-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:0
 output=$(notmuch show --format=json "json-show-message")
 test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"excluded\": false, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
 
+# This should be the same output as above.
+test_begin_subtest "Show message: json --include-text-content=true"
+output=$(notmuch show --format=json --include-text-content=true "json-show-message")
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"excluded\": false, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
+
+test_begin_subtest "Show message: json --include-text-content=false"
+output=$(notmuch show --format=json --include-text-content=false "json-show-message")
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"excluded\": false, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\"}]}, []]]]"
+
 test_begin_subtest "Search message: json"
 add_message "[subject]=\"json-search-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"json-search-message\""
 output=$(notmuch search --format=json "json-search-message" | notmuch_search_sanitize)
-- 
1.7.9.1

[PATCH 4/4] man: update man page for --include-text-option

[Mark Walters]

---
 man/man1/notmuch-show.1 |   12 ++++++++++++
 1 files changed, 12 insertions(+), 0 deletions(-)

diff --git a/man/man1/notmuch-show.1 b/man/man1/notmuch-show.1
index b51a54c..0226d3b 100644
--- a/man/man1/notmuch-show.1
+++ b/man/man1/notmuch-show.1
@@ -152,6 +152,18 @@ The default is
 
 .RE
 
+.RS 4
+.TP 4
+.B \-\-include-text-content=(true|false)
+
+By default the output includes the content structure together with all
+text/plain content parts. With --include-text-content=false this
+content is omitted from the output. This is useful if the caller is
+only interested in the headers as it is much faster and the amount of
+output data is substantially reduced. Note this option is not
+applicable to --format=raw or --format=mbox.
+.RE
+
 A common use of
 .B notmuch show
 is to display a single thread of email messages. For this, use a
-- 
1.7.9.1