[PATCH 0/8] Rewrite JSON show format

[PATCH 0/8] Rewrite JSON show format

[Austin Clements]

The saga continues.  As for the text format, this first shifts lots of
code around without changing its semantics, then it dives in and
simplifies a lot of things.  Don't be put off by the number of
patches; most of them are straightforward.

As an added bonus, I documented (!) the JSON format for both show and
search.

Thanks to Adam for the kick in the tail I needed to put the finishing
touches on this series.

[PATCH 1/8] Document the JSON schemata used by show and search

[Austin Clements]

---
 devel/schemata   |  135 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 notmuch-search.c |    3 +
 notmuch-show.c   |    2 +
 3 files changed, 140 insertions(+), 0 deletions(-)
 create mode 100644 devel/schemata

diff --git a/devel/schemata b/devel/schemata
new file mode 100644
index 0000000..d90d4c6
--- /dev/null
+++ b/devel/schemata
@@ -0,0 +1,135 @@
+This file describes the schemata used for notmuch's structured output
+format (currently JSON).
+
+[]'s indicate lists.  List items can be marked with a '?', meaning
+they are optional; or a '*', meaning there can be zero or more of that
+item.  {}'s indicate an object that maps from field identifiers to
+values.  An object field marked '?' is optional.  |'s indicate
+alternates (e.g., int|string means something can be an int or a
+string).
+
+Common non-terminals
+--------------------
+
+# Number of seconds since the Epoch
+unix_time = int
+
+# Thread ID, sans "thread:"
+threadid = string
+
+# Message ID, sans "id:"
+messageid = string
+
+notmuch show schema
+-------------------
+
+# A top-level set of threads (do_show)
+# Returned by notmuch show without a --part argument
+thread_set = [thread*]
+
+# Top-level messages in a thread (show_messages)
+thread = [thread_node*]
+
+# A message and its replies (show_messages)
+thread_node = [
+    message?,                 # present if --entire-thread or matched
+    [thread_node*]            # children of message
+]
+
+# A message (show_message)
+message = {
+    # (format_message_json)
+    id:             messageid,
+    match:          bool,
+    filename:	    string,
+    timestamp:      unix_time, # date header as unix time
+    date_relative:  string,   # user-friendly timestamp
+    tags:           [string*],
+
+    headers:        headers,
+    body:           [part]
+}
+
+# A MIME part (show_message_body)
+part = {
+    # format_part_start_json
+    id:             int|string, # part id (currently DFS part number)
+
+    # format_part_encstatus_json
+    encstatus?:     encstatus,
+
+    # format_part_sigstatus_json
+    sigstatus?:     sigstatus,
+
+    # format_part_content_json
+    content-type:   string,
+    content-id?:    string,
+    # if content-type starts with "multipart/":
+    content:        [part*],
+    # if content-type is "message/rfc822":
+    content:        [{headers: headers, body: [part]}],
+    # otherwise (leaf parts):
+    filename?:      string,
+    content-charset?: string,
+    content?:       string    # pre-fetched body content
+}
+
+# The headers of a message (format_headers_json with raw headers) or
+# a part (format_headers_message_part_json with pretty-printed headers)
+headers = {
+    Subject:        string,
+    From:           string,
+    To?:            string,
+    Cc?:            string,
+    Bcc?:           string,
+    Date:           string
+}
+
+# Encryption status (format_part_encstatus_json)
+encstatus = [{status: "good"|"bad"}]
+
+# Signature status (format_part_sigstatus_json)
+sigstatus = [signature*]
+
+signature = {
+    # signature_status_to_string
+    status:         "none"|"good"|"bad"|"error"|"unknown",
+    # if status is "good":
+    fingerprint?:   string,
+    created?:       unix_time,
+    expires?:       unix_time,
+    userid?:        string
+    # if status is not "good":
+    keyid?:         string
+    # if the signature has errors:    
+    errors?:        int
+}
+
+notmuch search schema
+---------------------
+
+# --output=summary
+summary = [thread*]
+
+# --output=threads
+threads = [threadid*]
+
+# --output=messages
+messages = [messageid*]
+
+# --output=files
+files = [string*]
+
+# --output=tags
+tags = [string*]
+
+thread = {
+    thread:         threadid,
+    timestamp:      unix_time,
+    date_relative:  string,   # user-friendly timestamp
+    matched:        int,      # number of matched messages
+    total:          int,      # total messages in thread
+    authors:        string,   # comma-separated names with | between 
+                              # matched and unmatched
+    subject:        string
+}
diff --git a/notmuch-search.c b/notmuch-search.c
index d504051..92ce38a 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -90,6 +90,9 @@ format_thread_json (const void *ctx,
 		    const int total,
 		    const char *authors,
 		    const char *subject);
+
+/* Any changes to the JSON format should be reflected in the file
+ * devel/schemata. */
 static const search_format_t format_json = {
     "[",
 	"{",
diff --git a/notmuch-show.c b/notmuch-show.c
index d930f94..93fb16f 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -65,6 +65,8 @@ format_part_content_json (GMimeObject *part);
 static void
 format_part_end_json (GMimeObject *part);
 
+/* Any changes to the JSON format should be reflected in the file
+ * devel/schemata. */
 static const notmuch_show_format_t format_json = {
     "[", NULL,
 	"{", format_message_json,
-- 
1.7.7.3

[PATCH 2/8] show: Convert JSON format to the new self-recursive style

[Austin Clements]

As before, this is all code movement and a smidgen of glue.  This
moves the existing JSON formatter code into one self-recursive
function, but doesn't change any of the logic to take advantage of the
new structure.

In general, "leafs" of the JSON structure are left in helper functions
(most of them untouched), so that it's easy to see the overall
structure of the format from the main recursive function.
---
 notmuch-show.c |  273 ++++++++++++++++++++++++++++----------------------------
 1 files changed, 135 insertions(+), 138 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 93fb16f..868b2cd 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -35,52 +35,14 @@ static const notmuch_show_format_t format_text = {
 };
 
 static void
-format_message_json (const void *ctx,
-		     notmuch_message_t *message,
-		     unused (int indent));
-static void
-format_headers_json (const void *ctx,
-		     notmuch_message_t *message);
+format_part_json_entry (const void *ctx, mime_node_t *node,
+			int indent, const notmuch_show_params_t *params);
 
-static void
-format_headers_message_part_json (GMimeMessage *message);
-
-static void
-format_part_start_json (unused (GMimeObject *part),
-			int *part_count);
-
-static void
-format_part_encstatus_json (int status);
-
-static void
-#ifdef GMIME_ATLEAST_26
-format_part_sigstatus_json (GMimeSignatureList* siglist);
-#else
-format_part_sigstatus_json (const GMimeSignatureValidity* validity);
-#endif
-
-static void
-format_part_content_json (GMimeObject *part);
-
-static void
-format_part_end_json (GMimeObject *part);
-
-/* Any changes to the JSON format should be reflected in the file
- * devel/schemata. */
 static const notmuch_show_format_t format_json = {
-    "[", NULL,
-	"{", format_message_json,
-	    "\"headers\": {", format_headers_json, format_headers_message_part_json, "}",
-	    ", \"body\": [",
-	        format_part_start_json,
-	        format_part_encstatus_json,
-	        format_part_sigstatus_json,
-	        format_part_content_json,
-	        format_part_end_json,
-	        ", ",
-	    "]",
-	"}", ", ",
-    "]"
+    .message_set_start = "[",
+    .part = format_part_json_entry,
+    .message_set_sep = ", ",
+    .message_set_end = "]"
 };
 
 static void
@@ -170,7 +132,7 @@ _get_one_line_summary (const void *ctx, notmuch_message_t *message)
 }
 
 static void
-format_message_json (const void *ctx, notmuch_message_t *message, unused (int indent))
+format_message_json (const void *ctx, notmuch_message_t *message)
 {
     notmuch_tags_t *tags;
     int first = 1;
@@ -471,24 +433,6 @@ signer_status_to_string (GMimeSignerStatus x)
 }
 #endif
 
-static void
-format_part_start_json (unused (GMimeObject *part), int *part_count)
-{
-    printf ("{\"id\": %d", *part_count);
-}
-
-static void
-format_part_encstatus_json (int status)
-{
-    printf (", \"encstatus\": [{\"status\": ");
-    if (status) {
-	printf ("\"good\"");
-    } else {
-	printf ("\"bad\"");
-    }
-    printf ("}]");
-}
-
 #ifdef GMIME_ATLEAST_26
 static void
 format_part_sigstatus_json (GMimeSignatureList *siglist)
@@ -619,81 +563,6 @@ format_part_sigstatus_json (const GMimeSignatureValidity* validity)
 #endif
 
 static void
-format_part_content_json (GMimeObject *part)
-{
-    GMimeContentType *content_type = g_mime_object_get_content_type (GMIME_OBJECT (part));
-    GMimeStream *stream_memory = g_mime_stream_mem_new ();
-    const char *cid = g_mime_object_get_content_id (part);
-    void *ctx = talloc_new (NULL);
-    GByteArray *part_content;
-
-    printf (", \"content-type\": %s",
-	    json_quote_str (ctx, g_mime_content_type_to_string (content_type)));
-
-    if (cid != NULL)
-	    printf(", \"content-id\": %s", json_quote_str (ctx, cid));
-
-    if (GMIME_IS_PART (part))
-    {
-	const char *filename = g_mime_part_get_filename (GMIME_PART (part));
-	if (filename)
-	    printf (", \"filename\": %s", json_quote_str (ctx, filename));
-    }
-
-    if (g_mime_content_type_is_type (content_type, "text", "*"))
-    {
-	/* For non-HTML text parts, we include the content in the
-	 * JSON. Since JSON must be Unicode, we handle charset
-	 * decoding here and do not report a charset to the caller.
-	 * For text/html parts, we do not include the content. If a
-	 * caller is interested in text/html parts, it should retrieve
-	 * them separately and they will not be decoded. Since this
-	 * makes charset decoding the responsibility on the caller, we
-	 * report the charset for text/html parts.
-	 */
-	if (g_mime_content_type_is_type (content_type, "text", "html"))
-	{
-	    const char *content_charset = g_mime_object_get_content_type_parameter (GMIME_OBJECT (part), "charset");
-
-	    if (content_charset != NULL)
-		printf (", \"content-charset\": %s", json_quote_str (ctx, content_charset));
-	}
-	else
-	{
-	    show_text_part_content (part, stream_memory);
-	    part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
-
-	    printf (", \"content\": %s", json_quote_chararray (ctx, (char *) part_content->data, part_content->len));
-	}
-    }
-    else if (g_mime_content_type_is_type (content_type, "multipart", "*"))
-    {
-	printf (", \"content\": [");
-    }
-    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
-    {
-	printf (", \"content\": [{");
-    }
-
-    talloc_free (ctx);
-    if (stream_memory)
-	g_object_unref (stream_memory);
-}
-
-static void
-format_part_end_json (GMimeObject *part)
-{
-    GMimeContentType *content_type = g_mime_object_get_content_type (GMIME_OBJECT (part));
-
-    if (g_mime_content_type_is_type (content_type, "multipart", "*"))
-	printf ("]");
-    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
-	printf ("}]");
-
-    printf ("}");
-}
-
-static void
 format_part_content_raw (GMimeObject *part)
 {
     if (! GMIME_IS_PART (part))
@@ -811,6 +680,134 @@ format_part_text (const void *ctx, mime_node_t *node,
 }
 
 static void
+format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
+{
+    /* Any changes to the JSON format should be reflected in the file
+     * devel/schemata. */
+
+    if (node->envelope_file) {
+	printf ("{");
+	format_message_json (ctx, node->envelope_file);
+
+	printf ("\"headers\": {");
+	format_headers_json (ctx, node->envelope_file);
+	printf ("}");
+
+	printf (", \"body\": [");
+	format_part_json (ctx, mime_node_child (node, 0), first);
+
+	printf ("]}");
+	return;
+    }
+
+    void *local = talloc_new (ctx);
+    /* The disposition and content-type metadata are associated with
+     * the envelope for message parts */
+    GMimeObject *meta = node->envelope_part ?
+	GMIME_OBJECT (node->envelope_part) : node->part;
+    GMimeContentType *content_type = g_mime_object_get_content_type (meta);
+    GMimeStream *stream_memory = g_mime_stream_mem_new ();
+    const char *cid = g_mime_object_get_content_id (meta);
+    GByteArray *part_content;
+    int i;
+
+    if (!first)
+	printf (", ");
+
+    printf ("{\"id\": %d", node->part_num);
+
+    if (node->decrypt_attempted) {
+	printf (", \"encstatus\": [{\"status\": ");
+	if (node->decrypt_success) {
+	    printf ("\"good\"");
+	} else {
+	    printf ("\"bad\"");
+	}
+	printf ("}]");
+    }
+
+    if (node->verify_attempted) {
+#ifdef GMIME_ATLEAST_26
+	format_part_sigstatus_json (node->sig_list);
+#else
+	format_part_sigstatus_json (node->sig_validity);
+#endif
+    }
+
+    printf (", \"content-type\": %s",
+	    json_quote_str (local, g_mime_content_type_to_string (content_type)));
+
+    if (cid != NULL)
+	    printf(", \"content-id\": %s", json_quote_str (local, cid));
+
+    if (GMIME_IS_PART (node->part)) {
+	const char *filename = g_mime_part_get_filename (GMIME_PART (node->part));
+	if (filename)
+	    printf (", \"filename\": %s", json_quote_str (local, filename));
+    }
+
+    if (g_mime_content_type_is_type (content_type, "text", "*")) {
+	/* For non-HTML text parts, we include the content in the
+	 * JSON. Since JSON must be Unicode, we handle charset
+	 * decoding here and do not report a charset to the caller.
+	 * For text/html parts, we do not include the content. If a
+	 * caller is interested in text/html parts, it should retrieve
+	 * them separately and they will not be decoded. Since this
+	 * makes charset decoding the responsibility on the caller, we
+	 * report the charset for text/html parts.
+	 */
+	if (g_mime_content_type_is_type (content_type, "text", "html")) {
+	    const char *content_charset = g_mime_object_get_content_type_parameter (meta, "charset");
+
+	    if (content_charset != NULL)
+		printf (", \"content-charset\": %s", json_quote_str (local, content_charset));
+	} else {
+	    show_text_part_content (node->part, stream_memory);
+	    part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
+
+	    printf (", \"content\": %s", json_quote_chararray (local, (char *) part_content->data, part_content->len));
+	}
+    } else if (g_mime_content_type_is_type (content_type, "multipart", "*")) {
+	printf (", \"content\": [");
+    } else if (g_mime_content_type_is_type (content_type, "message", "rfc822")) {
+	printf (", \"content\": [{");
+    }
+
+    if (stream_memory)
+	g_object_unref (stream_memory);
+
+    if (GMIME_IS_MESSAGE (node->part)) {
+	printf ("\"headers\": {");
+	format_headers_message_part_json (GMIME_MESSAGE (node->part));
+	printf ("}");
+
+	printf (", \"body\": [");
+    }
+
+    for (i = 0; i < node->nchildren; i++)
+	format_part_json (ctx, mime_node_child (node, i), i == 0);
+
+    if (GMIME_IS_MESSAGE (node->part))
+	printf ("]");
+
+    if (g_mime_content_type_is_type (content_type, "multipart", "*"))
+	printf ("]");
+    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
+	printf ("}]");
+
+    printf ("}");
+
+    talloc_free (local);
+}
+
+static void
+format_part_json_entry (const void *ctx, mime_node_t *node, unused (int indent),
+			unused (const notmuch_show_params_t *params))
+{
+    format_part_json (ctx, node, TRUE);
+}
+
+static void
 show_message (void *ctx,
 	      const notmuch_show_format_t *format,
 	      notmuch_message_t *message,
-- 
1.7.7.3

[PATCH 3/8] show: Use consistent header ordering in the JSON format

[Austin Clements]

Previously, top-level message headers were printed as Subject, From,
To, Date, while embedded message headers were printed From, To,
Subject, Date.  This makes both cases use the former order and updates
the tests accordingly.
---
 notmuch-show.c |    6 +++---
 test/multipart |    8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 868b2cd..9ca9882 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -328,6 +328,9 @@ format_headers_message_part_json (GMimeMessage *message)
     const char *recipients_string;
 
     printf ("%s: %s",
+	    json_quote_str (ctx_quote, "Subject"),
+	    json_quote_str (ctx_quote, g_mime_message_get_subject (message)));
+    printf (", %s: %s",
 	    json_quote_str (ctx_quote, "From"),
 	    json_quote_str (ctx_quote, g_mime_message_get_sender (message)));
     recipients = g_mime_message_get_recipients (message, GMIME_RECIPIENT_TYPE_TO);
@@ -343,9 +346,6 @@ format_headers_message_part_json (GMimeMessage *message)
 		json_quote_str (ctx_quote, "Cc"),
 		json_quote_str (ctx_quote, recipients_string));
     printf (", %s: %s",
-	    json_quote_str (ctx_quote, "Subject"),
-	    json_quote_str (ctx_quote, g_mime_message_get_subject (message)));
-    printf (", %s: %s",
 	    json_quote_str (ctx_quote, "Date"),
 	    json_quote_str (ctx_quote, g_mime_message_get_date_as_string (message)));
 
diff --git a/test/multipart b/test/multipart
index 2dd73f5..4d14804 100755
--- a/test/multipart
+++ b/test/multipart
@@ -325,7 +325,7 @@ cat <<EOF >EXPECTED
 {"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "${MAIL_DIR}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Cc": "", "Bcc": "", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
 {"id": 1, "content-type": "multipart/signed", "content": [
 {"id": 2, "content-type": "multipart/mixed", "content": [
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}, 
@@ -342,7 +342,7 @@ cat <<EOF >EXPECTED
 
 {"id": 1, "content-type": "multipart/signed", "content": [
 {"id": 2, "content-type": "multipart/mixed", "content": [
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}, 
@@ -358,7 +358,7 @@ echo >>OUTPUT # expect *no* newline at end of output
 cat <<EOF >EXPECTED
 
 {"id": 2, "content-type": "multipart/mixed", "content": [
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}, 
@@ -372,7 +372,7 @@ notmuch show --format=json --part=3 'id:87liy5ap00.fsf@yoom.home.cworth.org' | s
 echo >>OUTPUT # expect *no* newline at end of output
 cat <<EOF >EXPECTED
 
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}
-- 
1.7.7.3

[PATCH 4/8] show: Unify JSON header output for messages and message parts

[Austin Clements]

This has three ramifications:
- Blank To and Cc headers are no longer output for messages.
- Dates are now canonicalized for messages, which means they always
  have a day of the week and GMT is printed +0000 (never -0000)
- Invalid From message headers are handled slightly differently, since
  they get parsed by GMime now instead of notmuch.
---
 notmuch-show.c    |   35 +++--------------------------------
 test/crypto       |   35 ++++++++++++++---------------------
 test/emacs        |    4 ++--
 test/json         |    6 +++---
 test/maildir-sync |    2 --
 test/multipart    |    2 +-
 6 files changed, 23 insertions(+), 61 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 9ca9882..209ff45 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -291,36 +291,7 @@ format_headers_message_part_text (GMimeMessage *message)
 }
 
 static void
-format_headers_json (const void *ctx, notmuch_message_t *message)
-{
-    const char *headers[] = {
-	"Subject", "From", "To", "Cc", "Bcc", "Date"
-    };
-    const char *name, *value;
-    unsigned int i;
-    int first_header = 1;
-    void *ctx_quote = talloc_new (ctx);
-
-    for (i = 0; i < ARRAY_SIZE (headers); i++) {
-	name = headers[i];
-	value = notmuch_message_get_header (message, name);
-	if (value)
-	{
-	    if (!first_header)
-		fputs (", ", stdout);
-	    first_header = 0;
-
-	    printf ("%s: %s",
-		    json_quote_str (ctx_quote, name),
-		    json_quote_str (ctx_quote, value));
-	}
-    }
-
-    talloc_free (ctx_quote);
-}
-
-static void
-format_headers_message_part_json (GMimeMessage *message)
+format_headers_json (GMimeMessage *message)
 {
     void *ctx = talloc_new (NULL);
     void *ctx_quote = talloc_new (ctx);
@@ -690,7 +661,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	format_message_json (ctx, node->envelope_file);
 
 	printf ("\"headers\": {");
-	format_headers_json (ctx, node->envelope_file);
+	format_headers_json (GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
@@ -778,7 +749,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     if (GMIME_IS_MESSAGE (node->part)) {
 	printf ("\"headers\": {");
-	format_headers_message_part_json (GMIME_MESSAGE (node->part));
+	format_headers_json (GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
diff --git a/test/crypto b/test/crypto
index 1dbb60a..7e774c8 100755
--- a/test/crypto
+++ b/test/crypto
@@ -50,9 +50,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "good",
  "fingerprint": "'$FINGERPRINT'",
@@ -84,9 +83,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "good",
  "fingerprint": "'$FINGERPRINT'",
@@ -120,9 +118,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "error",
  "keyid": "'$(echo $FINGERPRINT | cut -c 25-)'",
@@ -194,9 +191,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test encrypted message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "encstatus": [{"status": "good"}],
  "sigstatus": [],
@@ -249,9 +245,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test encrypted message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "encstatus": [{"status": "bad"}],
  "content-type": "multipart/encrypted",
@@ -284,9 +279,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test encrypted message 002",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "encstatus": [{"status": "good"}],
  "sigstatus": [{"status": "good",
@@ -339,9 +333,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "error",
  "keyid": "6D92612D94E46381",
diff --git a/test/emacs b/test/emacs
index b74cfa9..d4a8d30 100755
--- a/test/emacs
+++ b/test/emacs
@@ -78,7 +78,7 @@ thread=$(notmuch search --output=threads subject:message-with-invalid-from)
 test_emacs "(notmuch-show \"$thread\")
 	    (test-output)"
 cat <<EOF >EXPECTED
-Invalid " From <test_suite@notmuchmail.org> (2001-01-05) (inbox)
+"Invalid " (2001-01-05) (inbox)
 Subject: message-with-invalid-from
 To: Notmuch Test Suite <test_suite@notmuchmail.org>
 Date: Fri, 05 Jan 2001 15:43:57 +0000
@@ -411,7 +411,7 @@ test_emacs '(notmuch-show "id:\"bought\"")
 	(reverse-region (point-min) (point-max))
 	    (test-output)'
 cat <<EOF >EXPECTED
-Sat, 01 Jan 2000 12:00:00 -0000
+Sat, 01 Jan 2000 12:00:00 +0000
 Some One <someone@somewhere.org>
 Some One Else <notsomeone@somewhere.org>
 Notmuch <notmuch@notmuchmail.org>
diff --git a/test/json b/test/json
index 7df4380..1bdffd2 100755
--- a/test/json
+++ b/test/json
@@ -5,7 +5,7 @@ test_description="--format=json output"
 test_begin_subtest "Show message: json"
 add_message "[subject]=\"json-show-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"json-show-message\""
 output=$(notmuch show --format=json "json-show-message")
-test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Cc\": \"\", \"Bcc\": \"\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 -0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
 
 test_begin_subtest "Search message: json"
 add_message "[subject]=\"json-search-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"json-search-message\""
@@ -22,7 +22,7 @@ test_expect_equal "$output" "[{\"thread\": \"XXX\",
 test_begin_subtest "Show message: json, utf-8"
 add_message "[subject]=\"json-show-utf8-body-sübjéct\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"jsön-show-méssage\""
 output=$(notmuch show --format=json "jsön-show-méssage")
-test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-utf8-body-sübjéct\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Cc\": \"\", \"Bcc\": \"\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 -0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"jsön-show-méssage\n\"}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-utf8-body-sübjéct\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"jsön-show-méssage\n\"}]}, []]]]"
 
 test_begin_subtest "Show message: json, inline attachment filename"
 subject='json-show-inline-attachment-filename'
@@ -35,7 +35,7 @@ emacs_deliver_message \
      (insert \"Message-ID: <$id>\n\")"
 output=$(notmuch show --format=json "id:$id")
 filename=$(notmuch search --output=files "id:$id")
-test_expect_equal "$output" "[[[{\"id\": \"$id\", \"match\": true, \"filename\": \"$filename\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\"], \"headers\": {\"Subject\": \"$subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"test_suite@notmuchmail.org\", \"Cc\": \"\", \"Bcc\": \"\", \"Date\": \"01 Jan 2000 12:00:00 -0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"multipart/mixed\", \"content\": [{\"id\": 2, \"content-type\": \"text/plain\", \"content\": \"This is a test message with inline attachment with a filename\"}, {\"id\": 3, \"content-type\": \"application/octet-stream\", \"filename\": \"README\"}]}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"$id\", \"match\": true, \"filename\": \"$filename\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\"], \"headers\": {\"Subject\": \"$subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"test_suite@notmuchmail.org\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"multipart/mixed\", \"content\": [{\"id\": 2, \"content-type\": \"text/plain\", \"content\": \"This is a test message with inline attachment with a filename\"}, {\"id\": 3, \"content-type\": \"application/octet-stream\", \"filename\": \"README\"}]}]}, []]]]"
 
 test_begin_subtest "Search message: json, utf-8"
 add_message "[subject]=\"json-search-utf8-body-sübjéct\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"jsön-search-méssage\""
diff --git a/test/maildir-sync b/test/maildir-sync
index d5872a5..1ee2db0 100755
--- a/test/maildir-sync
+++ b/test/maildir-sync
@@ -53,8 +53,6 @@ test_expect_equal "$output" '[[[{"id": "adding-replied-tag@notmuch-test-suite",
 "headers": {"Subject": "Adding replied tag",
 "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
 "To": "Notmuch Test Suite <test_suite@notmuchmail.org>",
-"Cc": "",
-"Bcc": "",
 "Date": "Fri,
 05 Jan 2001 15:43:57 +0000"},
 "body": [{"id": 1,
diff --git a/test/multipart b/test/multipart
index 4d14804..a3036b4 100755
--- a/test/multipart
+++ b/test/multipart
@@ -322,7 +322,7 @@ notmuch show --format=json --part=0 'id:87liy5ap00.fsf@yoom.home.cworth.org' | s
 echo >>OUTPUT # expect *no* newline at end of output
 cat <<EOF >EXPECTED
 
-{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "${MAIL_DIR}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Cc": "", "Bcc": "", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
+{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "${MAIL_DIR}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
 {"id": 1, "content-type": "multipart/signed", "content": [
 {"id": 2, "content-type": "multipart/mixed", "content": [
 {"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
-- 
1.7.7.3

[PATCH 5/8] show: Simplify talloc use in format_headers_json

[Austin Clements]

Previously there was an unnecessary talloc context.
---
 notmuch-show.c |   32 +++++++++++++++-----------------
 1 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 209ff45..6259d30 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -291,37 +291,35 @@ format_headers_message_part_text (GMimeMessage *message)
 }
 
 static void
-format_headers_json (GMimeMessage *message)
+format_headers_json (const void *ctx, GMimeMessage *message)
 {
-    void *ctx = talloc_new (NULL);
-    void *ctx_quote = talloc_new (ctx);
+    void *local = talloc_new (ctx);
     InternetAddressList *recipients;
     const char *recipients_string;
 
     printf ("%s: %s",
-	    json_quote_str (ctx_quote, "Subject"),
-	    json_quote_str (ctx_quote, g_mime_message_get_subject (message)));
+	    json_quote_str (local, "Subject"),
+	    json_quote_str (local, g_mime_message_get_subject (message)));
     printf (", %s: %s",
-	    json_quote_str (ctx_quote, "From"),
-	    json_quote_str (ctx_quote, g_mime_message_get_sender (message)));
+	    json_quote_str (local, "From"),
+	    json_quote_str (local, g_mime_message_get_sender (message)));
     recipients = g_mime_message_get_recipients (message, GMIME_RECIPIENT_TYPE_TO);
     recipients_string = internet_address_list_to_string (recipients, 0);
     if (recipients_string)
 	printf (", %s: %s",
-		json_quote_str (ctx_quote, "To"),
-		json_quote_str (ctx_quote, recipients_string));
+		json_quote_str (local, "To"),
+		json_quote_str (local, recipients_string));
     recipients = g_mime_message_get_recipients (message, GMIME_RECIPIENT_TYPE_CC);
     recipients_string = internet_address_list_to_string (recipients, 0);
     if (recipients_string)
 	printf (", %s: %s",
-		json_quote_str (ctx_quote, "Cc"),
-		json_quote_str (ctx_quote, recipients_string));
+		json_quote_str (local, "Cc"),
+		json_quote_str (local, recipients_string));
     printf (", %s: %s",
-	    json_quote_str (ctx_quote, "Date"),
-	    json_quote_str (ctx_quote, g_mime_message_get_date_as_string (message)));
+	    json_quote_str (local, "Date"),
+	    json_quote_str (local, g_mime_message_get_date_as_string (message)));
 
-    talloc_free (ctx_quote);
-    talloc_free (ctx);
+    talloc_free (local);
 }
 
 /* Write a MIME text part out to the given stream.
@@ -661,7 +659,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	format_message_json (ctx, node->envelope_file);
 
 	printf ("\"headers\": {");
-	format_headers_json (GMIME_MESSAGE (node->part));
+	format_headers_json (ctx, GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
@@ -749,7 +747,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     if (GMIME_IS_MESSAGE (node->part)) {
 	printf ("\"headers\": {");
-	format_headers_json (GMIME_MESSAGE (node->part));
+	format_headers_json (local, GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
-- 
1.7.7.3

[PATCH 6/8] show: Make JSON helper functions print complete objects

[Austin Clements]

This makes the main recursive function easier to follow because helper
functions don't add fields to the running object.
---
 notmuch-show.c |   15 +++++++--------
 1 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 6259d30..8fb6fa6 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -297,7 +297,7 @@ format_headers_json (const void *ctx, GMimeMessage *message)
     InternetAddressList *recipients;
     const char *recipients_string;
 
-    printf ("%s: %s",
+    printf ("{%s: %s",
 	    json_quote_str (local, "Subject"),
 	    json_quote_str (local, g_mime_message_get_subject (message)));
     printf (", %s: %s",
@@ -315,7 +315,7 @@ format_headers_json (const void *ctx, GMimeMessage *message)
 	printf (", %s: %s",
 		json_quote_str (local, "Cc"),
 		json_quote_str (local, recipients_string));
-    printf (", %s: %s",
+    printf (", %s: %s}",
 	    json_quote_str (local, "Date"),
 	    json_quote_str (local, g_mime_message_get_date_as_string (message)));
 
@@ -406,7 +406,7 @@ signer_status_to_string (GMimeSignerStatus x)
 static void
 format_part_sigstatus_json (GMimeSignatureList *siglist)
 {
-    printf (", \"sigstatus\": [");
+    printf ("[");
 
     if (!siglist) {
 	printf ("]");
@@ -472,7 +472,7 @@ format_part_sigstatus_json (GMimeSignatureList *siglist)
 static void
 format_part_sigstatus_json (const GMimeSignatureValidity* validity)
 {
-    printf (", \"sigstatus\": [");
+    printf ("[");
 
     if (!validity) {
 	printf ("]");
@@ -658,9 +658,8 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	printf ("{");
 	format_message_json (ctx, node->envelope_file);
 
-	printf ("\"headers\": {");
+	printf ("\"headers\": ");
 	format_headers_json (ctx, GMIME_MESSAGE (node->part));
-	printf ("}");
 
 	printf (", \"body\": [");
 	format_part_json (ctx, mime_node_child (node, 0), first);
@@ -696,6 +695,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
     }
 
     if (node->verify_attempted) {
+	printf (", \"sigstatus\": ");
 #ifdef GMIME_ATLEAST_26
 	format_part_sigstatus_json (node->sig_list);
 #else
@@ -746,9 +746,8 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	g_object_unref (stream_memory);
 
     if (GMIME_IS_MESSAGE (node->part)) {
-	printf ("\"headers\": {");
+	printf ("\"headers\": ");
 	format_headers_json (local, GMIME_MESSAGE (node->part));
-	printf ("}");
 
 	printf (", \"body\": [");
     }
-- 
1.7.7.3

[PATCH 7/8] show: Make format_part_sigstatus_json's API consistent between GMIME 2.4 and 2.6

[Austin Clements]

The implementation is still different for GMIME 2.4 and 2.6, but at
least now the caller doesn't have to be aware of this.
---
 notmuch-show.c |   14 +++++++-------
 1 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 8fb6fa6..07276c7 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -404,8 +404,10 @@ signer_status_to_string (GMimeSignerStatus x)
 
 #ifdef GMIME_ATLEAST_26
 static void
-format_part_sigstatus_json (GMimeSignatureList *siglist)
+format_part_sigstatus_json (mime_node_t *node)
 {
+    GMimeSignatureList *siglist = node->sig_list;
+
     printf ("[");
 
     if (!siglist) {
@@ -470,8 +472,10 @@ format_part_sigstatus_json (GMimeSignatureList *siglist)
 }
 #else
 static void
-format_part_sigstatus_json (const GMimeSignatureValidity* validity)
+format_part_sigstatus_json (mime_node_t *node)
 {
+    const GMimeSignatureValidity* validity = node->sig_validity;
+
     printf ("[");
 
     if (!validity) {
@@ -696,11 +700,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     if (node->verify_attempted) {
 	printf (", \"sigstatus\": ");
-#ifdef GMIME_ATLEAST_26
-	format_part_sigstatus_json (node->sig_list);
-#else
-	format_part_sigstatus_json (node->sig_validity);
-#endif
+	format_part_sigstatus_json (node);
     }
 
     printf (", \"content-type\": %s",
-- 
1.7.7.3

[PATCH 8/8] show: Further general simplifications of the JSON formatter

[Austin Clements]

---
 notmuch-show.c |   61 ++++++++++++++++++++-----------------------------------
 1 files changed, 22 insertions(+), 39 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 07276c7..6a171a4 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -678,9 +678,10 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
     GMimeObject *meta = node->envelope_part ?
 	GMIME_OBJECT (node->envelope_part) : node->part;
     GMimeContentType *content_type = g_mime_object_get_content_type (meta);
-    GMimeStream *stream_memory = g_mime_stream_mem_new ();
     const char *cid = g_mime_object_get_content_id (meta);
-    GByteArray *part_content;
+    const char *filename = GMIME_IS_PART (node->part) ?
+	g_mime_part_get_filename (GMIME_PART (node->part)) : NULL;
+    const char *terminator = "";
     int i;
 
     if (!first)
@@ -688,15 +689,9 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     printf ("{\"id\": %d", node->part_num);
 
-    if (node->decrypt_attempted) {
-	printf (", \"encstatus\": [{\"status\": ");
-	if (node->decrypt_success) {
-	    printf ("\"good\"");
-	} else {
-	    printf ("\"bad\"");
-	}
-	printf ("}]");
-    }
+    if (node->decrypt_attempted)
+	printf (", \"encstatus\": [{\"status\": \"%s\"}]",
+		node->decrypt_success ? "good" : "bad");
 
     if (node->verify_attempted) {
 	printf (", \"sigstatus\": ");
@@ -706,16 +701,13 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
     printf (", \"content-type\": %s",
 	    json_quote_str (local, g_mime_content_type_to_string (content_type)));
 
-    if (cid != NULL)
-	    printf(", \"content-id\": %s", json_quote_str (local, cid));
+    if (cid)
+	printf (", \"content-id\": %s", json_quote_str (local, cid));
+
+    if (filename)
+	printf (", \"filename\": %s", json_quote_str (local, filename));
 
     if (GMIME_IS_PART (node->part)) {
-	const char *filename = g_mime_part_get_filename (GMIME_PART (node->part));
-	if (filename)
-	    printf (", \"filename\": %s", json_quote_str (local, filename));
-    }
-
-    if (g_mime_content_type_is_type (content_type, "text", "*")) {
 	/* For non-HTML text parts, we include the content in the
 	 * JSON. Since JSON must be Unicode, we handle charset
 	 * decoding here and do not report a charset to the caller.
@@ -730,42 +722,33 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
 	    if (content_charset != NULL)
 		printf (", \"content-charset\": %s", json_quote_str (local, content_charset));
-	} else {
+	} else if (g_mime_content_type_is_type (content_type, "text", "*")) {
+	    GMimeStream *stream_memory = g_mime_stream_mem_new ();
+	    GByteArray *part_content;
 	    show_text_part_content (node->part, stream_memory);
 	    part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
 
 	    printf (", \"content\": %s", json_quote_chararray (local, (char *) part_content->data, part_content->len));
+	    g_object_unref (stream_memory);
 	}
-    } else if (g_mime_content_type_is_type (content_type, "multipart", "*")) {
+    } else if (GMIME_IS_MULTIPART (node->part)) {
 	printf (", \"content\": [");
-    } else if (g_mime_content_type_is_type (content_type, "message", "rfc822")) {
+	terminator = "]";
+    } else if (GMIME_IS_MESSAGE (node->part)) {
 	printf (", \"content\": [{");
-    }
-
-    if (stream_memory)
-	g_object_unref (stream_memory);
-
-    if (GMIME_IS_MESSAGE (node->part)) {
 	printf ("\"headers\": ");
 	format_headers_json (local, GMIME_MESSAGE (node->part));
 
 	printf (", \"body\": [");
+	terminator = "]}]";
     }
 
+    talloc_free (local);
+
     for (i = 0; i < node->nchildren; i++)
 	format_part_json (ctx, mime_node_child (node, i), i == 0);
 
-    if (GMIME_IS_MESSAGE (node->part))
-	printf ("]");
-
-    if (g_mime_content_type_is_type (content_type, "multipart", "*"))
-	printf ("]");
-    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
-	printf ("}]");
-
-    printf ("}");
-
-    talloc_free (local);
+    printf ("%s}", terminator);
 }
 
 static void
-- 
1.7.7.3

Re: [PATCH 0/8] Rewrite JSON show format

[Adam Wolfe Gordon]

Hi Austin,

On Tue, Feb 14, 2012 at 10:33, Austin Clements <amdragon@mit.edu> wrote:
> The saga continues.  As for the text format, this first shifts lots of
> code around without changing its semantics, then it dives in and
> simplifies a lot of things.  Don't be put off by the number of
> patches; most of them are straightforward.

I haven't looked thoroughly style-wise, but otherwise everything LGTM.
I really like the new formatters - their operation is much easier to
follow than the old ones.

> As an added bonus, I documented (!) the JSON format for both show and
> search.

Good idea. I'll add my JSON reply format to this once it seems final.

Re: [PATCH 0/8] Rewrite JSON show format

[Tomi Ollila]

On Tue, 14 Feb 2012 12:33:35 -0500, Austin Clements <amdragon@MIT.EDU> wrote:
> The saga continues.  As for the text format, this first shifts lots of
> code around without changing its semantics, then it dives in and
> simplifies a lot of things.  Don't be put off by the number of
> patches; most of them are straightforward.
> 
> As an added bonus, I documented (!) the JSON format for both show and
> search.

I got 2 trailing whitespace "errors" when applied (schemata) patches.

Emacs MUA works for me.

Most json-related tests pass (for me), but I get this:

 FAIL   notmuch-show for message with invalid From
        --- emacs.10.expected   2012-02-15 14:28:48.000000000 +0000
        +++ emacs.10.output     2012-02-15 14:28:48.000000000 +0000
        @@ -1,4 +1,4 @@
        -"Invalid " (2001-01-05) (inbox)
        +Invalid  From <test_suite@notmuchmail.org> <q> (2001-01-05) (inbox)

If this works for others then the problem is in my setup
(I have 8 other tests failing for various other reasons)

The patch in id:"1329240823-7856-5-git-send-email-amdragon@mit.edu"
changes this -- the + line is almost like it was (without that <q>).
Maybe the reason is within gmime-2.4.20-1.el5.rf I'm having in this
machine -- have to try on another...

...YES. other machine has (self-compiled) gmime 2.4.25 and there I got

All 370 tests behaved as expected (2 expected failures).

Code LGTM.

+1

Tomi

--8<----8<----8<----8<----8<----8<--

PS: I'm using code derived from dme's work to get patches. As it
was so convenient to get patched moved across machine I'll mention
it here -- maybe we get even better ways to do these things in the
future:

I have in my notmuch emacs configuration file:

(defun get-patches ()
  (interactive)
  (let ((search-terms-list (notmuch-show-get-message-ids-for-open-messages)))
    (with-temp-file "~/patch"
      (call-process notmuch-command nil t nil "show" "--format=mbox"
                    (mapconcat 'identity search-terms-list " OR ")))
    (with-temp-file "~/message-ids"
      (insert (mapconcat 'identity search-terms-list "\n") "\n"))))

This creates ~/patch which contains 'git am' -able mbox file and
~/message-ids for the message id's (if one has use for it).

Now I just scp:d the patch file to another machine, git am'd it
and started testing.
 

Re: [PATCH 0/8] Rewrite JSON show format

[Adam Wolfe Gordon]

On Wed, Feb 15, 2012 at 08:00, Tomi Ollila <tomi.ollila@iki.fi> wrote:
> Most json-related tests pass (for me), but I get this:
>
>  FAIL   notmuch-show for message with invalid From
>        --- emacs.10.expected   2012-02-15 14:28:48.000000000 +0000
>        +++ emacs.10.output     2012-02-15 14:28:48.000000000 +0000
>        @@ -1,4 +1,4 @@
>        -"Invalid " (2001-01-05) (inbox)
>        +Invalid  From <test_suite@notmuchmail.org> <q> (2001-01-05) (inbox)
>
> If this works for others then the problem is in my setup
> (I have 8 other tests failing for various other reasons)

This one works for me, but the expected output in my version looks
like: "Invalid " (2001-01-05) (inbox). It looks like that test changed
in part 4 of this series, so maybe something didn't apply properly on
your tree?

Re: [PATCH 0/8] Rewrite JSON show format

[Austin Clements]

Quoth Tomi Ollila on Feb 15 at  5:00 pm:
> On Tue, 14 Feb 2012 12:33:35 -0500, Austin Clements <amdragon@MIT.EDU> wrote:
> > The saga continues.  As for the text format, this first shifts lots of
> > code around without changing its semantics, then it dives in and
> > simplifies a lot of things.  Don't be put off by the number of
> > patches; most of them are straightforward.
> > 
> > As an added bonus, I documented (!) the JSON format for both show and
> > search.
> 
> I got 2 trailing whitespace "errors" when applied (schemata) patches.

Oops.  Will fix in v2.

> 
> Emacs MUA works for me.
> 
> Most json-related tests pass (for me), but I get this:
> 
>  FAIL   notmuch-show for message with invalid From
>         --- emacs.10.expected   2012-02-15 14:28:48.000000000 +0000
>         +++ emacs.10.output     2012-02-15 14:28:48.000000000 +0000
>         @@ -1,4 +1,4 @@
>         -"Invalid " (2001-01-05) (inbox)
>         +Invalid  From <test_suite@notmuchmail.org> <q> (2001-01-05) (inbox)
> 
> If this works for others then the problem is in my setup
> (I have 8 other tests failing for various other reasons)
> 
> The patch in id:"1329240823-7856-5-git-send-email-amdragon@mit.edu"
> changes this -- the + line is almost like it was (without that <q>).
> Maybe the reason is within gmime-2.4.20-1.el5.rf I'm having in this
> machine -- have to try on another...
> 
> ...YES. other machine has (self-compiled) gmime 2.4.25 and there I got

The from header is now being parsed by gmime, so if gmime's parsing of
invalid headers has changed, it's quite possible it would affect this
test.

> All 370 tests behaved as expected (2 expected failures).
> 
> Code LGTM.
> 
> +1
> 
> Tomi

[PATCH v2 0/8] Rewrite JSON show format

[Austin Clements]

v2 fixes some trivial whitespace issues in devel/schemata [1] and
beefs up one of the explanations in devel/schemata.  None of the other
patches have changed, so I'm marking this ready.

As a teaser, I've got mbox and raw converted to the new style, too, so
as soon as this goes in I'll send those patches (they aren't actually
dependent, but do share some diff context with this one).

diff from v1 to v2:

diff --git a/devel/schemata b/devel/schemata
index d90d4c6..24ad775 100644
--- a/devel/schemata
+++ b/devel/schemata
@@ -71,7 +71,10 @@ part = {
     # otherwise (leaf parts):
     filename?:      string,
     content-charset?: string,
-    content?:       string    # pre-fetched body content
+    # A leaf part's body content is optional, but may be included if
+    # it can be correctly encoded as a string.  Consumers should use
+    # this in preference to fetching the part content separately.
+    content?:       string
 }
 
 # The headers of a message (format_headers_json with raw headers) or
@@ -101,7 +104,7 @@ signature = {
     userid?:        string
     # if status is not "good":
     keyid?:         string
-    # if the signature has errors:    
+    # if the signature has errors:
     errors?:        int
 }
 
@@ -129,7 +132,7 @@ thread = {
     date_relative:  string,   # user-friendly timestamp
     matched:        int,      # number of matched messages
     total:          int,      # total messages in thread
-    authors:        string,   # comma-separated names with | between 
+    authors:        string,   # comma-separated names with | between
                               # matched and unmatched
     subject:        string
 }

[PATCH v2 1/8] Document the JSON schemata used by show and search

[Austin Clements]

---
 devel/schemata   |  138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 notmuch-search.c |    3 +
 notmuch-show.c   |    2 +
 3 files changed, 143 insertions(+), 0 deletions(-)
 create mode 100644 devel/schemata

diff --git a/devel/schemata b/devel/schemata
new file mode 100644
index 0000000..24ad775
--- /dev/null
+++ b/devel/schemata
@@ -0,0 +1,138 @@
+This file describes the schemata used for notmuch's structured output
+format (currently JSON).
+
+[]'s indicate lists.  List items can be marked with a '?', meaning
+they are optional; or a '*', meaning there can be zero or more of that
+item.  {}'s indicate an object that maps from field identifiers to
+values.  An object field marked '?' is optional.  |'s indicate
+alternates (e.g., int|string means something can be an int or a
+string).
+
+Common non-terminals
+--------------------
+
+# Number of seconds since the Epoch
+unix_time = int
+
+# Thread ID, sans "thread:"
+threadid = string
+
+# Message ID, sans "id:"
+messageid = string
+
+notmuch show schema
+-------------------
+
+# A top-level set of threads (do_show)
+# Returned by notmuch show without a --part argument
+thread_set = [thread*]
+
+# Top-level messages in a thread (show_messages)
+thread = [thread_node*]
+
+# A message and its replies (show_messages)
+thread_node = [
+    message?,                 # present if --entire-thread or matched
+    [thread_node*]            # children of message
+]
+
+# A message (show_message)
+message = {
+    # (format_message_json)
+    id:             messageid,
+    match:          bool,
+    filename:	    string,
+    timestamp:      unix_time, # date header as unix time
+    date_relative:  string,   # user-friendly timestamp
+    tags:           [string*],
+
+    headers:        headers,
+    body:           [part]
+}
+
+# A MIME part (show_message_body)
+part = {
+    # format_part_start_json
+    id:             int|string, # part id (currently DFS part number)
+
+    # format_part_encstatus_json
+    encstatus?:     encstatus,
+
+    # format_part_sigstatus_json
+    sigstatus?:     sigstatus,
+
+    # format_part_content_json
+    content-type:   string,
+    content-id?:    string,
+    # if content-type starts with "multipart/":
+    content:        [part*],
+    # if content-type is "message/rfc822":
+    content:        [{headers: headers, body: [part]}],
+    # otherwise (leaf parts):
+    filename?:      string,
+    content-charset?: string,
+    # A leaf part's body content is optional, but may be included if
+    # it can be correctly encoded as a string.  Consumers should use
+    # this in preference to fetching the part content separately.
+    content?:       string
+}
+
+# The headers of a message (format_headers_json with raw headers) or
+# a part (format_headers_message_part_json with pretty-printed headers)
+headers = {
+    Subject:        string,
+    From:           string,
+    To?:            string,
+    Cc?:            string,
+    Bcc?:           string,
+    Date:           string
+}
+
+# Encryption status (format_part_encstatus_json)
+encstatus = [{status: "good"|"bad"}]
+
+# Signature status (format_part_sigstatus_json)
+sigstatus = [signature*]
+
+signature = {
+    # signature_status_to_string
+    status:         "none"|"good"|"bad"|"error"|"unknown",
+    # if status is "good":
+    fingerprint?:   string,
+    created?:       unix_time,
+    expires?:       unix_time,
+    userid?:        string
+    # if status is not "good":
+    keyid?:         string
+    # if the signature has errors:
+    errors?:        int
+}
+
+notmuch search schema
+---------------------
+
+# --output=summary
+summary = [thread*]
+
+# --output=threads
+threads = [threadid*]
+
+# --output=messages
+messages = [messageid*]
+
+# --output=files
+files = [string*]
+
+# --output=tags
+tags = [string*]
+
+thread = {
+    thread:         threadid,
+    timestamp:      unix_time,
+    date_relative:  string,   # user-friendly timestamp
+    matched:        int,      # number of matched messages
+    total:          int,      # total messages in thread
+    authors:        string,   # comma-separated names with | between
+                              # matched and unmatched
+    subject:        string
+}
diff --git a/notmuch-search.c b/notmuch-search.c
index d504051..92ce38a 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -90,6 +90,9 @@ format_thread_json (const void *ctx,
 		    const int total,
 		    const char *authors,
 		    const char *subject);
+
+/* Any changes to the JSON format should be reflected in the file
+ * devel/schemata. */
 static const search_format_t format_json = {
     "[",
 	"{",
diff --git a/notmuch-show.c b/notmuch-show.c
index d930f94..93fb16f 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -65,6 +65,8 @@ format_part_content_json (GMimeObject *part);
 static void
 format_part_end_json (GMimeObject *part);
 
+/* Any changes to the JSON format should be reflected in the file
+ * devel/schemata. */
 static const notmuch_show_format_t format_json = {
     "[", NULL,
 	"{", format_message_json,
-- 
1.7.7.3

Re: [PATCH v2 1/8] Document the JSON schemata used by show and search

[David Bremner]

On Sun, 19 Feb 2012 19:26:23 -0500, Austin Clements <amdragon@MIT.EDU> wrote:
> ---
>  devel/schemata   |  138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  notmuch-search.c |    3 +
>  notmuch-show.c   |    2 +

Pushed just the first in the series.

d

[PATCH v2 2/8] show: Convert JSON format to the new self-recursive style

[Austin Clements]

As before, this is all code movement and a smidgen of glue.  This
moves the existing JSON formatter code into one self-recursive
function, but doesn't change any of the logic to take advantage of the
new structure.

In general, "leafs" of the JSON structure are left in helper functions
(most of them untouched), so that it's easy to see the overall
structure of the format from the main recursive function.
---
 notmuch-show.c |  273 ++++++++++++++++++++++++++++----------------------------
 1 files changed, 135 insertions(+), 138 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 93fb16f..868b2cd 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -35,52 +35,14 @@ static const notmuch_show_format_t format_text = {
 };
 
 static void
-format_message_json (const void *ctx,
-		     notmuch_message_t *message,
-		     unused (int indent));
-static void
-format_headers_json (const void *ctx,
-		     notmuch_message_t *message);
+format_part_json_entry (const void *ctx, mime_node_t *node,
+			int indent, const notmuch_show_params_t *params);
 
-static void
-format_headers_message_part_json (GMimeMessage *message);
-
-static void
-format_part_start_json (unused (GMimeObject *part),
-			int *part_count);
-
-static void
-format_part_encstatus_json (int status);
-
-static void
-#ifdef GMIME_ATLEAST_26
-format_part_sigstatus_json (GMimeSignatureList* siglist);
-#else
-format_part_sigstatus_json (const GMimeSignatureValidity* validity);
-#endif
-
-static void
-format_part_content_json (GMimeObject *part);
-
-static void
-format_part_end_json (GMimeObject *part);
-
-/* Any changes to the JSON format should be reflected in the file
- * devel/schemata. */
 static const notmuch_show_format_t format_json = {
-    "[", NULL,
-	"{", format_message_json,
-	    "\"headers\": {", format_headers_json, format_headers_message_part_json, "}",
-	    ", \"body\": [",
-	        format_part_start_json,
-	        format_part_encstatus_json,
-	        format_part_sigstatus_json,
-	        format_part_content_json,
-	        format_part_end_json,
-	        ", ",
-	    "]",
-	"}", ", ",
-    "]"
+    .message_set_start = "[",
+    .part = format_part_json_entry,
+    .message_set_sep = ", ",
+    .message_set_end = "]"
 };
 
 static void
@@ -170,7 +132,7 @@ _get_one_line_summary (const void *ctx, notmuch_message_t *message)
 }
 
 static void
-format_message_json (const void *ctx, notmuch_message_t *message, unused (int indent))
+format_message_json (const void *ctx, notmuch_message_t *message)
 {
     notmuch_tags_t *tags;
     int first = 1;
@@ -471,24 +433,6 @@ signer_status_to_string (GMimeSignerStatus x)
 }
 #endif
 
-static void
-format_part_start_json (unused (GMimeObject *part), int *part_count)
-{
-    printf ("{\"id\": %d", *part_count);
-}
-
-static void
-format_part_encstatus_json (int status)
-{
-    printf (", \"encstatus\": [{\"status\": ");
-    if (status) {
-	printf ("\"good\"");
-    } else {
-	printf ("\"bad\"");
-    }
-    printf ("}]");
-}
-
 #ifdef GMIME_ATLEAST_26
 static void
 format_part_sigstatus_json (GMimeSignatureList *siglist)
@@ -619,81 +563,6 @@ format_part_sigstatus_json (const GMimeSignatureValidity* validity)
 #endif
 
 static void
-format_part_content_json (GMimeObject *part)
-{
-    GMimeContentType *content_type = g_mime_object_get_content_type (GMIME_OBJECT (part));
-    GMimeStream *stream_memory = g_mime_stream_mem_new ();
-    const char *cid = g_mime_object_get_content_id (part);
-    void *ctx = talloc_new (NULL);
-    GByteArray *part_content;
-
-    printf (", \"content-type\": %s",
-	    json_quote_str (ctx, g_mime_content_type_to_string (content_type)));
-
-    if (cid != NULL)
-	    printf(", \"content-id\": %s", json_quote_str (ctx, cid));
-
-    if (GMIME_IS_PART (part))
-    {
-	const char *filename = g_mime_part_get_filename (GMIME_PART (part));
-	if (filename)
-	    printf (", \"filename\": %s", json_quote_str (ctx, filename));
-    }
-
-    if (g_mime_content_type_is_type (content_type, "text", "*"))
-    {
-	/* For non-HTML text parts, we include the content in the
-	 * JSON. Since JSON must be Unicode, we handle charset
-	 * decoding here and do not report a charset to the caller.
-	 * For text/html parts, we do not include the content. If a
-	 * caller is interested in text/html parts, it should retrieve
-	 * them separately and they will not be decoded. Since this
-	 * makes charset decoding the responsibility on the caller, we
-	 * report the charset for text/html parts.
-	 */
-	if (g_mime_content_type_is_type (content_type, "text", "html"))
-	{
-	    const char *content_charset = g_mime_object_get_content_type_parameter (GMIME_OBJECT (part), "charset");
-
-	    if (content_charset != NULL)
-		printf (", \"content-charset\": %s", json_quote_str (ctx, content_charset));
-	}
-	else
-	{
-	    show_text_part_content (part, stream_memory);
-	    part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
-
-	    printf (", \"content\": %s", json_quote_chararray (ctx, (char *) part_content->data, part_content->len));
-	}
-    }
-    else if (g_mime_content_type_is_type (content_type, "multipart", "*"))
-    {
-	printf (", \"content\": [");
-    }
-    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
-    {
-	printf (", \"content\": [{");
-    }
-
-    talloc_free (ctx);
-    if (stream_memory)
-	g_object_unref (stream_memory);
-}
-
-static void
-format_part_end_json (GMimeObject *part)
-{
-    GMimeContentType *content_type = g_mime_object_get_content_type (GMIME_OBJECT (part));
-
-    if (g_mime_content_type_is_type (content_type, "multipart", "*"))
-	printf ("]");
-    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
-	printf ("}]");
-
-    printf ("}");
-}
-
-static void
 format_part_content_raw (GMimeObject *part)
 {
     if (! GMIME_IS_PART (part))
@@ -811,6 +680,134 @@ format_part_text (const void *ctx, mime_node_t *node,
 }
 
 static void
+format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
+{
+    /* Any changes to the JSON format should be reflected in the file
+     * devel/schemata. */
+
+    if (node->envelope_file) {
+	printf ("{");
+	format_message_json (ctx, node->envelope_file);
+
+	printf ("\"headers\": {");
+	format_headers_json (ctx, node->envelope_file);
+	printf ("}");
+
+	printf (", \"body\": [");
+	format_part_json (ctx, mime_node_child (node, 0), first);
+
+	printf ("]}");
+	return;
+    }
+
+    void *local = talloc_new (ctx);
+    /* The disposition and content-type metadata are associated with
+     * the envelope for message parts */
+    GMimeObject *meta = node->envelope_part ?
+	GMIME_OBJECT (node->envelope_part) : node->part;
+    GMimeContentType *content_type = g_mime_object_get_content_type (meta);
+    GMimeStream *stream_memory = g_mime_stream_mem_new ();
+    const char *cid = g_mime_object_get_content_id (meta);
+    GByteArray *part_content;
+    int i;
+
+    if (!first)
+	printf (", ");
+
+    printf ("{\"id\": %d", node->part_num);
+
+    if (node->decrypt_attempted) {
+	printf (", \"encstatus\": [{\"status\": ");
+	if (node->decrypt_success) {
+	    printf ("\"good\"");
+	} else {
+	    printf ("\"bad\"");
+	}
+	printf ("}]");
+    }
+
+    if (node->verify_attempted) {
+#ifdef GMIME_ATLEAST_26
+	format_part_sigstatus_json (node->sig_list);
+#else
+	format_part_sigstatus_json (node->sig_validity);
+#endif
+    }
+
+    printf (", \"content-type\": %s",
+	    json_quote_str (local, g_mime_content_type_to_string (content_type)));
+
+    if (cid != NULL)
+	    printf(", \"content-id\": %s", json_quote_str (local, cid));
+
+    if (GMIME_IS_PART (node->part)) {
+	const char *filename = g_mime_part_get_filename (GMIME_PART (node->part));
+	if (filename)
+	    printf (", \"filename\": %s", json_quote_str (local, filename));
+    }
+
+    if (g_mime_content_type_is_type (content_type, "text", "*")) {
+	/* For non-HTML text parts, we include the content in the
+	 * JSON. Since JSON must be Unicode, we handle charset
+	 * decoding here and do not report a charset to the caller.
+	 * For text/html parts, we do not include the content. If a
+	 * caller is interested in text/html parts, it should retrieve
+	 * them separately and they will not be decoded. Since this
+	 * makes charset decoding the responsibility on the caller, we
+	 * report the charset for text/html parts.
+	 */
+	if (g_mime_content_type_is_type (content_type, "text", "html")) {
+	    const char *content_charset = g_mime_object_get_content_type_parameter (meta, "charset");
+
+	    if (content_charset != NULL)
+		printf (", \"content-charset\": %s", json_quote_str (local, content_charset));
+	} else {
+	    show_text_part_content (node->part, stream_memory);
+	    part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
+
+	    printf (", \"content\": %s", json_quote_chararray (local, (char *) part_content->data, part_content->len));
+	}
+    } else if (g_mime_content_type_is_type (content_type, "multipart", "*")) {
+	printf (", \"content\": [");
+    } else if (g_mime_content_type_is_type (content_type, "message", "rfc822")) {
+	printf (", \"content\": [{");
+    }
+
+    if (stream_memory)
+	g_object_unref (stream_memory);
+
+    if (GMIME_IS_MESSAGE (node->part)) {
+	printf ("\"headers\": {");
+	format_headers_message_part_json (GMIME_MESSAGE (node->part));
+	printf ("}");
+
+	printf (", \"body\": [");
+    }
+
+    for (i = 0; i < node->nchildren; i++)
+	format_part_json (ctx, mime_node_child (node, i), i == 0);
+
+    if (GMIME_IS_MESSAGE (node->part))
+	printf ("]");
+
+    if (g_mime_content_type_is_type (content_type, "multipart", "*"))
+	printf ("]");
+    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
+	printf ("}]");
+
+    printf ("}");
+
+    talloc_free (local);
+}
+
+static void
+format_part_json_entry (const void *ctx, mime_node_t *node, unused (int indent),
+			unused (const notmuch_show_params_t *params))
+{
+    format_part_json (ctx, node, TRUE);
+}
+
+static void
 show_message (void *ctx,
 	      const notmuch_show_format_t *format,
 	      notmuch_message_t *message,
-- 
1.7.7.3

Re: [PATCH v2 2/8] show: Convert JSON format to the new self-recursive style

[David Bremner]

On Sun, 19 Feb 2012 19:26:24 -0500, Austin Clements <amdragon@MIT.EDU> wrote:
> As before, this is all code movement and a smidgen of glue.  This
> moves the existing JSON formatter code into one self-recursive
> function, but doesn't change any of the logic to take advantage of the
> new structure.

Pushed the remainder of the series.

d

[PATCH v2 3/8] show: Use consistent header ordering in the JSON format

[Austin Clements]

Previously, top-level message headers were printed as Subject, From,
To, Date, while embedded message headers were printed From, To,
Subject, Date.  This makes both cases use the former order and updates
the tests accordingly.
---
 notmuch-show.c |    6 +++---
 test/multipart |    8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 868b2cd..9ca9882 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -328,6 +328,9 @@ format_headers_message_part_json (GMimeMessage *message)
     const char *recipients_string;
 
     printf ("%s: %s",
+	    json_quote_str (ctx_quote, "Subject"),
+	    json_quote_str (ctx_quote, g_mime_message_get_subject (message)));
+    printf (", %s: %s",
 	    json_quote_str (ctx_quote, "From"),
 	    json_quote_str (ctx_quote, g_mime_message_get_sender (message)));
     recipients = g_mime_message_get_recipients (message, GMIME_RECIPIENT_TYPE_TO);
@@ -343,9 +346,6 @@ format_headers_message_part_json (GMimeMessage *message)
 		json_quote_str (ctx_quote, "Cc"),
 		json_quote_str (ctx_quote, recipients_string));
     printf (", %s: %s",
-	    json_quote_str (ctx_quote, "Subject"),
-	    json_quote_str (ctx_quote, g_mime_message_get_subject (message)));
-    printf (", %s: %s",
 	    json_quote_str (ctx_quote, "Date"),
 	    json_quote_str (ctx_quote, g_mime_message_get_date_as_string (message)));
 
diff --git a/test/multipart b/test/multipart
index 2dd73f5..4d14804 100755
--- a/test/multipart
+++ b/test/multipart
@@ -325,7 +325,7 @@ cat <<EOF >EXPECTED
 {"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "${MAIL_DIR}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Cc": "", "Bcc": "", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
 {"id": 1, "content-type": "multipart/signed", "content": [
 {"id": 2, "content-type": "multipart/mixed", "content": [
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}, 
@@ -342,7 +342,7 @@ cat <<EOF >EXPECTED
 
 {"id": 1, "content-type": "multipart/signed", "content": [
 {"id": 2, "content-type": "multipart/mixed", "content": [
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}, 
@@ -358,7 +358,7 @@ echo >>OUTPUT # expect *no* newline at end of output
 cat <<EOF >EXPECTED
 
 {"id": 2, "content-type": "multipart/mixed", "content": [
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}, 
@@ -372,7 +372,7 @@ notmuch show --format=json --part=3 'id:87liy5ap00.fsf@yoom.home.cworth.org' | s
 echo >>OUTPUT # expect *no* newline at end of output
 cat <<EOF >EXPECTED
 
-{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Subject": "html message", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
+{"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
 {"id": 4, "content-type": "multipart/alternative", "content": [
 {"id": 5, "content-type": "text/html"}, 
 {"id": 6, "content-type": "text/plain", "content": "This is an embedded message, with a multipart/alternative part.\n"}]}]}]}
-- 
1.7.7.3

[PATCH v2 4/8] show: Unify JSON header output for messages and message parts

[Austin Clements]

This has three ramifications:
- Blank To and Cc headers are no longer output for messages.
- Dates are now canonicalized for messages, which means they always
  have a day of the week and GMT is printed +0000 (never -0000)
- Invalid From message headers are handled slightly differently, since
  they get parsed by GMime now instead of notmuch.
---
 notmuch-show.c    |   35 +++--------------------------------
 test/crypto       |   35 ++++++++++++++---------------------
 test/emacs        |    4 ++--
 test/json         |    6 +++---
 test/maildir-sync |    2 --
 test/multipart    |    2 +-
 6 files changed, 23 insertions(+), 61 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 9ca9882..209ff45 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -291,36 +291,7 @@ format_headers_message_part_text (GMimeMessage *message)
 }
 
 static void
-format_headers_json (const void *ctx, notmuch_message_t *message)
-{
-    const char *headers[] = {
-	"Subject", "From", "To", "Cc", "Bcc", "Date"
-    };
-    const char *name, *value;
-    unsigned int i;
-    int first_header = 1;
-    void *ctx_quote = talloc_new (ctx);
-
-    for (i = 0; i < ARRAY_SIZE (headers); i++) {
-	name = headers[i];
-	value = notmuch_message_get_header (message, name);
-	if (value)
-	{
-	    if (!first_header)
-		fputs (", ", stdout);
-	    first_header = 0;
-
-	    printf ("%s: %s",
-		    json_quote_str (ctx_quote, name),
-		    json_quote_str (ctx_quote, value));
-	}
-    }
-
-    talloc_free (ctx_quote);
-}
-
-static void
-format_headers_message_part_json (GMimeMessage *message)
+format_headers_json (GMimeMessage *message)
 {
     void *ctx = talloc_new (NULL);
     void *ctx_quote = talloc_new (ctx);
@@ -690,7 +661,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	format_message_json (ctx, node->envelope_file);
 
 	printf ("\"headers\": {");
-	format_headers_json (ctx, node->envelope_file);
+	format_headers_json (GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
@@ -778,7 +749,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     if (GMIME_IS_MESSAGE (node->part)) {
 	printf ("\"headers\": {");
-	format_headers_message_part_json (GMIME_MESSAGE (node->part));
+	format_headers_json (GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
diff --git a/test/crypto b/test/crypto
index 1dbb60a..7e774c8 100755
--- a/test/crypto
+++ b/test/crypto
@@ -50,9 +50,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "good",
  "fingerprint": "'$FINGERPRINT'",
@@ -84,9 +83,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "good",
  "fingerprint": "'$FINGERPRINT'",
@@ -120,9 +118,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "error",
  "keyid": "'$(echo $FINGERPRINT | cut -c 25-)'",
@@ -194,9 +191,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test encrypted message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "encstatus": [{"status": "good"}],
  "sigstatus": [],
@@ -249,9 +245,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test encrypted message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "encstatus": [{"status": "bad"}],
  "content-type": "multipart/encrypted",
@@ -284,9 +279,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test encrypted message 002",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "encstatus": [{"status": "good"}],
  "sigstatus": [{"status": "good",
@@ -339,9 +333,8 @@ expected='[[[{"id": "XXXXX",
  "headers": {"Subject": "test signed message 001",
  "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
  "To": "test_suite@notmuchmail.org",
- "Cc": "",
- "Bcc": "",
- "Date": "01 Jan 2000 12:00:00 -0000"},
+ "Date": "Sat,
+ 01 Jan 2000 12:00:00 +0000"},
  "body": [{"id": 1,
  "sigstatus": [{"status": "error",
  "keyid": "6D92612D94E46381",
diff --git a/test/emacs b/test/emacs
index b74cfa9..d4a8d30 100755
--- a/test/emacs
+++ b/test/emacs
@@ -78,7 +78,7 @@ thread=$(notmuch search --output=threads subject:message-with-invalid-from)
 test_emacs "(notmuch-show \"$thread\")
 	    (test-output)"
 cat <<EOF >EXPECTED
-Invalid " From <test_suite@notmuchmail.org> (2001-01-05) (inbox)
+"Invalid " (2001-01-05) (inbox)
 Subject: message-with-invalid-from
 To: Notmuch Test Suite <test_suite@notmuchmail.org>
 Date: Fri, 05 Jan 2001 15:43:57 +0000
@@ -411,7 +411,7 @@ test_emacs '(notmuch-show "id:\"bought\"")
 	(reverse-region (point-min) (point-max))
 	    (test-output)'
 cat <<EOF >EXPECTED
-Sat, 01 Jan 2000 12:00:00 -0000
+Sat, 01 Jan 2000 12:00:00 +0000
 Some One <someone@somewhere.org>
 Some One Else <notsomeone@somewhere.org>
 Notmuch <notmuch@notmuchmail.org>
diff --git a/test/json b/test/json
index 7df4380..1bdffd2 100755
--- a/test/json
+++ b/test/json
@@ -5,7 +5,7 @@ test_description="--format=json output"
 test_begin_subtest "Show message: json"
 add_message "[subject]=\"json-show-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"json-show-message\""
 output=$(notmuch show --format=json "json-show-message")
-test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Cc\": \"\", \"Bcc\": \"\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 -0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
 
 test_begin_subtest "Search message: json"
 add_message "[subject]=\"json-search-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"json-search-message\""
@@ -22,7 +22,7 @@ test_expect_equal "$output" "[{\"thread\": \"XXX\",
 test_begin_subtest "Show message: json, utf-8"
 add_message "[subject]=\"json-show-utf8-body-sübjéct\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"jsön-show-méssage\""
 output=$(notmuch show --format=json "jsön-show-méssage")
-test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-utf8-body-sübjéct\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Cc\": \"\", \"Bcc\": \"\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 -0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"jsön-show-méssage\n\"}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-utf8-body-sübjéct\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"jsön-show-méssage\n\"}]}, []]]]"
 
 test_begin_subtest "Show message: json, inline attachment filename"
 subject='json-show-inline-attachment-filename'
@@ -35,7 +35,7 @@ emacs_deliver_message \
      (insert \"Message-ID: <$id>\n\")"
 output=$(notmuch show --format=json "id:$id")
 filename=$(notmuch search --output=files "id:$id")
-test_expect_equal "$output" "[[[{\"id\": \"$id\", \"match\": true, \"filename\": \"$filename\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\"], \"headers\": {\"Subject\": \"$subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"test_suite@notmuchmail.org\", \"Cc\": \"\", \"Bcc\": \"\", \"Date\": \"01 Jan 2000 12:00:00 -0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"multipart/mixed\", \"content\": [{\"id\": 2, \"content-type\": \"text/plain\", \"content\": \"This is a test message with inline attachment with a filename\"}, {\"id\": 3, \"content-type\": \"application/octet-stream\", \"filename\": \"README\"}]}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"$id\", \"match\": true, \"filename\": \"$filename\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\"], \"headers\": {\"Subject\": \"$subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"test_suite@notmuchmail.org\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"multipart/mixed\", \"content\": [{\"id\": 2, \"content-type\": \"text/plain\", \"content\": \"This is a test message with inline attachment with a filename\"}, {\"id\": 3, \"content-type\": \"application/octet-stream\", \"filename\": \"README\"}]}]}, []]]]"
 
 test_begin_subtest "Search message: json, utf-8"
 add_message "[subject]=\"json-search-utf8-body-sübjéct\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"jsön-search-méssage\""
diff --git a/test/maildir-sync b/test/maildir-sync
index d5872a5..1ee2db0 100755
--- a/test/maildir-sync
+++ b/test/maildir-sync
@@ -53,8 +53,6 @@ test_expect_equal "$output" '[[[{"id": "adding-replied-tag@notmuch-test-suite",
 "headers": {"Subject": "Adding replied tag",
 "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
 "To": "Notmuch Test Suite <test_suite@notmuchmail.org>",
-"Cc": "",
-"Bcc": "",
 "Date": "Fri,
 05 Jan 2001 15:43:57 +0000"},
 "body": [{"id": 1,
diff --git a/test/multipart b/test/multipart
index 4d14804..a3036b4 100755
--- a/test/multipart
+++ b/test/multipart
@@ -322,7 +322,7 @@ notmuch show --format=json --part=0 'id:87liy5ap00.fsf@yoom.home.cworth.org' | s
 echo >>OUTPUT # expect *no* newline at end of output
 cat <<EOF >EXPECTED
 
-{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "${MAIL_DIR}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Cc": "", "Bcc": "", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
+{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "filename": "${MAIL_DIR}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
 {"id": 1, "content-type": "multipart/signed", "content": [
 {"id": 2, "content-type": "multipart/mixed", "content": [
 {"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
-- 
1.7.7.3

[PATCH v2 5/8] show: Simplify talloc use in format_headers_json

[Austin Clements]

Previously there was an unnecessary talloc context.
---
 notmuch-show.c |   32 +++++++++++++++-----------------
 1 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 209ff45..6259d30 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -291,37 +291,35 @@ format_headers_message_part_text (GMimeMessage *message)
 }
 
 static void
-format_headers_json (GMimeMessage *message)
+format_headers_json (const void *ctx, GMimeMessage *message)
 {
-    void *ctx = talloc_new (NULL);
-    void *ctx_quote = talloc_new (ctx);
+    void *local = talloc_new (ctx);
     InternetAddressList *recipients;
     const char *recipients_string;
 
     printf ("%s: %s",
-	    json_quote_str (ctx_quote, "Subject"),
-	    json_quote_str (ctx_quote, g_mime_message_get_subject (message)));
+	    json_quote_str (local, "Subject"),
+	    json_quote_str (local, g_mime_message_get_subject (message)));
     printf (", %s: %s",
-	    json_quote_str (ctx_quote, "From"),
-	    json_quote_str (ctx_quote, g_mime_message_get_sender (message)));
+	    json_quote_str (local, "From"),
+	    json_quote_str (local, g_mime_message_get_sender (message)));
     recipients = g_mime_message_get_recipients (message, GMIME_RECIPIENT_TYPE_TO);
     recipients_string = internet_address_list_to_string (recipients, 0);
     if (recipients_string)
 	printf (", %s: %s",
-		json_quote_str (ctx_quote, "To"),
-		json_quote_str (ctx_quote, recipients_string));
+		json_quote_str (local, "To"),
+		json_quote_str (local, recipients_string));
     recipients = g_mime_message_get_recipients (message, GMIME_RECIPIENT_TYPE_CC);
     recipients_string = internet_address_list_to_string (recipients, 0);
     if (recipients_string)
 	printf (", %s: %s",
-		json_quote_str (ctx_quote, "Cc"),
-		json_quote_str (ctx_quote, recipients_string));
+		json_quote_str (local, "Cc"),
+		json_quote_str (local, recipients_string));
     printf (", %s: %s",
-	    json_quote_str (ctx_quote, "Date"),
-	    json_quote_str (ctx_quote, g_mime_message_get_date_as_string (message)));
+	    json_quote_str (local, "Date"),
+	    json_quote_str (local, g_mime_message_get_date_as_string (message)));
 
-    talloc_free (ctx_quote);
-    talloc_free (ctx);
+    talloc_free (local);
 }
 
 /* Write a MIME text part out to the given stream.
@@ -661,7 +659,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	format_message_json (ctx, node->envelope_file);
 
 	printf ("\"headers\": {");
-	format_headers_json (GMIME_MESSAGE (node->part));
+	format_headers_json (ctx, GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
@@ -749,7 +747,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     if (GMIME_IS_MESSAGE (node->part)) {
 	printf ("\"headers\": {");
-	format_headers_json (GMIME_MESSAGE (node->part));
+	format_headers_json (local, GMIME_MESSAGE (node->part));
 	printf ("}");
 
 	printf (", \"body\": [");
-- 
1.7.7.3

[PATCH v2 6/8] show: Make JSON helper functions print complete objects

[Austin Clements]

This makes the main recursive function easier to follow because helper
functions don't add fields to the running object.
---
 notmuch-show.c |   15 +++++++--------
 1 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 6259d30..8fb6fa6 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -297,7 +297,7 @@ format_headers_json (const void *ctx, GMimeMessage *message)
     InternetAddressList *recipients;
     const char *recipients_string;
 
-    printf ("%s: %s",
+    printf ("{%s: %s",
 	    json_quote_str (local, "Subject"),
 	    json_quote_str (local, g_mime_message_get_subject (message)));
     printf (", %s: %s",
@@ -315,7 +315,7 @@ format_headers_json (const void *ctx, GMimeMessage *message)
 	printf (", %s: %s",
 		json_quote_str (local, "Cc"),
 		json_quote_str (local, recipients_string));
-    printf (", %s: %s",
+    printf (", %s: %s}",
 	    json_quote_str (local, "Date"),
 	    json_quote_str (local, g_mime_message_get_date_as_string (message)));
 
@@ -406,7 +406,7 @@ signer_status_to_string (GMimeSignerStatus x)
 static void
 format_part_sigstatus_json (GMimeSignatureList *siglist)
 {
-    printf (", \"sigstatus\": [");
+    printf ("[");
 
     if (!siglist) {
 	printf ("]");
@@ -472,7 +472,7 @@ format_part_sigstatus_json (GMimeSignatureList *siglist)
 static void
 format_part_sigstatus_json (const GMimeSignatureValidity* validity)
 {
-    printf (", \"sigstatus\": [");
+    printf ("[");
 
     if (!validity) {
 	printf ("]");
@@ -658,9 +658,8 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	printf ("{");
 	format_message_json (ctx, node->envelope_file);
 
-	printf ("\"headers\": {");
+	printf ("\"headers\": ");
 	format_headers_json (ctx, GMIME_MESSAGE (node->part));
-	printf ("}");
 
 	printf (", \"body\": [");
 	format_part_json (ctx, mime_node_child (node, 0), first);
@@ -696,6 +695,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
     }
 
     if (node->verify_attempted) {
+	printf (", \"sigstatus\": ");
 #ifdef GMIME_ATLEAST_26
 	format_part_sigstatus_json (node->sig_list);
 #else
@@ -746,9 +746,8 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 	g_object_unref (stream_memory);
 
     if (GMIME_IS_MESSAGE (node->part)) {
-	printf ("\"headers\": {");
+	printf ("\"headers\": ");
 	format_headers_json (local, GMIME_MESSAGE (node->part));
-	printf ("}");
 
 	printf (", \"body\": [");
     }
-- 
1.7.7.3

[PATCH v2 7/8] show: Make format_part_sigstatus_json's API consistent between GMIME 2.4 and 2.6

[Austin Clements]

The implementation is still different for GMIME 2.4 and 2.6, but at
least now the caller doesn't have to be aware of this.
---
 notmuch-show.c |   14 +++++++-------
 1 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 8fb6fa6..07276c7 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -404,8 +404,10 @@ signer_status_to_string (GMimeSignerStatus x)
 
 #ifdef GMIME_ATLEAST_26
 static void
-format_part_sigstatus_json (GMimeSignatureList *siglist)
+format_part_sigstatus_json (mime_node_t *node)
 {
+    GMimeSignatureList *siglist = node->sig_list;
+
     printf ("[");
 
     if (!siglist) {
@@ -470,8 +472,10 @@ format_part_sigstatus_json (GMimeSignatureList *siglist)
 }
 #else
 static void
-format_part_sigstatus_json (const GMimeSignatureValidity* validity)
+format_part_sigstatus_json (mime_node_t *node)
 {
+    const GMimeSignatureValidity* validity = node->sig_validity;
+
     printf ("[");
 
     if (!validity) {
@@ -696,11 +700,7 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     if (node->verify_attempted) {
 	printf (", \"sigstatus\": ");
-#ifdef GMIME_ATLEAST_26
-	format_part_sigstatus_json (node->sig_list);
-#else
-	format_part_sigstatus_json (node->sig_validity);
-#endif
+	format_part_sigstatus_json (node);
     }
 
     printf (", \"content-type\": %s",
-- 
1.7.7.3

[PATCH v2 8/8] show: Further general simplifications of the JSON formatter

[Austin Clements]

---
 notmuch-show.c |   61 ++++++++++++++++++++-----------------------------------
 1 files changed, 22 insertions(+), 39 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 07276c7..6a171a4 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -678,9 +678,10 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
     GMimeObject *meta = node->envelope_part ?
 	GMIME_OBJECT (node->envelope_part) : node->part;
     GMimeContentType *content_type = g_mime_object_get_content_type (meta);
-    GMimeStream *stream_memory = g_mime_stream_mem_new ();
     const char *cid = g_mime_object_get_content_id (meta);
-    GByteArray *part_content;
+    const char *filename = GMIME_IS_PART (node->part) ?
+	g_mime_part_get_filename (GMIME_PART (node->part)) : NULL;
+    const char *terminator = "";
     int i;
 
     if (!first)
@@ -688,15 +689,9 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
     printf ("{\"id\": %d", node->part_num);
 
-    if (node->decrypt_attempted) {
-	printf (", \"encstatus\": [{\"status\": ");
-	if (node->decrypt_success) {
-	    printf ("\"good\"");
-	} else {
-	    printf ("\"bad\"");
-	}
-	printf ("}]");
-    }
+    if (node->decrypt_attempted)
+	printf (", \"encstatus\": [{\"status\": \"%s\"}]",
+		node->decrypt_success ? "good" : "bad");
 
     if (node->verify_attempted) {
 	printf (", \"sigstatus\": ");
@@ -706,16 +701,13 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
     printf (", \"content-type\": %s",
 	    json_quote_str (local, g_mime_content_type_to_string (content_type)));
 
-    if (cid != NULL)
-	    printf(", \"content-id\": %s", json_quote_str (local, cid));
+    if (cid)
+	printf (", \"content-id\": %s", json_quote_str (local, cid));
+
+    if (filename)
+	printf (", \"filename\": %s", json_quote_str (local, filename));
 
     if (GMIME_IS_PART (node->part)) {
-	const char *filename = g_mime_part_get_filename (GMIME_PART (node->part));
-	if (filename)
-	    printf (", \"filename\": %s", json_quote_str (local, filename));
-    }
-
-    if (g_mime_content_type_is_type (content_type, "text", "*")) {
 	/* For non-HTML text parts, we include the content in the
 	 * JSON. Since JSON must be Unicode, we handle charset
 	 * decoding here and do not report a charset to the caller.
@@ -730,42 +722,33 @@ format_part_json (const void *ctx, mime_node_t *node, notmuch_bool_t first)
 
 	    if (content_charset != NULL)
 		printf (", \"content-charset\": %s", json_quote_str (local, content_charset));
-	} else {
+	} else if (g_mime_content_type_is_type (content_type, "text", "*")) {
+	    GMimeStream *stream_memory = g_mime_stream_mem_new ();
+	    GByteArray *part_content;
 	    show_text_part_content (node->part, stream_memory);
 	    part_content = g_mime_stream_mem_get_byte_array (GMIME_STREAM_MEM (stream_memory));
 
 	    printf (", \"content\": %s", json_quote_chararray (local, (char *) part_content->data, part_content->len));
+	    g_object_unref (stream_memory);
 	}
-    } else if (g_mime_content_type_is_type (content_type, "multipart", "*")) {
+    } else if (GMIME_IS_MULTIPART (node->part)) {
 	printf (", \"content\": [");
-    } else if (g_mime_content_type_is_type (content_type, "message", "rfc822")) {
+	terminator = "]";
+    } else if (GMIME_IS_MESSAGE (node->part)) {
 	printf (", \"content\": [{");
-    }
-
-    if (stream_memory)
-	g_object_unref (stream_memory);
-
-    if (GMIME_IS_MESSAGE (node->part)) {
 	printf ("\"headers\": ");
 	format_headers_json (local, GMIME_MESSAGE (node->part));
 
 	printf (", \"body\": [");
+	terminator = "]}]";
     }
 
+    talloc_free (local);
+
     for (i = 0; i < node->nchildren; i++)
 	format_part_json (ctx, mime_node_child (node, i), i == 0);
 
-    if (GMIME_IS_MESSAGE (node->part))
-	printf ("]");
-
-    if (g_mime_content_type_is_type (content_type, "multipart", "*"))
-	printf ("]");
-    else if (g_mime_content_type_is_type (content_type, "message", "rfc822"))
-	printf ("}]");
-
-    printf ("}");
-
-    talloc_free (local);
+    printf ("%s}", terminator);
 }
 
 static void
-- 
1.7.7.3

Re: [PATCH v2 0/8] Rewrite JSON show format

[Tomi Ollila]

On Sun, 19 Feb 2012 19:26:22 -0500, Austin Clements <amdragon@MIT.EDU> wrote:
> v2 fixes some trivial whitespace issues in devel/schemata [1] and
> beefs up one of the explanations in devel/schemata.  None of the other
> patches have changed, so I'm marking this ready.

+1

Tomi