support extra headers in notmuch show v2

support extra headers in notmuch show v2

[David Bremner]

This is a respin of [1], that takes advantage of changes to the config
system in the meantime and adds a couple tests. Like that patch, this
only updates the CLI. There are some related changes to the emacs UI
proposed in [2], but those would require adapting to work with these
changes. In particular the emacs UI probably needs to interrogate
notmuch to see which extra headers the user has configured.

[1]: id:20191116162723.18343-1-johan.parin@gmail.com
[2]: id:20191122230730.35712-2-johan.parin@gmail.com

[PATCH 1/3] CLI: stash pointer to database in sprinter structs

[David Bremner]

We already use an allocated (and presumably open) database as a talloc
context. Keeping the pointer in the allocated struct will allow us to
e.g. interrogate the configuration in a sprinter function without
threading the database all the way through the various levels of function.
---
 notmuch-client.h |  2 +-
 sprinter-json.c  |  5 +++--
 sprinter-sexp.c  |  5 +++--
 sprinter-text.c  |  9 +++++----
 sprinter.h       | 13 +++++++++----
 5 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/notmuch-client.h b/notmuch-client.h
index de318e1f..9f57ac5e 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -65,7 +65,7 @@ struct sprinter;
 struct notmuch_show_params;
 
 typedef struct notmuch_show_format {
-    struct sprinter *(*new_sprinter)(const void *ctx, FILE *stream);
+    struct sprinter *(*new_sprinter)(notmuch_database_t * db, FILE *stream);
     notmuch_status_t (*part)(const void *ctx, struct sprinter *sprinter,
 			     struct mime_node *node, int indent,
 			     const struct notmuch_show_params *params);
diff --git a/sprinter-json.c b/sprinter-json.c
index c7f4851c..502f89fb 100644
--- a/sprinter-json.c
+++ b/sprinter-json.c
@@ -172,7 +172,7 @@ json_separator (struct sprinter *sp)
 }
 
 struct sprinter *
-sprinter_json_create (const void *ctx, FILE *stream)
+sprinter_json_create (notmuch_database_t *db, FILE *stream)
 {
     static const struct sprinter_json template = {
 	.vtable = {
@@ -192,11 +192,12 @@ sprinter_json_create (const void *ctx, FILE *stream)
     };
     struct sprinter_json *res;
 
-    res = talloc (ctx, struct sprinter_json);
+    res = talloc (db, struct sprinter_json);
     if (! res)
 	return NULL;
 
     *res = template;
+    res->vtable.notmuch = db;
     res->stream = stream;
     return &res->vtable;
 }
diff --git a/sprinter-sexp.c b/sprinter-sexp.c
index 63b25428..e37cb1f9 100644
--- a/sprinter-sexp.c
+++ b/sprinter-sexp.c
@@ -207,7 +207,7 @@ sexp_separator (struct sprinter *sp)
 }
 
 struct sprinter *
-sprinter_sexp_create (const void *ctx, FILE *stream)
+sprinter_sexp_create (notmuch_database_t *db, FILE *stream)
 {
     static const struct sprinter_sexp template = {
 	.vtable = {
@@ -227,11 +227,12 @@ sprinter_sexp_create (const void *ctx, FILE *stream)
     };
     struct sprinter_sexp *res;
 
-    res = talloc (ctx, struct sprinter_sexp);
+    res = talloc (db, struct sprinter_sexp);
     if (! res)
 	return NULL;
 
     *res = template;
+    res->vtable.notmuch = db;
     res->stream = stream;
     return &res->vtable;
 }
diff --git a/sprinter-text.c b/sprinter-text.c
index c75ec5be..99330a94 100644
--- a/sprinter-text.c
+++ b/sprinter-text.c
@@ -114,7 +114,7 @@ text_map_key (unused (struct sprinter *sp), unused (const char *key))
 }
 
 struct sprinter *
-sprinter_text_create (const void *ctx, FILE *stream)
+sprinter_text_create (notmuch_database_t *db, FILE *stream)
 {
     static const struct sprinter_text template = {
 	.vtable = {
@@ -134,21 +134,22 @@ sprinter_text_create (const void *ctx, FILE *stream)
     };
     struct sprinter_text *res;
 
-    res = talloc (ctx, struct sprinter_text);
+    res = talloc (db, struct sprinter_text);
     if (! res)
 	return NULL;
 
     *res = template;
+    res->vtable.notmuch = db;
     res->stream = stream;
     return &res->vtable;
 }
 
 struct sprinter *
-sprinter_text0_create (const void *ctx, FILE *stream)
+sprinter_text0_create (notmuch_database_t *db, FILE *stream)
 {
     struct sprinter *sp;
 
-    sp = sprinter_text_create (ctx, stream);
+    sp = sprinter_text_create (db, stream);
     if (! sp)
 	return NULL;
 
diff --git a/sprinter.h b/sprinter.h
index 528d8a2d..fd08641c 100644
--- a/sprinter.h
+++ b/sprinter.h
@@ -9,6 +9,11 @@
  * (strings, integers and booleans).
  */
 typedef struct sprinter {
+    /*
+     * Open notmuch database
+     */
+    notmuch_database_t *notmuch;
+
     /* Start a new map/dictionary structure. This should be followed by
      * a sequence of alternating calls to map_key and one of the
      * value-printing functions until the map is ended by end.
@@ -65,20 +70,20 @@ typedef struct sprinter {
 /* Create a new unstructured printer that emits the default text format
  * for "notmuch search". */
 struct sprinter *
-sprinter_text_create (const void *ctx, FILE *stream);
+sprinter_text_create (notmuch_database_t *db, FILE *stream);
 
 /* Create a new unstructured printer that emits the text format for
  * "notmuch search", with each field separated by a null character
  * instead of the newline character. */
 struct sprinter *
-sprinter_text0_create (const void *ctx, FILE *stream);
+sprinter_text0_create (notmuch_database_t *db, FILE *stream);
 
 /* Create a new structure printer that emits JSON. */
 struct sprinter *
-sprinter_json_create (const void *ctx, FILE *stream);
+sprinter_json_create (notmuch_database_t *db, FILE *stream);
 
 /* Create a new structure printer that emits S-Expressions. */
 struct sprinter *
-sprinter_sexp_create (const void *ctx, FILE *stream);
+sprinter_sexp_create (notmuch_database_t *db, FILE *stream);
 
 #endif // NOTMUCH_SPRINTER_H
-- 
2.34.1

[PATCH 2/3] lib/config: add known config key "show.extra_headers"

[David Bremner]

Used in a following commit to enable including extra headers beyond
the default in structured output.
---
 lib/config.cc          | 3 +++
 lib/notmuch.h          | 1 +
 test/T590-libconfig.sh | 4 ++++
 3 files changed, 8 insertions(+)

diff --git a/lib/config.cc b/lib/config.cc
index 7a2882de..97060f8b 100644
--- a/lib/config.cc
+++ b/lib/config.cc
@@ -596,6 +596,8 @@ _notmuch_config_key_to_string (notmuch_config_key_t key)
 	return "user.name";
     case NOTMUCH_CONFIG_AUTOCOMMIT:
 	return "database.autocommit";
+    case NOTMUCH_CONFIG_EXTRA_HEADERS:
+	return "show.extra_headers";
     default:
 	return NULL;
     }
@@ -643,6 +645,7 @@ _notmuch_config_default (notmuch_database_t *notmuch, notmuch_config_key_t key)
 	return "";
     case NOTMUCH_CONFIG_AUTOCOMMIT:
 	return "8000";
+    case NOTMUCH_CONFIG_EXTRA_HEADERS:
     case NOTMUCH_CONFIG_HOOK_DIR:
     case NOTMUCH_CONFIG_BACKUP_DIR:
     case NOTMUCH_CONFIG_OTHER_EMAIL:
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 1b2bdf3f..a75a42be 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -2546,6 +2546,7 @@ typedef enum {
     NOTMUCH_CONFIG_OTHER_EMAIL,
     NOTMUCH_CONFIG_USER_NAME,
     NOTMUCH_CONFIG_AUTOCOMMIT,
+    NOTMUCH_CONFIG_EXTRA_HEADERS,
     NOTMUCH_CONFIG_LAST
 } notmuch_config_key_t;
 
diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
index eb303444..7ee4bb11 100755
--- a/test/T590-libconfig.sh
+++ b/test/T590-libconfig.sh
@@ -439,6 +439,7 @@ cat <<'EOF' >EXPECTED
 09: 'NULL'
 10: 'USER_FULL_NAME'
 11: '8000'
+12: 'NULL'
 == stderr ==
 EOF
 unset MAILDIR
@@ -749,6 +750,7 @@ cat <<'EOF' >EXPECTED
 09: 'test_suite_other@notmuchmail.org;test_suite@otherdomain.org'
 10: 'Notmuch Test Suite'
 11: '8000'
+12: 'NULL'
 == stderr ==
 EOF
 test_expect_equal_file EXPECTED OUTPUT
@@ -782,6 +784,7 @@ cat <<'EOF' >EXPECTED
 09: 'NULL'
 10: 'USER_FULL_NAME'
 11: '8000'
+12: 'NULL'
 == stderr ==
 EOF
 test_expect_equal_file EXPECTED OUTPUT.clean
@@ -858,6 +861,7 @@ maildir.synchronize_flags true
 new.ignore sekrit_junk
 new.tags unread;inbox
 search.exclude_tags foo;bar;fub
+show.extra_headers (null)
 test.key1 testvalue1
 test.key2 testvalue2
 user.name Notmuch Test Suite
-- 
2.34.1

[PATCH 3/3] CLI: print extra headers in structured output

[David Bremner]

This is based on a patch from Johan Parin [1], which is in turn
responding to a bug report / feature requiest from Jan Malkhovski.

The update to the structured output documented in schemata is intended
to be upward compatible, so the format version stays the same

[1]: id:20191116162723.18343-1-johan.parin@gmail.com
[2]: id:87h8sdemnr.fsf@oxij.org
---
 devel/schemata    |  4 +++-
 notmuch-show.c    | 26 ++++++++++++++++++++++++++
 test/T160-json.sh | 42 ++++++++++++++++++++++++++++++++++++++++++
 test/T170-sexp.sh | 14 ++++++++++++++
 4 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/devel/schemata b/devel/schemata
index 01e3a3df..01810888 100644
--- a/devel/schemata
+++ b/devel/schemata
@@ -145,9 +145,11 @@ headers = {
     Cc?:            string,
     Bcc?:           string,
     Reply-To?:      string,
-    Date:           string
+    Date:           string,
+    extra_header_pair*
 }
 
+extra_header_pair=  (header_name: string)
 # Encryption status (format_part_sprinter)
 encstatus = [{status: "good"|"bad"}]
 
diff --git a/notmuch-show.c b/notmuch-show.c
index 2848c9c3..136f4439 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -209,6 +209,30 @@ _is_from_line (const char *line)
 	return 0;
 }
 
+/* Output extra headers if configured with the `show.extra_headers'
+ * configuration option
+ */
+static void
+format_extra_headers_sprinter (sprinter_t *sp, GMimeMessage *message)
+{
+    GMimeHeaderList *header_list = g_mime_object_get_header_list (GMIME_OBJECT (message));
+
+    for (notmuch_config_values_t *extra_headers = notmuch_config_get_values (
+	     sp->notmuch, NOTMUCH_CONFIG_EXTRA_HEADERS);
+	 notmuch_config_values_valid (extra_headers);
+	 notmuch_config_values_move_to_next (extra_headers)) {
+	GMimeHeader *header;
+	const char *header_name = notmuch_config_values_get (extra_headers);
+
+	header = g_mime_header_list_get_header (header_list, header_name);
+	if (header == NULL)
+	    continue;
+
+	sp->map_key (sp, g_mime_header_get_name (header));
+	sp->string (sp, g_mime_header_get_value (header));
+    }
+}
+
 void
 format_headers_sprinter (sprinter_t *sp, GMimeMessage *message,
 			 bool reply, const _notmuch_message_crypto_t *msg_crypto)
@@ -269,6 +293,8 @@ format_headers_sprinter (sprinter_t *sp, GMimeMessage *message,
 	sp->string (sp, g_mime_message_get_date_string (sp, message));
     }
 
+    /* Output extra headers the user has configured, if any */
+    format_extra_headers_sprinter (sp, message);
     sp->end (sp);
     talloc_free (local);
 }
diff --git a/test/T160-json.sh b/test/T160-json.sh
index 6a3e5812..ec7b1461 100755
--- a/test/T160-json.sh
+++ b/test/T160-json.sh
@@ -156,4 +156,46 @@ EOF
 output=$(notmuch show --format=json --body=false --format-version=2 id:message-id@example.com)
 test_expect_equal_json "$output" "$(cat EXPECTED)"
 
+test_begin_subtest "show extra headers"
+add_message "[subject]=\"extra-headers\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[in-reply-to]=\"<parent@notmuch-test-suite>\"" "[body]=\"extra-headers test\""\
+	   "[header]=\"Received: from mail.example.com (mail.example.com [1.1.1.1])
+	by mail.notmuchmail.org (some MTA) with ESMTP id 12345678
+	for <test_suite_other@notmuchmail.org>; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)\"" \
+
+notmuch config set show.extra_headers "in-reply-to;received"
+output=$(notmuch show --format=json --body=false id:${gen_msg_id} | notmuch_json_show_sanitize)
+cat <<EOF > EXPECTED
+[
+    [
+        [
+            {
+                "crypto": {},
+                "date_relative": "2000-01-01",
+                "excluded": false,
+                "filename": [
+                    "YYYYY"
+                ],
+                "headers": {
+                    "Date": "Sat, 01 Jan 2000 12:00:00 +0000",
+                    "From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
+                    "In-Reply-To": "<parent@notmuch-test-suite>",
+                    "Received": "from mail.example.com (mail.example.com [1.1.1.1])\tby mail.notmuchmail.org (some MTA) with ESMTP id 12345678\tfor <test_suite_other@notmuchmail.org>; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)",
+                    "Subject": "extra-headers",
+                    "To": "Notmuch Test Suite <test_suite@notmuchmail.org>"
+                },
+                "id": "XXXXX",
+                "match": true,
+                "tags": [
+                    "inbox",
+                    "unread"
+                ],
+                "timestamp": 946728000
+            },
+            []
+        ]
+    ]
+]
+EOF
+test_expect_equal_json "${output}" "$(cat EXPECTED)"
+
 test_done
diff --git a/test/T170-sexp.sh b/test/T170-sexp.sh
index 18084273..0d32560c 100755
--- a/test/T170-sexp.sh
+++ b/test/T170-sexp.sh
@@ -47,4 +47,18 @@ filename=$(notmuch search --output=files "id:$id")
 attachment_length=$(( $(base64 $NOTMUCH_SRCDIR/test/README | wc -c) - 1 ))
 test_expect_equal "$output" "((((:id \"$id\" :match t :excluded nil :filename (\"$filename\") :timestamp 946728000 :date_relative \"2000-01-01\" :tags (\"inbox\") :body ((:id 1 :content-type \"multipart/mixed\" :content ((:id 2 :content-type \"text/plain\" :content \"This is a test message with inline attachment with a filename\") (:id 3 :content-type \"application/octet-stream\" :content-disposition \"inline\" :filename \"README\" :content-transfer-encoding \"base64\" :content-length $attachment_length)))) :crypto () :headers (:Subject \"sexp-show-inline-attachment-filename\" :From \"Notmuch Test Suite <test_suite@notmuchmail.org>\" :To \"test_suite@notmuchmail.org\" :Date \"Sat, 01 Jan 2000 12:00:00 +0000\")) ())))"
 
+test_begin_subtest "show extra headers"
+add_message "[subject]=\"extra-headers\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[in-reply-to]=\"<parent@notmuch-test-suite>\"" "[body]=\"extra-headers test\""\
+	   "[header]=\"Received: from mail.example.com (mail.example.com [1.1.1.1])
+	by mail.notmuchmail.org (some MTA) with ESMTP id 12345678
+	for <test_suite_other@notmuchmail.org>; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)\"" \
+
+notmuch config set show.extra_headers "in-reply-to;received"
+notmuch show --format=sexp --body=false id:${gen_msg_id} | \
+    notmuch_dir_sanitize | sed 's/msg-[0-9]*/MSG/g'> OUTPUT
+cat <<EOF > EXPECTED
+((((:id "MSG@notmuch-test-suite" :match t :excluded nil :filename ("MAIL_DIR/MSG") :timestamp 946728000 :date_relative "2000-01-01" :tags ("inbox" "unread") :crypto () :headers (:Subject "extra-headers" :From "Notmuch Test Suite <test_suite@notmuchmail.org>" :To "Notmuch Test Suite <test_suite@notmuchmail.org>" :Date "Sat, 01 Jan 2000 12:00:00 +0000" :In-Reply-To "<parent@notmuch-test-suite>" :Received "from mail.example.com (mail.example.com [1.1.1.1])\011by mail.notmuchmail.org (some MTA) with ESMTP id 12345678\011for <test_suite_other@notmuchmail.org>; Sat, 10 Apr 2010 07:54:51 -0400 (EDT)")) ())))
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done
-- 
2.34.1

Re: [PATCH 1/3] CLI: stash pointer to database in sprinter structs

[David Bremner]

David Bremner <david@tethera.net> writes:

> We already use an allocated (and presumably open) database as a talloc
> context. Keeping the pointer in the allocated struct will allow us to
> e.g. interrogate the configuration in a sprinter function without
> threading the database all the way through the various levels of function.

Series applied to master.

d

[PATCH] doc: document new option `show.extra_headers`

[David Bremner]

Increase discoverability by cross referencing from the notmuch-show
manual entry to the notmuch-config manual entry.
---
 doc/man1/notmuch-config.rst | 14 ++++++++++++++
 doc/man1/notmuch-show.rst   |  7 +++++++
 2 files changed, 21 insertions(+)

diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index ed188a25..41e1338b 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -145,6 +145,20 @@ search.exclude\_tags
     Default: empty list. Note that :any:`notmuch-setup(1)` puts
     ``deleted;spam`` here when creating new configuration file.
 
+.. _show.extra_headers:
+
+show.extra\_headers
+
+    By default :any:`notmuch-show(1)` includes the following headers
+    in structured output if they are present in the message:
+    `Subject`, `From`, `To`, `Cc`, `Bcc`, `Reply-To`, `Date`. This
+    option allows the specification of a list of further
+    headers to output.
+
+    History: This configuration value was introduced in notmuch 0.35.
+
+    Default: empty list.
+
 maildir.synchronize\_flags
     If true, then the following maildir flags (in message filenames)
     will be synchronized with the corresponding notmuch tags:
diff --git a/doc/man1/notmuch-show.rst b/doc/man1/notmuch-show.rst
index 3d2a2c41..1b02d407 100644
--- a/doc/man1/notmuch-show.rst
+++ b/doc/man1/notmuch-show.rst
@@ -221,6 +221,13 @@ email messages. For this, use a search term of "thread:<thread-id>" as
 can be seen in the first column of output from the
 :any:`notmuch-search(1)` command.
 
+CONFIGURATION
+=============
+
+Structured output (json / sexp) is influenced by the configuration
+option :ref:`show.extra_headers <show.extra_headers>`. See
+:any:`notmuch-config(1)` for details.
+
 EXIT STATUS
 ===========
 
-- 
2.34.1

Re: [PATCH] doc: document new option `show.extra_headers`

[David Bremner]

David Bremner <david@tethera.net> writes:

> Increase discoverability by cross referencing from the notmuch-show
> manual entry to the notmuch-config manual entry.

applied to master.

d