[PATCH 0/4] lib: signed and encrypted part content-type indexing

[PATCH 0/4] lib: signed and encrypted part content-type indexing

[Jani Nikula]

v2 of signed and encrypted part content-type indexing, including both my
patches [1] and Daniel's [2], now with tests. Tests added first to prove
we don't currently index this stuff and the patches make a
difference. No other changes.

BR,
Jani.

[1] id:cover.1505329740.git.jani@nikula.org 
[1] id:20170915053852.23663-1-dkg@fifthhorseman.net

Daniel Kahn Gillmor (1):
  lib: index the content-type of the parts of encrypted messages

Jani Nikula (3):
  lib: abstract content type indexing
  test: signed and encrypted part content type indexing
  lib: index the content type of signature parts

 lib/index.cc        | 46 ++++++++++++++++++++++++++++++++--------------
 test/T350-crypto.sh |  8 ++++++++
 2 files changed, 40 insertions(+), 14 deletions(-)

-- 
2.11.0

[PATCH 1/4] lib: abstract content type indexing

[Jani Nikula]

Make the follow-up change of indexing signature content types
easier. No functional changes.
---
 lib/index.cc | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 2b98b588d38d..64bc92a5b56d 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -351,6 +351,19 @@ _index_address_list (notmuch_message_t *message,
     }
 }
 
+static void
+_index_content_type (notmuch_message_t *message, GMimeObject *part)
+{
+    GMimeContentType *content_type = g_mime_object_get_content_type (part);
+    if (content_type) {
+	char *mime_string = g_mime_content_type_to_string (content_type);
+	if (mime_string) {
+	    _notmuch_message_gen_terms (message, "mimetype", mime_string);
+	    g_free (mime_string);
+	}
+    }
+}
+
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
@@ -361,6 +374,7 @@ _index_mime_part (notmuch_message_t *message,
     GMimeDataWrapper *wrapper;
     GByteArray *byte_array;
     GMimeContentDisposition *disposition;
+    GMimeContentType *content_type;
     char *body;
     const char *charset;
 
@@ -370,15 +384,7 @@ _index_mime_part (notmuch_message_t *message,
 	return;
     }
 
-    GMimeContentType *content_type = g_mime_object_get_content_type(part);
-    if (content_type) {
-	char *mime_string = g_mime_content_type_to_string(content_type);
-	if (mime_string)
-	{
-	    _notmuch_message_gen_terms (message, "mimetype", mime_string);
-	    g_free(mime_string);
-	}
-    }
+    _index_content_type (message, part);
 
     if (GMIME_IS_MULTIPART (part)) {
 	GMimeMultipart *multipart = GMIME_MULTIPART (part);
@@ -447,6 +453,8 @@ _index_mime_part (notmuch_message_t *message,
     g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
 
     filter = g_mime_stream_filter_new (stream);
+
+    content_type = g_mime_object_get_content_type (part);
     discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type);
 
     g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
-- 
2.11.0

[PATCH 2/4] test: signed and encrypted part content type indexing

[Jani Nikula]

Add known broken subtests for searching signed and encrypted messages
using mimetype: prefix search for the content-types of signed and
encrypted parts.
---
 test/T350-crypto.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh
index 1d408af71dd7..2f1f9ce4913c 100755
--- a/test/T350-crypto.sh
+++ b/test/T350-crypto.sh
@@ -37,6 +37,11 @@ test_expect_success \
     "This is a test signed message." \
     "(mml-secure-message-sign)"'
 
+test_begin_subtest "signed part content-type indexing"
+test_subtest_known_broken
+output=$(notmuch search mimetype:multipart/signed and mimetype:application/pgp-signature | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test signed message 001 (inbox signed)"
+
 test_begin_subtest "signature verification"
 output=$(notmuch show --format=json --verify subject:"test signed message 001" \
     | notmuch_json_show_sanitize \
@@ -231,6 +236,11 @@ test_expect_success \
     "This is a test encrypted message.\n" \
     "(mml-attach-file \"TESTATTACHMENT\") (mml-secure-message-encrypt)"'
 
+test_begin_subtest "encrypted part content-type indexing"
+test_subtest_known_broken
+output=$(notmuch search mimetype:multipart/encrypted and mimetype:application/pgp-encrypted and mimetype:application/octet-stream | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message 001 (encrypted inbox)"
+
 test_begin_subtest "decryption, --format=text"
 output=$(notmuch show --format=text --decrypt subject:"test encrypted message 001" \
     | notmuch_show_sanitize_all \
-- 
2.11.0

[PATCH 3/4] lib: index the content type of signature parts

[Jani Nikula]

It's useful (*) to be able to easily find messages with certain types
of signatures. Having the mimetype: prefix searches fail for some
content types is also genuinely surprising (*). Index the content type
of signature parts.

While at it, switch to the gmime convenience constants for content and
signature part indexes.

*) At least for developers of email software!
---
 lib/index.cc        | 11 +++++++----
 test/T350-crypto.sh |  1 -
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 64bc92a5b56d..0beaae62f048 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -398,12 +398,15 @@ _index_mime_part (notmuch_message_t *message,
 
 	for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
 	    if (GMIME_IS_MULTIPART_SIGNED (multipart)) {
-		/* Don't index the signature. */
-		if (i == 1)
+		/* Don't index the signature, but index its content type. */
+		if (i == GMIME_MULTIPART_SIGNED_SIGNATURE) {
+		    _index_content_type (message,
+					 g_mime_multipart_get_part (multipart, i));
 		    continue;
-		if (i > 1)
+		} else if (i != GMIME_MULTIPART_SIGNED_CONTENT) {
 		    _notmuch_database_log (_notmuch_message_database (message),
-					  "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
+					   "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
+		}
 	    }
 	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
 		/* Don't index encrypted parts. */
diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh
index 2f1f9ce4913c..b8f862560006 100755
--- a/test/T350-crypto.sh
+++ b/test/T350-crypto.sh
@@ -38,7 +38,6 @@ test_expect_success \
     "(mml-secure-message-sign)"'
 
 test_begin_subtest "signed part content-type indexing"
-test_subtest_known_broken
 output=$(notmuch search mimetype:multipart/signed and mimetype:application/pgp-signature | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test signed message 001 (inbox signed)"
 
-- 
2.11.0

[PATCH 4/4] lib: index the content-type of the parts of encrypted messages

[Jani Nikula]

From: Daniel Kahn Gillmor <dkg@fifthhorseman.net>

This is a logical followup to "lib: index the content type of
signature parts", which will make it easier to record the message
structure of all messages.
---
 lib/index.cc        | 9 ++++++++-
 test/T350-crypto.sh | 1 -
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 0beaae62f048..ceb444df60bf 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -409,7 +409,14 @@ _index_mime_part (notmuch_message_t *message,
 		}
 	    }
 	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
-		/* Don't index encrypted parts. */
+		/* Don't index encrypted parts, but index their content type. */
+		_index_content_type (message,
+				     g_mime_multipart_get_part (multipart, i));
+		if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) &&
+		    (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) {
+		    _notmuch_database_log (_notmuch_message_database (message),
+					   "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		}
 		continue;
 	    }
 	    _index_mime_part (message,
diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh
index b8f862560006..53bf4113d622 100755
--- a/test/T350-crypto.sh
+++ b/test/T350-crypto.sh
@@ -236,7 +236,6 @@ test_expect_success \
     "(mml-attach-file \"TESTATTACHMENT\") (mml-secure-message-encrypt)"'
 
 test_begin_subtest "encrypted part content-type indexing"
-test_subtest_known_broken
 output=$(notmuch search mimetype:multipart/encrypted and mimetype:application/pgp-encrypted and mimetype:application/octet-stream | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message 001 (encrypted inbox)"
 
-- 
2.11.0

Re: [PATCH 4/4] lib: index the content-type of the parts of encrypted messages

[Daniel Kahn Gillmor]

this 4-patch series LGTM.

Thanks, Jani!

        --dkg

Re: [PATCH 0/4] lib: signed and encrypted part content-type indexing

[David Bremner]

Jani Nikula <jani@nikula.org> writes:

> v2 of signed and encrypted part content-type indexing, including both my
> patches [1] and Daniel's [2], now with tests. Tests added first to prove
> we don't currently index this stuff and the patches make a
> difference. No other changes.
>

series pushed.

d