cleartext indexing version 5

cleartext indexing version 5

[Daniel Kahn Gillmor]

Thanks to everyone for their reviews of this series that provides
indexing of the cleartext of encrypted messages (and thanks to bremner
for already pushing some of the less controversial patches of the
previous revisions of this series).  I hope you'll agree that your
suggestions have improved the propsoal :)

In particular, the following significant changes have been made since
the previous revision:

 * all properties whose key is prefixed with "index." are cleared (and
   possibly re-set) upon reindexing, not just a registered list of
   "autoproperties".

 * handling of command-line options for indexing (i.e. for the "new",
   "reindex", and "insert" subcommands) has been consolidated into a
   centralized location, which not only reduces total code footprint
   but will make it easier to add shared indexing options in the
   future.

I welcome review and feedback.

      --dkg

Series History
--------------

the first version of this series was sent starting at:

id:20170912230153.4175-1-dkg@fifthhorseman.net

version 2 of this series was sent as:

id:20170915055359.24123-1-dkg@fifthhorseman.net

version 3 of this series was sent as:

id:20171010054916.23925-1-dkg@fifthhorseman.net

and version 4 was only partially sent to the list.

[PATCH v5 01/12] crypto: move into libnotmuch_util

[Daniel Kahn Gillmor]

This prepares us for using the crypto object in both libnotmuch and
the client.
---
 Makefile.local            |  1 -
 notmuch-client.h          | 22 +---------------------
 util/Makefile.local       |  2 +-
 crypto.c => util/crypto.c |  7 ++++++-
 util/crypto.h             | 28 ++++++++++++++++++++++++++++
 5 files changed, 36 insertions(+), 24 deletions(-)
 rename crypto.c => util/crypto.c (96%)
 create mode 100644 util/crypto.h

diff --git a/Makefile.local b/Makefile.local
index 9d9c52c2..9505b7fe 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -246,7 +246,6 @@ notmuch_client_srcs =		\
 	sprinter-text.c		\
 	query-string.c		\
 	mime-node.c		\
-	crypto.c		\
 	tag-util.c
 
 notmuch_client_modules = $(notmuch_client_srcs:.c=.o)
diff --git a/notmuch-client.h b/notmuch-client.h
index bdcfd893..d17cdf01 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -32,9 +32,6 @@
 
 #include "gmime-extra.h"
 
-/* This is automatically included only since gmime 2.6.10 */
-#include <gmime/gmime-pkcs7-context.h>
-
 #include "notmuch.h"
 
 /* This is separate from notmuch-private.h because we're trying to
@@ -54,6 +51,7 @@
 #include <ctype.h>
 
 #include "talloc-extra.h"
+#include "crypto.h"
 
 #define unused(x) x __attribute__ ((unused))
 
@@ -71,16 +69,6 @@ typedef struct notmuch_show_format {
 			      const struct notmuch_show_params *params);
 } notmuch_show_format_t;
 
-typedef struct _notmuch_crypto {
-    bool verify;
-    bool decrypt;
-#if (GMIME_MAJOR_VERSION < 3)
-    GMimeCryptoContext* gpgctx;
-    GMimeCryptoContext* pkcs7ctx;
-    const char *gpgpath;
-#endif
-} _notmuch_crypto_t;
-
 typedef struct notmuch_show_params {
     bool entire_thread;
     bool omit_excluded;
@@ -180,14 +168,6 @@ typedef struct _notmuch_config notmuch_config_t;
 void
 notmuch_exit_if_unsupported_format (void);
 
-#if (GMIME_MAJOR_VERSION <3)
-GMimeCryptoContext *
-_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol);
-#endif
-
-void
-_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto);
-
 int
 notmuch_count_command (notmuch_config_t *config, int argc, char *argv[]);
 
diff --git a/util/Makefile.local b/util/Makefile.local
index 3027880b..ba03230e 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -5,7 +5,7 @@ extra_cflags += -I$(srcdir)/$(dir)
 
 libnotmuch_util_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \
 		  $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \
-		$(dir)/util.c $(dir)/gmime-extra.c
+		$(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c
 
 libnotmuch_util_modules := $(libnotmuch_util_c_srcs:.c=.o)
 
diff --git a/crypto.c b/util/crypto.c
similarity index 96%
rename from crypto.c
rename to util/crypto.c
index 4c1b7eec..c51d67ed 100644
--- a/crypto.c
+++ b/util/crypto.c
@@ -18,7 +18,12 @@
  * Authors: Jameson Rollins <jrollins@finestructure.net>
  */
 
-#include "notmuch-client.h"
+#include "crypto.h"
+#include <strings.h>
+#define unused(x) x __attribute__ ((unused))
+
+#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
+
 #if (GMIME_MAJOR_VERSION < 3)
 /* Create a GPG context (GMime 2.6) */
 static GMimeCryptoContext *
diff --git a/util/crypto.h b/util/crypto.h
new file mode 100644
index 00000000..80628dc5
--- /dev/null
+++ b/util/crypto.h
@@ -0,0 +1,28 @@
+#ifndef _CRYPTO_H
+#define _CRYPTO_H
+
+#include <stdbool.h>
+#if (GMIME_MAJOR_VERSION < 3)
+#include "gmime-extra.h"
+#endif
+
+typedef struct _notmuch_crypto {
+    bool verify;
+    bool decrypt;
+#if (GMIME_MAJOR_VERSION < 3)
+    GMimeCryptoContext* gpgctx;
+    GMimeCryptoContext* pkcs7ctx;
+    const char *gpgpath;
+#endif
+} _notmuch_crypto_t;
+
+
+#if (GMIME_MAJOR_VERSION < 3)
+GMimeCryptoContext *
+_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol);
+#endif
+
+void
+_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto);
+
+#endif
-- 
2.14.2

[PATCH v5 02/12] crypto: make shared crypto code behave library-like

[Daniel Kahn Gillmor]

If we're going to reuse the crypto code across both the library and
the client, then it needs to report error states properly and not
write to stderr.
---
 lib/database.cc |  6 ++++
 lib/notmuch.h   | 17 +++++++++++
 mime-node.c     |  7 ++++-
 util/crypto.c   | 92 ++++++++++++++++++++++++++++-----------------------------
 util/crypto.h   |  7 +++--
 5 files changed, 79 insertions(+), 50 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index 35c66939..02444e09 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -413,6 +413,12 @@ notmuch_status_to_string (notmuch_status_t status)
 	return "Operation requires a database upgrade";
     case NOTMUCH_STATUS_PATH_ERROR:
 	return "Path supplied is illegal for this function";
+    case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL:
+	return "Crypto protocol missing, malformed, or unintelligible";
+    case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION:
+	return "Crypto engine initialization failure";
+    case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL:
+	return "Unknown crypto protocol";
     default:
     case NOTMUCH_STATUS_LAST_STATUS:
 	return "Unknown error status value";
diff --git a/lib/notmuch.h b/lib/notmuch.h
index e8e0cc12..669e01b1 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -191,6 +191,23 @@ typedef enum _notmuch_status {
      * function, in a way not covered by a more specific argument.
      */
     NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+    /**
+     * A MIME object claimed to have cryptographic protection which
+     * notmuch tried to handle, but the protocol was not specified in
+     * an intelligible way.
+     */
+    NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL,
+    /**
+     * Notmuch attempted to do crypto processing, but could not
+     * initialize the engine needed to do so.
+     */
+    NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION,
+    /**
+     * A MIME object claimed to have cryptographic protection, and
+     * notmuch attempted to process it, but the specific protocol was
+     * something that notmuch doesn't know how to handle.
+     */
+    NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL,
     /**
      * Not an actual status value. Just a way to find out how many
      * valid status values there are.
diff --git a/mime-node.c b/mime-node.c
index d48be4c4..c3d5cb9b 100644
--- a/mime-node.c
+++ b/mime-node.c
@@ -269,7 +269,12 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part)
 	|| (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) {
 	GMimeContentType *content_type = g_mime_object_get_content_type (part);
 	const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol");
-	cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol);
+	notmuch_status_t status;
+	status = _notmuch_crypto_get_gmime_ctx_for_protocol (node->ctx->crypto,
+							     protocol, &cryptoctx);
+	if (status) /* this is a warning, not an error */
+	    fprintf (stderr, "Warning: %s (%s).\n", notmuch_status_to_string (status),
+		     protocol ? protocol : "NULL");
 	if (!cryptoctx)
 	    return NULL;
     }
diff --git a/util/crypto.c b/util/crypto.c
index c51d67ed..5c84282e 100644
--- a/util/crypto.c
+++ b/util/crypto.c
@@ -25,85 +25,86 @@
 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
 
 #if (GMIME_MAJOR_VERSION < 3)
-/* Create a GPG context (GMime 2.6) */
-static GMimeCryptoContext *
-create_gpg_context (_notmuch_crypto_t *crypto)
+/* Create or pass on a GPG context (GMime 2.6) */
+static notmuch_status_t
+get_gpg_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx)
 {
-    GMimeCryptoContext *gpgctx;
+    if (ctx == NULL || crypto == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
 
-    if (crypto->gpgctx)
-	return crypto->gpgctx;
+    if (crypto->gpgctx) {
+	*ctx = crypto->gpgctx;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
 
     /* TODO: GMimePasswordRequestFunc */
-    gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg");
-    if (! gpgctx) {
-	fprintf (stderr, "Failed to construct gpg context.\n");
-	return NULL;
+    crypto->gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg");
+    if (! crypto->gpgctx) {
+	return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION;
     }
-    crypto->gpgctx = gpgctx;
 
-    g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, true);
-    g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, false);
+    g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) crypto->gpgctx, true);
+    g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) crypto->gpgctx, false);
 
-    return gpgctx;
+    *ctx = crypto->gpgctx;
+    return NOTMUCH_STATUS_SUCCESS;
 }
 
-/* Create a PKCS7 context (GMime 2.6) */
-static GMimeCryptoContext *
-create_pkcs7_context (_notmuch_crypto_t *crypto)
+/* Create or pass on a PKCS7 context (GMime 2.6) */
+static notmuch_status_t
+get_pkcs7_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx)
 {
-    GMimeCryptoContext *pkcs7ctx;
+    if (ctx == NULL || crypto == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
 
-    if (crypto->pkcs7ctx)
-	return crypto->pkcs7ctx;
+    if (crypto->pkcs7ctx) {
+	*ctx = crypto->pkcs7ctx;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
 
     /* TODO: GMimePasswordRequestFunc */
-    pkcs7ctx = g_mime_pkcs7_context_new (NULL);
-    if (! pkcs7ctx) {
-	fprintf (stderr, "Failed to construct pkcs7 context.\n");
-	return NULL;
+    crypto->pkcs7ctx = g_mime_pkcs7_context_new (NULL);
+    if (! crypto->pkcs7ctx) {
+	return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION;
     }
-    crypto->pkcs7ctx = pkcs7ctx;
 
-    g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx,
+    g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) crypto->pkcs7ctx,
 					   false);
 
-    return pkcs7ctx;
+    *ctx = crypto->pkcs7ctx;
+    return NOTMUCH_STATUS_SUCCESS;
 }
 static const struct {
     const char *protocol;
-    GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto);
+    notmuch_status_t (*get_context) (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx);
 } protocols[] = {
     {
 	.protocol = "application/pgp-signature",
-	.get_context = create_gpg_context,
+	.get_context = get_gpg_context,
     },
     {
 	.protocol = "application/pgp-encrypted",
-	.get_context = create_gpg_context,
+	.get_context = get_gpg_context,
     },
     {
 	.protocol = "application/pkcs7-signature",
-	.get_context = create_pkcs7_context,
+	.get_context = get_pkcs7_context,
     },
     {
 	.protocol = "application/x-pkcs7-signature",
-	.get_context = create_pkcs7_context,
+	.get_context = get_pkcs7_context,
     },
 };
 
 /* for the specified protocol return the context pointer (initializing
  * if needed) */
-GMimeCryptoContext *
-_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol)
+notmuch_status_t
+_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto,
+					    const char *protocol,
+					    GMimeCryptoContext **ctx)
 {
-    GMimeCryptoContext *cryptoctx = NULL;
-    size_t i;
-
-    if (! protocol) {
-	fprintf (stderr, "Cryptographic protocol is empty.\n");
-	return cryptoctx;
-    }
+    if (! protocol)
+	return NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL;
 
     /* As per RFC 1847 section 2.1: "the [protocol] value token is
      * comprised of the type and sub-type tokens of the Content-Type".
@@ -111,15 +112,12 @@ _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protoc
      * parameter names as defined in this document are
      * case-insensitive."  Thus, we use strcasecmp for the protocol.
      */
-    for (i = 0; i < ARRAY_SIZE (protocols); i++) {
+    for (size_t i = 0; i < ARRAY_SIZE (protocols); i++) {
 	if (strcasecmp (protocol, protocols[i].protocol) == 0)
-	    return protocols[i].get_context (crypto);
+	    return protocols[i].get_context (crypto, ctx);
     }
 
-    fprintf (stderr, "Unknown or unsupported cryptographic protocol %s.\n",
-	     protocol);
-
-    return NULL;
+    return NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL;
 }
 
 void
diff --git a/util/crypto.h b/util/crypto.h
index 80628dc5..1ff0297d 100644
--- a/util/crypto.h
+++ b/util/crypto.h
@@ -4,6 +4,7 @@
 #include <stdbool.h>
 #if (GMIME_MAJOR_VERSION < 3)
 #include "gmime-extra.h"
+#include "notmuch.h"
 #endif
 
 typedef struct _notmuch_crypto {
@@ -18,8 +19,10 @@ typedef struct _notmuch_crypto {
 
 
 #if (GMIME_MAJOR_VERSION < 3)
-GMimeCryptoContext *
-_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol);
+notmuch_status_t
+_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto,
+					    const char *protocol,
+					    GMimeCryptoContext **ctx);
 #endif
 
 void
-- 
2.14.2

[PATCH v5 03/12] index: implement notmuch_indexopts_t with try_decrypt

[Daniel Kahn Gillmor]

This is currently mostly a wrapper around _notmuch_crypto_t that keeps
its internals private and doesn't expose any of the GMime API.
However, non-crypto indexing options might also be added later
(e.g. filters or other transformations).
---
 lib/add-message.cc    | 11 ++++++++++-
 lib/indexopts.c       | 22 ++++++++++++++++++++--
 lib/notmuch-private.h |  7 +++++++
 lib/notmuch.h         | 20 ++++++++++++++++++++
 4 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index bce10a0f..34099ed5 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch,
 notmuch_status_t
 notmuch_database_index_file (notmuch_database_t *notmuch,
 			     const char *filename,
-			     notmuch_indexopts_t unused (*indexopts),
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_t **message_ret)
 {
     notmuch_message_file_t *message_file;
@@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
     notmuch_private_status_t private_status;
     bool is_ghost = false, is_new = false;
+    notmuch_indexopts_t *def_indexopts = NULL;
 
     const char *date;
     const char *from, *to, *subject;
@@ -540,6 +541,11 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
 	if (is_new || is_ghost)
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
+	if (!indexopts) {
+	    def_indexopts = notmuch_database_get_default_indexopts (notmuch);
+	    indexopts = def_indexopts;
+	}
+
 	ret = _notmuch_message_index_file (message, message_file);
 	if (ret)
 	    goto DONE;
@@ -557,6 +563,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
     }
 
   DONE:
+    if (def_indexopts)
+	notmuch_indexopts_destroy (def_indexopts);
+
     if (message) {
 	if ((ret == NOTMUCH_STATUS_SUCCESS ||
 	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
diff --git a/lib/indexopts.c b/lib/indexopts.c
index 2f9b841b..cc1d6422 100644
--- a/lib/indexopts.c
+++ b/lib/indexopts.c
@@ -21,9 +21,27 @@
 #include "notmuch-private.h"
 
 notmuch_indexopts_t *
-notmuch_database_get_default_indexopts (notmuch_database_t unused (*db))
+notmuch_database_get_default_indexopts (notmuch_database_t *db)
 {
-    return NULL;
+    return talloc_zero (db, notmuch_indexopts_t);
+}
+
+notmuch_status_t
+notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts,
+				   bool try_decrypt)
+{
+    if (!indexopts)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    indexopts->crypto.decrypt = try_decrypt;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+bool
+notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts)
+{
+    if (!indexopts)
+	return false;
+    return indexopts->crypto.decrypt;
 }
 
 void
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index e86f4582..4c408396 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -52,6 +52,7 @@ NOTMUCH_BEGIN_DECLS
 #include "xutil.h"
 #include "error_util.h"
 #include "string-util.h"
+#include "crypto.h"
 
 #ifdef DEBUG
 # define DEBUG_DATABASE_SANITY 1
@@ -633,6 +634,12 @@ _notmuch_thread_create (void *ctx,
 			notmuch_exclude_t omit_exclude,
 			notmuch_sort_t sort);
 
+/* indexopts.c */
+
+struct _notmuch_indexopts {
+    _notmuch_crypto_t crypto;
+};
+
 NOTMUCH_END_DECLS
 
 #ifdef __cplusplus
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 669e01b1..0b2e8305 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -42,6 +42,7 @@
 NOTMUCH_BEGIN_DECLS
 
 #include <time.h>
+#include <stdbool.h>
 
 #pragma GCC visibility push(default)
 
@@ -2214,6 +2215,25 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list);
 notmuch_indexopts_t *
 notmuch_database_get_default_indexopts (notmuch_database_t *db);
 
+/**
+ * Specify whether to decrypt encrypted parts while indexing.
+ *
+ * Be aware that the index is likely sufficient to reconstruct the
+ * cleartext of the message itself, so please ensure that the notmuch
+ * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE
+ * without considering the security of your index.
+ */
+notmuch_status_t
+notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts,
+				   bool try_decrypt);
+
+/**
+ * Return whether to decrypt encrypted parts while indexing.
+ * see notmuch_indexopts_set_try_decrypt.
+ */
+bool
+notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts);
+
 /**
  * Destroy a notmuch_indexopts_t object.
  *
-- 
2.14.2

[PATCH v5 04/12] properties: add notmuch_message_remove_all_properties_with_prefix()

[Daniel Kahn Gillmor]

Subsequent patches will introduce a convention that properties whose
name starts with "index." will be stripped (and possibly re-added)
during re-indexing.  This patch lays the groundwork for doing that.
---
 lib/message-property.cc | 18 ++++++++++++++++--
 lib/notmuch.h           | 16 ++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/lib/message-property.cc b/lib/message-property.cc
index d72c74c3..35eaf3c6 100644
--- a/lib/message-property.cc
+++ b/lib/message-property.cc
@@ -85,8 +85,9 @@ notmuch_message_remove_property (notmuch_message_t *message, const char *key, co
     return _notmuch_message_modify_property (message, key, value, true);
 }
 
+static
 notmuch_status_t
-notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+_notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key, bool prefix)
 {
     notmuch_status_t status;
     const char * term_prefix;
@@ -97,7 +98,8 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
 
     _notmuch_message_invalidate_metadata (message, "property");
     if (key)
-	term_prefix = talloc_asprintf (message, "%s%s=", _find_prefix ("property"), key);
+	term_prefix = talloc_asprintf (message, "%s%s%s", _find_prefix ("property"), key,
+				       prefix ? "" : "=");
     else
 	term_prefix = _find_prefix ("property");
 
@@ -107,6 +109,18 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
     return NOTMUCH_STATUS_SUCCESS;
 }
 
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+    return _notmuch_message_remove_all_properties (message, key, false);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix)
+{
+    return _notmuch_message_remove_all_properties (message, prefix, true);
+}
+
 notmuch_message_properties_t *
 notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
 {
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 0b2e8305..817f357f 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1824,6 +1824,22 @@ notmuch_message_remove_property (notmuch_message_t *message, const char *key, co
 notmuch_status_t
 notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
 
+/**
+ * Remove all (prefix*,value) pairs from the given message
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     prefix   delete properties with keys that start with prefix.
+ *			   If NULL, delete all properties
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix);
+
 /**
  * Opaque message property iterator
  */
-- 
2.14.2

[PATCH v5 05/12] reindex: drop all properties named with prefix "index."

[Daniel Kahn Gillmor]

This allows us to create new properties that will be automatically set
during indexing, and cleared during re-indexing, just by choice of
property name.
---
 lib/message.cc | 6 ++++++
 lib/notmuch.h  | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/lib/message.cc b/lib/message.cc
index 4ab0ed26..77bb6c76 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -1999,6 +1999,12 @@ notmuch_message_reindex (notmuch_message_t *message,
 	goto DONE;
     }
 
+    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
+    if (ret) {
+	INTERNAL_ERROR ("failed to remove index.* properties");
+	goto DONE;
+    }
+
     /* re-add the filenames with the associated indexopts */
     for (; notmuch_filenames_valid (orig_filenames);
 	 notmuch_filenames_move_to_next (orig_filenames)) {
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 817f357f..fd6ead81 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1765,6 +1765,12 @@ notmuch_message_destroy (notmuch_message_t *message);
  * add or delete values for, as other subsystems or extensions may
  * depend on these properties.
  *
+ * Notmuch has some conventions about how certain properties are
+ * treated.  Those conventions include:
+ *
+ * - preperties whose name begins with "index." are cleared (and
+ *   possibly re-set) upon reindexing.
+ *
  */
 /**@{*/
 /**
-- 
2.14.2

[PATCH v5 06/12] crypto: index encrypted parts when indexopts try_decrypt is set.

[Daniel Kahn Gillmor]

If we see index options that ask us to decrypt when indexing a
message, and we encounter an encrypted part, we'll try to descend into
it.

If we can decrypt, we add the property index.decryption=success.

If we can't decrypt (or recognize the encrypted type of mail), we add
the property index.decryption=failure.

Note that a single message may have both values of the
"index-decryption" property: "success" and "failure".  For example,
consider a message that includes multiple layers of encryption.  If we
manage to decrypt the outer layer ("index.decryption=success"), but
fail on the inner layer ("index.decryption=failure").

Because of the property name, this will be automatically cleared (and
possibly re-set) during re-indexing.  This means it will subsequently
correspond to the actual semantics of the stored index.
---
 lib/add-message.cc    |  2 +-
 lib/index.cc          | 91 ++++++++++++++++++++++++++++++++++++++++++++++-----
 lib/message.cc        |  4 +--
 lib/notmuch-private.h |  1 +
 4 files changed, 86 insertions(+), 12 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 34099ed5..f5fac8be 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -546,7 +546,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
 	    indexopts = def_indexopts;
 	}
 
-	ret = _notmuch_message_index_file (message, message_file);
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
 	if (ret)
 	    goto DONE;
 
diff --git a/lib/index.cc b/lib/index.cc
index e5ae2ba7..6e684f5f 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -364,9 +364,15 @@ _index_content_type (notmuch_message_t *message, GMimeObject *part)
     }
 }
 
+static void
+_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts,
+			    GMimeContentType *content_type,
+			    GMimeMultipartEncrypted *part);
+
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
+		  notmuch_indexopts_t *indexopts,
 		  GMimeObject *part)
 {
     GMimeStream *stream, *filter;
@@ -385,6 +391,7 @@ _index_mime_part (notmuch_message_t *message,
     }
 
     _index_content_type (message, part);
+    content_type = g_mime_object_get_content_type (part);
 
     if (GMIME_IS_MULTIPART (part)) {
 	GMimeMultipart *multipart = GMIME_MULTIPART (part);
@@ -409,17 +416,21 @@ _index_mime_part (notmuch_message_t *message,
 		}
 	    }
 	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
-		/* Don't index encrypted parts, but index their content type. */
 		_index_content_type (message,
 				     g_mime_multipart_get_part (multipart, i));
-		if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) &&
-		    (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) {
-		    _notmuch_database_log (_notmuch_message_database (message),
-					   "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) {
+		    _index_encrypted_mime_part(message, indexopts,
+					       content_type,
+					       GMIME_MULTIPART_ENCRYPTED (part));
+		} else {
+		    if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) {
+			_notmuch_database_log (_notmuch_message_database (message),
+					       "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		    }
 		}
 		continue;
 	    }
-	    _index_mime_part (message,
+	    _index_mime_part (message, indexopts,
 			      g_mime_multipart_get_part (multipart, i));
 	}
 	return;
@@ -430,7 +441,7 @@ _index_mime_part (notmuch_message_t *message,
 
 	mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part));
 
-	_index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+	_index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
 	return;
     }
@@ -464,7 +475,6 @@ _index_mime_part (notmuch_message_t *message,
 
     filter = g_mime_stream_filter_new (stream);
 
-    content_type = g_mime_object_get_content_type (part);
     discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type);
 
     g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
@@ -502,8 +512,71 @@ _index_mime_part (notmuch_message_t *message,
     }
 }
 
+/* descend (if desired) into the cleartext part of an encrypted MIME
+ * part while indexing. */
+static void
+_index_encrypted_mime_part (notmuch_message_t *message,
+			    notmuch_indexopts_t *indexopts,
+			    g_mime_3_unused(GMimeContentType *content_type),
+			    GMimeMultipartEncrypted *encrypted_data)
+{
+    notmuch_status_t status;
+    GError *err = NULL;
+    notmuch_database_t * notmuch = NULL;
+    GMimeObject *clear = NULL;
+
+    if (!indexopts || !notmuch_indexopts_get_try_decrypt (indexopts))
+	return;
+
+    notmuch = _notmuch_message_database (message);
+
+#if (GMIME_MAJOR_VERSION < 3)
+    {
+	GMimeCryptoContext* crypto_ctx = NULL;
+	const char *protocol = NULL;
+	protocol = g_mime_content_type_get_parameter (content_type, "protocol");
+	status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto),
+							 protocol, &crypto_ctx);
+	if (status) {
+	    _notmuch_database_log (notmuch, "Warning: setup failed for decrypting "
+				   "during indexing. (%d)\n", status);
+	    status = notmuch_message_add_property (message, "index.decryption", "failure");
+	    if (status)
+		_notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					      "property (%d)\n", status);
+	    return;
+	}
+	clear = g_mime_multipart_encrypted_decrypt(encrypted_data, crypto_ctx,
+						   NULL, &err);
+    }
+#else
+    clear = g_mime_multipart_encrypted_decrypt(encrypted_data, GMIME_DECRYPT_NONE, NULL,
+					       NULL, &err);
+#endif
+    if (err) {
+	_notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n",
+			       err->domain, err->code, err->message);
+	g_error_free(err);
+	/* Indicate that we failed to decrypt during indexing */
+	status = notmuch_message_add_property (message, "index.decryption", "failure");
+	if (status)
+	    _notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					  "property (%d)\n", status);
+	return;
+    }
+    _index_mime_part (message, indexopts, clear);
+    g_object_unref (clear);
+
+    status = notmuch_message_add_property (message, "index.decryption", "success");
+    if (status)
+	_notmuch_database_log (notmuch, "failed to add index.decryption "
+			       "property (%d)\n", status);
+
+}
+
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file)
 {
     GMimeMessage *mime_message;
@@ -531,7 +604,7 @@ _notmuch_message_index_file (notmuch_message_t *message,
     subject = g_mime_message_get_subject (mime_message);
     _notmuch_message_gen_terms (message, "subject", subject);
 
-    _index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+    _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
     return NOTMUCH_STATUS_SUCCESS;
 }
diff --git a/lib/message.cc b/lib/message.cc
index 77bb6c76..a47590c6 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -1961,7 +1961,7 @@ _notmuch_message_frozen (notmuch_message_t *message)
 
 notmuch_status_t
 notmuch_message_reindex (notmuch_message_t *message,
-			 notmuch_indexopts_t unused (*indexopts))
+			 notmuch_indexopts_t *indexopts)
 {
     notmuch_database_t *notmuch = NULL;
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
@@ -2044,7 +2044,7 @@ notmuch_message_reindex (notmuch_message_t *message,
 	if (found == 0)
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
-	ret = _notmuch_message_index_file (message, message_file);
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
 
 	if (ret == NOTMUCH_STATUS_FILE_ERROR)
 	    continue;
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 4c408396..1093429c 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -448,6 +448,7 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
 
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file);
 
 /* messages.c */
-- 
2.14.2

[PATCH v5 07/12] config: indexing defaults will be stored in the database.

[Daniel Kahn Gillmor]

At indexing time, the database needs to know its internal defaults.
It shouldn't be contingent on an external config file (since that
can't be retrieved from the database object itself).

This behaves the same as the query.* configurations, which are also
stored in the database itself, so we're not introducing any new
dependencies.
---
 notmuch-config.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/notmuch-config.c b/notmuch-config.c
index 8fb59f96..e82f7dd7 100644
--- a/notmuch-config.c
+++ b/notmuch-config.c
@@ -809,6 +809,7 @@ _item_split (char *item, char **group, char **key)
 
 #define BUILT_WITH_PREFIX "built_with."
 #define QUERY_PREFIX "query."
+#define INDEX_PREFIX "index."
 
 static int
 _print_db_config(notmuch_config_t *config, const char *name)
@@ -859,6 +860,8 @@ notmuch_config_command_get (notmuch_config_t *config, char *item)
 		notmuch_built_with (item + strlen (BUILT_WITH_PREFIX)) ? "true" : "false");
     } else if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) {
 	return _print_db_config (config, item);
+    } else if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) {
+	return _print_db_config (config, item);
     } else {
 	char **value;
 	size_t i, length;
@@ -931,6 +934,9 @@ notmuch_config_command_set (notmuch_config_t *config, char *item, int argc, char
     if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) {
 	return _set_db_config (config, item, argc, argv);
     }
+    if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) {
+	return _set_db_config (config, item, argc, argv);
+    }
 
     if (_item_split (item, &group, &key))
 	return 1;
-- 
2.14.2

[PATCH v5 08/12] config: define new option index.try_decrypt

[Daniel Kahn Gillmor]

By default, notmuch won't try to decrypt on indexing.  With this
patch, we make it possible to indicate a per-database preference using
the config variable "index.try_decrypt", which by default will be
false.
---
 doc/man1/notmuch-config.rst | 12 ++++++++++++
 lib/indexopts.c             | 18 +++++++++++++++++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 6a51e64f..6f35d127 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -134,6 +134,18 @@ The available configuration items are described below.
 
         Default: ``gpg``.
 
+    **index.try_decrypt**
+
+        When indexing an encrypted e-mail message, if this variable is
+        set to true, notmuch will try to decrypt the message and index
+        the cleartext.  Be aware that the index is likely sufficient
+        to reconstruct the cleartext of the message itself, so please
+        ensure that the notmuch message index is adequately protected.
+        DO NOT USE ``index.try_decrypt=true`` without considering the
+        security of your index.
+
+        Default: ``false``.
+
     **built_with.<name>**
 
         Compile time feature <name>. Current possibilities include
diff --git a/lib/indexopts.c b/lib/indexopts.c
index cc1d6422..987d8952 100644
--- a/lib/indexopts.c
+++ b/lib/indexopts.c
@@ -23,7 +23,23 @@
 notmuch_indexopts_t *
 notmuch_database_get_default_indexopts (notmuch_database_t *db)
 {
-    return talloc_zero (db, notmuch_indexopts_t);
+    notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t);
+    if (!ret)
+	return ret;
+
+    char * try_decrypt;
+    notmuch_status_t err = notmuch_database_get_config (db, "index.try_decrypt", &try_decrypt);
+    if (err)
+	return ret;
+
+    if (try_decrypt &&
+	((!(strcasecmp(try_decrypt, "true"))) ||
+	 (!(strcasecmp(try_decrypt, "yes"))) ||
+	 (!(strcasecmp(try_decrypt, "1")))))
+	notmuch_indexopts_set_try_decrypt (ret, true);
+
+    free (try_decrypt);
+    return ret;
 }
 
 notmuch_status_t
-- 
2.14.2

[PATCH v5 09/12] cli: set up shared command-line arguments for indexing

[Daniel Kahn Gillmor]

We have an indexopts structure for manipulating indexing in different
ways, but we also have three command-line invocations that can trigger
indexing: new, insert, and reindex.

This changeset prepares a common parser that these subcommands can
share.

At the moment, it's just --try-decrypt, but others will likely follow.
---
 notmuch-client.h | 12 ++++++++++++
 notmuch.c        | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/notmuch-client.h b/notmuch-client.h
index d17cdf01..27852360 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -495,4 +495,16 @@ void notmuch_exit_if_unmatched_db_uuid (notmuch_database_t *notmuch);
 void notmuch_process_shared_options (const char* subcommand_name);
 int notmuch_minimal_options (const char* subcommand_name,
 			     int argc, char **argv);
+
+
+struct _notmuch_client_index_options {
+    bool try_decrypt;
+    bool try_decrypt_set;
+    notmuch_indexopts_t * opts;
+};
+extern struct _notmuch_client_index_options index_options;
+extern const notmuch_opt_desc_t  notmuch_index_options [];
+notmuch_status_t
+notmuch_process_index_options (notmuch_database_t *notmuch, notmuch_config_t *config);
+
 #endif
diff --git a/notmuch.c b/notmuch.c
index 02148f2e..f83b5a6f 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -96,6 +96,45 @@ int notmuch_minimal_options (const char *subcommand_name,
     return opt_index;
 }
 
+
+struct _notmuch_client_index_options index_options = { };
+const notmuch_opt_desc_t  notmuch_index_options [] = {
+    { .opt_bool = &index_options.try_decrypt,
+      .present = &index_options.try_decrypt_set,
+      .name = "try-decrypt" },
+    { }
+};
+
+
+notmuch_status_t
+notmuch_process_index_options (notmuch_database_t *notmuch, g_mime_3_unused(notmuch_config_t *config))
+{
+    if (index_options.opts == NULL)
+	index_options.opts = notmuch_database_get_default_indexopts (notmuch);
+    if (index_options.try_decrypt_set) {
+	notmuch_status_t status;
+	if (index_options.opts == NULL)
+	    return NOTMUCH_STATUS_OUT_OF_MEMORY;
+	status = notmuch_indexopts_set_try_decrypt (index_options.opts, index_options.try_decrypt);
+	if (status != NOTMUCH_STATUS_SUCCESS) {
+	    fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n",
+		     index_options.try_decrypt ? "True" : "False", notmuch_status_to_string (status));
+	    notmuch_indexopts_destroy (index_options.opts);
+	    return status;
+	}
+    }
+#if (GMIME_MAJOR_VERSION < 3)
+    if (index_options.opts && notmuch_indexopts_get_try_decrypt (index_options.opts)) {
+	const char* gpg_path = notmuch_config_get_crypto_gpg_path (config);
+	if (gpg_path && strcmp(gpg_path, "gpg"))
+	    fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n"
+		     "\tbut ignoring (use $PATH instead)\n", gpg_path);
+    }
+#endif
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+
 static command_t commands[] = {
     { NULL, notmuch_command, NOTMUCH_CONFIG_OPEN | NOTMUCH_CONFIG_CREATE,
       "Notmuch main command." },
-- 
2.14.2

[PATCH v5 10/12] cli/new: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Try to decrypt any encrypted parts of newly-discovered messages while
indexing them.  The cleartext of any successfully-decrypted messages
will be indexed, with tags applied in the same form as from notmuch
insert --try-decrypt=true.

Note: if the deprecated crypto.gpg_path configuration option is set to
anything other than "gpg", we ignore it (and print a warning on
stderr, if built against gmime < 3.0).

We also add a new test making use of this functionality.  This
requires a bit of reorganization, because we need to allow passing
--long-arguments to "notmuch new" via emacs_fcc_message
---
 completion/notmuch-completion.bash | 13 ++++++++--
 doc/man1/notmuch-new.rst           | 12 +++++++++
 notmuch-new.c                      | 10 +++++++-
 test/T357-index-decryption.sh      | 51 ++++++++++++++++++++++++++++++++++++++
 test/test-lib.sh                   | 11 +++++++-
 5 files changed, 93 insertions(+), 4 deletions(-)
 create mode 100755 test/T357-index-decryption.sh

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 5201be63..17be6b8f 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -311,11 +311,20 @@ _notmuch_insert()
 _notmuch_new()
 {
     local cur prev words cword split
-    _init_completion || return
+    _init_completion -s || return
+
+    $split &&
+    case "${prev}" in
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
+    esac
 
+    ! $split &&
     case "${cur}" in
 	-*)
-	    local options="--no-hooks --quiet ${_notmuch_shared_options}"
+	    local options="--no-hooks --try-decrypt= --quiet ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "${options}" -- ${cur}) )
 	    ;;
diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst
index 6acfa112..c255f980 100644
--- a/doc/man1/notmuch-new.rst
+++ b/doc/man1/notmuch-new.rst
@@ -43,6 +43,18 @@ Supported options for **new** include
     ``--quiet``
         Do not print progress or results.
 
+    ``--try-decrypt=(true|false)``
+
+        If true, when encountering an encrypted message, try to
+        decrypt it while indexing.  If decryption is successful, index
+        the cleartext itself.  Be aware that the index is likely
+        sufficient to reconstruct the cleartext of the message itself,
+        so please ensure that the notmuch message index is adequately
+        protected.  DO NOT USE ``--try-decrypt=true`` without
+        considering the security of your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 EXIT STATUS
 ===========
 
diff --git a/notmuch-new.c b/notmuch-new.c
index 0f50457e..9b14baf5 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -267,7 +267,7 @@ add_file (notmuch_database_t *notmuch, const char *filename,
     if (status)
 	goto DONE;
 
-    status = notmuch_database_index_file (notmuch, filename, NULL, &message);
+    status = notmuch_database_index_file (notmuch, filename, index_options.opts, &message);
     switch (status) {
     /* Success. */
     case NOTMUCH_STATUS_SUCCESS:
@@ -963,6 +963,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
 	{ .opt_bool = &verbose, .name = "verbose" },
 	{ .opt_bool = &add_files_state.debug, .name = "debug" },
 	{ .opt_bool = &no_hooks, .name = "no-hooks" },
+	{ .opt_inherit = notmuch_index_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -1080,6 +1081,13 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     if (notmuch == NULL)
 	return EXIT_FAILURE;
 
+    status = notmuch_process_index_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
+
     /* Set up our handler for SIGINT. We do this after having
      * potentially done a database upgrade we this interrupt handler
      * won't support. */
diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh
new file mode 100755
index 00000000..d7180797
--- /dev/null
+++ b/test/T357-index-decryption.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# TODO: test index.decryption=failed
+
+test_description='indexing decrypted mail'
+. ./test-lib.sh || exit 1
+
+##################################################
+
+add_gnupg_home
+# get key fingerprint
+FINGERPRINT=$(gpg --no-tty --list-secret-keys --with-colons --fingerprint | grep '^fpr:' | cut -d: -f10)
+
+# create a test encrypted message
+test_begin_subtest 'emacs delivery of encrypted message'
+test_expect_success \
+'emacs_fcc_message \
+    "test encrypted message for cleartext index 001" \
+    "This is a test encrypted message with a wumpus.\n" \
+    "(mml-secure-message-encrypt)"'
+
+test_begin_subtest "search for unindexed cleartext"
+output=$(notmuch search wumpus)
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# create a test encrypted message that is indexed in the clear
+test_begin_subtest 'emacs delivery of encrypted message'
+test_expect_success \
+'emacs_fcc_message --try-decrypt=true \
+    "test encrypted message for cleartext index 002" \
+    "This is a test encrypted message with a wumpus.\n" \
+    "(mml-secure-message-encrypt)"'
+
+test_begin_subtest "emacs delivery of encrypted message, indexed cleartext"
+output=$(notmuch search wumpus)
+expected='thread:0000000000000002   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property for one message"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+test_done
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 84051bc9..9c336662 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -340,8 +340,17 @@ emacs_deliver_message ()
 # Accepts arbitrary extra emacs/elisp functions to modify the message
 # before sending, which is useful to doing things like attaching files
 # to the message and encrypting/signing.
+#
+# If any GNU-style long-arguments (like --quiet or --try-decrypt=true) are
+# at the head of the argument list, they are sent directly to "notmuch
+# new" after message delivery
 emacs_fcc_message ()
 {
+    local nmn_args=''
+    while [[ "$1" =~ ^-- ]]; do
+        nmn_args="$nmn_args $1"
+        shift
+    done
     local subject="$1"
     local body="$2"
     shift 2
@@ -360,7 +369,7 @@ emacs_fcc_message ()
 	   (insert \"${body}\")
 	   $@
 	   (notmuch-mua-send-and-exit))" || return 1
-    notmuch new >/dev/null
+    notmuch new $nmn_args >/dev/null
 }
 
 # Add an existing, fixed corpus of email to the database.
-- 
2.14.2

[PATCH v5 11/12] cli/insert: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Allow an incoming message to be delivered while indexing the
cleartext, on a per-message basis.

This requires the secret keys for the message to be available.  For
the moment, the most functional approach is to ensure that gpg-agent
is running and knows about any secret keys that might be useful to
decrypt incoming mail.

Any additional recommendations for how to phrase the caveat for this
option are welcome.

Note: if the deprecated crypto.gpg_path is set to anything other than
"gpg", we ignore it (and print a warning on stderr, if built against
gmime < 3.0).
---
 completion/notmuch-completion.bash |  6 +++++-
 doc/man1/notmuch-insert.rst        | 14 ++++++++++++++
 notmuch-insert.c                   | 14 +++++++++++---
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 17be6b8f..72a75a94 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -287,12 +287,16 @@ _notmuch_insert()
 		sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) )
 	    return
 	    ;;
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
     esac
 
     ! $split &&
     case "${cur}" in
 	--*)
-	    local options="--create-folder --folder= --keep --no-hooks ${_notmuch_shared_options}"
+	    local options="--create-folder --folder= --keep --no-hooks --try-decrypt= ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
 	    return
diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst
index f79600d6..647dac06 100644
--- a/doc/man1/notmuch-insert.rst
+++ b/doc/man1/notmuch-insert.rst
@@ -50,6 +50,20 @@ Supported options for **insert** include
     ``--no-hooks``
         Prevent hooks from being run.
 
+    ``--try-decrypt=(true|false)``
+
+        If true and the message is encrypted, try to decrypt the
+        message while indexing.  If decryption is successful, index
+        the cleartext itself.  Either way, the message is always
+        stored to disk in its original form (ciphertext).  Be aware
+        that the index is likely sufficient to reconstruct the
+        cleartext of the message itself, so please ensure that the
+        notmuch message index is adequately protected. DO NOT USE
+        ``--try-decrypt=true`` without considering the security of
+        your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 EXIT STATUS
 ===========
 
diff --git a/notmuch-insert.c b/notmuch-insert.c
index 32be7419..e64655cc 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -379,12 +379,13 @@ FAIL:
  */
 static notmuch_status_t
 add_file (notmuch_database_t *notmuch, const char *path, tag_op_list_t *tag_ops,
-	  bool synchronize_flags, bool keep)
+	  bool synchronize_flags, bool keep,
+	  notmuch_indexopts_t *indexopts)
 {
     notmuch_message_t *message;
     notmuch_status_t status;
 
-    status = notmuch_database_index_file (notmuch, path, NULL, &message);
+    status = notmuch_database_index_file (notmuch, path, indexopts, &message);
     if (status == NOTMUCH_STATUS_SUCCESS) {
 	status = tag_op_list_apply (message, tag_ops, 0);
 	if (status) {
@@ -467,6 +468,7 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
 	{ .opt_bool = &create_folder, .name = "create-folder" },
 	{ .opt_bool = &keep, .name = "keep" },
 	{ .opt_bool =  &no_hooks, .name = "no-hooks" },
+	{ .opt_inherit = notmuch_index_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -545,9 +547,15 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
 
     notmuch_exit_if_unmatched_db_uuid (notmuch);
 
+    status = notmuch_process_index_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
 
     /* Index the message. */
-    status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep);
+    status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, index_options.opts);
 
     /* Commit changes. */
     close_status = notmuch_database_destroy (notmuch);
-- 
2.14.2

[PATCH v5 12/12] cli/reindex: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Try to decrypt any encrypted parts of newly-discovered messages while
re-indexing them.  The cleartext of any successfully-decrypted
messages will be indexed, with tags applied in the same form as from
notmuch insert --try-decrypt=true.

Note: if the deprecated crypto.gpg_path configuration option is set to
anything other than "gpg", we ignore it (and print a warning on
stderr, if built against gmime < 3.0).
---
 completion/notmuch-completion.bash | 10 +++++-
 doc/man1/notmuch-reindex.rst       | 14 ++++++++
 notmuch-reindex.c                  | 12 +++++--
 test/T357-index-decryption.sh      | 65 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 98 insertions(+), 3 deletions(-)

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 72a75a94..7aae4297 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -435,10 +435,18 @@ _notmuch_reindex()
     local cur prev words cword split
     _init_completion -s || return
 
+    $split &&
+    case "${prev}" in
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
+    esac
+
     ! $split &&
     case "${cur}" in
 	-*)
-	    local options="${_notmuch_shared_options}"
+	    local options="--try-decrypt= ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
 	    ;;
diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst
index e39cc4ee..60a060a7 100644
--- a/doc/man1/notmuch-reindex.rst
+++ b/doc/man1/notmuch-reindex.rst
@@ -19,6 +19,20 @@ The **reindex** command searches for all messages matching the
 supplied search terms, and re-creates the full-text index on these
 messages using the supplied options.
 
+Supported options for **reindex** include
+
+    ``--try-decrypt=(true|false)``
+
+        If true, when encountering an encrypted message, try to
+        decrypt it while reindexing.  If decryption is successful,
+        index the cleartext itself.  Be aware that the index is likely
+        sufficient to reconstruct the cleartext of the message itself,
+        so please ensure that the notmuch message index is adequately
+        protected. DO NOT USE ``--try-decrypt=true`` without
+        considering the security of your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 SEE ALSO
 ========
 
diff --git a/notmuch-reindex.c b/notmuch-reindex.c
index 57ff5904..2b06ec6b 100644
--- a/notmuch-reindex.c
+++ b/notmuch-reindex.c
@@ -89,7 +89,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
     struct sigaction action;
     int opt_index;
     int ret;
-    notmuch_indexopts_t *indexopts = NULL;
+    notmuch_status_t status;
 
     /* Set up our handler for SIGINT */
     memset (&action, 0, sizeof (struct sigaction));
@@ -99,6 +99,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
     sigaction (SIGINT, &action, NULL);
 
     notmuch_opt_desc_t options[] = {
+	{ .opt_inherit = notmuch_index_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -115,6 +116,13 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
 
     notmuch_exit_if_unmatched_db_uuid (notmuch);
 
+    status = notmuch_process_index_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
+
     query_string = query_string_from_args (config, argc-opt_index, argv+opt_index);
     if (query_string == NULL) {
 	fprintf (stderr, "Out of memory\n");
@@ -126,7 +134,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
 	return EXIT_FAILURE;
     }
     
-    ret = reindex_query (notmuch, query_string, indexopts);
+    ret = reindex_query (notmuch, query_string, index_options.opts);
 
     notmuch_database_destroy (notmuch);
 
diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh
index d7180797..cd97c48f 100755
--- a/test/T357-index-decryption.sh
+++ b/test/T357-index-decryption.sh
@@ -48,4 +48,69 @@ test_expect_equal \
     "$output" \
     "$expected"
 
+# add a tag to all messages to ensure that it stays after reindexing
+test_begin_subtest 'tagging all messages'
+test_expect_success 'notmuch tag +blarney "encrypted message"'
+test_begin_subtest "verify that tags have not changed"
+output=$(notmuch search tag:blarney)
+expected='thread:0000000000000001   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox)
+thread:0000000000000002   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# see if first message shows up after reindexing with --try-decrypt=true (same $expected, untouched):
+test_begin_subtest 'reindex old messages'
+test_expect_success 'notmuch reindex --try-decrypt=true tag:encrypted and not property:index.decryption=success'
+test_begin_subtest "reindexed encrypted message, including cleartext"
+output=$(notmuch search wumpus)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property for both messages"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+
+# try to remove cleartext indexing
+test_begin_subtest 'reindex without cleartext'
+test_expect_success 'notmuch reindex tag:encrypted and property:index.decryption=success'
+test_begin_subtest "reindexed encrypted messages, without cleartext"
+output=$(notmuch search wumpus)
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property with both messages unindexed"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# ensure that the tags remain even when we are dropping the cleartext.
+test_begin_subtest "verify that tags remain without cleartext"
+output=$(notmuch search tag:blarney)
+expected='thread:0000000000000001   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox)
+thread:0000000000000002   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+
+# TODO: test removal of a message from the message store between
+# indexing and reindexing.
+
+# TODO: insert the same message into the message store twice, index,
+# remove one of them from the message store, and then reindex.
+# reindexing should return a failure but the message should still be
+# present? -- or what should the semantics be if you ask to reindex a
+# message whose underlying files have been renamed or moved or
+# removed?
+
 test_done
-- 
2.14.2

Re: cleartext indexing version 5

[Jameson Graef Rollins]

[-- Attachment #1: Type: text/plain, Size: 769 bytes --]

On Sun, Oct 15 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote:
> I welcome review and feedback.

I have reviewed this patch series and everything looks in order.  Coding
style is consistent with notmuch standards.  The patch is
feature-complete, with appropriate documentation.  All tests pass.  I am
now using this series by default, with config set to automatically index
encrypted mail ("index.try_decrypt=true"), and have experienced no
issues.

I am strongly in support of this patch series.  Daniel is doing crucial
work pushing for widespread use of email encryption.  This series is
necessary in the context of that larger goal.  He's been pushing this
series for a year and a half now, so I think it's appropriate that it
final be integrated.

jamie.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 832 bytes --]