cleartext indexing, revision 6

cleartext indexing, revision 6

[Daniel Kahn Gillmor]

This is the 6th revision of the cleartext indexing series.

It differs from the 5th mainly in minor edits: fixing typos and
improving comments, documentation, and commit messages.  It also has
the following (slightly more) significant changes:

 * improvement of the test suite to permit passing --long-arguments to
   "notmuch new" is now a separate patch (see 11/14 here)

 * only the "index.try_decrypt" config setting is stored in the
   database, as an attempt to (partially) address the concerns raised
   by bremner in id:878tgdy53g.fsf@tethera.net

 * I've introduced a new man page, notmuch-properties(7), as a place
   to collect and reflect on how properties are used within notmuch.
   Hopefully this central place for property documentation will make
   it easier to coordinate conventions and standards used in the
   project in the future.

This series should be applied on top of my two-part trivial
documentation cleanup series starting at
id:20171017165721.1244-1-dkg@fifthhorseman.net

I welcome review and feedback.

Regards,

        --dkg

Series History
--------------

the first version of this series was sent starting at:

id:20170912230153.4175-1-dkg@fifthhorseman.net

version 2 of this series was sent as:

id:20170915055359.24123-1-dkg@fifthhorseman.net

version 3 of this series was sent as:

id:20171010054916.23925-1-dkg@fifthhorseman.net

and version 4 was only partially sent to the list.

version 5 was sent as:

id:20171015064807.14205-1-dkg@fifthhorseman.net

[PATCH v6 01/14] crypto: move into libnotmuch_util

[Daniel Kahn Gillmor]

This prepares us for using the crypto object in both libnotmuch and
the client.
---
 Makefile.local            |  1 -
 notmuch-client.h          | 22 +---------------------
 util/Makefile.local       |  2 +-
 crypto.c => util/crypto.c |  7 ++++++-
 util/crypto.h             | 28 ++++++++++++++++++++++++++++
 5 files changed, 36 insertions(+), 24 deletions(-)
 rename crypto.c => util/crypto.c (96%)
 create mode 100644 util/crypto.h

diff --git a/Makefile.local b/Makefile.local
index 9d9c52c2..9505b7fe 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -246,7 +246,6 @@ notmuch_client_srcs =		\
 	sprinter-text.c		\
 	query-string.c		\
 	mime-node.c		\
-	crypto.c		\
 	tag-util.c
 
 notmuch_client_modules = $(notmuch_client_srcs:.c=.o)
diff --git a/notmuch-client.h b/notmuch-client.h
index bdcfd893..d17cdf01 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -32,9 +32,6 @@
 
 #include "gmime-extra.h"
 
-/* This is automatically included only since gmime 2.6.10 */
-#include <gmime/gmime-pkcs7-context.h>
-
 #include "notmuch.h"
 
 /* This is separate from notmuch-private.h because we're trying to
@@ -54,6 +51,7 @@
 #include <ctype.h>
 
 #include "talloc-extra.h"
+#include "crypto.h"
 
 #define unused(x) x __attribute__ ((unused))
 
@@ -71,16 +69,6 @@ typedef struct notmuch_show_format {
 			      const struct notmuch_show_params *params);
 } notmuch_show_format_t;
 
-typedef struct _notmuch_crypto {
-    bool verify;
-    bool decrypt;
-#if (GMIME_MAJOR_VERSION < 3)
-    GMimeCryptoContext* gpgctx;
-    GMimeCryptoContext* pkcs7ctx;
-    const char *gpgpath;
-#endif
-} _notmuch_crypto_t;
-
 typedef struct notmuch_show_params {
     bool entire_thread;
     bool omit_excluded;
@@ -180,14 +168,6 @@ typedef struct _notmuch_config notmuch_config_t;
 void
 notmuch_exit_if_unsupported_format (void);
 
-#if (GMIME_MAJOR_VERSION <3)
-GMimeCryptoContext *
-_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol);
-#endif
-
-void
-_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto);
-
 int
 notmuch_count_command (notmuch_config_t *config, int argc, char *argv[]);
 
diff --git a/util/Makefile.local b/util/Makefile.local
index 3027880b..ba03230e 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -5,7 +5,7 @@ extra_cflags += -I$(srcdir)/$(dir)
 
 libnotmuch_util_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \
 		  $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \
-		$(dir)/util.c $(dir)/gmime-extra.c
+		$(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c
 
 libnotmuch_util_modules := $(libnotmuch_util_c_srcs:.c=.o)
 
diff --git a/crypto.c b/util/crypto.c
similarity index 96%
rename from crypto.c
rename to util/crypto.c
index 4c1b7eec..c51d67ed 100644
--- a/crypto.c
+++ b/util/crypto.c
@@ -18,7 +18,12 @@
  * Authors: Jameson Rollins <jrollins@finestructure.net>
  */
 
-#include "notmuch-client.h"
+#include "crypto.h"
+#include <strings.h>
+#define unused(x) x __attribute__ ((unused))
+
+#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
+
 #if (GMIME_MAJOR_VERSION < 3)
 /* Create a GPG context (GMime 2.6) */
 static GMimeCryptoContext *
diff --git a/util/crypto.h b/util/crypto.h
new file mode 100644
index 00000000..80628dc5
--- /dev/null
+++ b/util/crypto.h
@@ -0,0 +1,28 @@
+#ifndef _CRYPTO_H
+#define _CRYPTO_H
+
+#include <stdbool.h>
+#if (GMIME_MAJOR_VERSION < 3)
+#include "gmime-extra.h"
+#endif
+
+typedef struct _notmuch_crypto {
+    bool verify;
+    bool decrypt;
+#if (GMIME_MAJOR_VERSION < 3)
+    GMimeCryptoContext* gpgctx;
+    GMimeCryptoContext* pkcs7ctx;
+    const char *gpgpath;
+#endif
+} _notmuch_crypto_t;
+
+
+#if (GMIME_MAJOR_VERSION < 3)
+GMimeCryptoContext *
+_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol);
+#endif
+
+void
+_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto);
+
+#endif
-- 
2.14.2

[PATCH v6 02/14] crypto: make shared crypto code behave library-like

[Daniel Kahn Gillmor]

If we're going to reuse the crypto code across both the library and
the client, then it needs to report error states properly and not
write to stderr.
---
 lib/database.cc |  6 ++++
 lib/notmuch.h   | 17 +++++++++++
 mime-node.c     |  7 ++++-
 util/crypto.c   | 92 ++++++++++++++++++++++++++++-----------------------------
 util/crypto.h   |  7 +++--
 5 files changed, 79 insertions(+), 50 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index 35c66939..02444e09 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -413,6 +413,12 @@ notmuch_status_to_string (notmuch_status_t status)
 	return "Operation requires a database upgrade";
     case NOTMUCH_STATUS_PATH_ERROR:
 	return "Path supplied is illegal for this function";
+    case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL:
+	return "Crypto protocol missing, malformed, or unintelligible";
+    case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION:
+	return "Crypto engine initialization failure";
+    case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL:
+	return "Unknown crypto protocol";
     default:
     case NOTMUCH_STATUS_LAST_STATUS:
 	return "Unknown error status value";
diff --git a/lib/notmuch.h b/lib/notmuch.h
index e8e0cc12..669e01b1 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -191,6 +191,23 @@ typedef enum _notmuch_status {
      * function, in a way not covered by a more specific argument.
      */
     NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+    /**
+     * A MIME object claimed to have cryptographic protection which
+     * notmuch tried to handle, but the protocol was not specified in
+     * an intelligible way.
+     */
+    NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL,
+    /**
+     * Notmuch attempted to do crypto processing, but could not
+     * initialize the engine needed to do so.
+     */
+    NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION,
+    /**
+     * A MIME object claimed to have cryptographic protection, and
+     * notmuch attempted to process it, but the specific protocol was
+     * something that notmuch doesn't know how to handle.
+     */
+    NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL,
     /**
      * Not an actual status value. Just a way to find out how many
      * valid status values there are.
diff --git a/mime-node.c b/mime-node.c
index d48be4c4..c3d5cb9b 100644
--- a/mime-node.c
+++ b/mime-node.c
@@ -269,7 +269,12 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part)
 	|| (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) {
 	GMimeContentType *content_type = g_mime_object_get_content_type (part);
 	const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol");
-	cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol);
+	notmuch_status_t status;
+	status = _notmuch_crypto_get_gmime_ctx_for_protocol (node->ctx->crypto,
+							     protocol, &cryptoctx);
+	if (status) /* this is a warning, not an error */
+	    fprintf (stderr, "Warning: %s (%s).\n", notmuch_status_to_string (status),
+		     protocol ? protocol : "NULL");
 	if (!cryptoctx)
 	    return NULL;
     }
diff --git a/util/crypto.c b/util/crypto.c
index c51d67ed..5c84282e 100644
--- a/util/crypto.c
+++ b/util/crypto.c
@@ -25,85 +25,86 @@
 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
 
 #if (GMIME_MAJOR_VERSION < 3)
-/* Create a GPG context (GMime 2.6) */
-static GMimeCryptoContext *
-create_gpg_context (_notmuch_crypto_t *crypto)
+/* Create or pass on a GPG context (GMime 2.6) */
+static notmuch_status_t
+get_gpg_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx)
 {
-    GMimeCryptoContext *gpgctx;
+    if (ctx == NULL || crypto == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
 
-    if (crypto->gpgctx)
-	return crypto->gpgctx;
+    if (crypto->gpgctx) {
+	*ctx = crypto->gpgctx;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
 
     /* TODO: GMimePasswordRequestFunc */
-    gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg");
-    if (! gpgctx) {
-	fprintf (stderr, "Failed to construct gpg context.\n");
-	return NULL;
+    crypto->gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg");
+    if (! crypto->gpgctx) {
+	return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION;
     }
-    crypto->gpgctx = gpgctx;
 
-    g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, true);
-    g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, false);
+    g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) crypto->gpgctx, true);
+    g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) crypto->gpgctx, false);
 
-    return gpgctx;
+    *ctx = crypto->gpgctx;
+    return NOTMUCH_STATUS_SUCCESS;
 }
 
-/* Create a PKCS7 context (GMime 2.6) */
-static GMimeCryptoContext *
-create_pkcs7_context (_notmuch_crypto_t *crypto)
+/* Create or pass on a PKCS7 context (GMime 2.6) */
+static notmuch_status_t
+get_pkcs7_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx)
 {
-    GMimeCryptoContext *pkcs7ctx;
+    if (ctx == NULL || crypto == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
 
-    if (crypto->pkcs7ctx)
-	return crypto->pkcs7ctx;
+    if (crypto->pkcs7ctx) {
+	*ctx = crypto->pkcs7ctx;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
 
     /* TODO: GMimePasswordRequestFunc */
-    pkcs7ctx = g_mime_pkcs7_context_new (NULL);
-    if (! pkcs7ctx) {
-	fprintf (stderr, "Failed to construct pkcs7 context.\n");
-	return NULL;
+    crypto->pkcs7ctx = g_mime_pkcs7_context_new (NULL);
+    if (! crypto->pkcs7ctx) {
+	return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION;
     }
-    crypto->pkcs7ctx = pkcs7ctx;
 
-    g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx,
+    g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) crypto->pkcs7ctx,
 					   false);
 
-    return pkcs7ctx;
+    *ctx = crypto->pkcs7ctx;
+    return NOTMUCH_STATUS_SUCCESS;
 }
 static const struct {
     const char *protocol;
-    GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto);
+    notmuch_status_t (*get_context) (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx);
 } protocols[] = {
     {
 	.protocol = "application/pgp-signature",
-	.get_context = create_gpg_context,
+	.get_context = get_gpg_context,
     },
     {
 	.protocol = "application/pgp-encrypted",
-	.get_context = create_gpg_context,
+	.get_context = get_gpg_context,
     },
     {
 	.protocol = "application/pkcs7-signature",
-	.get_context = create_pkcs7_context,
+	.get_context = get_pkcs7_context,
     },
     {
 	.protocol = "application/x-pkcs7-signature",
-	.get_context = create_pkcs7_context,
+	.get_context = get_pkcs7_context,
     },
 };
 
 /* for the specified protocol return the context pointer (initializing
  * if needed) */
-GMimeCryptoContext *
-_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol)
+notmuch_status_t
+_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto,
+					    const char *protocol,
+					    GMimeCryptoContext **ctx)
 {
-    GMimeCryptoContext *cryptoctx = NULL;
-    size_t i;
-
-    if (! protocol) {
-	fprintf (stderr, "Cryptographic protocol is empty.\n");
-	return cryptoctx;
-    }
+    if (! protocol)
+	return NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL;
 
     /* As per RFC 1847 section 2.1: "the [protocol] value token is
      * comprised of the type and sub-type tokens of the Content-Type".
@@ -111,15 +112,12 @@ _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protoc
      * parameter names as defined in this document are
      * case-insensitive."  Thus, we use strcasecmp for the protocol.
      */
-    for (i = 0; i < ARRAY_SIZE (protocols); i++) {
+    for (size_t i = 0; i < ARRAY_SIZE (protocols); i++) {
 	if (strcasecmp (protocol, protocols[i].protocol) == 0)
-	    return protocols[i].get_context (crypto);
+	    return protocols[i].get_context (crypto, ctx);
     }
 
-    fprintf (stderr, "Unknown or unsupported cryptographic protocol %s.\n",
-	     protocol);
-
-    return NULL;
+    return NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL;
 }
 
 void
diff --git a/util/crypto.h b/util/crypto.h
index 80628dc5..1ff0297d 100644
--- a/util/crypto.h
+++ b/util/crypto.h
@@ -4,6 +4,7 @@
 #include <stdbool.h>
 #if (GMIME_MAJOR_VERSION < 3)
 #include "gmime-extra.h"
+#include "notmuch.h"
 #endif
 
 typedef struct _notmuch_crypto {
@@ -18,8 +19,10 @@ typedef struct _notmuch_crypto {
 
 
 #if (GMIME_MAJOR_VERSION < 3)
-GMimeCryptoContext *
-_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol);
+notmuch_status_t
+_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto,
+					    const char *protocol,
+					    GMimeCryptoContext **ctx);
 #endif
 
 void
-- 
2.14.2

[PATCH v6 03/14] index: implement notmuch_indexopts_t with try_decrypt

[Daniel Kahn Gillmor]

This is currently mostly a wrapper around _notmuch_crypto_t that keeps
its internals private and doesn't expose any of the GMime API.
However, non-crypto indexing options might also be added later
(e.g. filters or other transformations).
---
 lib/add-message.cc    | 11 ++++++++++-
 lib/indexopts.c       | 22 ++++++++++++++++++++--
 lib/notmuch-private.h |  7 +++++++
 lib/notmuch.h         | 20 ++++++++++++++++++++
 4 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index bce10a0f..34099ed5 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch,
 notmuch_status_t
 notmuch_database_index_file (notmuch_database_t *notmuch,
 			     const char *filename,
-			     notmuch_indexopts_t unused (*indexopts),
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_t **message_ret)
 {
     notmuch_message_file_t *message_file;
@@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
     notmuch_private_status_t private_status;
     bool is_ghost = false, is_new = false;
+    notmuch_indexopts_t *def_indexopts = NULL;
 
     const char *date;
     const char *from, *to, *subject;
@@ -540,6 +541,11 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
 	if (is_new || is_ghost)
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
+	if (!indexopts) {
+	    def_indexopts = notmuch_database_get_default_indexopts (notmuch);
+	    indexopts = def_indexopts;
+	}
+
 	ret = _notmuch_message_index_file (message, message_file);
 	if (ret)
 	    goto DONE;
@@ -557,6 +563,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
     }
 
   DONE:
+    if (def_indexopts)
+	notmuch_indexopts_destroy (def_indexopts);
+
     if (message) {
 	if ((ret == NOTMUCH_STATUS_SUCCESS ||
 	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
diff --git a/lib/indexopts.c b/lib/indexopts.c
index 2f9b841b..cc1d6422 100644
--- a/lib/indexopts.c
+++ b/lib/indexopts.c
@@ -21,9 +21,27 @@
 #include "notmuch-private.h"
 
 notmuch_indexopts_t *
-notmuch_database_get_default_indexopts (notmuch_database_t unused (*db))
+notmuch_database_get_default_indexopts (notmuch_database_t *db)
 {
-    return NULL;
+    return talloc_zero (db, notmuch_indexopts_t);
+}
+
+notmuch_status_t
+notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts,
+				   bool try_decrypt)
+{
+    if (!indexopts)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    indexopts->crypto.decrypt = try_decrypt;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+bool
+notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts)
+{
+    if (!indexopts)
+	return false;
+    return indexopts->crypto.decrypt;
 }
 
 void
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index e86f4582..4c408396 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -52,6 +52,7 @@ NOTMUCH_BEGIN_DECLS
 #include "xutil.h"
 #include "error_util.h"
 #include "string-util.h"
+#include "crypto.h"
 
 #ifdef DEBUG
 # define DEBUG_DATABASE_SANITY 1
@@ -633,6 +634,12 @@ _notmuch_thread_create (void *ctx,
 			notmuch_exclude_t omit_exclude,
 			notmuch_sort_t sort);
 
+/* indexopts.c */
+
+struct _notmuch_indexopts {
+    _notmuch_crypto_t crypto;
+};
+
 NOTMUCH_END_DECLS
 
 #ifdef __cplusplus
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 669e01b1..0b2e8305 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -42,6 +42,7 @@
 NOTMUCH_BEGIN_DECLS
 
 #include <time.h>
+#include <stdbool.h>
 
 #pragma GCC visibility push(default)
 
@@ -2214,6 +2215,25 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list);
 notmuch_indexopts_t *
 notmuch_database_get_default_indexopts (notmuch_database_t *db);
 
+/**
+ * Specify whether to decrypt encrypted parts while indexing.
+ *
+ * Be aware that the index is likely sufficient to reconstruct the
+ * cleartext of the message itself, so please ensure that the notmuch
+ * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE
+ * without considering the security of your index.
+ */
+notmuch_status_t
+notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts,
+				   bool try_decrypt);
+
+/**
+ * Return whether to decrypt encrypted parts while indexing.
+ * see notmuch_indexopts_set_try_decrypt.
+ */
+bool
+notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts);
+
 /**
  * Destroy a notmuch_indexopts_t object.
  *
-- 
2.14.2

[PATCH v6 04/14] doc: add notmuch-properties(7)

[Daniel Kahn Gillmor]

We will want a user-facing place to record details about the use of
notmuch properties shortly.  This establishes a new manual page for
that purpose.
---
 doc/conf.py                       |  4 +++
 doc/index.rst                     |  1 +
 doc/man1/notmuch-dump.rst         |  5 ++--
 doc/man1/notmuch-restore.rst      |  4 ++-
 doc/man1/notmuch.rst              |  1 +
 doc/man7/notmuch-properties.rst   | 53 +++++++++++++++++++++++++++++++++++++++
 doc/man7/notmuch-search-terms.rst |  4 ++-
 7 files changed, 68 insertions(+), 4 deletions(-)
 create mode 100644 doc/man7/notmuch-properties.rst

diff --git a/doc/conf.py b/doc/conf.py
index 0e65413d..c7013bec 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -99,6 +99,10 @@ man_pages = [
      u'incorporate new mail into the notmuch database',
      [notmuch_authors], 1),
 
+    ('man7/notmuch-properties', 'notmuch-properties',
+     u'notmuch message property conventions and documentation',
+     [notmuch_authors], 7),
+
     ('man1/notmuch-reindex', 'notmuch-reindex',
      u're-index matching messages',
      [notmuch_authors], 1),
diff --git a/doc/index.rst b/doc/index.rst
index aa6c9f40..4440d93a 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -18,6 +18,7 @@ Contents:
    man5/notmuch-hooks
    man1/notmuch-insert
    man1/notmuch-new
+   man7/notmuch-properties
    man1/notmuch-reindex
    man1/notmuch-reply
    man1/notmuch-restore
diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index 883e454d..7bc57d29 100644
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -85,8 +85,8 @@ Supported options for **dump** include
 
         Output per-message (key,value) metadata.  Each line starts
         with "#= ", followed by a message id, and a space separated
-        list of key=value pairs.  Ids, keys and values are hex
-        encoded if needed.
+        list of key=value pairs.  Ids, keys and values are hex encoded
+        if needed.  See **notmuch-properties(7)** for more details.
 
       **tags**
 
@@ -116,6 +116,7 @@ SEE ALSO
 **notmuch-hooks(5)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
+**notmuch-properties(7)**,
 **notmuch-reply(1)**,
 **notmuch-restore(1)**,
 **notmuch-search(1)**,
diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst
index 1cfaaaed..b578af1f 100644
--- a/doc/man1/notmuch-restore.rst
+++ b/doc/man1/notmuch-restore.rst
@@ -65,7 +65,8 @@ Supported options for **restore** include
           Restore per-message (key,value) metadata.  Each line starts
           with "#= ", followed by a message id, and a space separated
           list of key=value pairs.  Ids, keys and values are hex
-          encoded if needed.
+          encoded if needed.  See **notmuch-properties(7)** for more
+          details.
 
         **tags**
 
@@ -96,6 +97,7 @@ SEE ALSO
 **notmuch-hooks(5)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
+**notmuch-properties(7)**,
 **notmuch-reply(1)**,
 **notmuch-search(1)**,
 **notmuch-search-terms(7)**,
diff --git a/doc/man1/notmuch.rst b/doc/man1/notmuch.rst
index 5e994746..358f42e8 100644
--- a/doc/man1/notmuch.rst
+++ b/doc/man1/notmuch.rst
@@ -169,6 +169,7 @@ SEE ALSO
 **notmuch-hooks(5)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
+**notmuch-properties(7)**,
 **notmuch-reindex(1)**,
 **notmuch-reply(1)**,
 **notmuch-restore(1)**,
diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
new file mode 100644
index 00000000..ffaa2418
--- /dev/null
+++ b/doc/man7/notmuch-properties.rst
@@ -0,0 +1,53 @@
+==================
+notmuch-properties
+==================
+
+SYNOPSIS
+========
+
+**notmuch** **count** **property:**<*key*>=<*value*>
+
+**notmuch** **search** **property:**<*key*>=<*value*>
+
+**notmuch** **show** **property:**<*key*>=<*value*>
+
+**notmuch** **reindex** **property:**<*key*>=<*value*>
+
+**notmuch** **tag** +<*tag*> **property:**<*key*>=<*value*>
+
+
+**notmuch** **dump** **--include=properties**
+
+**notmuch** **restore** **--include=properties**
+
+DESCRIPTION
+===========
+
+Several notmuch commands can search for, modify, add or remove
+properties associated with specific messages.  Properties are
+key/value pairs, and a message can have more than one key/value pair
+for the same key.
+
+While users can select based on a specific property in their search
+terms with the prefix **property:**, the notmuch command-line
+interface does not provide mechanisms for modifying properties
+directly to the user.
+
+Instead, message properties are expected to be set and used
+programmatically, according to logic in notmuch itself, or in
+extensions to it.
+
+Extensions to notmuch which make use of properties are encouraged to
+report the specific properties used to the upstream notmuch project,
+as a way of avoiding collisions in the property namespace.
+
+SEE ALSO
+========
+
+**notmuch(1)**,
+**notmuch-dump(1)**,
+**notmuch-insert(1)**,
+**notmuch-new(1)**,
+**notmuch-reindex(1)**,
+**notmuch-restore(1)**,
+***notmuch-search-terms(7)**
diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index c602eadb..637f7777 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -159,7 +159,8 @@ below).
 The **property:** prefix searches for messages with a particular
 <key>=<value> property pair. Properties are used internally by notmuch
 (and extensions) to add metadata to messages. A given key can be
-present on a given message with several different values.
+present on a given message with several different values.  See
+**notmuch-properties(7)** for more details.
 
 Operators
 ---------
@@ -429,6 +430,7 @@ SEE ALSO
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
 **notmuch-reindex(1)**,
+**notmuch-properties(1)**,
 ***notmuch-reply(1)**,
 **notmuch-restore(1)**,
 **notmuch-search(1)**,
-- 
2.14.2

[PATCH v6 05/14] properties: add notmuch_message_remove_all_properties_with_prefix()

[Daniel Kahn Gillmor]

Subsequent patches will introduce a convention that properties whose
name starts with "index." will be stripped (and possibly re-added)
during re-indexing.  This patch lays the groundwork for doing that.
---
 lib/message-property.cc | 18 ++++++++++++++++--
 lib/notmuch.h           | 16 ++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/lib/message-property.cc b/lib/message-property.cc
index d72c74c3..35eaf3c6 100644
--- a/lib/message-property.cc
+++ b/lib/message-property.cc
@@ -85,8 +85,9 @@ notmuch_message_remove_property (notmuch_message_t *message, const char *key, co
     return _notmuch_message_modify_property (message, key, value, true);
 }
 
+static
 notmuch_status_t
-notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+_notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key, bool prefix)
 {
     notmuch_status_t status;
     const char * term_prefix;
@@ -97,7 +98,8 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
 
     _notmuch_message_invalidate_metadata (message, "property");
     if (key)
-	term_prefix = talloc_asprintf (message, "%s%s=", _find_prefix ("property"), key);
+	term_prefix = talloc_asprintf (message, "%s%s%s", _find_prefix ("property"), key,
+				       prefix ? "" : "=");
     else
 	term_prefix = _find_prefix ("property");
 
@@ -107,6 +109,18 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
     return NOTMUCH_STATUS_SUCCESS;
 }
 
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+    return _notmuch_message_remove_all_properties (message, key, false);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix)
+{
+    return _notmuch_message_remove_all_properties (message, prefix, true);
+}
+
 notmuch_message_properties_t *
 notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
 {
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 0b2e8305..817f357f 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1824,6 +1824,22 @@ notmuch_message_remove_property (notmuch_message_t *message, const char *key, co
 notmuch_status_t
 notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
 
+/**
+ * Remove all (prefix*,value) pairs from the given message
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     prefix   delete properties with keys that start with prefix.
+ *			   If NULL, delete all properties
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix);
+
 /**
  * Opaque message property iterator
  */
-- 
2.14.2

[PATCH v6 06/14] reindex: drop all properties named with prefix "index."

[Daniel Kahn Gillmor]

This allows us to create new properties that will be automatically set
during indexing, and cleared during re-indexing, just by choice of
property name.
---
 doc/man7/notmuch-properties.rst | 6 ++++++
 lib/message.cc                  | 6 ++++++
 lib/notmuch.h                   | 6 ++++++
 3 files changed, 18 insertions(+)

diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
index ffaa2418..2ab4f3f4 100644
--- a/doc/man7/notmuch-properties.rst
+++ b/doc/man7/notmuch-properties.rst
@@ -41,6 +41,12 @@ Extensions to notmuch which make use of properties are encouraged to
 report the specific properties used to the upstream notmuch project,
 as a way of avoiding collisions in the property namespace.
 
+CONVENTIONS
+===========
+
+Any property with a key that starts with "index." will be removed (and
+possibly re-set) upon reindexing (see **notmuch-reindex(1)**).
+
 SEE ALSO
 ========
 
diff --git a/lib/message.cc b/lib/message.cc
index 4ab0ed26..77bb6c76 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -1999,6 +1999,12 @@ notmuch_message_reindex (notmuch_message_t *message,
 	goto DONE;
     }
 
+    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
+    if (ret) {
+	INTERNAL_ERROR ("failed to remove index.* properties");
+	goto DONE;
+    }
+
     /* re-add the filenames with the associated indexopts */
     for (; notmuch_filenames_valid (orig_filenames);
 	 notmuch_filenames_move_to_next (orig_filenames)) {
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 817f357f..01191a41 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1765,6 +1765,12 @@ notmuch_message_destroy (notmuch_message_t *message);
  * add or delete values for, as other subsystems or extensions may
  * depend on these properties.
  *
+ * Notmuch has some conventions about how certain properties are
+ * treated.  Those conventions include:
+ *
+ * - properties whose name begins with "index." are cleared (and
+ *   possibly re-set) upon reindexing.
+ *
  */
 /**@{*/
 /**
-- 
2.14.2

[PATCH v6 07/14] crypto: index encrypted parts when indexopts try_decrypt is set.

[Daniel Kahn Gillmor]

If we see index options that ask us to decrypt when indexing a
message, and we encounter an encrypted part, we'll try to descend into
it.

If we can decrypt, we add the property index.decryption=success.

If we can't decrypt (or recognize the encrypted type of mail), we add
the property index.decryption=failure.

Note that a single message may have both values of the
"index.decryption" property: "success" and "failure".  For example,
consider a message that includes multiple layers of encryption.  If we
manage to decrypt the outer layer ("index.decryption=success"), but
fail on the inner layer ("index.decryption=failure").

Because of the property name, this will be automatically cleared (and
possibly re-set) during re-indexing.  This means it will subsequently
correspond to the actual semantics of the stored index.
---
 doc/man7/notmuch-properties.rst | 26 ++++++++++++
 lib/add-message.cc              |  2 +-
 lib/index.cc                    | 91 +++++++++++++++++++++++++++++++++++++----
 lib/message.cc                  |  4 +-
 lib/notmuch-private.h           |  1 +
 5 files changed, 112 insertions(+), 12 deletions(-)

diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
index 2ab4f3f4..8d4cebf7 100644
--- a/doc/man7/notmuch-properties.rst
+++ b/doc/man7/notmuch-properties.rst
@@ -47,6 +47,32 @@ CONVENTIONS
 Any property with a key that starts with "index." will be removed (and
 possibly re-set) upon reindexing (see **notmuch-reindex(1)**).
 
+MESSAGE PROPERTIES
+==================
+
+The following properties are set by notmuch internally in the course
+of its normal activity.
+
+**index.decryption**
+
+    If a message contains encrypted content, and notmuch tries to
+    decrypt that content during indexing, it will add the property
+    ``index.decryption=success`` when the cleartext was successfully
+    indexed.  If notmuch attempts to decrypt any part of a message
+    during indexing and that decryption attempt fails, it will add the
+    property ``index.decryption=failure`` to the message.
+
+    Note that it's possible for a single message to have both
+    ``index.decryption=success`` and ``index.decryption=failure``.
+    Consider an encrypted e-mail message that contains another
+    encrypted e-mail message as an attachment -- if the outer message
+    can be decrypted, but the attached part cannot, then both
+    properties will be set on the message as a whole.
+
+    If notmuch never tried to decrypt an encrypted message during
+    indexing (which is the default), then this property will not be
+    set on that message.
+
 SEE ALSO
 ========
 
diff --git a/lib/add-message.cc b/lib/add-message.cc
index 34099ed5..f5fac8be 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -546,7 +546,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
 	    indexopts = def_indexopts;
 	}
 
-	ret = _notmuch_message_index_file (message, message_file);
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
 	if (ret)
 	    goto DONE;
 
diff --git a/lib/index.cc b/lib/index.cc
index e5ae2ba7..6e684f5f 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -364,9 +364,15 @@ _index_content_type (notmuch_message_t *message, GMimeObject *part)
     }
 }
 
+static void
+_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts,
+			    GMimeContentType *content_type,
+			    GMimeMultipartEncrypted *part);
+
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
+		  notmuch_indexopts_t *indexopts,
 		  GMimeObject *part)
 {
     GMimeStream *stream, *filter;
@@ -385,6 +391,7 @@ _index_mime_part (notmuch_message_t *message,
     }
 
     _index_content_type (message, part);
+    content_type = g_mime_object_get_content_type (part);
 
     if (GMIME_IS_MULTIPART (part)) {
 	GMimeMultipart *multipart = GMIME_MULTIPART (part);
@@ -409,17 +416,21 @@ _index_mime_part (notmuch_message_t *message,
 		}
 	    }
 	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
-		/* Don't index encrypted parts, but index their content type. */
 		_index_content_type (message,
 				     g_mime_multipart_get_part (multipart, i));
-		if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) &&
-		    (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) {
-		    _notmuch_database_log (_notmuch_message_database (message),
-					   "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) {
+		    _index_encrypted_mime_part(message, indexopts,
+					       content_type,
+					       GMIME_MULTIPART_ENCRYPTED (part));
+		} else {
+		    if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) {
+			_notmuch_database_log (_notmuch_message_database (message),
+					       "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		    }
 		}
 		continue;
 	    }
-	    _index_mime_part (message,
+	    _index_mime_part (message, indexopts,
 			      g_mime_multipart_get_part (multipart, i));
 	}
 	return;
@@ -430,7 +441,7 @@ _index_mime_part (notmuch_message_t *message,
 
 	mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part));
 
-	_index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+	_index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
 	return;
     }
@@ -464,7 +475,6 @@ _index_mime_part (notmuch_message_t *message,
 
     filter = g_mime_stream_filter_new (stream);
 
-    content_type = g_mime_object_get_content_type (part);
     discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type);
 
     g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
@@ -502,8 +512,71 @@ _index_mime_part (notmuch_message_t *message,
     }
 }
 
+/* descend (if desired) into the cleartext part of an encrypted MIME
+ * part while indexing. */
+static void
+_index_encrypted_mime_part (notmuch_message_t *message,
+			    notmuch_indexopts_t *indexopts,
+			    g_mime_3_unused(GMimeContentType *content_type),
+			    GMimeMultipartEncrypted *encrypted_data)
+{
+    notmuch_status_t status;
+    GError *err = NULL;
+    notmuch_database_t * notmuch = NULL;
+    GMimeObject *clear = NULL;
+
+    if (!indexopts || !notmuch_indexopts_get_try_decrypt (indexopts))
+	return;
+
+    notmuch = _notmuch_message_database (message);
+
+#if (GMIME_MAJOR_VERSION < 3)
+    {
+	GMimeCryptoContext* crypto_ctx = NULL;
+	const char *protocol = NULL;
+	protocol = g_mime_content_type_get_parameter (content_type, "protocol");
+	status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto),
+							 protocol, &crypto_ctx);
+	if (status) {
+	    _notmuch_database_log (notmuch, "Warning: setup failed for decrypting "
+				   "during indexing. (%d)\n", status);
+	    status = notmuch_message_add_property (message, "index.decryption", "failure");
+	    if (status)
+		_notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					      "property (%d)\n", status);
+	    return;
+	}
+	clear = g_mime_multipart_encrypted_decrypt(encrypted_data, crypto_ctx,
+						   NULL, &err);
+    }
+#else
+    clear = g_mime_multipart_encrypted_decrypt(encrypted_data, GMIME_DECRYPT_NONE, NULL,
+					       NULL, &err);
+#endif
+    if (err) {
+	_notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n",
+			       err->domain, err->code, err->message);
+	g_error_free(err);
+	/* Indicate that we failed to decrypt during indexing */
+	status = notmuch_message_add_property (message, "index.decryption", "failure");
+	if (status)
+	    _notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					  "property (%d)\n", status);
+	return;
+    }
+    _index_mime_part (message, indexopts, clear);
+    g_object_unref (clear);
+
+    status = notmuch_message_add_property (message, "index.decryption", "success");
+    if (status)
+	_notmuch_database_log (notmuch, "failed to add index.decryption "
+			       "property (%d)\n", status);
+
+}
+
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file)
 {
     GMimeMessage *mime_message;
@@ -531,7 +604,7 @@ _notmuch_message_index_file (notmuch_message_t *message,
     subject = g_mime_message_get_subject (mime_message);
     _notmuch_message_gen_terms (message, "subject", subject);
 
-    _index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+    _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
     return NOTMUCH_STATUS_SUCCESS;
 }
diff --git a/lib/message.cc b/lib/message.cc
index 77bb6c76..a47590c6 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -1961,7 +1961,7 @@ _notmuch_message_frozen (notmuch_message_t *message)
 
 notmuch_status_t
 notmuch_message_reindex (notmuch_message_t *message,
-			 notmuch_indexopts_t unused (*indexopts))
+			 notmuch_indexopts_t *indexopts)
 {
     notmuch_database_t *notmuch = NULL;
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
@@ -2044,7 +2044,7 @@ notmuch_message_reindex (notmuch_message_t *message,
 	if (found == 0)
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
-	ret = _notmuch_message_index_file (message, message_file);
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
 
 	if (ret == NOTMUCH_STATUS_FILE_ERROR)
 	    continue;
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 4c408396..1093429c 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -448,6 +448,7 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
 
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file);
 
 /* messages.c */
-- 
2.14.2

[PATCH v6 08/14] config: store index.try_default in the database.

[Daniel Kahn Gillmor]

At indexing time, the database needs some way to know its internal
defaults for how to index encrypted parts.  It shouldn't be contingent
on an external config file (since that can't be retrieved from the
database object itself).

This behaves similarly to the query.* configurations, which are also
stored in the database itself, so we're not introducing any new
dependencies.

In some future series, it's conceivable that other indexing settings
(or all indexing settings) should be stored in the database, but for
the moment we'll just store/retrieve this one.
---
 notmuch-config.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/notmuch-config.c b/notmuch-config.c
index 8fb59f96..20e448f9 100644
--- a/notmuch-config.c
+++ b/notmuch-config.c
@@ -859,6 +859,8 @@ notmuch_config_command_get (notmuch_config_t *config, char *item)
 		notmuch_built_with (item + strlen (BUILT_WITH_PREFIX)) ? "true" : "false");
     } else if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) {
 	return _print_db_config (config, item);
+    } else if (strcmp (item, "index.try_decrypt") == 0) {
+	return _print_db_config (config, item);
     } else {
 	char **value;
 	size_t i, length;
@@ -931,6 +933,9 @@ notmuch_config_command_set (notmuch_config_t *config, char *item, int argc, char
     if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) {
 	return _set_db_config (config, item, argc, argv);
     }
+    if (strcmp (item, "index.try_decrypt") == 0) {
+	return _set_db_config (config, item, argc, argv);
+    }
 
     if (_item_split (item, &group, &key))
 	return 1;
-- 
2.14.2

[PATCH v6 09/14] config: define new option index.try_decrypt

[Daniel Kahn Gillmor]

By default, notmuch won't try to decrypt on indexing.  With this
patch, we make it possible to indicate a per-database preference using
the config variable "index.try_decrypt", which by default will be
false.
---
 doc/man1/notmuch-config.rst     | 12 ++++++++++++
 doc/man7/notmuch-properties.rst |  6 ++++--
 lib/indexopts.c                 | 18 +++++++++++++++++-
 3 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 71294554..9ebceb59 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -134,6 +134,18 @@ The available configuration items are described below.
 
         Default: ``gpg``.
 
+    **index.try_decrypt**
+
+        When indexing an encrypted e-mail message, if this variable is
+        set to true, notmuch will try to decrypt the message and index
+        the cleartext.  Be aware that the index is likely sufficient
+        to reconstruct the cleartext of the message itself, so please
+        ensure that the notmuch message index is adequately protected.
+        DO NOT USE ``index.try_decrypt=true`` without considering the
+        security of your index.
+
+        Default: ``false``.
+
     **built_with.<name>**
 
         Compile time feature <name>. Current possibilities include
diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
index 8d4cebf7..7e771288 100644
--- a/doc/man7/notmuch-properties.rst
+++ b/doc/man7/notmuch-properties.rst
@@ -70,13 +70,15 @@ of its normal activity.
     properties will be set on the message as a whole.
 
     If notmuch never tried to decrypt an encrypted message during
-    indexing (which is the default), then this property will not be
-    set on that message.
+    indexing (which is the default, see ``index.try_decrypt`` in
+    **notmuch-config(1)**), then this property will not be set on that
+    message.
 
 SEE ALSO
 ========
 
 **notmuch(1)**,
+**notmuch-config(1)**,
 **notmuch-dump(1)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
diff --git a/lib/indexopts.c b/lib/indexopts.c
index cc1d6422..987d8952 100644
--- a/lib/indexopts.c
+++ b/lib/indexopts.c
@@ -23,7 +23,23 @@
 notmuch_indexopts_t *
 notmuch_database_get_default_indexopts (notmuch_database_t *db)
 {
-    return talloc_zero (db, notmuch_indexopts_t);
+    notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t);
+    if (!ret)
+	return ret;
+
+    char * try_decrypt;
+    notmuch_status_t err = notmuch_database_get_config (db, "index.try_decrypt", &try_decrypt);
+    if (err)
+	return ret;
+
+    if (try_decrypt &&
+	((!(strcasecmp(try_decrypt, "true"))) ||
+	 (!(strcasecmp(try_decrypt, "yes"))) ||
+	 (!(strcasecmp(try_decrypt, "1")))))
+	notmuch_indexopts_set_try_decrypt (ret, true);
+
+    free (try_decrypt);
+    return ret;
 }
 
 notmuch_status_t
-- 
2.14.2

[PATCH v6 10/14] cli: set up shared command-line arguments for indexing

[Daniel Kahn Gillmor]

We have an indexopts structure for manipulating indexing in different
ways, but we also have three command-line invocations that can trigger
indexing: new, insert, and reindex.

This changeset prepares a common parser that these subcommands can
share.

Note: if the deprecated crypto.gpg_path configuration option is set to
anything other than "gpg", we ignore it (and print a warning on
stderr, if built against gmime < 3.0).

At the moment, it's just --try-decrypt, but others will likely follow.
---
 notmuch-client.h | 12 ++++++++++++
 notmuch.c        | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/notmuch-client.h b/notmuch-client.h
index d17cdf01..27852360 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -495,4 +495,16 @@ void notmuch_exit_if_unmatched_db_uuid (notmuch_database_t *notmuch);
 void notmuch_process_shared_options (const char* subcommand_name);
 int notmuch_minimal_options (const char* subcommand_name,
 			     int argc, char **argv);
+
+
+struct _notmuch_client_index_options {
+    bool try_decrypt;
+    bool try_decrypt_set;
+    notmuch_indexopts_t * opts;
+};
+extern struct _notmuch_client_index_options index_options;
+extern const notmuch_opt_desc_t  notmuch_index_options [];
+notmuch_status_t
+notmuch_process_index_options (notmuch_database_t *notmuch, notmuch_config_t *config);
+
 #endif
diff --git a/notmuch.c b/notmuch.c
index 02148f2e..f83b5a6f 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -96,6 +96,45 @@ int notmuch_minimal_options (const char *subcommand_name,
     return opt_index;
 }
 
+
+struct _notmuch_client_index_options index_options = { };
+const notmuch_opt_desc_t  notmuch_index_options [] = {
+    { .opt_bool = &index_options.try_decrypt,
+      .present = &index_options.try_decrypt_set,
+      .name = "try-decrypt" },
+    { }
+};
+
+
+notmuch_status_t
+notmuch_process_index_options (notmuch_database_t *notmuch, g_mime_3_unused(notmuch_config_t *config))
+{
+    if (index_options.opts == NULL)
+	index_options.opts = notmuch_database_get_default_indexopts (notmuch);
+    if (index_options.try_decrypt_set) {
+	notmuch_status_t status;
+	if (index_options.opts == NULL)
+	    return NOTMUCH_STATUS_OUT_OF_MEMORY;
+	status = notmuch_indexopts_set_try_decrypt (index_options.opts, index_options.try_decrypt);
+	if (status != NOTMUCH_STATUS_SUCCESS) {
+	    fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n",
+		     index_options.try_decrypt ? "True" : "False", notmuch_status_to_string (status));
+	    notmuch_indexopts_destroy (index_options.opts);
+	    return status;
+	}
+    }
+#if (GMIME_MAJOR_VERSION < 3)
+    if (index_options.opts && notmuch_indexopts_get_try_decrypt (index_options.opts)) {
+	const char* gpg_path = notmuch_config_get_crypto_gpg_path (config);
+	if (gpg_path && strcmp(gpg_path, "gpg"))
+	    fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n"
+		     "\tbut ignoring (use $PATH instead)\n", gpg_path);
+    }
+#endif
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+
 static command_t commands[] = {
     { NULL, notmuch_command, NOTMUCH_CONFIG_OPEN | NOTMUCH_CONFIG_CREATE,
       "Notmuch main command." },
-- 
2.14.2

[PATCH v6 11/14] tests: emacs_fcc_message: allow passing --arguments to notmuch new

[Daniel Kahn Gillmor]

Subsequent patches may want to send GNU-style --long-arguments to
notmuch new in the test suite, in particular when invoking
emacs_fcc_message.  This changeset makes that possible.
---
 test/test-lib.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/test-lib.sh b/test/test-lib.sh
index 84051bc9..9c336662 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -340,8 +340,17 @@ emacs_deliver_message ()
 # Accepts arbitrary extra emacs/elisp functions to modify the message
 # before sending, which is useful to doing things like attaching files
 # to the message and encrypting/signing.
+#
+# If any GNU-style long-arguments (like --quiet or --try-decrypt=true) are
+# at the head of the argument list, they are sent directly to "notmuch
+# new" after message delivery
 emacs_fcc_message ()
 {
+    local nmn_args=''
+    while [[ "$1" =~ ^-- ]]; do
+        nmn_args="$nmn_args $1"
+        shift
+    done
     local subject="$1"
     local body="$2"
     shift 2
@@ -360,7 +369,7 @@ emacs_fcc_message ()
 	   (insert \"${body}\")
 	   $@
 	   (notmuch-mua-send-and-exit))" || return 1
-    notmuch new >/dev/null
+    notmuch new $nmn_args >/dev/null
 }
 
 # Add an existing, fixed corpus of email to the database.
-- 
2.14.2

[PATCH v6 12/14] cli/new: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Enable override of the index.try_decrypt setting during "notmuch new"
on a per-invocation basis.

We update the documentation and tab completion, and also add a test.
---
 completion/notmuch-completion.bash | 13 ++++++++--
 doc/man1/notmuch-new.rst           | 12 +++++++++
 notmuch-new.c                      | 10 +++++++-
 test/T357-index-decryption.sh      | 51 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 83 insertions(+), 3 deletions(-)
 create mode 100755 test/T357-index-decryption.sh

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 5201be63..17be6b8f 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -311,11 +311,20 @@ _notmuch_insert()
 _notmuch_new()
 {
     local cur prev words cword split
-    _init_completion || return
+    _init_completion -s || return
+
+    $split &&
+    case "${prev}" in
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
+    esac
 
+    ! $split &&
     case "${cur}" in
 	-*)
-	    local options="--no-hooks --quiet ${_notmuch_shared_options}"
+	    local options="--no-hooks --try-decrypt= --quiet ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "${options}" -- ${cur}) )
 	    ;;
diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst
index 14bc5da4..bc26aa48 100644
--- a/doc/man1/notmuch-new.rst
+++ b/doc/man1/notmuch-new.rst
@@ -43,6 +43,18 @@ Supported options for **new** include
     ``--quiet``
         Do not print progress or results.
 
+    ``--try-decrypt=(true|false)``
+
+        If true, when encountering an encrypted message, try to
+        decrypt it while indexing.  If decryption is successful, index
+        the cleartext itself.  Be aware that the index is likely
+        sufficient to reconstruct the cleartext of the message itself,
+        so please ensure that the notmuch message index is adequately
+        protected.  DO NOT USE ``--try-decrypt=true`` without
+        considering the security of your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 EXIT STATUS
 ===========
 
diff --git a/notmuch-new.c b/notmuch-new.c
index 0f50457e..9b14baf5 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -267,7 +267,7 @@ add_file (notmuch_database_t *notmuch, const char *filename,
     if (status)
 	goto DONE;
 
-    status = notmuch_database_index_file (notmuch, filename, NULL, &message);
+    status = notmuch_database_index_file (notmuch, filename, index_options.opts, &message);
     switch (status) {
     /* Success. */
     case NOTMUCH_STATUS_SUCCESS:
@@ -963,6 +963,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
 	{ .opt_bool = &verbose, .name = "verbose" },
 	{ .opt_bool = &add_files_state.debug, .name = "debug" },
 	{ .opt_bool = &no_hooks, .name = "no-hooks" },
+	{ .opt_inherit = notmuch_index_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -1080,6 +1081,13 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     if (notmuch == NULL)
 	return EXIT_FAILURE;
 
+    status = notmuch_process_index_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
+
     /* Set up our handler for SIGINT. We do this after having
      * potentially done a database upgrade we this interrupt handler
      * won't support. */
diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh
new file mode 100755
index 00000000..d7180797
--- /dev/null
+++ b/test/T357-index-decryption.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# TODO: test index.decryption=failed
+
+test_description='indexing decrypted mail'
+. ./test-lib.sh || exit 1
+
+##################################################
+
+add_gnupg_home
+# get key fingerprint
+FINGERPRINT=$(gpg --no-tty --list-secret-keys --with-colons --fingerprint | grep '^fpr:' | cut -d: -f10)
+
+# create a test encrypted message
+test_begin_subtest 'emacs delivery of encrypted message'
+test_expect_success \
+'emacs_fcc_message \
+    "test encrypted message for cleartext index 001" \
+    "This is a test encrypted message with a wumpus.\n" \
+    "(mml-secure-message-encrypt)"'
+
+test_begin_subtest "search for unindexed cleartext"
+output=$(notmuch search wumpus)
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# create a test encrypted message that is indexed in the clear
+test_begin_subtest 'emacs delivery of encrypted message'
+test_expect_success \
+'emacs_fcc_message --try-decrypt=true \
+    "test encrypted message for cleartext index 002" \
+    "This is a test encrypted message with a wumpus.\n" \
+    "(mml-secure-message-encrypt)"'
+
+test_begin_subtest "emacs delivery of encrypted message, indexed cleartext"
+output=$(notmuch search wumpus)
+expected='thread:0000000000000002   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property for one message"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+test_done
-- 
2.14.2

[PATCH v6 13/14] cli/insert: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Enable override of the index.try_decrypt setting on a per-message
basis when invoking "notmuch insert".

We also update the documentation and tab completion.
---
 completion/notmuch-completion.bash |  6 +++++-
 doc/man1/notmuch-insert.rst        | 14 ++++++++++++++
 notmuch-insert.c                   | 14 +++++++++++---
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 17be6b8f..72a75a94 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -287,12 +287,16 @@ _notmuch_insert()
 		sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) )
 	    return
 	    ;;
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
     esac
 
     ! $split &&
     case "${cur}" in
 	--*)
-	    local options="--create-folder --folder= --keep --no-hooks ${_notmuch_shared_options}"
+	    local options="--create-folder --folder= --keep --no-hooks --try-decrypt= ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
 	    return
diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst
index f83a7385..e2bf37d0 100644
--- a/doc/man1/notmuch-insert.rst
+++ b/doc/man1/notmuch-insert.rst
@@ -50,6 +50,20 @@ Supported options for **insert** include
     ``--no-hooks``
         Prevent hooks from being run.
 
+    ``--try-decrypt=(true|false)``
+
+        If true and the message is encrypted, try to decrypt the
+        message while indexing.  If decryption is successful, index
+        the cleartext itself.  Either way, the message is always
+        stored to disk in its original form (ciphertext).  Be aware
+        that the index is likely sufficient to reconstruct the
+        cleartext of the message itself, so please ensure that the
+        notmuch message index is adequately protected. DO NOT USE
+        ``--try-decrypt=true`` without considering the security of
+        your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 EXIT STATUS
 ===========
 
diff --git a/notmuch-insert.c b/notmuch-insert.c
index 32be7419..e64655cc 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -379,12 +379,13 @@ FAIL:
  */
 static notmuch_status_t
 add_file (notmuch_database_t *notmuch, const char *path, tag_op_list_t *tag_ops,
-	  bool synchronize_flags, bool keep)
+	  bool synchronize_flags, bool keep,
+	  notmuch_indexopts_t *indexopts)
 {
     notmuch_message_t *message;
     notmuch_status_t status;
 
-    status = notmuch_database_index_file (notmuch, path, NULL, &message);
+    status = notmuch_database_index_file (notmuch, path, indexopts, &message);
     if (status == NOTMUCH_STATUS_SUCCESS) {
 	status = tag_op_list_apply (message, tag_ops, 0);
 	if (status) {
@@ -467,6 +468,7 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
 	{ .opt_bool = &create_folder, .name = "create-folder" },
 	{ .opt_bool = &keep, .name = "keep" },
 	{ .opt_bool =  &no_hooks, .name = "no-hooks" },
+	{ .opt_inherit = notmuch_index_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -545,9 +547,15 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
 
     notmuch_exit_if_unmatched_db_uuid (notmuch);
 
+    status = notmuch_process_index_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
 
     /* Index the message. */
-    status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep);
+    status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, index_options.opts);
 
     /* Commit changes. */
     close_status = notmuch_database_destroy (notmuch);
-- 
2.14.2

[PATCH v6 14/14] cli/reindex: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Enable override of the index.try_decrypt setting on a per-run basis
when invoking "notmuch reindex".  This allows the possibility of (for
example) an emacs keybinding that adds the cleartext of the currently
shown decrypted message to the index, making it searchable in the
future.

It also enables one-time indexing of all messages matching some query,
like so:

    notmuch reindex tag:encrypted and\
       not property:index.decryption=success and\
       from:alice@example.org

We also update the documentation and tab completion, and add a few
more tests.
---
 completion/notmuch-completion.bash | 10 +++++-
 doc/man1/notmuch-reindex.rst       | 14 ++++++++
 notmuch-reindex.c                  | 12 +++++--
 test/T357-index-decryption.sh      | 65 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 98 insertions(+), 3 deletions(-)

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 72a75a94..7aae4297 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -435,10 +435,18 @@ _notmuch_reindex()
     local cur prev words cword split
     _init_completion -s || return
 
+    $split &&
+    case "${prev}" in
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
+    esac
+
     ! $split &&
     case "${cur}" in
 	-*)
-	    local options="${_notmuch_shared_options}"
+	    local options="--try-decrypt= ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
 	    ;;
diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst
index 1ca10b60..21f6c7a9 100644
--- a/doc/man1/notmuch-reindex.rst
+++ b/doc/man1/notmuch-reindex.rst
@@ -19,6 +19,20 @@ The **reindex** command searches for all messages matching the
 supplied search terms, and re-creates the full-text index on these
 messages using the supplied options.
 
+Supported options for **reindex** include
+
+    ``--try-decrypt=(true|false)``
+
+        If true, when encountering an encrypted message, try to
+        decrypt it while reindexing.  If decryption is successful,
+        index the cleartext itself.  Be aware that the index is likely
+        sufficient to reconstruct the cleartext of the message itself,
+        so please ensure that the notmuch message index is adequately
+        protected. DO NOT USE ``--try-decrypt=true`` without
+        considering the security of your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 SEE ALSO
 ========
 
diff --git a/notmuch-reindex.c b/notmuch-reindex.c
index 57ff5904..2b06ec6b 100644
--- a/notmuch-reindex.c
+++ b/notmuch-reindex.c
@@ -89,7 +89,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
     struct sigaction action;
     int opt_index;
     int ret;
-    notmuch_indexopts_t *indexopts = NULL;
+    notmuch_status_t status;
 
     /* Set up our handler for SIGINT */
     memset (&action, 0, sizeof (struct sigaction));
@@ -99,6 +99,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
     sigaction (SIGINT, &action, NULL);
 
     notmuch_opt_desc_t options[] = {
+	{ .opt_inherit = notmuch_index_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -115,6 +116,13 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
 
     notmuch_exit_if_unmatched_db_uuid (notmuch);
 
+    status = notmuch_process_index_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
+
     query_string = query_string_from_args (config, argc-opt_index, argv+opt_index);
     if (query_string == NULL) {
 	fprintf (stderr, "Out of memory\n");
@@ -126,7 +134,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
 	return EXIT_FAILURE;
     }
     
-    ret = reindex_query (notmuch, query_string, indexopts);
+    ret = reindex_query (notmuch, query_string, index_options.opts);
 
     notmuch_database_destroy (notmuch);
 
diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh
index d7180797..cd97c48f 100755
--- a/test/T357-index-decryption.sh
+++ b/test/T357-index-decryption.sh
@@ -48,4 +48,69 @@ test_expect_equal \
     "$output" \
     "$expected"
 
+# add a tag to all messages to ensure that it stays after reindexing
+test_begin_subtest 'tagging all messages'
+test_expect_success 'notmuch tag +blarney "encrypted message"'
+test_begin_subtest "verify that tags have not changed"
+output=$(notmuch search tag:blarney)
+expected='thread:0000000000000001   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox)
+thread:0000000000000002   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# see if first message shows up after reindexing with --try-decrypt=true (same $expected, untouched):
+test_begin_subtest 'reindex old messages'
+test_expect_success 'notmuch reindex --try-decrypt=true tag:encrypted and not property:index.decryption=success'
+test_begin_subtest "reindexed encrypted message, including cleartext"
+output=$(notmuch search wumpus)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property for both messages"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+
+# try to remove cleartext indexing
+test_begin_subtest 'reindex without cleartext'
+test_expect_success 'notmuch reindex tag:encrypted and property:index.decryption=success'
+test_begin_subtest "reindexed encrypted messages, without cleartext"
+output=$(notmuch search wumpus)
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property with both messages unindexed"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# ensure that the tags remain even when we are dropping the cleartext.
+test_begin_subtest "verify that tags remain without cleartext"
+output=$(notmuch search tag:blarney)
+expected='thread:0000000000000001   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox)
+thread:0000000000000002   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+
+# TODO: test removal of a message from the message store between
+# indexing and reindexing.
+
+# TODO: insert the same message into the message store twice, index,
+# remove one of them from the message store, and then reindex.
+# reindexing should return a failure but the message should still be
+# present? -- or what should the semantics be if you ask to reindex a
+# message whose underlying files have been renamed or moved or
+# removed?
+
 test_done
-- 
2.14.2

Re: [PATCH v6 03/14] index: implement notmuch_indexopts_t with try_decrypt

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:


> +/**
> + * Specify whether to decrypt encrypted parts while indexing.
> + *
> + * Be aware that the index is likely sufficient to reconstruct the
> + * cleartext of the message itself, so please ensure that the notmuch
> + * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE
> + * without considering the security of your index.
> + */
> +notmuch_status_t
> +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts,
> +				   bool try_decrypt);
> +
> +/**
> + * Return whether to decrypt encrypted parts while indexing.
> + * see notmuch_indexopts_set_try_decrypt.
> + */
> +bool
> +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts);
> +
>  /**
>   * Destroy a notmuch_indexopts_t object.
>   *

This is updated in dkg's git branch to use notmuch_bool_t 

Re: [PATCH v6 06/14] reindex: drop all properties named with prefix "index."

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

> +    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
> +    if (ret) {
> +	INTERNAL_ERROR ("failed to remove index.* properties");
> +	goto DONE;
> +    }

1) INTERNAL_ERROR is fatal, so the goto is unneeded / confusing.

2) Is there no sensible return value here? The function already has an
error return path. I see there is one INTERNAL_ERROR there so maybe
similar reasoning applies and a TODO is enough.

Re: [PATCH v6 06/14] reindex: drop all properties named with prefix "index."

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

>   *
> + * Notmuch has some conventions about how certain properties are
> + * treated.  Those conventions include:
> + *
> + * - properties whose name begins with "index." are cleared (and
> + *   possibly re-set) upon reindexing.
> + *
>   */

I wonder if this duplication of documentation is a good idea. I agree
it's conceptually part of the API, but I worry that the two copies will
drift apart. One option would be to have notmuch.h comment point to the
man page.

Re: [PATCH v6 04/14] doc: add notmuch-properties(7)

[Daniel Kahn Gillmor]

[-- Attachment #1: Type: text/plain, Size: 638 bytes --]

On Tue 2017-10-17 15:09:58 -0400, Daniel Kahn Gillmor wrote:
> diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
> new file mode 100644
 [...]
> +**notmuch** **count** **property:**<*key*>=<*value*>
> +
> +**notmuch** **search** **property:**<*key*>=<*value*>
> +
> +**notmuch** **show** **property:**<*key*>=<*value*>
> +
> +**notmuch** **reindex** **property:**<*key*>=<*value*>
> +
> +**notmuch** **tag** +<*tag*> **property:**<*key*>=<*value*>

on my git branch, these now look like:

      **property:**\ <*key*>=<*value*>

The escaped whitespace was necessary to get them to render properly.

    --dkg

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 832 bytes --]

Re: [PATCH v6 06/14] reindex: drop all properties named with prefix "index."

[Daniel Kahn Gillmor]

On Thu 2017-10-19 08:54:25 -0300, David Bremner wrote:
> Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:
>
>>   *
>> + * Notmuch has some conventions about how certain properties are
>> + * treated.  Those conventions include:
>> + *
>> + * - properties whose name begins with "index." are cleared (and
>> + *   possibly re-set) upon reindexing.
>> + *
>>   */
>
> I wonder if this duplication of documentation is a good idea. I agree
> it's conceptually part of the API, but I worry that the two copies will
> drift apart. One option would be to have notmuch.h comment point to the
> man page.

Good catch.  I've updated this in my series on gitlab.

     --dkg

Re: [PATCH v6 06/14] reindex: drop all properties named with prefix "index."

[Daniel Kahn Gillmor]

[-- Attachment #1: Type: text/plain, Size: 1145 bytes --]

On Thu 2017-10-19 08:52:46 -0300, David Bremner wrote:
> Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:
>
>> +    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
>> +    if (ret) {
>> +	INTERNAL_ERROR ("failed to remove index.* properties");
>> +	goto DONE;
>> +    }
>
> 1) INTERNAL_ERROR is fatal, so the goto is unneeded / confusing.
>
> 2) Is there no sensible return value here? The function already has an
> error return path. I see there is one INTERNAL_ERROR there so maybe
> similar reasoning applies and a TODO is enough.

I suppose the error returned by
notmuch_message_remove_all_properties_with_prefix() is the value that we
should return.  I've switched this to a simple goto DONE in my gitlab
repo.

However, i note that an error here will be (from the outside)
indistinguishable from an error in _notmuch_database_ensure_writable or
_notmuch_message_remove_indexed_terms() immediately above, so i've added
a small TODO anyway.

thanks for the reviews, bremner.  please keep them coming!  (or tell me
when you're done so i can send v7 to the list with resolutions to all
these nitpicks)

  --dkg

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 832 bytes --]

Re: [PATCH v6 10/14] cli: set up shared command-line arguments for indexing

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

> We have an indexopts structure for manipulating indexing in different
> ways, but we also have three command-line invocations that can trigger
> indexing: new, insert, and reindex.
>
> This changeset prepares a common parser that these subcommands can
> share.
>
> Note: if the deprecated crypto.gpg_path configuration option is set to
> anything other than "gpg", we ignore it (and print a warning on
> stderr, if built against gmime < 3.0).
>
> At the moment, it's just --try-decrypt, but others will likely follow.
> ---
>  notmuch-client.h | 12 ++++++++++++
>  notmuch.c        | 39 +++++++++++++++++++++++++++++++++++++++
>  2 files changed, 51 insertions(+)
>
> diff --git a/notmuch-client.h b/notmuch-client.h
> index d17cdf01..27852360 100644
> --- a/notmuch-client.h
> +++ b/notmuch-client.h
> @@ -495,4 +495,16 @@ void notmuch_exit_if_unmatched_db_uuid (notmuch_database_t *notmuch);
>  void notmuch_process_shared_options (const char* subcommand_name);
>  int notmuch_minimal_options (const char* subcommand_name,
>  			     int argc, char **argv);
> +
> +
> +struct _notmuch_client_index_options {
> +    bool try_decrypt;
> +    bool try_decrypt_set;
> +    notmuch_indexopts_t * opts;
> +};
> +extern struct _notmuch_client_index_options index_options;
> +extern const notmuch_opt_desc_t  notmuch_index_options [];
> +notmuch_status_t
> +notmuch_process_index_options (notmuch_database_t *notmuch, notmuch_config_t *config);
> +
>  #endif
> diff --git a/notmuch.c b/notmuch.c
> index 02148f2e..f83b5a6f 100644
> --- a/notmuch.c
> +++ b/notmuch.c
> @@ -96,6 +96,45 @@ int notmuch_minimal_options (const char *subcommand_name,
>      return opt_index;
>  }
>  
> +
> +struct _notmuch_client_index_options index_options = { };
> +const notmuch_opt_desc_t  notmuch_index_options [] = {
> +    { .opt_bool = &index_options.try_decrypt,
> +      .present = &index_options.try_decrypt_set,
> +      .name = "try-decrypt" },
> +    { }
> +};
> +

I can live with this if you can't think of an improvement, but I did 
find it confusing how similar the following 3 names are

- notmuch_indexopts_t
- _notuch_client_index_options
- notmuch_index_options

Re: [PATCH v6 13/14] cli/insert: add --try-decrypt=(true|false)

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

> Enable override of the index.try_decrypt setting on a per-message
> basis when invoking "notmuch insert".
>
> We also update the documentation and tab completion.

I'm curious why the "notmuch new" and "notmuch reindex" versions have tests, but not the
"notmuch insert" version?.

d

Re: cleartext indexing, revision 6

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

> This is the 6th revision of the cleartext indexing series.
>
> It differs from the 5th mainly in minor edits: fixing typos and
> improving comments, documentation, and commit messages.  It also has
> the following (slightly more) significant changes:

My review of this series is complete.

I've pushed 3 of these patches, marked 5 as notmuch::moreinfo, meaning
some (possibly ignorable) comment sent, and marked 6 as notmuch::stale,
meaning they have been reviewed, but don't apply because of missing
patches.

d