cleartext indexing, revision 7

cleartext indexing, revision 7

[Daniel Kahn Gillmor]

What follows is the 7th revision of the cleartext indexing series.

As mentioned in the previous discussions, there are a handful of minor
cleanups (spelling, formatting).  Beyond those, the main differences
between this series and the 6th are:

 * changed names of variables that support command-line indexing
   arguments.  Hopefully this makes it slightly less confusing.

 * introduced documentation about which config variables are stored in
   the database and which are in the config file, which wasn't
   previously documented.

 * added some tests for "notmuch insert --try-decrypt"

 * added python bindings to enable explicit control over cleartext
   indexing (this is in the novel final patch in the series)

Thanks to Jani, bremner, jrollins, and Tomi for the earlier reviews.

       --dkg

Series History
--------------

the first version of this series was sent starting at:

id:20170912230153.4175-1-dkg@fifthhorseman.net

version 2 of this series was sent as:

id:20170915055359.24123-1-dkg@fifthhorseman.net

version 3 of this series was sent as:

id:20171010054916.23925-1-dkg@fifthhorseman.net

and version 4 was only partially sent to the list.

version 5 was sent as:

id:20171015064807.14205-1-dkg@fifthhorseman.net

version 6 was sent as:

id:20171017191008.8742-1-dkg@fifthhorseman.net

[PATCH 01/12] index: implement notmuch_indexopts_t with try_decrypt

[Daniel Kahn Gillmor]

This is currently mostly a wrapper around _notmuch_crypto_t that keeps
its internals private and doesn't expose any of the GMime API.
However, non-crypto indexing options might also be added later
(e.g. filters or other transformations).
---
 lib/add-message.cc    | 11 ++++++++++-
 lib/indexopts.c       | 22 ++++++++++++++++++++--
 lib/notmuch-private.h |  7 +++++++
 lib/notmuch.h         | 23 +++++++++++++++++++++++
 4 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index bce10a0f..34099ed5 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch,
 notmuch_status_t
 notmuch_database_index_file (notmuch_database_t *notmuch,
 			     const char *filename,
-			     notmuch_indexopts_t unused (*indexopts),
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_t **message_ret)
 {
     notmuch_message_file_t *message_file;
@@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
     notmuch_private_status_t private_status;
     bool is_ghost = false, is_new = false;
+    notmuch_indexopts_t *def_indexopts = NULL;
 
     const char *date;
     const char *from, *to, *subject;
@@ -540,6 +541,11 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
 	if (is_new || is_ghost)
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
+	if (!indexopts) {
+	    def_indexopts = notmuch_database_get_default_indexopts (notmuch);
+	    indexopts = def_indexopts;
+	}
+
 	ret = _notmuch_message_index_file (message, message_file);
 	if (ret)
 	    goto DONE;
@@ -557,6 +563,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
     }
 
   DONE:
+    if (def_indexopts)
+	notmuch_indexopts_destroy (def_indexopts);
+
     if (message) {
 	if ((ret == NOTMUCH_STATUS_SUCCESS ||
 	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
diff --git a/lib/indexopts.c b/lib/indexopts.c
index 2f9b841b..51b56dd7 100644
--- a/lib/indexopts.c
+++ b/lib/indexopts.c
@@ -21,9 +21,27 @@
 #include "notmuch-private.h"
 
 notmuch_indexopts_t *
-notmuch_database_get_default_indexopts (notmuch_database_t unused (*db))
+notmuch_database_get_default_indexopts (notmuch_database_t *db)
 {
-    return NULL;
+    return talloc_zero (db, notmuch_indexopts_t);
+}
+
+notmuch_status_t
+notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts,
+				   notmuch_bool_t try_decrypt)
+{
+    if (!indexopts)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    indexopts->crypto.decrypt = try_decrypt;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_bool_t
+notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts)
+{
+    if (!indexopts)
+	return false;
+    return indexopts->crypto.decrypt;
 }
 
 void
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index e86f4582..4c408396 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -52,6 +52,7 @@ NOTMUCH_BEGIN_DECLS
 #include "xutil.h"
 #include "error_util.h"
 #include "string-util.h"
+#include "crypto.h"
 
 #ifdef DEBUG
 # define DEBUG_DATABASE_SANITY 1
@@ -633,6 +634,12 @@ _notmuch_thread_create (void *ctx,
 			notmuch_exclude_t omit_exclude,
 			notmuch_sort_t sort);
 
+/* indexopts.c */
+
+struct _notmuch_indexopts {
+    _notmuch_crypto_t crypto;
+};
+
 NOTMUCH_END_DECLS
 
 #ifdef __cplusplus
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 89afb6d9..98f6e91a 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -2230,6 +2230,29 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list);
 notmuch_indexopts_t *
 notmuch_database_get_default_indexopts (notmuch_database_t *db);
 
+/**
+ * Specify whether to decrypt encrypted parts while indexing.
+ *
+ * Be aware that the index is likely sufficient to reconstruct the
+ * cleartext of the message itself, so please ensure that the notmuch
+ * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE
+ * without considering the security of your index.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts,
+				   notmuch_bool_t try_decrypt);
+
+/**
+ * Return whether to decrypt encrypted parts while indexing.
+ * see notmuch_indexopts_set_try_decrypt.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_bool_t
+notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts);
+
 /**
  * Destroy a notmuch_indexopts_t object.
  *
-- 
2.14.2

[PATCH 02/12] doc: add notmuch-properties(7)

[Daniel Kahn Gillmor]

We will want a user-facing place to record details about the use of
notmuch properties shortly.  This establishes a new manual page for
that purpose.
---
 doc/conf.py                       |  4 +++
 doc/index.rst                     |  1 +
 doc/man1/notmuch-dump.rst         |  5 ++--
 doc/man1/notmuch-restore.rst      |  4 ++-
 doc/man1/notmuch.rst              |  1 +
 doc/man7/notmuch-properties.rst   | 53 +++++++++++++++++++++++++++++++++++++++
 doc/man7/notmuch-search-terms.rst |  4 ++-
 7 files changed, 68 insertions(+), 4 deletions(-)
 create mode 100644 doc/man7/notmuch-properties.rst

diff --git a/doc/conf.py b/doc/conf.py
index 0e65413d..c7013bec 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -99,6 +99,10 @@ man_pages = [
      u'incorporate new mail into the notmuch database',
      [notmuch_authors], 1),
 
+    ('man7/notmuch-properties', 'notmuch-properties',
+     u'notmuch message property conventions and documentation',
+     [notmuch_authors], 7),
+
     ('man1/notmuch-reindex', 'notmuch-reindex',
      u're-index matching messages',
      [notmuch_authors], 1),
diff --git a/doc/index.rst b/doc/index.rst
index aa6c9f40..4440d93a 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -18,6 +18,7 @@ Contents:
    man5/notmuch-hooks
    man1/notmuch-insert
    man1/notmuch-new
+   man7/notmuch-properties
    man1/notmuch-reindex
    man1/notmuch-reply
    man1/notmuch-restore
diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index 883e454d..7bc57d29 100644
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -85,8 +85,8 @@ Supported options for **dump** include
 
         Output per-message (key,value) metadata.  Each line starts
         with "#= ", followed by a message id, and a space separated
-        list of key=value pairs.  Ids, keys and values are hex
-        encoded if needed.
+        list of key=value pairs.  Ids, keys and values are hex encoded
+        if needed.  See **notmuch-properties(7)** for more details.
 
       **tags**
 
@@ -116,6 +116,7 @@ SEE ALSO
 **notmuch-hooks(5)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
+**notmuch-properties(7)**,
 **notmuch-reply(1)**,
 **notmuch-restore(1)**,
 **notmuch-search(1)**,
diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst
index 1cfaaaed..b578af1f 100644
--- a/doc/man1/notmuch-restore.rst
+++ b/doc/man1/notmuch-restore.rst
@@ -65,7 +65,8 @@ Supported options for **restore** include
           Restore per-message (key,value) metadata.  Each line starts
           with "#= ", followed by a message id, and a space separated
           list of key=value pairs.  Ids, keys and values are hex
-          encoded if needed.
+          encoded if needed.  See **notmuch-properties(7)** for more
+          details.
 
         **tags**
 
@@ -96,6 +97,7 @@ SEE ALSO
 **notmuch-hooks(5)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
+**notmuch-properties(7)**,
 **notmuch-reply(1)**,
 **notmuch-search(1)**,
 **notmuch-search-terms(7)**,
diff --git a/doc/man1/notmuch.rst b/doc/man1/notmuch.rst
index 5e994746..358f42e8 100644
--- a/doc/man1/notmuch.rst
+++ b/doc/man1/notmuch.rst
@@ -169,6 +169,7 @@ SEE ALSO
 **notmuch-hooks(5)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
+**notmuch-properties(7)**,
 **notmuch-reindex(1)**,
 **notmuch-reply(1)**,
 **notmuch-restore(1)**,
diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
new file mode 100644
index 00000000..8654077c
--- /dev/null
+++ b/doc/man7/notmuch-properties.rst
@@ -0,0 +1,53 @@
+==================
+notmuch-properties
+==================
+
+SYNOPSIS
+========
+
+**notmuch** **count** **property:**\ <*key*>=<*value*>
+
+**notmuch** **search** **property:**\ <*key*>=<*value*>
+
+**notmuch** **show** **property:**\ <*key*>=<*value*>
+
+**notmuch** **reindex** **property:**\ <*key*>=<*value*>
+
+**notmuch** **tag** +<*tag*> **property:**\ <*key*>=<*value*>
+
+
+**notmuch** **dump** **--include=properties**
+
+**notmuch** **restore** **--include=properties**
+
+DESCRIPTION
+===========
+
+Several notmuch commands can search for, modify, add or remove
+properties associated with specific messages.  Properties are
+key/value pairs, and a message can have more than one key/value pair
+for the same key.
+
+While users can select based on a specific property in their search
+terms with the prefix **property:**, the notmuch command-line
+interface does not provide mechanisms for modifying properties
+directly to the user.
+
+Instead, message properties are expected to be set and used
+programmatically, according to logic in notmuch itself, or in
+extensions to it.
+
+Extensions to notmuch which make use of properties are encouraged to
+report the specific properties used to the upstream notmuch project,
+as a way of avoiding collisions in the property namespace.
+
+SEE ALSO
+========
+
+**notmuch(1)**,
+**notmuch-dump(1)**,
+**notmuch-insert(1)**,
+**notmuch-new(1)**,
+**notmuch-reindex(1)**,
+**notmuch-restore(1)**,
+***notmuch-search-terms(7)**
diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index c602eadb..637f7777 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -159,7 +159,8 @@ below).
 The **property:** prefix searches for messages with a particular
 <key>=<value> property pair. Properties are used internally by notmuch
 (and extensions) to add metadata to messages. A given key can be
-present on a given message with several different values.
+present on a given message with several different values.  See
+**notmuch-properties(7)** for more details.
 
 Operators
 ---------
@@ -429,6 +430,7 @@ SEE ALSO
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
 **notmuch-reindex(1)**,
+**notmuch-properties(1)**,
 ***notmuch-reply(1)**,
 **notmuch-restore(1)**,
 **notmuch-search(1)**,
-- 
2.14.2

[PATCH 03/12] reindex: drop all properties named with prefix "index."

[Daniel Kahn Gillmor]

This allows us to create new properties that will be automatically set
during indexing, and cleared during re-indexing, just by choice of
property name.
---
 doc/man7/notmuch-properties.rst | 6 ++++++
 lib/message.cc                  | 4 ++++
 lib/notmuch.h                   | 3 +++
 3 files changed, 13 insertions(+)

diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
index 8654077c..f70ffb3c 100644
--- a/doc/man7/notmuch-properties.rst
+++ b/doc/man7/notmuch-properties.rst
@@ -41,6 +41,12 @@ Extensions to notmuch which make use of properties are encouraged to
 report the specific properties used to the upstream notmuch project,
 as a way of avoiding collisions in the property namespace.
 
+CONVENTIONS
+===========
+
+Any property with a key that starts with "index." will be removed (and
+possibly re-set) upon reindexing (see **notmuch-reindex(1)**).
+
 SEE ALSO
 ========
 
diff --git a/lib/message.cc b/lib/message.cc
index 4ab0ed26..e819f27a 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -1999,6 +1999,10 @@ notmuch_message_reindex (notmuch_message_t *message,
 	goto DONE;
     }
 
+    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
+    if (ret)
+	goto DONE; /* XXX TODO: distinguish from other error returns above? */
+
     /* re-add the filenames with the associated indexopts */
     for (; notmuch_filenames_valid (orig_filenames);
 	 notmuch_filenames_move_to_next (orig_filenames)) {
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 98f6e91a..2c5dcab5 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1764,6 +1764,9 @@ notmuch_message_destroy (notmuch_message_t *message);
  * add or delete values for, as other subsystems or extensions may
  * depend on these properties.
  *
+ * Please see notmuch-properties(7) for more details about specific
+ * properties and conventions around their use.
+ *
  */
 /**@{*/
 /**
-- 
2.14.2

[PATCH 04/12] crypto: index encrypted parts when indexopts try_decrypt is set.

[Daniel Kahn Gillmor]

If we see index options that ask us to decrypt when indexing a
message, and we encounter an encrypted part, we'll try to descend into
it.

If we can decrypt, we add the property index.decryption=success.

If we can't decrypt (or recognize the encrypted type of mail), we add
the property index.decryption=failure.

Note that a single message may have both values of the
"index.decryption" property: "success" and "failure".  For example,
consider a message that includes multiple layers of encryption.  If we
manage to decrypt the outer layer ("index.decryption=success"), but
fail on the inner layer ("index.decryption=failure").

Because of the property name, this will be automatically cleared (and
possibly re-set) during re-indexing.  This means it will subsequently
correspond to the actual semantics of the stored index.
---
 doc/man7/notmuch-properties.rst | 26 ++++++++++++
 lib/add-message.cc              |  2 +-
 lib/index.cc                    | 91 +++++++++++++++++++++++++++++++++++++----
 lib/message.cc                  |  4 +-
 lib/notmuch-private.h           |  1 +
 5 files changed, 112 insertions(+), 12 deletions(-)

diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
index f70ffb3c..4b47e8d7 100644
--- a/doc/man7/notmuch-properties.rst
+++ b/doc/man7/notmuch-properties.rst
@@ -47,6 +47,32 @@ CONVENTIONS
 Any property with a key that starts with "index." will be removed (and
 possibly re-set) upon reindexing (see **notmuch-reindex(1)**).
 
+MESSAGE PROPERTIES
+==================
+
+The following properties are set by notmuch internally in the course
+of its normal activity.
+
+**index.decryption**
+
+    If a message contains encrypted content, and notmuch tries to
+    decrypt that content during indexing, it will add the property
+    ``index.decryption=success`` when the cleartext was successfully
+    indexed.  If notmuch attempts to decrypt any part of a message
+    during indexing and that decryption attempt fails, it will add the
+    property ``index.decryption=failure`` to the message.
+
+    Note that it's possible for a single message to have both
+    ``index.decryption=success`` and ``index.decryption=failure``.
+    Consider an encrypted e-mail message that contains another
+    encrypted e-mail message as an attachment -- if the outer message
+    can be decrypted, but the attached part cannot, then both
+    properties will be set on the message as a whole.
+
+    If notmuch never tried to decrypt an encrypted message during
+    indexing (which is the default), then this property will not be
+    set on that message.
+
 SEE ALSO
 ========
 
diff --git a/lib/add-message.cc b/lib/add-message.cc
index 34099ed5..f5fac8be 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -546,7 +546,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch,
 	    indexopts = def_indexopts;
 	}
 
-	ret = _notmuch_message_index_file (message, message_file);
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
 	if (ret)
 	    goto DONE;
 
diff --git a/lib/index.cc b/lib/index.cc
index e5ae2ba7..6e684f5f 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -364,9 +364,15 @@ _index_content_type (notmuch_message_t *message, GMimeObject *part)
     }
 }
 
+static void
+_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts,
+			    GMimeContentType *content_type,
+			    GMimeMultipartEncrypted *part);
+
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
+		  notmuch_indexopts_t *indexopts,
 		  GMimeObject *part)
 {
     GMimeStream *stream, *filter;
@@ -385,6 +391,7 @@ _index_mime_part (notmuch_message_t *message,
     }
 
     _index_content_type (message, part);
+    content_type = g_mime_object_get_content_type (part);
 
     if (GMIME_IS_MULTIPART (part)) {
 	GMimeMultipart *multipart = GMIME_MULTIPART (part);
@@ -409,17 +416,21 @@ _index_mime_part (notmuch_message_t *message,
 		}
 	    }
 	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
-		/* Don't index encrypted parts, but index their content type. */
 		_index_content_type (message,
 				     g_mime_multipart_get_part (multipart, i));
-		if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) &&
-		    (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) {
-		    _notmuch_database_log (_notmuch_message_database (message),
-					   "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) {
+		    _index_encrypted_mime_part(message, indexopts,
+					       content_type,
+					       GMIME_MULTIPART_ENCRYPTED (part));
+		} else {
+		    if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) {
+			_notmuch_database_log (_notmuch_message_database (message),
+					       "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		    }
 		}
 		continue;
 	    }
-	    _index_mime_part (message,
+	    _index_mime_part (message, indexopts,
 			      g_mime_multipart_get_part (multipart, i));
 	}
 	return;
@@ -430,7 +441,7 @@ _index_mime_part (notmuch_message_t *message,
 
 	mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part));
 
-	_index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+	_index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
 	return;
     }
@@ -464,7 +475,6 @@ _index_mime_part (notmuch_message_t *message,
 
     filter = g_mime_stream_filter_new (stream);
 
-    content_type = g_mime_object_get_content_type (part);
     discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type);
 
     g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
@@ -502,8 +512,71 @@ _index_mime_part (notmuch_message_t *message,
     }
 }
 
+/* descend (if desired) into the cleartext part of an encrypted MIME
+ * part while indexing. */
+static void
+_index_encrypted_mime_part (notmuch_message_t *message,
+			    notmuch_indexopts_t *indexopts,
+			    g_mime_3_unused(GMimeContentType *content_type),
+			    GMimeMultipartEncrypted *encrypted_data)
+{
+    notmuch_status_t status;
+    GError *err = NULL;
+    notmuch_database_t * notmuch = NULL;
+    GMimeObject *clear = NULL;
+
+    if (!indexopts || !notmuch_indexopts_get_try_decrypt (indexopts))
+	return;
+
+    notmuch = _notmuch_message_database (message);
+
+#if (GMIME_MAJOR_VERSION < 3)
+    {
+	GMimeCryptoContext* crypto_ctx = NULL;
+	const char *protocol = NULL;
+	protocol = g_mime_content_type_get_parameter (content_type, "protocol");
+	status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto),
+							 protocol, &crypto_ctx);
+	if (status) {
+	    _notmuch_database_log (notmuch, "Warning: setup failed for decrypting "
+				   "during indexing. (%d)\n", status);
+	    status = notmuch_message_add_property (message, "index.decryption", "failure");
+	    if (status)
+		_notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					      "property (%d)\n", status);
+	    return;
+	}
+	clear = g_mime_multipart_encrypted_decrypt(encrypted_data, crypto_ctx,
+						   NULL, &err);
+    }
+#else
+    clear = g_mime_multipart_encrypted_decrypt(encrypted_data, GMIME_DECRYPT_NONE, NULL,
+					       NULL, &err);
+#endif
+    if (err) {
+	_notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n",
+			       err->domain, err->code, err->message);
+	g_error_free(err);
+	/* Indicate that we failed to decrypt during indexing */
+	status = notmuch_message_add_property (message, "index.decryption", "failure");
+	if (status)
+	    _notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					  "property (%d)\n", status);
+	return;
+    }
+    _index_mime_part (message, indexopts, clear);
+    g_object_unref (clear);
+
+    status = notmuch_message_add_property (message, "index.decryption", "success");
+    if (status)
+	_notmuch_database_log (notmuch, "failed to add index.decryption "
+			       "property (%d)\n", status);
+
+}
+
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file)
 {
     GMimeMessage *mime_message;
@@ -531,7 +604,7 @@ _notmuch_message_index_file (notmuch_message_t *message,
     subject = g_mime_message_get_subject (mime_message);
     _notmuch_message_gen_terms (message, "subject", subject);
 
-    _index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+    _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
     return NOTMUCH_STATUS_SUCCESS;
 }
diff --git a/lib/message.cc b/lib/message.cc
index e819f27a..12743460 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -1961,7 +1961,7 @@ _notmuch_message_frozen (notmuch_message_t *message)
 
 notmuch_status_t
 notmuch_message_reindex (notmuch_message_t *message,
-			 notmuch_indexopts_t unused (*indexopts))
+			 notmuch_indexopts_t *indexopts)
 {
     notmuch_database_t *notmuch = NULL;
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
@@ -2042,7 +2042,7 @@ notmuch_message_reindex (notmuch_message_t *message,
 	if (found == 0)
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
-	ret = _notmuch_message_index_file (message, message_file);
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
 
 	if (ret == NOTMUCH_STATUS_FILE_ERROR)
 	    continue;
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 4c408396..1093429c 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -448,6 +448,7 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
 
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file);
 
 /* messages.c */
-- 
2.14.2

[PATCH 05/12] config: test whether an item is stored in the database by name

[Daniel Kahn Gillmor]

QUERY_STRING was only used in two places, both to test whether a
variable should be stored in (or retrieved from) the database.

Since other configuration variables might be stored in the database in
the future, consolidate that test into a single function.

We also document that these configuration options should not be placed
in the config file.
---
 doc/man1/notmuch-config.rst |  7 ++++++-
 notmuch-config.c            | 13 ++++++++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 71294554..539199c2 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -15,7 +15,11 @@ DESCRIPTION
 ===========
 
 The **config** command can be used to get or set settings in the notmuch
-configuration file.
+configuration file and corresponding database.
+
+Items marked **[STORED IN DATABASE]** are only in the database.  They
+should not be placed in the configuration file, and should be accessed
+programmatically as described in the SYNOPSIS above.
 
     **get**
         The value of the specified configuration item is printed to
@@ -142,6 +146,7 @@ The available configuration items are described below.
 
     **query.<name>**
 
+        **[STORED IN DATABASE]**
         Expansion for named query called <name>. See
         **notmuch-search-terms(7)** for more information about named
         queries.
diff --git a/notmuch-config.c b/notmuch-config.c
index 8fb59f96..74668718 100644
--- a/notmuch-config.c
+++ b/notmuch-config.c
@@ -808,7 +808,14 @@ _item_split (char *item, char **group, char **key)
 }
 
 #define BUILT_WITH_PREFIX "built_with."
-#define QUERY_PREFIX "query."
+
+static bool
+_stored_in_db (const char *item)
+{
+    if (STRNCMP_LITERAL (item, "query.") == 0)
+	return true;
+    return false;
+}
 
 static int
 _print_db_config(notmuch_config_t *config, const char *name)
@@ -857,7 +864,7 @@ notmuch_config_command_get (notmuch_config_t *config, char *item)
     } else if (STRNCMP_LITERAL (item, BUILT_WITH_PREFIX) == 0) {
 	printf ("%s\n",
 		notmuch_built_with (item + strlen (BUILT_WITH_PREFIX)) ? "true" : "false");
-    } else if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) {
+    } else if (_stored_in_db (item)) {
 	return _print_db_config (config, item);
     } else {
 	char **value;
@@ -928,7 +935,7 @@ notmuch_config_command_set (notmuch_config_t *config, char *item, int argc, char
 	return 1;
     }
 
-    if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) {
+    if (_stored_in_db (item)) {
 	return _set_db_config (config, item, argc, argv);
     }
 
-- 
2.14.2

[PATCH 06/12] config: define new option index.try_decrypt

[Daniel Kahn Gillmor]

By default, notmuch won't try to decrypt on indexing.  With this
patch, we make it possible to indicate a per-database preference using
the config variable "index.try_decrypt", which by default will be
false.

At indexing time, the database needs some way to know its internal
defaults for how to index encrypted parts.  It shouldn't be contingent
on an external config file (since that can't be retrieved from the
database object itself), so we store it in the database.

This behaves similarly to the query.* configurations, which are also
stored in the database itself, so we're not introducing any new
dependencies by requiring that it be stored in the database.
---
 doc/man1/notmuch-config.rst     | 13 +++++++++++++
 doc/man7/notmuch-properties.rst |  6 ++++--
 lib/indexopts.c                 | 18 +++++++++++++++++-
 notmuch-config.c                |  6 ++++++
 4 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 539199c2..6961737f 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -138,6 +138,19 @@ The available configuration items are described below.
 
         Default: ``gpg``.
 
+    **index.try_decrypt**
+
+        **[STORED IN DATABASE]**
+        When indexing an encrypted e-mail message, if this variable is
+        set to true, notmuch will try to decrypt the message and index
+        the cleartext.  Be aware that the index is likely sufficient
+        to reconstruct the cleartext of the message itself, so please
+        ensure that the notmuch message index is adequately protected.
+        DO NOT USE ``index.try_decrypt=true`` without considering the
+        security of your index.
+
+        Default: ``false``.
+
     **built_with.<name>**
 
         Compile time feature <name>. Current possibilities include
diff --git a/doc/man7/notmuch-properties.rst b/doc/man7/notmuch-properties.rst
index 4b47e8d7..68121359 100644
--- a/doc/man7/notmuch-properties.rst
+++ b/doc/man7/notmuch-properties.rst
@@ -70,13 +70,15 @@ of its normal activity.
     properties will be set on the message as a whole.
 
     If notmuch never tried to decrypt an encrypted message during
-    indexing (which is the default), then this property will not be
-    set on that message.
+    indexing (which is the default, see ``index.try_decrypt`` in
+    **notmuch-config(1)**), then this property will not be set on that
+    message.
 
 SEE ALSO
 ========
 
 **notmuch(1)**,
+**notmuch-config(1)**,
 **notmuch-dump(1)**,
 **notmuch-insert(1)**,
 **notmuch-new(1)**,
diff --git a/lib/indexopts.c b/lib/indexopts.c
index 51b56dd7..15c31d24 100644
--- a/lib/indexopts.c
+++ b/lib/indexopts.c
@@ -23,7 +23,23 @@
 notmuch_indexopts_t *
 notmuch_database_get_default_indexopts (notmuch_database_t *db)
 {
-    return talloc_zero (db, notmuch_indexopts_t);
+    notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t);
+    if (!ret)
+	return ret;
+
+    char * try_decrypt;
+    notmuch_status_t err = notmuch_database_get_config (db, "index.try_decrypt", &try_decrypt);
+    if (err)
+	return ret;
+
+    if (try_decrypt &&
+	((!(strcasecmp(try_decrypt, "true"))) ||
+	 (!(strcasecmp(try_decrypt, "yes"))) ||
+	 (!(strcasecmp(try_decrypt, "1")))))
+	notmuch_indexopts_set_try_decrypt (ret, true);
+
+    free (try_decrypt);
+    return ret;
 }
 
 notmuch_status_t
diff --git a/notmuch-config.c b/notmuch-config.c
index 74668718..1cba2661 100644
--- a/notmuch-config.c
+++ b/notmuch-config.c
@@ -812,8 +812,14 @@ _item_split (char *item, char **group, char **key)
 static bool
 _stored_in_db (const char *item)
 {
+    const char * db_configs[] = {
+	"index.try_decrypt",
+    };
     if (STRNCMP_LITERAL (item, "query.") == 0)
 	return true;
+    for (size_t i = 0; i < ARRAY_SIZE (db_configs); i++)
+	if (strcmp (item, db_configs[i]) == 0)
+	    return true;
     return false;
 }
 
-- 
2.14.2

[PATCH 07/12] cli: set up shared command-line arguments for indexing

[Daniel Kahn Gillmor]

We have an indexopts structure for manipulating indexing in different
ways, but we also have three command-line invocations that can trigger
indexing: new, insert, and reindex.

This changeset prepares a common parser that these subcommands can
share.

Note: if the deprecated crypto.gpg_path configuration option is set to
anything other than "gpg", we ignore it (and print a warning on
stderr, if built against gmime < 3.0).

At the moment, it's just --try-decrypt, but others will likely follow.
---
 notmuch-client.h | 14 ++++++++++++++
 notmuch.c        | 40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/notmuch-client.h b/notmuch-client.h
index d17cdf01..f7524e59 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -495,4 +495,18 @@ void notmuch_exit_if_unmatched_db_uuid (notmuch_database_t *notmuch);
 void notmuch_process_shared_options (const char* subcommand_name);
 int notmuch_minimal_options (const char* subcommand_name,
 			     int argc, char **argv);
+
+
+/* the state chosen by the user invoking one of the notmuch
+ * subcommands that does indexing */
+struct _notmuch_client_indexing_cli_choices {
+    bool try_decrypt;
+    bool try_decrypt_set;
+    notmuch_indexopts_t * opts;
+};
+extern struct _notmuch_client_indexing_cli_choices indexing_cli_choices;
+extern const notmuch_opt_desc_t  notmuch_shared_indexing_options [];
+notmuch_status_t
+notmuch_process_shared_indexing_options (notmuch_database_t *notmuch, notmuch_config_t *config);
+
 #endif
diff --git a/notmuch.c b/notmuch.c
index 02148f2e..539ac58c 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -96,6 +96,46 @@ int notmuch_minimal_options (const char *subcommand_name,
     return opt_index;
 }
 
+
+struct _notmuch_client_indexing_cli_choices indexing_cli_choices = { };
+const notmuch_opt_desc_t  notmuch_shared_indexing_options [] = {
+    { .opt_bool = &indexing_cli_choices.try_decrypt,
+      .present = &indexing_cli_choices.try_decrypt_set,
+      .name = "try-decrypt" },
+    { }
+};
+
+
+notmuch_status_t
+notmuch_process_shared_indexing_options (notmuch_database_t *notmuch, g_mime_3_unused(notmuch_config_t *config))
+{
+    if (indexing_cli_choices.opts == NULL)
+	indexing_cli_choices.opts = notmuch_database_get_default_indexopts (notmuch);
+    if (indexing_cli_choices.try_decrypt_set) {
+	notmuch_status_t status;
+	if (indexing_cli_choices.opts == NULL)
+	    return NOTMUCH_STATUS_OUT_OF_MEMORY;
+	status = notmuch_indexopts_set_try_decrypt (indexing_cli_choices.opts, indexing_cli_choices.try_decrypt);
+	if (status != NOTMUCH_STATUS_SUCCESS) {
+	    fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n",
+		     indexing_cli_choices.try_decrypt ? "True" : "False", notmuch_status_to_string (status));
+	    notmuch_indexopts_destroy (indexing_cli_choices.opts);
+	    indexing_cli_choices.opts = NULL;
+	    return status;
+	}
+    }
+#if (GMIME_MAJOR_VERSION < 3)
+    if (indexing_cli_choices.opts && notmuch_indexopts_get_try_decrypt (indexing_cli_choices.opts)) {
+	const char* gpg_path = notmuch_config_get_crypto_gpg_path (config);
+	if (gpg_path && strcmp(gpg_path, "gpg"))
+	    fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n"
+		     "\tbut ignoring (use $PATH instead)\n", gpg_path);
+    }
+#endif
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+
 static command_t commands[] = {
     { NULL, notmuch_command, NOTMUCH_CONFIG_OPEN | NOTMUCH_CONFIG_CREATE,
       "Notmuch main command." },
-- 
2.14.2

[PATCH 08/12] tests: emacs_fcc_message: allow passing --arguments to notmuch new

[Daniel Kahn Gillmor]

Subsequent patches may want to send GNU-style --long-arguments to
notmuch new in the test suite, in particular when invoking
emacs_fcc_message.  This changeset makes that possible.
---
 test/test-lib.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/test-lib.sh b/test/test-lib.sh
index 4619c327..01891bf8 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -349,8 +349,17 @@ emacs_deliver_message ()
 # Accepts arbitrary extra emacs/elisp functions to modify the message
 # before sending, which is useful to doing things like attaching files
 # to the message and encrypting/signing.
+#
+# If any GNU-style long-arguments (like --quiet or --try-decrypt=true) are
+# at the head of the argument list, they are sent directly to "notmuch
+# new" after message delivery
 emacs_fcc_message ()
 {
+    local nmn_args=''
+    while [[ "$1" =~ ^-- ]]; do
+        nmn_args="$nmn_args $1"
+        shift
+    done
     local subject="$1"
     local body="$2"
     shift 2
@@ -369,7 +378,7 @@ emacs_fcc_message ()
 	   (insert \"${body}\")
 	   $@
 	   (notmuch-mua-send-and-exit))" || return 1
-    notmuch new >/dev/null
+    notmuch new $nmn_args >/dev/null
 }
 
 # Add an existing, fixed corpus of email to the database.
-- 
2.14.2

[PATCH 09/12] cli/new: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Enable override of the index.try_decrypt setting during "notmuch new"
on a per-invocation basis.

We update the documentation and tab completion, and also add a test.
---
 completion/notmuch-completion.bash | 13 ++++++++--
 doc/man1/notmuch-new.rst           | 12 +++++++++
 notmuch-new.c                      | 10 +++++++-
 test/T357-index-decryption.sh      | 51 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 83 insertions(+), 3 deletions(-)
 create mode 100755 test/T357-index-decryption.sh

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 5201be63..17be6b8f 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -311,11 +311,20 @@ _notmuch_insert()
 _notmuch_new()
 {
     local cur prev words cword split
-    _init_completion || return
+    _init_completion -s || return
+
+    $split &&
+    case "${prev}" in
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
+    esac
 
+    ! $split &&
     case "${cur}" in
 	-*)
-	    local options="--no-hooks --quiet ${_notmuch_shared_options}"
+	    local options="--no-hooks --try-decrypt= --quiet ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "${options}" -- ${cur}) )
 	    ;;
diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst
index 14bc5da4..bc26aa48 100644
--- a/doc/man1/notmuch-new.rst
+++ b/doc/man1/notmuch-new.rst
@@ -43,6 +43,18 @@ Supported options for **new** include
     ``--quiet``
         Do not print progress or results.
 
+    ``--try-decrypt=(true|false)``
+
+        If true, when encountering an encrypted message, try to
+        decrypt it while indexing.  If decryption is successful, index
+        the cleartext itself.  Be aware that the index is likely
+        sufficient to reconstruct the cleartext of the message itself,
+        so please ensure that the notmuch message index is adequately
+        protected.  DO NOT USE ``--try-decrypt=true`` without
+        considering the security of your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 EXIT STATUS
 ===========
 
diff --git a/notmuch-new.c b/notmuch-new.c
index 0f50457e..fb021b18 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -267,7 +267,7 @@ add_file (notmuch_database_t *notmuch, const char *filename,
     if (status)
 	goto DONE;
 
-    status = notmuch_database_index_file (notmuch, filename, NULL, &message);
+    status = notmuch_database_index_file (notmuch, filename, indexing_cli_choices.opts, &message);
     switch (status) {
     /* Success. */
     case NOTMUCH_STATUS_SUCCESS:
@@ -963,6 +963,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
 	{ .opt_bool = &verbose, .name = "verbose" },
 	{ .opt_bool = &add_files_state.debug, .name = "debug" },
 	{ .opt_bool = &no_hooks, .name = "no-hooks" },
+	{ .opt_inherit = notmuch_shared_indexing_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -1080,6 +1081,13 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     if (notmuch == NULL)
 	return EXIT_FAILURE;
 
+    status = notmuch_process_shared_indexing_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
+
     /* Set up our handler for SIGINT. We do this after having
      * potentially done a database upgrade we this interrupt handler
      * won't support. */
diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh
new file mode 100755
index 00000000..547a3c7e
--- /dev/null
+++ b/test/T357-index-decryption.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# TODO: test index.decryption=failed
+
+test_description='indexing decrypted mail'
+. $(dirname "$0")/test-lib.sh || exit 1
+
+##################################################
+
+add_gnupg_home
+# get key fingerprint
+FINGERPRINT=$(gpg --no-tty --list-secret-keys --with-colons --fingerprint | grep '^fpr:' | cut -d: -f10)
+
+# create a test encrypted message
+test_begin_subtest 'emacs delivery of encrypted message'
+test_expect_success \
+'emacs_fcc_message \
+    "test encrypted message for cleartext index 001" \
+    "This is a test encrypted message with a wumpus.\n" \
+    "(mml-secure-message-encrypt)"'
+
+test_begin_subtest "search for unindexed cleartext"
+output=$(notmuch search wumpus)
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# create a test encrypted message that is indexed in the clear
+test_begin_subtest 'emacs delivery of encrypted message'
+test_expect_success \
+'emacs_fcc_message --try-decrypt=true \
+    "test encrypted message for cleartext index 002" \
+    "This is a test encrypted message with a wumpus.\n" \
+    "(mml-secure-message-encrypt)"'
+
+test_begin_subtest "emacs delivery of encrypted message, indexed cleartext"
+output=$(notmuch search wumpus)
+expected='thread:0000000000000002   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property for one message"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+test_done
-- 
2.14.2

[PATCH 10/12] cli/insert: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Enable override of the index.try_decrypt setting on a per-message
basis when invoking "notmuch insert".

We also update the documentation and tab completion, and add more tests.
---
 completion/notmuch-completion.bash |  6 +++-
 doc/man1/notmuch-insert.rst        | 14 ++++++++++
 notmuch-insert.c                   | 14 ++++++++--
 test/T357-index-decryption.sh      | 56 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 86 insertions(+), 4 deletions(-)

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 17be6b8f..72a75a94 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -287,12 +287,16 @@ _notmuch_insert()
 		sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) )
 	    return
 	    ;;
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
     esac
 
     ! $split &&
     case "${cur}" in
 	--*)
-	    local options="--create-folder --folder= --keep --no-hooks ${_notmuch_shared_options}"
+	    local options="--create-folder --folder= --keep --no-hooks --try-decrypt= ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
 	    return
diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst
index f83a7385..e2bf37d0 100644
--- a/doc/man1/notmuch-insert.rst
+++ b/doc/man1/notmuch-insert.rst
@@ -50,6 +50,20 @@ Supported options for **insert** include
     ``--no-hooks``
         Prevent hooks from being run.
 
+    ``--try-decrypt=(true|false)``
+
+        If true and the message is encrypted, try to decrypt the
+        message while indexing.  If decryption is successful, index
+        the cleartext itself.  Either way, the message is always
+        stored to disk in its original form (ciphertext).  Be aware
+        that the index is likely sufficient to reconstruct the
+        cleartext of the message itself, so please ensure that the
+        notmuch message index is adequately protected. DO NOT USE
+        ``--try-decrypt=true`` without considering the security of
+        your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 EXIT STATUS
 ===========
 
diff --git a/notmuch-insert.c b/notmuch-insert.c
index 32be7419..4d6b0a7f 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -379,12 +379,13 @@ FAIL:
  */
 static notmuch_status_t
 add_file (notmuch_database_t *notmuch, const char *path, tag_op_list_t *tag_ops,
-	  bool synchronize_flags, bool keep)
+	  bool synchronize_flags, bool keep,
+	  notmuch_indexopts_t *indexopts)
 {
     notmuch_message_t *message;
     notmuch_status_t status;
 
-    status = notmuch_database_index_file (notmuch, path, NULL, &message);
+    status = notmuch_database_index_file (notmuch, path, indexopts, &message);
     if (status == NOTMUCH_STATUS_SUCCESS) {
 	status = tag_op_list_apply (message, tag_ops, 0);
 	if (status) {
@@ -467,6 +468,7 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
 	{ .opt_bool = &create_folder, .name = "create-folder" },
 	{ .opt_bool = &keep, .name = "keep" },
 	{ .opt_bool =  &no_hooks, .name = "no-hooks" },
+	{ .opt_inherit = notmuch_shared_indexing_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -545,9 +547,15 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
 
     notmuch_exit_if_unmatched_db_uuid (notmuch);
 
+    status = notmuch_process_shared_indexing_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
 
     /* Index the message. */
-    status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep);
+    status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, indexing_cli_choices.opts);
 
     /* Commit changes. */
     close_status = notmuch_database_destroy (notmuch);
diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh
index 547a3c7e..1e60df04 100755
--- a/test/T357-index-decryption.sh
+++ b/test/T357-index-decryption.sh
@@ -48,4 +48,60 @@ test_expect_equal \
     "$output" \
     "$expected"
 
+
+test_begin_subtest "message should go away after deletion"
+# cache the message in an env var and remove it:
+fname=$(notmuch search --output=files wumpus)
+contents="$(notmuch show --format=raw wumpus)"
+rm -f "$fname"
+notmuch new
+output=$(notmuch search wumpus)
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# try reinserting it without decryption, should stay the same:
+test_begin_subtest "message cleartext not present after insert"
+notmuch insert --folder=sent <<<"$contents"
+output=$(notmuch search wumpus)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# try reinserting it with decryption, should appear again, but now we
+# have two copies of the message:
+test_begin_subtest "message cleartext is present after reinserting with --try-decrypt"
+notmuch insert --folder=sent --try-decrypt <<<"$contents"
+output=$(notmuch search wumpus)
+expected='thread:0000000000000003   2000-01-01 [1/1(2)] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# remove all copies
+test_begin_subtest "delete all copies of the message"
+mid="$(notmuch search --output=messages wumpus)"
+rm -f $(notmuch search --output=files wumpus)
+notmuch new
+output=$(notmuch search "id:$mid")
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# try inserting it with decryption, should appear as a single copy
+# (note: i think thread id skips 4 because of duplicate message-id
+# insertion, above)
+test_begin_subtest "message cleartext is present with insert --try-decrypt"
+notmuch insert --folder=sent --try-decrypt <<<"$contents"
+output=$(notmuch search wumpus)
+expected='thread:0000000000000005   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox unread)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+
+
+
 test_done
-- 
2.14.2

[PATCH 11/12] cli/reindex: add --try-decrypt=(true|false)

[Daniel Kahn Gillmor]

Enable override of the index.try_decrypt setting on a per-run basis
when invoking "notmuch reindex".  This allows the possibility of (for
example) an emacs keybinding that adds the cleartext of the currently
shown decrypted message to the index, making it searchable in the
future.

It also enables one-time indexing of all messages matching some query,
like so:

    notmuch reindex tag:encrypted and\
       not property:index.decryption=success and\
       from:alice@example.org

We also update the documentation and tab completion, and add a few
more tests.
---
 completion/notmuch-completion.bash | 10 +++++-
 doc/man1/notmuch-reindex.rst       | 14 +++++++++
 notmuch-reindex.c                  | 12 ++++++--
 test/T357-index-decryption.sh      | 63 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 96 insertions(+), 3 deletions(-)

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 72a75a94..7aae4297 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -435,10 +435,18 @@ _notmuch_reindex()
     local cur prev words cword split
     _init_completion -s || return
 
+    $split &&
+    case "${prev}" in
+	--try-decrypt)
+	    COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) )
+	    return
+	    ;;
+    esac
+
     ! $split &&
     case "${cur}" in
 	-*)
-	    local options="${_notmuch_shared_options}"
+	    local options="--try-decrypt= ${_notmuch_shared_options}"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
 	    ;;
diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst
index 1ca10b60..21f6c7a9 100644
--- a/doc/man1/notmuch-reindex.rst
+++ b/doc/man1/notmuch-reindex.rst
@@ -19,6 +19,20 @@ The **reindex** command searches for all messages matching the
 supplied search terms, and re-creates the full-text index on these
 messages using the supplied options.
 
+Supported options for **reindex** include
+
+    ``--try-decrypt=(true|false)``
+
+        If true, when encountering an encrypted message, try to
+        decrypt it while reindexing.  If decryption is successful,
+        index the cleartext itself.  Be aware that the index is likely
+        sufficient to reconstruct the cleartext of the message itself,
+        so please ensure that the notmuch message index is adequately
+        protected. DO NOT USE ``--try-decrypt=true`` without
+        considering the security of your index.
+
+        See also ``index.try_decrypt`` in **notmuch-config(1)**.
+
 SEE ALSO
 ========
 
diff --git a/notmuch-reindex.c b/notmuch-reindex.c
index 57ff5904..5d702510 100644
--- a/notmuch-reindex.c
+++ b/notmuch-reindex.c
@@ -89,7 +89,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
     struct sigaction action;
     int opt_index;
     int ret;
-    notmuch_indexopts_t *indexopts = NULL;
+    notmuch_status_t status;
 
     /* Set up our handler for SIGINT */
     memset (&action, 0, sizeof (struct sigaction));
@@ -99,6 +99,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
     sigaction (SIGINT, &action, NULL);
 
     notmuch_opt_desc_t options[] = {
+	{ .opt_inherit = notmuch_shared_indexing_options },
 	{ .opt_inherit = notmuch_shared_options },
 	{ }
     };
@@ -115,6 +116,13 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
 
     notmuch_exit_if_unmatched_db_uuid (notmuch);
 
+    status = notmuch_process_shared_indexing_options (notmuch, config);
+    if (status != NOTMUCH_STATUS_SUCCESS) {
+	fprintf (stderr, "Error: Failed to process index options. (%s)\n",
+		 notmuch_status_to_string (status));
+	return EXIT_FAILURE;
+    }
+
     query_string = query_string_from_args (config, argc-opt_index, argv+opt_index);
     if (query_string == NULL) {
 	fprintf (stderr, "Out of memory\n");
@@ -126,7 +134,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[])
 	return EXIT_FAILURE;
     }
     
-    ret = reindex_query (notmuch, query_string, indexopts);
+    ret = reindex_query (notmuch, query_string, indexing_cli_choices.opts);
 
     notmuch_database_destroy (notmuch);
 
diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh
index 1e60df04..22e716c6 100755
--- a/test/T357-index-decryption.sh
+++ b/test/T357-index-decryption.sh
@@ -102,6 +102,69 @@ test_expect_equal \
     "$expected"
 
 
+# add a tag to all messages to ensure that it stays after reindexing
+test_begin_subtest 'tagging all messages'
+test_expect_success 'notmuch tag +blarney "encrypted message"'
+test_begin_subtest "verify that tags have not changed"
+output=$(notmuch search tag:blarney)
+expected='thread:0000000000000001   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox)
+thread:0000000000000005   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# see if first message shows up after reindexing with --try-decrypt=true (same $expected, untouched):
+test_begin_subtest 'reindex old messages'
+test_expect_success 'notmuch reindex --try-decrypt=true tag:encrypted and not property:index.decryption=success'
+test_begin_subtest "reindexed encrypted message, including cleartext"
+output=$(notmuch search wumpus)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property for both messages"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+
+# try to remove cleartext indexing
+test_begin_subtest 'reindex without cleartext'
+test_expect_success 'notmuch reindex tag:encrypted and property:index.decryption=success'
+test_begin_subtest "reindexed encrypted messages, without cleartext"
+output=$(notmuch search wumpus)
+expected=''
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# and the same search, but by property ($expected is untouched):
+test_begin_subtest "emacs search by property with both messages unindexed"
+output=$(notmuch search property:index.decryption=success)
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+# ensure that the tags remain even when we are dropping the cleartext.
+test_begin_subtest "verify that tags remain without cleartext"
+output=$(notmuch search tag:blarney)
+expected='thread:0000000000000001   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox)
+thread:0000000000000005   2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox unread)'
+test_expect_equal \
+    "$output" \
+    "$expected"
+
+
+# TODO: test removal of a message from the message store between
+# indexing and reindexing.
 
+# TODO: insert the same message into the message store twice, index,
+# remove one of them from the message store, and then reindex.
+# reindexing should return a failure but the message should still be
+# present? -- or what should the semantics be if you ask to reindex a
+# message whose underlying files have been renamed or moved or
+# removed?
 
 test_done
-- 
2.14.2

[PATCH 12/12] python: add try_decrypt argument to Database.index_file()

[Daniel Kahn Gillmor]

We adopt a pythonic idiom here with an optional argument, rather than
exposing the user to the C indexopts object directly.
---
 bindings/python/notmuch/database.py | 37 ++++++++++++++++++++++++++++++++++---
 bindings/python/notmuch/globals.py  |  5 +++++
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/bindings/python/notmuch/database.py b/bindings/python/notmuch/database.py
index 1279804a..ce80c15e 100644
--- a/bindings/python/notmuch/database.py
+++ b/bindings/python/notmuch/database.py
@@ -20,7 +20,7 @@ Copyright 2010 Sebastian Spaeth <Sebastian@SSpaeth.de>
 import os
 import codecs
 import warnings
-from ctypes import c_char_p, c_void_p, c_uint, byref, POINTER
+from ctypes import c_char_p, c_void_p, c_uint, c_int, byref, POINTER
 from .compat import SafeConfigParser
 from .globals import (
     nmlib,
@@ -28,6 +28,7 @@ from .globals import (
     _str,
     NotmuchDatabaseP,
     NotmuchDirectoryP,
+    NotmuchIndexoptsP,
     NotmuchMessageP,
     NotmuchTagsP,
 )
@@ -400,13 +401,25 @@ class Database(object):
         # return the Directory, init it with the absolute path
         return Directory(abs_dirpath, dir_p, self)
 
+    _get_default_indexopts = nmlib.notmuch_database_get_default_indexopts
+    _get_default_indexopts.argtypes = [NotmuchDatabaseP]
+    _get_default_indexopts.restype = NotmuchIndexoptsP
+
+    _indexopts_set_try_decrypt = nmlib.notmuch_indexopts_set_try_decrypt
+    _indexopts_set_try_decrypt.argtypes = [NotmuchIndexoptsP, c_int]
+    _indexopts_set_try_decrypt.restype = None
+
+    _indexopts_destroy = nmlib.notmuch_indexopts_destroy
+    _indexopts_destroy.argtypes = [NotmuchIndexoptsP]
+    _indexopts_destroy.restype = None
+
     _index_file = nmlib.notmuch_database_index_file
     _index_file.argtypes = [NotmuchDatabaseP, c_char_p,
                              c_void_p,
                              POINTER(NotmuchMessageP)]
     _index_file.restype = c_uint
 
-    def index_file(self, filename, sync_maildir_flags=False):
+    def index_file(self, filename, sync_maildir_flags=False, try_decrypt=None):
         """Adds a new message to the database
 
         :param filename: should be a path relative to the path of the
@@ -427,6 +440,15 @@ class Database(object):
             API. You might want to look into the underlying method
             :meth:`Message.maildir_flags_to_tags`.
 
+        :param try_decrypt: If the message contains any encrypted
+            parts, and try_decrypt is set to true, notmuch will try to
+            decrypt the message and index the cleartext.  if
+            try_decrypt is set to false, it will never try to decrypt
+            during indexing.  If it is set to None (the default), then
+            the database itself will decide whether to decrypt, based
+            on the `index.try_decrypt` configuration setting (which is
+            stored in the database).
+
         :returns: On success, we return
 
            1) a :class:`Message` object that can be used for things
@@ -454,10 +476,19 @@ class Database(object):
               :attr:`STATUS`.READ_ONLY_DATABASE
                       Database was opened in read-only mode so no message can
                       be added.
+
         """
         self._assert_db_is_initialized()
         msg_p = NotmuchMessageP()
-        status = self._index_file(self._db, _str(filename), c_void_p(None), byref(msg_p))
+        indexopts = c_void_p(None)
+        if try_decrypt is not None:
+            indexopts = self._get_default_indexopts(self._db)
+            self._indexopts_set_try_decrypt(indexopts, try_decrypt)
+
+        status = self._index_file(self._db, _str(filename), indexopts, byref(msg_p))
+
+        if indexopts:
+            self._indexopts_destroy(indexopts)
 
         if not status in [STATUS.SUCCESS, STATUS.DUPLICATE_MESSAGE_ID]:
             raise NotmuchError(status)
diff --git a/bindings/python/notmuch/globals.py b/bindings/python/notmuch/globals.py
index b1eec2cf..71426c84 100644
--- a/bindings/python/notmuch/globals.py
+++ b/bindings/python/notmuch/globals.py
@@ -88,3 +88,8 @@ NotmuchDirectoryP = POINTER(NotmuchDirectoryS)
 class NotmuchFilenamesS(Structure):
     pass
 NotmuchFilenamesP = POINTER(NotmuchFilenamesS)
+
+
+class NotmuchIndexoptsS(Structure):
+    pass
+NotmuchIndexoptsP = POINTER(NotmuchIndexoptsS)
-- 
2.14.2

Re: [PATCH 12/12] python: add try_decrypt argument to Database.index_file()

[Daniel Kahn Gillmor]

[-- Attachment #1: Type: text/plain, Size: 398 bytes --]

On Fri 2017-10-20 22:25:49 -0400, Daniel Kahn Gillmor wrote:
> We adopt a pythonic idiom here with an optional argument, rather than
> exposing the user to the C indexopts object directly.

Please ignore this patch to the python bindings -- i plan to publish a
new series shortly ("stashed session keys") than includes better python
bindings for message indexing decryption policy.

         --dkg

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 832 bytes --]

Re: cleartext indexing, revision 7

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

> What follows is the 7th revision of the cleartext indexing series.
>
>
>  * added python bindings to enable explicit control over cleartext
>    indexing (this is in the novel final patch in the series)

I pushed all but the python bindings change, which I haven't had a
chance to look at yet.

d