* [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted
@ 2017-09-12 23:01 Daniel Kahn Gillmor
2017-09-12 23:01 ` [PATCH 02/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor
` (8 more replies)
0 siblings, 9 replies; 67+ messages in thread
From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw)
To: Notmuch Mail
This makes no functional changes, but prepares the codebase for a
cleaner changeset for dealing with indexing some encrypted messages in
the clear.
---
lib/index.cc | 42 ++++++++++++++++++++++--------------------
1 file changed, 22 insertions(+), 20 deletions(-)
diff --git a/lib/index.cc b/lib/index.cc
index 2b98b588..3ed3b2c6 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -384,27 +384,29 @@ _index_mime_part (notmuch_message_t *message,
GMimeMultipart *multipart = GMIME_MULTIPART (part);
int i;
- if (GMIME_IS_MULTIPART_SIGNED (multipart))
- _notmuch_message_add_term (message, "tag", "signed");
-
- if (GMIME_IS_MULTIPART_ENCRYPTED (multipart))
- _notmuch_message_add_term (message, "tag", "encrypted");
-
- for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
- if (GMIME_IS_MULTIPART_SIGNED (multipart)) {
- /* Don't index the signature. */
- if (i == 1)
- continue;
- if (i > 1)
- _notmuch_database_log (_notmuch_message_database (message),
- "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
- }
- if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
- /* Don't index encrypted parts. */
- continue;
- }
+ if (GMIME_IS_MULTIPART_SIGNED (multipart)) {
+ _notmuch_message_add_term (message, "tag", "signed");
+ /* FIXME: should we try to validate the signature? */
+
+ /* FIXME: is it always just the first part that is signed
+ in all multipart/signed messages? it is for
+ application/pgp-signature and
+ application/pkcs7-signature, which is all gmime can
+ handle anyway. */
_index_mime_part (message,
- g_mime_multipart_get_part (multipart, i));
+ g_mime_multipart_get_part (multipart, 0));
+
+ if (g_mime_multipart_get_count (multipart) > 2)
+ _notmuch_database_log (_notmuch_message_database (message),
+ "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
+ } else if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
+ /* Don't index encrypted parts */
+ _notmuch_message_add_term (message, "tag", "encrypted");
+ } else {
+ for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
+ _index_mime_part (message,
+ g_mime_multipart_get_part (multipart, i));
+ }
}
return;
}
--
2.14.1
^ permalink raw reply related [flat|nested] 67+ messages in thread* [PATCH 02/10] crypto: Move crypto.c into libutil 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 03/10] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor ` (7 subsequent siblings) 8 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail This prepares us for using the crypto object in both the library and the client. i've prefixed notmuch_crypto with _ to indicate that while this can be built into the library when needed, it's not something to be exported or used externally. --- Makefile.local | 1 - mime-node.c | 12 ++++++------ notmuch-client.h | 27 +++------------------------ notmuch-reply.c | 2 +- notmuch-show.c | 2 +- util/Makefile.local | 2 +- crypto.c => util/crypto.c | 45 +++++++++++++++++++++++++-------------------- util/crypto.h | 31 +++++++++++++++++++++++++++++++ 8 files changed, 68 insertions(+), 54 deletions(-) rename crypto.c => util/crypto.c (79%) create mode 100644 util/crypto.h diff --git a/Makefile.local b/Makefile.local index 9d9c52c2..9505b7fe 100644 --- a/Makefile.local +++ b/Makefile.local @@ -246,7 +246,6 @@ notmuch_client_srcs = \ sprinter-text.c \ query-string.c \ mime-node.c \ - crypto.c \ tag-util.c notmuch_client_modules = $(notmuch_client_srcs:.c=.o) diff --git a/mime-node.c b/mime-node.c index 24d73afa..d9ff7de1 100644 --- a/mime-node.c +++ b/mime-node.c @@ -33,7 +33,7 @@ typedef struct mime_node_context { GMimeMessage *mime_message; /* Context provided by the caller. */ - notmuch_crypto_t *crypto; + _notmuch_crypto_t *crypto; } mime_node_context_t; static int @@ -56,7 +56,7 @@ _mime_node_context_free (mime_node_context_t *res) notmuch_status_t mime_node_open (const void *ctx, notmuch_message_t *message, - notmuch_crypto_t *crypto, mime_node_t **root_out) + _notmuch_crypto_t *crypto, mime_node_t **root_out) { const char *filename = notmuch_message_get_filename (message); mime_node_context_t *mctx; @@ -171,7 +171,7 @@ set_signature_list_destructor (mime_node_t *node) /* Verify a signed mime node (GMime 2.6) */ static void node_verify (mime_node_t *node, GMimeObject *part, - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) { GError *err = NULL; @@ -192,7 +192,7 @@ node_verify (mime_node_t *node, GMimeObject *part, /* Decrypt and optionally verify an encrypted mime node (GMime 2.6) */ static void node_decrypt_and_verify (mime_node_t *node, GMimeObject *part, - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) { GError *err = NULL; GMimeDecryptResult *decrypt_result = NULL; @@ -227,7 +227,7 @@ static mime_node_t * _mime_node_create (mime_node_t *parent, GMimeObject *part) { mime_node_t *node = talloc_zero (parent, mime_node_t); - notmuch_crypto_context_t *cryptoctx = NULL; + GMimeCryptoContext *cryptoctx = NULL; /* Set basic node properties */ node->part = part; @@ -265,7 +265,7 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { GMimeContentType *content_type = g_mime_object_get_content_type (part); const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); - cryptoctx = notmuch_crypto_get_context (node->ctx->crypto, protocol); + cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); if (!cryptoctx) return NULL; } diff --git a/notmuch-client.h b/notmuch-client.h index 9d0f367d..76e69501 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -31,10 +31,6 @@ #include "gmime-extra.h" -typedef GMimeCryptoContext notmuch_crypto_context_t; -/* This is automatically included only since gmime 2.6.10 */ -#include <gmime/gmime-pkcs7-context.h> - #include "notmuch.h" /* This is separate from notmuch-private.h because we're trying to @@ -54,6 +50,7 @@ typedef GMimeCryptoContext notmuch_crypto_context_t; #include <ctype.h> #include "talloc-extra.h" +#include "crypto.h" #define unused(x) x __attribute__ ((unused)) @@ -71,22 +68,12 @@ typedef struct notmuch_show_format { const struct notmuch_show_params *params); } notmuch_show_format_t; -typedef struct notmuch_crypto { - notmuch_bool_t verify; - notmuch_bool_t decrypt; -#if (GMIME_MAJOR_VERSION < 3) - notmuch_crypto_context_t* gpgctx; - notmuch_crypto_context_t* pkcs7ctx; - const char *gpgpath; -#endif -} notmuch_crypto_t; - typedef struct notmuch_show_params { notmuch_bool_t entire_thread; notmuch_bool_t omit_excluded; notmuch_bool_t output_body; int part; - notmuch_crypto_t crypto; + _notmuch_crypto_t crypto; notmuch_bool_t include_html; GMimeStream *out_stream; } notmuch_show_params_t; @@ -180,14 +167,6 @@ typedef struct _notmuch_config notmuch_config_t; void notmuch_exit_if_unsupported_format (void); -#if (GMIME_MAJOR_VERSION <3) -notmuch_crypto_context_t * -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol); -#endif - -int -notmuch_crypto_cleanup (notmuch_crypto_t *crypto); - int notmuch_count_command (notmuch_config_t *config, int argc, char *argv[]); @@ -448,7 +427,7 @@ struct mime_node { */ notmuch_status_t mime_node_open (const void *ctx, notmuch_message_t *message, - notmuch_crypto_t *crypto, mime_node_t **node_out); + _notmuch_crypto_t *crypto, mime_node_t **node_out); /* Return a new MIME node for the requested child part of parent. * parent will be used as the talloc context for the returned child diff --git a/notmuch-reply.c b/notmuch-reply.c index 929f3077..00fff3ca 100644 --- a/notmuch-reply.c +++ b/notmuch-reply.c @@ -759,7 +759,7 @@ notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[]) if (do_reply (config, query, ¶ms, format, reply_all) != 0) return EXIT_FAILURE; - notmuch_crypto_cleanup (¶ms.crypto); + _notmuch_crypto_cleanup (¶ms.crypto); notmuch_query_destroy (query); notmuch_database_destroy (notmuch); diff --git a/notmuch-show.c b/notmuch-show.c index cdcc2a98..a35b11ad 100644 --- a/notmuch-show.c +++ b/notmuch-show.c @@ -1241,7 +1241,7 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[]) g_mime_stream_flush (params.out_stream); g_object_unref (params.out_stream); - notmuch_crypto_cleanup (¶ms.crypto); + _notmuch_crypto_cleanup (¶ms.crypto); notmuch_query_destroy (query); notmuch_database_destroy (notmuch); diff --git a/util/Makefile.local b/util/Makefile.local index 3027880b..ba03230e 100644 --- a/util/Makefile.local +++ b/util/Makefile.local @@ -5,7 +5,7 @@ extra_cflags += -I$(srcdir)/$(dir) libnotmuch_util_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \ $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \ - $(dir)/util.c $(dir)/gmime-extra.c + $(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c libnotmuch_util_modules := $(libnotmuch_util_c_srcs:.c=.o) diff --git a/crypto.c b/util/crypto.c similarity index 79% rename from crypto.c rename to util/crypto.c index cc45b885..97e8c8f4 100644 --- a/crypto.c +++ b/util/crypto.c @@ -16,18 +16,26 @@ * along with this program. If not, see https://www.gnu.org/licenses/ . * * Authors: Jameson Rollins <jrollins@finestructure.net> + * Daniel Kahn Gillmor <dkg@fifthhorseman.net> */ -#include "notmuch-client.h" +#include "notmuch.h" +#include "crypto.h" +#include "lib/notmuch-private.h" +#include <string.h> + +#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) + #if (GMIME_MAJOR_VERSION < 3) /* Create a GPG context (GMime 2.6) */ -static notmuch_crypto_context_t * -create_gpg_context (notmuch_crypto_t *crypto) +static GMimeCryptoContext* +create_gpg_context (_notmuch_crypto_t *crypto) { - notmuch_crypto_context_t *gpgctx; + GMimeCryptoContext *gpgctx; - if (crypto->gpgctx) + if (crypto->gpgctx) { return crypto->gpgctx; + } /* TODO: GMimePasswordRequestFunc */ gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); @@ -40,14 +48,14 @@ create_gpg_context (notmuch_crypto_t *crypto) g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, TRUE); g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, FALSE); - return gpgctx; + return crypto->gpgctx; } /* Create a PKCS7 context (GMime 2.6) */ -static notmuch_crypto_context_t * -create_pkcs7_context (notmuch_crypto_t *crypto) +static GMimeCryptoContext* +create_pkcs7_context (_notmuch_crypto_t *crypto) { - notmuch_crypto_context_t *pkcs7ctx; + GMimeCryptoContext *pkcs7ctx; if (crypto->pkcs7ctx) return crypto->pkcs7ctx; @@ -63,11 +71,11 @@ create_pkcs7_context (notmuch_crypto_t *crypto) g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx, FALSE); - return pkcs7ctx; + return crypto->pkcs7ctx; } static const struct { const char *protocol; - notmuch_crypto_context_t *(*get_context) (notmuch_crypto_t *crypto); + GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); } protocols[] = { { .protocol = "application/pgp-signature", @@ -89,10 +97,10 @@ static const struct { /* for the specified protocol return the context pointer (initializing * if needed) */ -notmuch_crypto_context_t * -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) +GMimeCryptoContext * +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) { - notmuch_crypto_context_t *cryptoctx = NULL; + GMimeCryptoContext *cryptoctx = NULL; size_t i; if (! protocol) { @@ -117,8 +125,8 @@ notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) return NULL; } -int -notmuch_crypto_cleanup (notmuch_crypto_t *crypto) +void +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto) { if (crypto->gpgctx) { g_object_unref (crypto->gpgctx); @@ -129,12 +137,9 @@ notmuch_crypto_cleanup (notmuch_crypto_t *crypto) g_object_unref (crypto->pkcs7ctx); crypto->pkcs7ctx = NULL; } - - return 0; } #else -int notmuch_crypto_cleanup (unused(notmuch_crypto_t *crypto)) +void _notmuch_crypto_cleanup (unused(_notmuch_crypto_t *crypto)) { - return 0; } #endif diff --git a/util/crypto.h b/util/crypto.h new file mode 100644 index 00000000..6d15a6ae --- /dev/null +++ b/util/crypto.h @@ -0,0 +1,31 @@ +#ifndef _CRYPTO_H +#define _CRYPTO_H + +#include "notmuch.h" +#if (GMIME_MAJOR_VERSION < 3) +#include "gmime-extra.h" +#include <gmime/gmime.h> +/* This is automatically included only since gmime 2.6.10 */ +#include <gmime/gmime-pkcs7-context.h> +#endif + +typedef struct _notmuch_crypto { + notmuch_bool_t verify; + notmuch_bool_t decrypt; +#if (GMIME_MAJOR_VERSION < 3) + GMimeCryptoContext* gpgctx; + GMimeCryptoContext* pkcs7ctx; + const char *gpgpath; +#endif +} _notmuch_crypto_t; + + +#if (GMIME_MAJOR_VERSION < 3) +GMimeCryptoContext * +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); +#endif + +void +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto); + +#endif -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH 03/10] crypto: make shared crypto code behave library-like 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 02/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 04/10] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor ` (6 subsequent siblings) 8 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail If we're going to reuse the crypto code across both the library and the client, then it needs to report error states properly and not write to stderr. --- lib/database.cc | 6 ++++ lib/notmuch.h | 17 +++++++++++ mime-node.c | 7 ++++- util/crypto.c | 89 ++++++++++++++++++++++++++++----------------------------- util/crypto.h | 6 ++-- 5 files changed, 76 insertions(+), 49 deletions(-) diff --git a/lib/database.cc b/lib/database.cc index 79eb3d69..82a3d463 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -413,6 +413,12 @@ notmuch_status_to_string (notmuch_status_t status) return "Operation requires a database upgrade"; case NOTMUCH_STATUS_PATH_ERROR: return "Path supplied is illegal for this function"; + case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL: + return "Crypto protocol missing, malformed, or unintelligible"; + case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION: + return "Crypto engine initialization failure"; + case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL: + return "Unknown crypto protocol"; default: case NOTMUCH_STATUS_LAST_STATUS: return "Unknown error status value"; diff --git a/lib/notmuch.h b/lib/notmuch.h index f26565f3..6c76fb40 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -191,6 +191,23 @@ typedef enum _notmuch_status { * function, in a way not covered by a more specific argument. */ NOTMUCH_STATUS_ILLEGAL_ARGUMENT, + /** + * A MIME object claimed to have cryptographic protection which + * notmuch tried to handle, but the protocol was not specified in + * an intelligible way. + */ + NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL, + /** + * Notmuch attempted to do crypto processing, but could not + * initialize the engine needed to do so. + */ + NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION, + /** + * A MIME object claimed to have cryptographic protection, and + * notmuch attempted to process it, but the specific protocol was + * something that notmuch doesn't know how to handle. + */ + NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL, /** * Not an actual status value. Just a way to find out how many * valid status values there are. diff --git a/mime-node.c b/mime-node.c index d9ff7de1..6cd7d2de 100644 --- a/mime-node.c +++ b/mime-node.c @@ -265,7 +265,12 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { GMimeContentType *content_type = g_mime_object_get_content_type (part); const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); - cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); + notmuch_status_t status; + status = _notmuch_crypto_get_gmime_ctx_for_protocol (node->ctx->crypto, + protocol, &cryptoctx); + if (status) /* this is a warning, not an error */ + fprintf (stderr, "Warning: %s (%s).\n", notmuch_status_to_string (status), + protocol ? protocol : "(NULL)"); if (!cryptoctx) return NULL; } diff --git a/util/crypto.c b/util/crypto.c index 97e8c8f4..e7908197 100644 --- a/util/crypto.c +++ b/util/crypto.c @@ -27,86 +27,86 @@ #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) #if (GMIME_MAJOR_VERSION < 3) -/* Create a GPG context (GMime 2.6) */ -static GMimeCryptoContext* -create_gpg_context (_notmuch_crypto_t *crypto) +/* Create or pass on a GPG context (GMime 2.6) */ +static notmuch_status_t +get_gpg_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) { - GMimeCryptoContext *gpgctx; + if (ctx == NULL || crypto == NULL) + return NOTMUCH_STATUS_NULL_POINTER; if (crypto->gpgctx) { - return crypto->gpgctx; + *ctx = crypto->gpgctx; + return NOTMUCH_STATUS_SUCCESS; } /* TODO: GMimePasswordRequestFunc */ - gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); - if (! gpgctx) { - fprintf (stderr, "Failed to construct gpg context.\n"); - return NULL; + crypto->gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); + if (! crypto->gpgctx) { + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; } - crypto->gpgctx = gpgctx; - g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, TRUE); - g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, FALSE); + g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) crypto->gpgctx, TRUE); + g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) crypto->gpgctx, FALSE); - return crypto->gpgctx; + *ctx = crypto->gpgctx; + return NOTMUCH_STATUS_SUCCESS; } -/* Create a PKCS7 context (GMime 2.6) */ -static GMimeCryptoContext* -create_pkcs7_context (_notmuch_crypto_t *crypto) +/* Create or pass on a PKCS7 context (GMime 2.6) */ +static notmuch_status_t +get_pkcs7_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) { - GMimeCryptoContext *pkcs7ctx; + if (ctx == NULL || crypto == NULL) + return NOTMUCH_STATUS_NULL_POINTER; - if (crypto->pkcs7ctx) - return crypto->pkcs7ctx; + if (crypto->pkcs7ctx) { + *ctx = crypto->pkcs7ctx; + return NOTMUCH_STATUS_SUCCESS; + } /* TODO: GMimePasswordRequestFunc */ - pkcs7ctx = g_mime_pkcs7_context_new (NULL); - if (! pkcs7ctx) { - fprintf (stderr, "Failed to construct pkcs7 context.\n"); - return NULL; + crypto->pkcs7ctx = g_mime_pkcs7_context_new (NULL); + if (! crypto->pkcs7ctx) { + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; } - crypto->pkcs7ctx = pkcs7ctx; - g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx, + g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) crypto->pkcs7ctx, FALSE); - return crypto->pkcs7ctx; + *ctx = crypto->pkcs7ctx; + return NOTMUCH_STATUS_SUCCESS; } static const struct { const char *protocol; - GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); + notmuch_status_t (*get_context) (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx); } protocols[] = { { .protocol = "application/pgp-signature", - .get_context = create_gpg_context, + .get_context = get_gpg_context, }, { .protocol = "application/pgp-encrypted", - .get_context = create_gpg_context, + .get_context = get_gpg_context, }, { .protocol = "application/pkcs7-signature", - .get_context = create_pkcs7_context, + .get_context = get_pkcs7_context, }, { .protocol = "application/x-pkcs7-signature", - .get_context = create_pkcs7_context, + .get_context = get_pkcs7_context, }, }; /* for the specified protocol return the context pointer (initializing * if needed) */ -GMimeCryptoContext * -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) +notmuch_status_t +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, + const char *protocol, + GMimeCryptoContext **ctx) { - GMimeCryptoContext *cryptoctx = NULL; - size_t i; - - if (! protocol) { - fprintf (stderr, "Cryptographic protocol is empty.\n"); - return cryptoctx; - } + if (! protocol) + return NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL; /* As per RFC 1847 section 2.1: "the [protocol] value token is * comprised of the type and sub-type tokens of the Content-Type". @@ -114,15 +114,12 @@ _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protoc * parameter names as defined in this document are * case-insensitive." Thus, we use strcasecmp for the protocol. */ - for (i = 0; i < ARRAY_SIZE (protocols); i++) { + for (size_t i = 0; i < ARRAY_SIZE (protocols); i++) { if (strcasecmp (protocol, protocols[i].protocol) == 0) - return protocols[i].get_context (crypto); + return protocols[i].get_context (crypto, ctx); } - fprintf (stderr, "Unknown or unsupported cryptographic protocol %s.\n", - protocol); - - return NULL; + return NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL; } void diff --git a/util/crypto.h b/util/crypto.h index 6d15a6ae..d653ffb4 100644 --- a/util/crypto.h +++ b/util/crypto.h @@ -21,8 +21,10 @@ typedef struct _notmuch_crypto { #if (GMIME_MAJOR_VERSION < 3) -GMimeCryptoContext * -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); +notmuch_status_t +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, + const char *protocol, + GMimeCryptoContext **ctx); #endif void -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH 04/10] tests: prepare for more crypto tests (using add_gnupg_home) 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 02/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 03/10] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 05/10] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor ` (5 subsequent siblings) 8 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail Move add_gnupg_home to test-lib.sh to prepare it for reuse. --- test/T350-crypto.sh | 17 ----------------- test/test-lib.sh | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh index 1d408af7..e1b8fd83 100755 --- a/test/T350-crypto.sh +++ b/test/T350-crypto.sh @@ -7,23 +7,6 @@ test_description='PGP/MIME signature verification and decryption' . ./test-lib.sh || exit 1 -add_gnupg_home () -{ - local output - [ -d ${GNUPGHOME} ] && return - _gnupg_exit () { gpgconf --kill all 2>/dev/null || true; } - at_exit_function _gnupg_exit - mkdir -m 0700 "$GNUPGHOME" - gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 - test_debug "cat $GNUPGHOME/import.log" - if (gpg --quick-random --version >/dev/null 2>&1) ; then - echo quick-random >> "$GNUPGHOME"/gpg.conf - elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then - echo debug-quick-random >> "$GNUPGHOME"/gpg.conf - fi - echo no-emit-version >> "$GNUPGHOME"/gpg.conf -} - ################################################## add_gnupg_home diff --git a/test/test-lib.sh b/test/test-lib.sh index 35024649..b8427d97 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -93,6 +93,21 @@ unset GREP_OPTIONS # For emacsclient unset ALTERNATE_EDITOR +add_gnupg_home () +{ + local output + [ -d ${GNUPGHOME} ] && return + mkdir -m 0700 "$GNUPGHOME" + gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 + test_debug "cat $GNUPGHOME/import.log" + if (gpg --quick-random --version >/dev/null 2>&1) ; then + echo quick-random >> "$GNUPGHOME"/gpg.conf + elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then + echo debug-quick-random >> "$GNUPGHOME"/gpg.conf + fi + echo no-emit-version >> "$GNUPGHOME"/gpg.conf +} + # Each test should start with something like this, after copyright notices: # # test_description='Description of this test... -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH 05/10] index: implement notmuch_indexopts_t with try_decrypt 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor ` (2 preceding siblings ...) 2017-09-12 23:01 ` [PATCH 04/10] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 06/10] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor ` (4 subsequent siblings) 8 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail This is currently mostly a wrapper around _notmuch_crypto_t that keeps its internals private and doesn't expose any of the GMime API. However, non-crypto indexing options might also be added later (e.g. filters or other transformations). --- lib/add-message.cc | 9 ++++++++- lib/indexopts.c | 22 ++++++++++++++++++++-- lib/notmuch-private.h | 7 +++++++ lib/notmuch.h | 19 +++++++++++++++++++ 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index 73bde7fa..1fd91c14 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_status_t notmuch_database_index_file (notmuch_database_t *notmuch, const char *filename, - notmuch_indexopts_t unused (*indexopts), + notmuch_indexopts_t *indexopts, notmuch_message_t **message_ret) { notmuch_message_file_t *message_file; @@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2; notmuch_private_status_t private_status; notmuch_bool_t is_ghost = FALSE, is_new = FALSE; + notmuch_indexopts_t *def_indexopts = NULL; const char *date; const char *from, *to, *subject; @@ -540,6 +541,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, if (is_new || is_ghost) _notmuch_message_set_header_values (message, date, from, subject); + if (!indexopts) + indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); + ret = _notmuch_message_index_file (message, message_file); if (ret) goto DONE; @@ -557,6 +561,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, } DONE: + if (def_indexopts) + notmuch_indexopts_destroy (def_indexopts); + if (message) { if ((ret == NOTMUCH_STATUS_SUCCESS || ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret) diff --git a/lib/indexopts.c b/lib/indexopts.c index 2f9b841b..1162900c 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -21,9 +21,27 @@ #include "notmuch-private.h" notmuch_indexopts_t * -notmuch_database_get_default_indexopts (notmuch_database_t unused (*db)) +notmuch_database_get_default_indexopts (notmuch_database_t *db) { - return NULL; + return talloc_zero (db, notmuch_indexopts_t); +} + +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + notmuch_bool_t try_decrypt) +{ + if (!indexopts) + return NOTMUCH_STATUS_NULL_POINTER; + indexopts->crypto.decrypt = try_decrypt; + return NOTMUCH_STATUS_SUCCESS; +} + +notmuch_bool_t +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts) +{ + if (!indexopts) + return FALSE; + return indexopts->crypto.decrypt; } void diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index b187a80f..3168cf3c 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -51,6 +51,7 @@ NOTMUCH_BEGIN_DECLS #include "xutil.h" #include "error_util.h" #include "string-util.h" +#include "crypto.h" #ifdef DEBUG # define DEBUG_DATABASE_SANITY 1 @@ -632,6 +633,12 @@ _notmuch_thread_create (void *ctx, notmuch_exclude_t omit_exclude, notmuch_sort_t sort); +/* param.c */ + +typedef struct _notmuch_indexopts { + _notmuch_crypto_t crypto; +} notmuch_indexopts_t; + NOTMUCH_END_DECLS #ifdef __cplusplus diff --git a/lib/notmuch.h b/lib/notmuch.h index 6c76fb40..8baa76ab 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -2214,6 +2214,25 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list); notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db); +/** + * Specify whether to decrypt encrypted parts while indexing. + * + * Be aware that the index is likely sufficient to reconstruct the + * cleartext of the message itself, so please ensure that the notmuch + * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE + * without considering the security of your index. + */ +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + notmuch_bool_t try_decrypt); + +/** + * Return whether to decrypt encrypted parts while indexing. + * see notmuch_indexopts_set_try_decrypt. + */ +notmuch_bool_t +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts); + /** * Destroy a notmuch_indexopts_t object. * -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH 06/10] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor ` (3 preceding siblings ...) 2017-09-12 23:01 ` [PATCH 05/10] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 07/10] Define new config option index.try_decrypt Daniel Kahn Gillmor ` (3 subsequent siblings) 8 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail If we see index options that ask us to decrypt when indexing a message, and we encounter an encrypted part, we'll try to descend into it. If we can decrypt, we add the property index-decryption=success. If we can't decrypt (or recognize the encrypted type of mail), we add the property index-decryption=failure. Note that a single message may have both values of the "index-decryption" property: "success" and "failure". For example, consider a message that includes multiple layers of encryption. If we manage to decrypt the outer layer ("index-decryption=success"), but fail on the inner layer ("index-decryption=failure"). Before re-indexing, we wipe this automatically-added property, so that it will subsequently correspond to the actual semantics of the stored index. --- lib/add-message.cc | 2 +- lib/index.cc | 86 +++++++++++++++++++++++++++++++++++++++++++++++++-- lib/message.cc | 14 +++++++-- lib/notmuch-private.h | 1 + 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index 1fd91c14..66eb0a1f 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -544,7 +544,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, if (!indexopts) indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); - ret = _notmuch_message_index_file (message, message_file); + ret = _notmuch_message_index_file (message, indexopts, message_file); if (ret) goto DONE; diff --git a/lib/index.cc b/lib/index.cc index 3ed3b2c6..2efac307 100644 --- a/lib/index.cc +++ b/lib/index.cc @@ -351,9 +351,17 @@ _index_address_list (notmuch_message_t *message, } } +static void +_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts, +#if (GMIME_MAJOR_VERSION < 3) + GMimeContentType *content_type, +#endif + GMimeMultipartEncrypted *part); + /* Callback to generate terms for each mime part of a message. */ static void _index_mime_part (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, GMimeObject *part) { GMimeStream *stream, *filter; @@ -394,17 +402,23 @@ _index_mime_part (notmuch_message_t *message, application/pkcs7-signature, which is all gmime can handle anyway. */ _index_mime_part (message, + indexopts, g_mime_multipart_get_part (multipart, 0)); if (g_mime_multipart_get_count (multipart) > 2) _notmuch_database_log (_notmuch_message_database (message), "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n"); } else if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) { - /* Don't index encrypted parts */ _notmuch_message_add_term (message, "tag", "encrypted"); + _index_encrypted_mime_part(message, indexopts, +#if (GMIME_MAJOR_VERSION < 3) + content_type, +#endif + GMIME_MULTIPART_ENCRYPTED (part)); } else { for (i = 0; i < g_mime_multipart_get_count (multipart); i++) { _index_mime_part (message, + indexopts, g_mime_multipart_get_part (multipart, i)); } } @@ -416,7 +430,7 @@ _index_mime_part (notmuch_message_t *message, mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part)); - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); return; } @@ -486,8 +500,74 @@ _index_mime_part (notmuch_message_t *message, } } +/* descend (if desired) into the cleartext part of an encrypted MIME + * part while indexing. */ +static void +_index_encrypted_mime_part (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, +#if (GMIME_MAJOR_VERSION < 3) + GMimeContentType *content_type, +#endif + GMimeMultipartEncrypted *encrypted_data) +{ + notmuch_status_t status; +#if (GMIME_MAJOR_VERSION < 3) + GMimeCryptoContext* crypto_ctx = NULL; + const char *protocol = NULL; +#endif + GError *err = NULL; + notmuch_database_t * notmuch = NULL; + GMimeObject *clear = NULL; + + if (!indexopts || !notmuch_indexopts_get_try_decrypt (indexopts)) + return; + + notmuch = _notmuch_message_database (message); + +#if (GMIME_MAJOR_VERSION < 3) + protocol = g_mime_content_type_get_parameter (content_type, "protocol"); + status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto), + protocol, &crypto_ctx); + if (status) { + _notmuch_database_log (notmuch, "Warning: setup failed for decrypting " + "during indexing. (%d)\n", status); + status = notmuch_message_add_property (message, "index-decryption", "failure"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + return; + } +#endif + /* we don't need the GMimeDecryptResult, because we're not looking + * at validating signatures, and we don't care about indexing who + * the message was ostensibly encrypted to. + */ + clear = g_mime_multipart_encrypted_decrypt(encrypted_data, crypto_ctx, + NULL, &err); + if (err) { + _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n", + err->domain, err->code, err->message); + g_error_free(err); + /* Indicate that we failed to decrypt during indexing */ + status = notmuch_message_add_property (message, "index-decryption", "failure"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + return; + } + _index_mime_part (message, indexopts, clear); + g_object_unref (clear); + + status = notmuch_message_add_property (message, "index-decryption", "success"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + +} + notmuch_status_t _notmuch_message_index_file (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, notmuch_message_file_t *message_file) { GMimeMessage *mime_message; @@ -515,7 +595,7 @@ _notmuch_message_index_file (notmuch_message_t *message, subject = g_mime_message_get_subject (mime_message); _notmuch_message_gen_terms (message, "subject", subject); - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); return NOTMUCH_STATUS_SUCCESS; } diff --git a/lib/message.cc b/lib/message.cc index 0e3b5a4f..2ffa25a3 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -1961,7 +1961,7 @@ _notmuch_message_frozen (notmuch_message_t *message) notmuch_status_t notmuch_message_reindex (notmuch_message_t *message, - notmuch_indexopts_t unused (*indexopts)) + notmuch_indexopts_t *indexopts) { notmuch_database_t *notmuch = NULL; notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; @@ -1969,6 +1969,7 @@ notmuch_message_reindex (notmuch_message_t *message, notmuch_filenames_t *orig_filenames = NULL; const char *orig_thread_id = NULL; notmuch_message_file_t *message_file = NULL; + const char *autoproperties[] = { "index-decryption" }; int found = 0; @@ -1999,6 +2000,15 @@ notmuch_message_reindex (notmuch_message_t *message, goto DONE; } + /* all automatically-added properties should be removed before re-indexing */ + for (size_t i = 0; i < ARRAY_SIZE (autoproperties); i++) { + ret = notmuch_message_remove_all_properties (message, autoproperties[i]); + if (ret) { + INTERNAL_ERROR ("failed to remove automatically-added property '%s'", autoproperties[i]); + goto DONE; + } + } + /* re-add the filenames with the associated indexopts */ for (; notmuch_filenames_valid (orig_filenames); notmuch_filenames_move_to_next (orig_filenames)) { @@ -2038,7 +2048,7 @@ notmuch_message_reindex (notmuch_message_t *message, if (found == 0) _notmuch_message_set_header_values (message, date, from, subject); - ret = _notmuch_message_index_file (message, message_file); + ret = _notmuch_message_index_file (message, indexopts, message_file); if (ret == NOTMUCH_STATUS_FILE_ERROR) continue; diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index 3168cf3c..362106c8 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -447,6 +447,7 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch, notmuch_status_t _notmuch_message_index_file (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, notmuch_message_file_t *message_file); /* messages.c */ -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH 07/10] Define new config option index.try_decrypt 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor ` (4 preceding siblings ...) 2017-09-12 23:01 ` [PATCH 06/10] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:29 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 08/10] add --try-decrypt=(true|false) to notmuch new Daniel Kahn Gillmor ` (2 subsequent siblings) 8 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail By default, notmuch won't try to decrypt on indexing. With this patch, we make it possible to indicate a per-database preference using the config variable "index.try_decrypt", which by default will be false. --- doc/man1/notmuch-config.rst | 12 ++++++++++++ lib/indexopts.c | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index 6a51e64f..6f35d127 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -134,6 +134,18 @@ The available configuration items are described below. Default: ``gpg``. + **index.try_decrypt** + + When indexing an encrypted e-mail message, if this variable is + set to true, notmuch will try to decrypt the message and index + the cleartext. Be aware that the index is likely sufficient + to reconstruct the cleartext of the message itself, so please + ensure that the notmuch message index is adequately protected. + DO NOT USE ``index.try_decrypt=true`` without considering the + security of your index. + + Default: ``false``. + **built_with.<name>** Compile time feature <name>. Current possibilities include diff --git a/lib/indexopts.c b/lib/indexopts.c index 1162900c..5bd396ff 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -23,7 +23,21 @@ notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db) { - return talloc_zero (db, notmuch_indexopts_t); + notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t); + if (ret) { + char * try_decrypt; + notmuch_status_t err; + if (!(err = notmuch_database_get_config (db, "index.try_decrypt", &try_decrypt))) { + if (try_decrypt && + ((!(strcasecmp(try_decrypt, "true"))) || + (!(strcasecmp(try_decrypt, "yes"))) || + (!(strcasecmp(try_decrypt, "1"))))) + notmuch_indexopts_set_try_decrypt (ret, TRUE); + + free (try_decrypt); + } + } + return ret; } notmuch_status_t -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH 07/10] Define new config option index.try_decrypt 2017-09-12 23:01 ` [PATCH 07/10] Define new config option index.try_decrypt Daniel Kahn Gillmor @ 2017-09-12 23:29 ` Daniel Kahn Gillmor 0 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:29 UTC (permalink / raw) To: Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 731 bytes --] On Tue 2017-09-12 19:01:50 -0400, Daniel Kahn Gillmor wrote: > By default, notmuch won't try to decrypt on indexing. With this > patch, we make it possible to indicate a per-database preference using > the config variable "index.try_decrypt", which by default will be > false. I've just discovered that this particular patch doesn't work properly. Despite doing "notmuch config set index.try_decrypt true", i get the empty string when i invoke (in C): notmuch_config_get(db, "index.try_decrypt"). I think that means i've misunderstood the notmuch config subsystem, unfortunately. I would be happy to hear any suggestions for what i should be doing differently! The rest of the series should still hold up, though. --dkg [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH 08/10] add --try-decrypt=(true|false) to notmuch new 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor ` (5 preceding siblings ...) 2017-09-12 23:01 ` [PATCH 07/10] Define new config option index.try_decrypt Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 09/10] add --try-decrypt=(true|false) to notmuch insert Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 10/10] add --try-decrypt=(true|false) to notmuch reindex Daniel Kahn Gillmor 8 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail Try to decrypt any encrypted parts of newly-discovered messages while indexing them. The cleartext of any successfully-decrypted messages will be indexed, with tags applied in the same form as from notmuch insert --try-decrypt=true. Note: if the deprecated crypto.gpg_path configuration option is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). We also add a new test making use of this functionality. This requires a bit of reorganization, because we need to allow passing --long-arguments to "notmuch new" via emacs_fcc_message --- completion/notmuch-completion.bash | 13 ++++++++-- doc/man1/notmuch-new.rst | 12 +++++++++ notmuch-new.c | 29 +++++++++++++++++++++- test/T357-index-decryption.sh | 51 ++++++++++++++++++++++++++++++++++++++ test/test-lib.sh | 11 +++++++- 5 files changed, 112 insertions(+), 4 deletions(-) create mode 100755 test/T357-index-decryption.sh diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 5201be63..17be6b8f 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -311,11 +311,20 @@ _notmuch_insert() _notmuch_new() { local cur prev words cword split - _init_completion || return + _init_completion -s || return + + $split && + case "${prev}" in + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; + esac + ! $split && case "${cur}" in -*) - local options="--no-hooks --quiet ${_notmuch_shared_options}" + local options="--no-hooks --try-decrypt= --quiet ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "${options}" -- ${cur}) ) ;; diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst index 6acfa112..c255f980 100644 --- a/doc/man1/notmuch-new.rst +++ b/doc/man1/notmuch-new.rst @@ -43,6 +43,18 @@ Supported options for **new** include ``--quiet`` Do not print progress or results. + ``--try-decrypt=(true|false)`` + + If true, when encountering an encrypted message, try to + decrypt it while indexing. If decryption is successful, index + the cleartext itself. Be aware that the index is likely + sufficient to reconstruct the cleartext of the message itself, + so please ensure that the notmuch message index is adequately + protected. DO NOT USE ``--try-decrypt=true`` without + considering the security of your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + EXIT STATUS =========== diff --git a/notmuch-new.c b/notmuch-new.c index e011788d..b5405996 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -49,6 +49,7 @@ typedef struct { size_t new_tags_length; const char **new_ignore; size_t new_ignore_length; + notmuch_indexopts_t *indexopts; int total_files; int processed_files; @@ -261,7 +262,7 @@ add_file (notmuch_database_t *notmuch, const char *filename, if (status) goto DONE; - status = notmuch_database_index_file (notmuch, filename, NULL, &message); + status = notmuch_database_index_file (notmuch, filename, state->indexopts, &message); switch (status) { /* Success. */ case NOTMUCH_STATUS_SUCCESS: @@ -952,6 +953,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) unsigned int i; notmuch_bool_t timer_is_active = FALSE; notmuch_bool_t no_hooks = FALSE; + int try_decrypt = -1; notmuch_bool_t quiet = FALSE, verbose = FALSE; notmuch_status_t status; @@ -960,6 +962,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) { NOTMUCH_OPT_BOOLEAN, &verbose, "verbose", 'v', 0 }, { NOTMUCH_OPT_BOOLEAN, &add_files_state.debug, "debug", 'd', 0 }, { NOTMUCH_OPT_BOOLEAN, &no_hooks, "no-hooks", 'n', 0 }, + { NOTMUCH_OPT_BOOLEAN, &try_decrypt, "try-decrypt", 0, 0 }, { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -1077,6 +1080,30 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) if (notmuch == NULL) return EXIT_FAILURE; + add_files_state.indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!add_files_state.indexopts) { + fprintf (stderr, "Error: could not create index options.\n"); + return EXIT_FAILURE; + } + if (try_decrypt == TRUE || try_decrypt == FALSE) { + status = notmuch_indexopts_set_try_decrypt (add_files_state.indexopts, try_decrypt); + if (status != NOTMUCH_STATUS_SUCCESS) { + fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n", + try_decrypt ? "True" : "False", notmuch_status_to_string (status)); + notmuch_indexopts_destroy (add_files_state.indexopts); + return EXIT_FAILURE; + } + } +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (add_files_state.indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif + + /* Set up our handler for SIGINT. We do this after having * potentially done a database upgrade we this interrupt handler * won't support. */ diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh new file mode 100755 index 00000000..7bbd81f6 --- /dev/null +++ b/test/T357-index-decryption.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# TODO: test index-decryption-failed + +test_description='indexing decrypted mail' +. ./test-lib.sh || exit 1 + +################################################## + +add_gnupg_home +# get key fingerprint +FINGERPRINT=$(gpg --no-tty --list-secret-keys --with-colons --fingerprint | grep '^fpr:' | cut -d: -f10) + +# create a test encrypted message +test_begin_subtest 'emacs delivery of encrypted message' +test_expect_success \ +'emacs_fcc_message \ + "test encrypted message for cleartext index 001" \ + "This is a test encrypted message with a wumpus.\n" \ + "(mml-secure-message-encrypt)"' + +test_begin_subtest "search for unindexed cleartext" +output=$(notmuch search wumpus) +expected='' +test_expect_equal \ + "$output" \ + "$expected" + +# create a test encrypted message that is indexed in the clear +test_begin_subtest 'emacs delivery of encrypted message' +test_expect_success \ +'emacs_fcc_message --try-decrypt=true \ + "test encrypted message for cleartext index 002" \ + "This is a test encrypted message with a wumpus.\n" \ + "(mml-secure-message-encrypt)"' + +test_begin_subtest "emacs delivery of encrypted message, indexed cleartext" +output=$(notmuch search wumpus) +expected='thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property for one message" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + +test_done diff --git a/test/test-lib.sh b/test/test-lib.sh index b8427d97..6a47354f 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -338,8 +338,17 @@ emacs_deliver_message () # Accepts arbitrary extra emacs/elisp functions to modify the message # before sending, which is useful to doing things like attaching files # to the message and encrypting/signing. +# +# If any GNU-style long-arguments (like --quiet or --try-decrypt=true) are +# at the head of the argument list, they are sent directly to "notmuch +# new" after message delivery emacs_fcc_message () { + local nmn_args='' + while [[ "$1" =~ ^-- ]]; do + nmn_args="$nmn_args $1" + shift + done local subject="$1" local body="$2" shift 2 @@ -358,7 +367,7 @@ emacs_fcc_message () (insert \"${body}\") $@ (notmuch-mua-send-and-exit))" || return 1 - notmuch new >/dev/null + notmuch new $nmn_args >/dev/null } # Add an existing, fixed corpus of email to the database. -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH 09/10] add --try-decrypt=(true|false) to notmuch insert 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor ` (6 preceding siblings ...) 2017-09-12 23:01 ` [PATCH 08/10] add --try-decrypt=(true|false) to notmuch new Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 10/10] add --try-decrypt=(true|false) to notmuch reindex Daniel Kahn Gillmor 8 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail Allow an incoming message to be delivered while indexing the cleartext, on a per-message basis. This requires the secret keys for the message to be available. For the moment, the most functional approach is to ensure that gpg-agent is running and knows about any secret keys that might be useful to decrypt incoming mail. Any additional recommendations for how to phrase the caveat for this option are welcome. Note: if the deprecated crypto.gpg_path is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). --- completion/notmuch-completion.bash | 6 +++++- doc/man1/notmuch-insert.rst | 14 ++++++++++++++ notmuch-insert.c | 32 +++++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 17be6b8f..72a75a94 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -287,12 +287,16 @@ _notmuch_insert() sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) ) return ;; + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; esac ! $split && case "${cur}" in --*) - local options="--create-folder --folder= --keep --no-hooks ${_notmuch_shared_options}" + local options="--create-folder --folder= --keep --no-hooks --try-decrypt= ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) return diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst index f79600d6..647dac06 100644 --- a/doc/man1/notmuch-insert.rst +++ b/doc/man1/notmuch-insert.rst @@ -50,6 +50,20 @@ Supported options for **insert** include ``--no-hooks`` Prevent hooks from being run. + ``--try-decrypt=(true|false)`` + + If true and the message is encrypted, try to decrypt the + message while indexing. If decryption is successful, index + the cleartext itself. Either way, the message is always + stored to disk in its original form (ciphertext). Be aware + that the index is likely sufficient to reconstruct the + cleartext of the message itself, so please ensure that the + notmuch message index is adequately protected. DO NOT USE + ``--try-decrypt=true`` without considering the security of + your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + EXIT STATUS =========== diff --git a/notmuch-insert.c b/notmuch-insert.c index 648bd944..c46a3278 100644 --- a/notmuch-insert.c +++ b/notmuch-insert.c @@ -379,12 +379,13 @@ FAIL: */ static notmuch_status_t add_file (notmuch_database_t *notmuch, const char *path, tag_op_list_t *tag_ops, - notmuch_bool_t synchronize_flags, notmuch_bool_t keep) + notmuch_bool_t synchronize_flags, notmuch_bool_t keep, + notmuch_indexopts_t *indexopts) { notmuch_message_t *message; notmuch_status_t status; - status = notmuch_database_index_file (notmuch, path, NULL, &message); + status = notmuch_database_index_file (notmuch, path, indexopts, &message); if (status == NOTMUCH_STATUS_SUCCESS) { status = tag_op_list_apply (message, tag_ops, 0); if (status) { @@ -456,17 +457,20 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_bool_t create_folder = FALSE; notmuch_bool_t keep = FALSE; notmuch_bool_t no_hooks = FALSE; + int try_decrypt = -1; notmuch_bool_t synchronize_flags; const char *maildir; char *newpath; int opt_index; unsigned int i; + notmuch_indexopts_t *indexopts; notmuch_opt_desc_t options[] = { { NOTMUCH_OPT_STRING, &folder, "folder", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &create_folder, "create-folder", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &keep, "keep", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &no_hooks, "no-hooks", 'n', 0 }, + { NOTMUCH_OPT_BOOLEAN, &try_decrypt, "try-decrypt", 0, 0 }, { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, { NOTMUCH_OPT_END, 0, 0, 0, 0 } }; @@ -547,9 +551,31 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_exit_if_unmatched_db_uuid (notmuch); + indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!indexopts) { + fprintf (stderr, "Error: could not create index options.\n"); + return EXIT_FAILURE; + } + if (try_decrypt == TRUE || try_decrypt == FALSE) { + status = notmuch_indexopts_set_try_decrypt (indexopts, try_decrypt); + if (status != NOTMUCH_STATUS_SUCCESS) { + fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n", + try_decrypt ? "True" : "False", notmuch_status_to_string (status)); + notmuch_indexopts_destroy (indexopts); + return EXIT_FAILURE; + } + } +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif /* Index the message. */ - status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep); + status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, indexopts); /* Commit changes. */ close_status = notmuch_database_destroy (notmuch); -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH 10/10] add --try-decrypt=(true|false) to notmuch reindex 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor ` (7 preceding siblings ...) 2017-09-12 23:01 ` [PATCH 09/10] add --try-decrypt=(true|false) to notmuch insert Daniel Kahn Gillmor @ 2017-09-12 23:01 ` Daniel Kahn Gillmor 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor 8 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-12 23:01 UTC (permalink / raw) To: Notmuch Mail Try to decrypt any encrypted parts of newly-discovered messages while re-indexing them. The cleartext of any successfully-decrypted messages will be indexed, with tags applied in the same form as from notmuch insert --try-decrypt=true. Note: if the deprecated crypto.gpg_path configuration option is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). --- completion/notmuch-completion.bash | 10 +++++- doc/man1/notmuch-reindex.rst | 14 ++++++++ notmuch-reindex.c | 23 ++++++++++++++ test/T357-index-decryption.sh | 65 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 72a75a94..ef79affe 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -435,10 +435,18 @@ _notmuch_reindex() local cur prev words cword split _init_completion -s || return + $split && + case "${prev}" in + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; + esac + ! $split && case "${cur}" in -*) - local options="${_notmuch_shared_options}" + local options="--try-decrypt= ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) ;; diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst index e39cc4ee..60a060a7 100644 --- a/doc/man1/notmuch-reindex.rst +++ b/doc/man1/notmuch-reindex.rst @@ -19,6 +19,20 @@ The **reindex** command searches for all messages matching the supplied search terms, and re-creates the full-text index on these messages using the supplied options. +Supported options for **reindex** include + + ``--try-decrypt=(true|false)`` + + If true, when encountering an encrypted message, try to + decrypt it while reindexing. If decryption is successful, + index the cleartext itself. Be aware that the index is likely + sufficient to reconstruct the cleartext of the message itself, + so please ensure that the notmuch message index is adequately + protected. DO NOT USE ``--try-decrypt=true`` without + considering the security of your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + SEE ALSO ======== diff --git a/notmuch-reindex.c b/notmuch-reindex.c index bceac722..83cd0a57 100644 --- a/notmuch-reindex.c +++ b/notmuch-reindex.c @@ -90,6 +90,8 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) int opt_index; int ret; notmuch_indexopts_t *indexopts = NULL; + int try_decrypt = -1; + notmuch_status_t status; /* Set up our handler for SIGINT */ memset (&action, 0, sizeof (struct sigaction)); @@ -99,6 +101,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) sigaction (SIGINT, &action, NULL); notmuch_opt_desc_t options[] = { + { NOTMUCH_OPT_BOOLEAN, &try_decrypt, "try-decrypt", 0, 0 }, { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -115,6 +118,26 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_exit_if_unmatched_db_uuid (notmuch); + indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!indexopts) + return EXIT_FAILURE; + + if (try_decrypt == TRUE || try_decrypt == FALSE) { + status = notmuch_indexopts_set_try_decrypt (indexopts, try_decrypt); + if (status) + fprintf (stderr, "Warning: failed to set --try-decrypt to %d (%s)\n", + try_decrypt, notmuch_status_to_string (status)); + } + +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif + query_string = query_string_from_args (config, argc-opt_index, argv+opt_index); if (query_string == NULL) { fprintf (stderr, "Out of memory\n"); diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh index 7bbd81f6..3d18f5af 100755 --- a/test/T357-index-decryption.sh +++ b/test/T357-index-decryption.sh @@ -48,4 +48,69 @@ test_expect_equal \ "$output" \ "$expected" +# add a tag to all messages to ensure that it stays after reindexing +test_begin_subtest 'tagging all messages' +test_expect_success 'notmuch tag +blarney "encrypted message"' +test_begin_subtest "verify that tags have not changed" +output=$(notmuch search tag:blarney) +expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + +# see if first message shows up after reindexing with --try-decrypt=true (same $expected, untouched): +test_begin_subtest 'reindex old messages' +test_expect_success 'notmuch reindex --try-decrypt=true tag:encrypted and not property:index-decryption=success' +test_begin_subtest "reindexed encrypted message, including cleartext" +output=$(notmuch search wumpus) +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property for both messages" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + + +# try to remove cleartext indexing +test_begin_subtest 'reindex without cleartext' +test_expect_success 'notmuch reindex tag:encrypted and property:index-decryption=success' +test_begin_subtest "reindexed encrypted messages, without cleartext" +output=$(notmuch search wumpus) +expected='' +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property with both messages unindexed" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + +# ensure that the tags remain even when we are dropping the cleartext. +test_begin_subtest "verify that tags remain without cleartext" +output=$(notmuch search tag:blarney) +expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + + +# TODO: test removal of a message from the message store between +# indexing and reindexing. + +# TODO: insert the same message into the message store twice, index, +# remove one of them from the message store, and then reindex. +# reindexing should return a failure but the message should still be +# present? -- or what should the semantics be if you ask to reindex a +# message whose underlying files have been renamed or moved or +# removed? + test_done -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* cleartext-indexing 2017-09-12 23:01 ` [PATCH 10/10] add --try-decrypt=(true|false) to notmuch reindex Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 01/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor ` (10 more replies) 0 siblings, 11 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail This series is now rebased on top of Jani's unobjectionable 2-commit "lib: signature content type indexing" series (as well as my third commit on that thread), id:cover.1505329740.git.jani@nikula.org I welcome review -- some flavor of this series has been kicking around for over a year now, and it would be great to get more parts of it merged. Regards, --dkg ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 01/10] crypto: Move crypto.c into libutil 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-23 15:23 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 02/10] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor ` (9 subsequent siblings) 10 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail This prepares us for using the crypto object in both the library and the client. i've prefixed notmuch_crypto with _ to indicate that while this can be built into the library when needed, it's not something to be exported or used externally. --- Makefile.local | 1 - mime-node.c | 12 ++++++------ notmuch-client.h | 27 +++------------------------ notmuch-reply.c | 2 +- notmuch-show.c | 2 +- util/Makefile.local | 2 +- crypto.c => util/crypto.c | 45 +++++++++++++++++++++++++-------------------- util/crypto.h | 31 +++++++++++++++++++++++++++++++ 8 files changed, 68 insertions(+), 54 deletions(-) rename crypto.c => util/crypto.c (79%) create mode 100644 util/crypto.h diff --git a/Makefile.local b/Makefile.local index 9d9c52c2..9505b7fe 100644 --- a/Makefile.local +++ b/Makefile.local @@ -246,7 +246,6 @@ notmuch_client_srcs = \ sprinter-text.c \ query-string.c \ mime-node.c \ - crypto.c \ tag-util.c notmuch_client_modules = $(notmuch_client_srcs:.c=.o) diff --git a/mime-node.c b/mime-node.c index 24d73afa..d9ff7de1 100644 --- a/mime-node.c +++ b/mime-node.c @@ -33,7 +33,7 @@ typedef struct mime_node_context { GMimeMessage *mime_message; /* Context provided by the caller. */ - notmuch_crypto_t *crypto; + _notmuch_crypto_t *crypto; } mime_node_context_t; static int @@ -56,7 +56,7 @@ _mime_node_context_free (mime_node_context_t *res) notmuch_status_t mime_node_open (const void *ctx, notmuch_message_t *message, - notmuch_crypto_t *crypto, mime_node_t **root_out) + _notmuch_crypto_t *crypto, mime_node_t **root_out) { const char *filename = notmuch_message_get_filename (message); mime_node_context_t *mctx; @@ -171,7 +171,7 @@ set_signature_list_destructor (mime_node_t *node) /* Verify a signed mime node (GMime 2.6) */ static void node_verify (mime_node_t *node, GMimeObject *part, - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) { GError *err = NULL; @@ -192,7 +192,7 @@ node_verify (mime_node_t *node, GMimeObject *part, /* Decrypt and optionally verify an encrypted mime node (GMime 2.6) */ static void node_decrypt_and_verify (mime_node_t *node, GMimeObject *part, - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) { GError *err = NULL; GMimeDecryptResult *decrypt_result = NULL; @@ -227,7 +227,7 @@ static mime_node_t * _mime_node_create (mime_node_t *parent, GMimeObject *part) { mime_node_t *node = talloc_zero (parent, mime_node_t); - notmuch_crypto_context_t *cryptoctx = NULL; + GMimeCryptoContext *cryptoctx = NULL; /* Set basic node properties */ node->part = part; @@ -265,7 +265,7 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { GMimeContentType *content_type = g_mime_object_get_content_type (part); const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); - cryptoctx = notmuch_crypto_get_context (node->ctx->crypto, protocol); + cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); if (!cryptoctx) return NULL; } diff --git a/notmuch-client.h b/notmuch-client.h index 9d0f367d..76e69501 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -31,10 +31,6 @@ #include "gmime-extra.h" -typedef GMimeCryptoContext notmuch_crypto_context_t; -/* This is automatically included only since gmime 2.6.10 */ -#include <gmime/gmime-pkcs7-context.h> - #include "notmuch.h" /* This is separate from notmuch-private.h because we're trying to @@ -54,6 +50,7 @@ typedef GMimeCryptoContext notmuch_crypto_context_t; #include <ctype.h> #include "talloc-extra.h" +#include "crypto.h" #define unused(x) x __attribute__ ((unused)) @@ -71,22 +68,12 @@ typedef struct notmuch_show_format { const struct notmuch_show_params *params); } notmuch_show_format_t; -typedef struct notmuch_crypto { - notmuch_bool_t verify; - notmuch_bool_t decrypt; -#if (GMIME_MAJOR_VERSION < 3) - notmuch_crypto_context_t* gpgctx; - notmuch_crypto_context_t* pkcs7ctx; - const char *gpgpath; -#endif -} notmuch_crypto_t; - typedef struct notmuch_show_params { notmuch_bool_t entire_thread; notmuch_bool_t omit_excluded; notmuch_bool_t output_body; int part; - notmuch_crypto_t crypto; + _notmuch_crypto_t crypto; notmuch_bool_t include_html; GMimeStream *out_stream; } notmuch_show_params_t; @@ -180,14 +167,6 @@ typedef struct _notmuch_config notmuch_config_t; void notmuch_exit_if_unsupported_format (void); -#if (GMIME_MAJOR_VERSION <3) -notmuch_crypto_context_t * -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol); -#endif - -int -notmuch_crypto_cleanup (notmuch_crypto_t *crypto); - int notmuch_count_command (notmuch_config_t *config, int argc, char *argv[]); @@ -448,7 +427,7 @@ struct mime_node { */ notmuch_status_t mime_node_open (const void *ctx, notmuch_message_t *message, - notmuch_crypto_t *crypto, mime_node_t **node_out); + _notmuch_crypto_t *crypto, mime_node_t **node_out); /* Return a new MIME node for the requested child part of parent. * parent will be used as the talloc context for the returned child diff --git a/notmuch-reply.c b/notmuch-reply.c index 929f3077..00fff3ca 100644 --- a/notmuch-reply.c +++ b/notmuch-reply.c @@ -759,7 +759,7 @@ notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[]) if (do_reply (config, query, ¶ms, format, reply_all) != 0) return EXIT_FAILURE; - notmuch_crypto_cleanup (¶ms.crypto); + _notmuch_crypto_cleanup (¶ms.crypto); notmuch_query_destroy (query); notmuch_database_destroy (notmuch); diff --git a/notmuch-show.c b/notmuch-show.c index cdcc2a98..a35b11ad 100644 --- a/notmuch-show.c +++ b/notmuch-show.c @@ -1241,7 +1241,7 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[]) g_mime_stream_flush (params.out_stream); g_object_unref (params.out_stream); - notmuch_crypto_cleanup (¶ms.crypto); + _notmuch_crypto_cleanup (¶ms.crypto); notmuch_query_destroy (query); notmuch_database_destroy (notmuch); diff --git a/util/Makefile.local b/util/Makefile.local index 3027880b..ba03230e 100644 --- a/util/Makefile.local +++ b/util/Makefile.local @@ -5,7 +5,7 @@ extra_cflags += -I$(srcdir)/$(dir) libnotmuch_util_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \ $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \ - $(dir)/util.c $(dir)/gmime-extra.c + $(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c libnotmuch_util_modules := $(libnotmuch_util_c_srcs:.c=.o) diff --git a/crypto.c b/util/crypto.c similarity index 79% rename from crypto.c rename to util/crypto.c index cc45b885..97e8c8f4 100644 --- a/crypto.c +++ b/util/crypto.c @@ -16,18 +16,26 @@ * along with this program. If not, see https://www.gnu.org/licenses/ . * * Authors: Jameson Rollins <jrollins@finestructure.net> + * Daniel Kahn Gillmor <dkg@fifthhorseman.net> */ -#include "notmuch-client.h" +#include "notmuch.h" +#include "crypto.h" +#include "lib/notmuch-private.h" +#include <string.h> + +#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) + #if (GMIME_MAJOR_VERSION < 3) /* Create a GPG context (GMime 2.6) */ -static notmuch_crypto_context_t * -create_gpg_context (notmuch_crypto_t *crypto) +static GMimeCryptoContext* +create_gpg_context (_notmuch_crypto_t *crypto) { - notmuch_crypto_context_t *gpgctx; + GMimeCryptoContext *gpgctx; - if (crypto->gpgctx) + if (crypto->gpgctx) { return crypto->gpgctx; + } /* TODO: GMimePasswordRequestFunc */ gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); @@ -40,14 +48,14 @@ create_gpg_context (notmuch_crypto_t *crypto) g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, TRUE); g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, FALSE); - return gpgctx; + return crypto->gpgctx; } /* Create a PKCS7 context (GMime 2.6) */ -static notmuch_crypto_context_t * -create_pkcs7_context (notmuch_crypto_t *crypto) +static GMimeCryptoContext* +create_pkcs7_context (_notmuch_crypto_t *crypto) { - notmuch_crypto_context_t *pkcs7ctx; + GMimeCryptoContext *pkcs7ctx; if (crypto->pkcs7ctx) return crypto->pkcs7ctx; @@ -63,11 +71,11 @@ create_pkcs7_context (notmuch_crypto_t *crypto) g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx, FALSE); - return pkcs7ctx; + return crypto->pkcs7ctx; } static const struct { const char *protocol; - notmuch_crypto_context_t *(*get_context) (notmuch_crypto_t *crypto); + GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); } protocols[] = { { .protocol = "application/pgp-signature", @@ -89,10 +97,10 @@ static const struct { /* for the specified protocol return the context pointer (initializing * if needed) */ -notmuch_crypto_context_t * -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) +GMimeCryptoContext * +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) { - notmuch_crypto_context_t *cryptoctx = NULL; + GMimeCryptoContext *cryptoctx = NULL; size_t i; if (! protocol) { @@ -117,8 +125,8 @@ notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) return NULL; } -int -notmuch_crypto_cleanup (notmuch_crypto_t *crypto) +void +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto) { if (crypto->gpgctx) { g_object_unref (crypto->gpgctx); @@ -129,12 +137,9 @@ notmuch_crypto_cleanup (notmuch_crypto_t *crypto) g_object_unref (crypto->pkcs7ctx); crypto->pkcs7ctx = NULL; } - - return 0; } #else -int notmuch_crypto_cleanup (unused(notmuch_crypto_t *crypto)) +void _notmuch_crypto_cleanup (unused(_notmuch_crypto_t *crypto)) { - return 0; } #endif diff --git a/util/crypto.h b/util/crypto.h new file mode 100644 index 00000000..6d15a6ae --- /dev/null +++ b/util/crypto.h @@ -0,0 +1,31 @@ +#ifndef _CRYPTO_H +#define _CRYPTO_H + +#include "notmuch.h" +#if (GMIME_MAJOR_VERSION < 3) +#include "gmime-extra.h" +#include <gmime/gmime.h> +/* This is automatically included only since gmime 2.6.10 */ +#include <gmime/gmime-pkcs7-context.h> +#endif + +typedef struct _notmuch_crypto { + notmuch_bool_t verify; + notmuch_bool_t decrypt; +#if (GMIME_MAJOR_VERSION < 3) + GMimeCryptoContext* gpgctx; + GMimeCryptoContext* pkcs7ctx; + const char *gpgpath; +#endif +} _notmuch_crypto_t; + + +#if (GMIME_MAJOR_VERSION < 3) +GMimeCryptoContext * +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); +#endif + +void +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto); + +#endif -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v2 01/10] crypto: Move crypto.c into libutil 2017-09-15 5:53 ` [PATCH v2 01/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor @ 2017-09-23 15:23 ` Jani Nikula 0 siblings, 0 replies; 67+ messages in thread From: Jani Nikula @ 2017-09-23 15:23 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Fri, 15 Sep 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > This prepares us for using the crypto object in both the library and > the client. > > i've prefixed notmuch_crypto with _ to indicate that while this can be > built into the library when needed, it's not something to be exported > or used externally. You know, this would be considerably easier to review if this were split to separate patches: - prefixing notmuch_crypto_t and friends with _ - dropping notmuch_crypto_context_t in favour of using GMimeCryptoContext directly - moving the stuff to util - changing the notmuch_crypto_cleanup() return type I think the patch is fine, but I'd have much more confidence in each individual patch if this were split up than I have in everything together. BR, Jani. > --- > Makefile.local | 1 - > mime-node.c | 12 ++++++------ > notmuch-client.h | 27 +++------------------------ > notmuch-reply.c | 2 +- > notmuch-show.c | 2 +- > util/Makefile.local | 2 +- > crypto.c => util/crypto.c | 45 +++++++++++++++++++++++++-------------------- > util/crypto.h | 31 +++++++++++++++++++++++++++++++ > 8 files changed, 68 insertions(+), 54 deletions(-) > rename crypto.c => util/crypto.c (79%) > create mode 100644 util/crypto.h > > diff --git a/Makefile.local b/Makefile.local > index 9d9c52c2..9505b7fe 100644 > --- a/Makefile.local > +++ b/Makefile.local > @@ -246,7 +246,6 @@ notmuch_client_srcs = \ > sprinter-text.c \ > query-string.c \ > mime-node.c \ > - crypto.c \ > tag-util.c > > notmuch_client_modules = $(notmuch_client_srcs:.c=.o) > diff --git a/mime-node.c b/mime-node.c > index 24d73afa..d9ff7de1 100644 > --- a/mime-node.c > +++ b/mime-node.c > @@ -33,7 +33,7 @@ typedef struct mime_node_context { > GMimeMessage *mime_message; > > /* Context provided by the caller. */ > - notmuch_crypto_t *crypto; > + _notmuch_crypto_t *crypto; > } mime_node_context_t; > > static int > @@ -56,7 +56,7 @@ _mime_node_context_free (mime_node_context_t *res) > > notmuch_status_t > mime_node_open (const void *ctx, notmuch_message_t *message, > - notmuch_crypto_t *crypto, mime_node_t **root_out) > + _notmuch_crypto_t *crypto, mime_node_t **root_out) > { > const char *filename = notmuch_message_get_filename (message); > mime_node_context_t *mctx; > @@ -171,7 +171,7 @@ set_signature_list_destructor (mime_node_t *node) > /* Verify a signed mime node (GMime 2.6) */ > static void > node_verify (mime_node_t *node, GMimeObject *part, > - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) > + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) > { > GError *err = NULL; > > @@ -192,7 +192,7 @@ node_verify (mime_node_t *node, GMimeObject *part, > /* Decrypt and optionally verify an encrypted mime node (GMime 2.6) */ > static void > node_decrypt_and_verify (mime_node_t *node, GMimeObject *part, > - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) > + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) > { > GError *err = NULL; > GMimeDecryptResult *decrypt_result = NULL; > @@ -227,7 +227,7 @@ static mime_node_t * > _mime_node_create (mime_node_t *parent, GMimeObject *part) > { > mime_node_t *node = talloc_zero (parent, mime_node_t); > - notmuch_crypto_context_t *cryptoctx = NULL; > + GMimeCryptoContext *cryptoctx = NULL; > > /* Set basic node properties */ > node->part = part; > @@ -265,7 +265,7 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) > || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { > GMimeContentType *content_type = g_mime_object_get_content_type (part); > const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); > - cryptoctx = notmuch_crypto_get_context (node->ctx->crypto, protocol); > + cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); > if (!cryptoctx) > return NULL; > } > diff --git a/notmuch-client.h b/notmuch-client.h > index 9d0f367d..76e69501 100644 > --- a/notmuch-client.h > +++ b/notmuch-client.h > @@ -31,10 +31,6 @@ > > #include "gmime-extra.h" > > -typedef GMimeCryptoContext notmuch_crypto_context_t; > -/* This is automatically included only since gmime 2.6.10 */ > -#include <gmime/gmime-pkcs7-context.h> > - > #include "notmuch.h" > > /* This is separate from notmuch-private.h because we're trying to > @@ -54,6 +50,7 @@ typedef GMimeCryptoContext notmuch_crypto_context_t; > #include <ctype.h> > > #include "talloc-extra.h" > +#include "crypto.h" > > #define unused(x) x __attribute__ ((unused)) > > @@ -71,22 +68,12 @@ typedef struct notmuch_show_format { > const struct notmuch_show_params *params); > } notmuch_show_format_t; > > -typedef struct notmuch_crypto { > - notmuch_bool_t verify; > - notmuch_bool_t decrypt; > -#if (GMIME_MAJOR_VERSION < 3) > - notmuch_crypto_context_t* gpgctx; > - notmuch_crypto_context_t* pkcs7ctx; > - const char *gpgpath; > -#endif > -} notmuch_crypto_t; > - > typedef struct notmuch_show_params { > notmuch_bool_t entire_thread; > notmuch_bool_t omit_excluded; > notmuch_bool_t output_body; > int part; > - notmuch_crypto_t crypto; > + _notmuch_crypto_t crypto; > notmuch_bool_t include_html; > GMimeStream *out_stream; > } notmuch_show_params_t; > @@ -180,14 +167,6 @@ typedef struct _notmuch_config notmuch_config_t; > void > notmuch_exit_if_unsupported_format (void); > > -#if (GMIME_MAJOR_VERSION <3) > -notmuch_crypto_context_t * > -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol); > -#endif > - > -int > -notmuch_crypto_cleanup (notmuch_crypto_t *crypto); > - > int > notmuch_count_command (notmuch_config_t *config, int argc, char *argv[]); > > @@ -448,7 +427,7 @@ struct mime_node { > */ > notmuch_status_t > mime_node_open (const void *ctx, notmuch_message_t *message, > - notmuch_crypto_t *crypto, mime_node_t **node_out); > + _notmuch_crypto_t *crypto, mime_node_t **node_out); > > /* Return a new MIME node for the requested child part of parent. > * parent will be used as the talloc context for the returned child > diff --git a/notmuch-reply.c b/notmuch-reply.c > index 929f3077..00fff3ca 100644 > --- a/notmuch-reply.c > +++ b/notmuch-reply.c > @@ -759,7 +759,7 @@ notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[]) > if (do_reply (config, query, ¶ms, format, reply_all) != 0) > return EXIT_FAILURE; > > - notmuch_crypto_cleanup (¶ms.crypto); > + _notmuch_crypto_cleanup (¶ms.crypto); > notmuch_query_destroy (query); > notmuch_database_destroy (notmuch); > > diff --git a/notmuch-show.c b/notmuch-show.c > index cdcc2a98..a35b11ad 100644 > --- a/notmuch-show.c > +++ b/notmuch-show.c > @@ -1241,7 +1241,7 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[]) > g_mime_stream_flush (params.out_stream); > g_object_unref (params.out_stream); > > - notmuch_crypto_cleanup (¶ms.crypto); > + _notmuch_crypto_cleanup (¶ms.crypto); > notmuch_query_destroy (query); > notmuch_database_destroy (notmuch); > > diff --git a/util/Makefile.local b/util/Makefile.local > index 3027880b..ba03230e 100644 > --- a/util/Makefile.local > +++ b/util/Makefile.local > @@ -5,7 +5,7 @@ extra_cflags += -I$(srcdir)/$(dir) > > libnotmuch_util_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \ > $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \ > - $(dir)/util.c $(dir)/gmime-extra.c > + $(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c > > libnotmuch_util_modules := $(libnotmuch_util_c_srcs:.c=.o) > > diff --git a/crypto.c b/util/crypto.c > similarity index 79% > rename from crypto.c > rename to util/crypto.c > index cc45b885..97e8c8f4 100644 > --- a/crypto.c > +++ b/util/crypto.c > @@ -16,18 +16,26 @@ > * along with this program. If not, see https://www.gnu.org/licenses/ . > * > * Authors: Jameson Rollins <jrollins@finestructure.net> > + * Daniel Kahn Gillmor <dkg@fifthhorseman.net> > */ > > -#include "notmuch-client.h" > +#include "notmuch.h" > +#include "crypto.h" > +#include "lib/notmuch-private.h" > +#include <string.h> > + > +#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) > + > #if (GMIME_MAJOR_VERSION < 3) > /* Create a GPG context (GMime 2.6) */ > -static notmuch_crypto_context_t * > -create_gpg_context (notmuch_crypto_t *crypto) > +static GMimeCryptoContext* > +create_gpg_context (_notmuch_crypto_t *crypto) > { > - notmuch_crypto_context_t *gpgctx; > + GMimeCryptoContext *gpgctx; > > - if (crypto->gpgctx) > + if (crypto->gpgctx) { > return crypto->gpgctx; > + } > > /* TODO: GMimePasswordRequestFunc */ > gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); > @@ -40,14 +48,14 @@ create_gpg_context (notmuch_crypto_t *crypto) > g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, TRUE); > g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, FALSE); > > - return gpgctx; > + return crypto->gpgctx; > } > > /* Create a PKCS7 context (GMime 2.6) */ > -static notmuch_crypto_context_t * > -create_pkcs7_context (notmuch_crypto_t *crypto) > +static GMimeCryptoContext* > +create_pkcs7_context (_notmuch_crypto_t *crypto) > { > - notmuch_crypto_context_t *pkcs7ctx; > + GMimeCryptoContext *pkcs7ctx; > > if (crypto->pkcs7ctx) > return crypto->pkcs7ctx; > @@ -63,11 +71,11 @@ create_pkcs7_context (notmuch_crypto_t *crypto) > g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx, > FALSE); > > - return pkcs7ctx; > + return crypto->pkcs7ctx; > } > static const struct { > const char *protocol; > - notmuch_crypto_context_t *(*get_context) (notmuch_crypto_t *crypto); > + GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); > } protocols[] = { > { > .protocol = "application/pgp-signature", > @@ -89,10 +97,10 @@ static const struct { > > /* for the specified protocol return the context pointer (initializing > * if needed) */ > -notmuch_crypto_context_t * > -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) > +GMimeCryptoContext * > +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) > { > - notmuch_crypto_context_t *cryptoctx = NULL; > + GMimeCryptoContext *cryptoctx = NULL; > size_t i; > > if (! protocol) { > @@ -117,8 +125,8 @@ notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) > return NULL; > } > > -int > -notmuch_crypto_cleanup (notmuch_crypto_t *crypto) > +void > +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto) > { > if (crypto->gpgctx) { > g_object_unref (crypto->gpgctx); > @@ -129,12 +137,9 @@ notmuch_crypto_cleanup (notmuch_crypto_t *crypto) > g_object_unref (crypto->pkcs7ctx); > crypto->pkcs7ctx = NULL; > } > - > - return 0; > } > #else > -int notmuch_crypto_cleanup (unused(notmuch_crypto_t *crypto)) > +void _notmuch_crypto_cleanup (unused(_notmuch_crypto_t *crypto)) > { > - return 0; > } > #endif > diff --git a/util/crypto.h b/util/crypto.h > new file mode 100644 > index 00000000..6d15a6ae > --- /dev/null > +++ b/util/crypto.h > @@ -0,0 +1,31 @@ > +#ifndef _CRYPTO_H > +#define _CRYPTO_H > + > +#include "notmuch.h" > +#if (GMIME_MAJOR_VERSION < 3) > +#include "gmime-extra.h" > +#include <gmime/gmime.h> > +/* This is automatically included only since gmime 2.6.10 */ > +#include <gmime/gmime-pkcs7-context.h> > +#endif > + > +typedef struct _notmuch_crypto { > + notmuch_bool_t verify; > + notmuch_bool_t decrypt; > +#if (GMIME_MAJOR_VERSION < 3) > + GMimeCryptoContext* gpgctx; > + GMimeCryptoContext* pkcs7ctx; > + const char *gpgpath; > +#endif > +} _notmuch_crypto_t; > + > + > +#if (GMIME_MAJOR_VERSION < 3) > +GMimeCryptoContext * > +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); > +#endif > + > +void > +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto); > + > +#endif > -- > 2.14.1 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 02/10] crypto: make shared crypto code behave library-like 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 01/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-23 15:36 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 03/10] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor ` (8 subsequent siblings) 10 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail If we're going to reuse the crypto code across both the library and the client, then it needs to report error states properly and not write to stderr. --- lib/database.cc | 6 ++++ lib/notmuch.h | 17 +++++++++++ mime-node.c | 7 ++++- util/crypto.c | 89 ++++++++++++++++++++++++++++----------------------------- util/crypto.h | 6 ++-- 5 files changed, 76 insertions(+), 49 deletions(-) diff --git a/lib/database.cc b/lib/database.cc index 79eb3d69..82a3d463 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -413,6 +413,12 @@ notmuch_status_to_string (notmuch_status_t status) return "Operation requires a database upgrade"; case NOTMUCH_STATUS_PATH_ERROR: return "Path supplied is illegal for this function"; + case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL: + return "Crypto protocol missing, malformed, or unintelligible"; + case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION: + return "Crypto engine initialization failure"; + case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL: + return "Unknown crypto protocol"; default: case NOTMUCH_STATUS_LAST_STATUS: return "Unknown error status value"; diff --git a/lib/notmuch.h b/lib/notmuch.h index f26565f3..6c76fb40 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -191,6 +191,23 @@ typedef enum _notmuch_status { * function, in a way not covered by a more specific argument. */ NOTMUCH_STATUS_ILLEGAL_ARGUMENT, + /** + * A MIME object claimed to have cryptographic protection which + * notmuch tried to handle, but the protocol was not specified in + * an intelligible way. + */ + NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL, + /** + * Notmuch attempted to do crypto processing, but could not + * initialize the engine needed to do so. + */ + NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION, + /** + * A MIME object claimed to have cryptographic protection, and + * notmuch attempted to process it, but the specific protocol was + * something that notmuch doesn't know how to handle. + */ + NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL, /** * Not an actual status value. Just a way to find out how many * valid status values there are. diff --git a/mime-node.c b/mime-node.c index d9ff7de1..6cd7d2de 100644 --- a/mime-node.c +++ b/mime-node.c @@ -265,7 +265,12 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { GMimeContentType *content_type = g_mime_object_get_content_type (part); const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); - cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); + notmuch_status_t status; + status = _notmuch_crypto_get_gmime_ctx_for_protocol (node->ctx->crypto, + protocol, &cryptoctx); + if (status) /* this is a warning, not an error */ + fprintf (stderr, "Warning: %s (%s).\n", notmuch_status_to_string (status), + protocol ? protocol : "(NULL)"); if (!cryptoctx) return NULL; } diff --git a/util/crypto.c b/util/crypto.c index 97e8c8f4..e7908197 100644 --- a/util/crypto.c +++ b/util/crypto.c @@ -27,86 +27,86 @@ #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) #if (GMIME_MAJOR_VERSION < 3) -/* Create a GPG context (GMime 2.6) */ -static GMimeCryptoContext* -create_gpg_context (_notmuch_crypto_t *crypto) +/* Create or pass on a GPG context (GMime 2.6) */ +static notmuch_status_t +get_gpg_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) { - GMimeCryptoContext *gpgctx; + if (ctx == NULL || crypto == NULL) + return NOTMUCH_STATUS_NULL_POINTER; if (crypto->gpgctx) { - return crypto->gpgctx; + *ctx = crypto->gpgctx; + return NOTMUCH_STATUS_SUCCESS; } /* TODO: GMimePasswordRequestFunc */ - gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); - if (! gpgctx) { - fprintf (stderr, "Failed to construct gpg context.\n"); - return NULL; + crypto->gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); + if (! crypto->gpgctx) { + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; } - crypto->gpgctx = gpgctx; - g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, TRUE); - g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, FALSE); + g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) crypto->gpgctx, TRUE); + g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) crypto->gpgctx, FALSE); - return crypto->gpgctx; + *ctx = crypto->gpgctx; + return NOTMUCH_STATUS_SUCCESS; } -/* Create a PKCS7 context (GMime 2.6) */ -static GMimeCryptoContext* -create_pkcs7_context (_notmuch_crypto_t *crypto) +/* Create or pass on a PKCS7 context (GMime 2.6) */ +static notmuch_status_t +get_pkcs7_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) { - GMimeCryptoContext *pkcs7ctx; + if (ctx == NULL || crypto == NULL) + return NOTMUCH_STATUS_NULL_POINTER; - if (crypto->pkcs7ctx) - return crypto->pkcs7ctx; + if (crypto->pkcs7ctx) { + *ctx = crypto->pkcs7ctx; + return NOTMUCH_STATUS_SUCCESS; + } /* TODO: GMimePasswordRequestFunc */ - pkcs7ctx = g_mime_pkcs7_context_new (NULL); - if (! pkcs7ctx) { - fprintf (stderr, "Failed to construct pkcs7 context.\n"); - return NULL; + crypto->pkcs7ctx = g_mime_pkcs7_context_new (NULL); + if (! crypto->pkcs7ctx) { + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; } - crypto->pkcs7ctx = pkcs7ctx; - g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx, + g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) crypto->pkcs7ctx, FALSE); - return crypto->pkcs7ctx; + *ctx = crypto->pkcs7ctx; + return NOTMUCH_STATUS_SUCCESS; } static const struct { const char *protocol; - GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); + notmuch_status_t (*get_context) (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx); } protocols[] = { { .protocol = "application/pgp-signature", - .get_context = create_gpg_context, + .get_context = get_gpg_context, }, { .protocol = "application/pgp-encrypted", - .get_context = create_gpg_context, + .get_context = get_gpg_context, }, { .protocol = "application/pkcs7-signature", - .get_context = create_pkcs7_context, + .get_context = get_pkcs7_context, }, { .protocol = "application/x-pkcs7-signature", - .get_context = create_pkcs7_context, + .get_context = get_pkcs7_context, }, }; /* for the specified protocol return the context pointer (initializing * if needed) */ -GMimeCryptoContext * -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) +notmuch_status_t +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, + const char *protocol, + GMimeCryptoContext **ctx) { - GMimeCryptoContext *cryptoctx = NULL; - size_t i; - - if (! protocol) { - fprintf (stderr, "Cryptographic protocol is empty.\n"); - return cryptoctx; - } + if (! protocol) + return NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL; /* As per RFC 1847 section 2.1: "the [protocol] value token is * comprised of the type and sub-type tokens of the Content-Type". @@ -114,15 +114,12 @@ _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protoc * parameter names as defined in this document are * case-insensitive." Thus, we use strcasecmp for the protocol. */ - for (i = 0; i < ARRAY_SIZE (protocols); i++) { + for (size_t i = 0; i < ARRAY_SIZE (protocols); i++) { if (strcasecmp (protocol, protocols[i].protocol) == 0) - return protocols[i].get_context (crypto); + return protocols[i].get_context (crypto, ctx); } - fprintf (stderr, "Unknown or unsupported cryptographic protocol %s.\n", - protocol); - - return NULL; + return NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL; } void diff --git a/util/crypto.h b/util/crypto.h index 6d15a6ae..d653ffb4 100644 --- a/util/crypto.h +++ b/util/crypto.h @@ -21,8 +21,10 @@ typedef struct _notmuch_crypto { #if (GMIME_MAJOR_VERSION < 3) -GMimeCryptoContext * -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); +notmuch_status_t +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, + const char *protocol, + GMimeCryptoContext **ctx); #endif void -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v2 02/10] crypto: make shared crypto code behave library-like 2017-09-15 5:53 ` [PATCH v2 02/10] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor @ 2017-09-23 15:36 ` Jani Nikula 2017-10-10 3:33 ` Daniel Kahn Gillmor 0 siblings, 1 reply; 67+ messages in thread From: Jani Nikula @ 2017-09-23 15:36 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Fri, 15 Sep 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > If we're going to reuse the crypto code across both the library and > the client, then it needs to report error states properly and not > write to stderr. > --- > lib/database.cc | 6 ++++ > lib/notmuch.h | 17 +++++++++++ > mime-node.c | 7 ++++- > util/crypto.c | 89 ++++++++++++++++++++++++++++----------------------------- > util/crypto.h | 6 ++-- > 5 files changed, 76 insertions(+), 49 deletions(-) > > diff --git a/lib/database.cc b/lib/database.cc > index 79eb3d69..82a3d463 100644 > --- a/lib/database.cc > +++ b/lib/database.cc > @@ -413,6 +413,12 @@ notmuch_status_to_string (notmuch_status_t status) > return "Operation requires a database upgrade"; > case NOTMUCH_STATUS_PATH_ERROR: > return "Path supplied is illegal for this function"; > + case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL: > + return "Crypto protocol missing, malformed, or unintelligible"; > + case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION: > + return "Crypto engine initialization failure"; > + case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL: > + return "Unknown crypto protocol"; > default: > case NOTMUCH_STATUS_LAST_STATUS: > return "Unknown error status value"; > diff --git a/lib/notmuch.h b/lib/notmuch.h > index f26565f3..6c76fb40 100644 > --- a/lib/notmuch.h > +++ b/lib/notmuch.h > @@ -191,6 +191,23 @@ typedef enum _notmuch_status { > * function, in a way not covered by a more specific argument. > */ > NOTMUCH_STATUS_ILLEGAL_ARGUMENT, > + /** > + * A MIME object claimed to have cryptographic protection which > + * notmuch tried to handle, but the protocol was not specified in > + * an intelligible way. > + */ > + NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL, > + /** > + * Notmuch attempted to do crypto processing, but could not > + * initialize the engine needed to do so. > + */ > + NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION, > + /** > + * A MIME object claimed to have cryptographic protection, and > + * notmuch attempted to process it, but the specific protocol was > + * something that notmuch doesn't know how to handle. > + */ > + NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL, > /** > * Not an actual status value. Just a way to find out how many > * valid status values there are. > diff --git a/mime-node.c b/mime-node.c > index d9ff7de1..6cd7d2de 100644 > --- a/mime-node.c > +++ b/mime-node.c > @@ -265,7 +265,12 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) > || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { > GMimeContentType *content_type = g_mime_object_get_content_type (part); > const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); > - cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); > + notmuch_status_t status; > + status = _notmuch_crypto_get_gmime_ctx_for_protocol (node->ctx->crypto, > + protocol, &cryptoctx); > + if (status) /* this is a warning, not an error */ > + fprintf (stderr, "Warning: %s (%s).\n", notmuch_status_to_string (status), > + protocol ? protocol : "(NULL)"); For NULL protocol this will print "((NULL))". > if (!cryptoctx) > return NULL; I guess this will work because we initialize cryptoctx to NULL, but if we return the status, I think we should trust status == success means cryptoctx is fine, and otherwise we shouldn't touch or look at cryptoctx. Other than that, LGTM. BR, Jani. > } > diff --git a/util/crypto.c b/util/crypto.c > index 97e8c8f4..e7908197 100644 > --- a/util/crypto.c > +++ b/util/crypto.c > @@ -27,86 +27,86 @@ > #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) > > #if (GMIME_MAJOR_VERSION < 3) > -/* Create a GPG context (GMime 2.6) */ > -static GMimeCryptoContext* > -create_gpg_context (_notmuch_crypto_t *crypto) > +/* Create or pass on a GPG context (GMime 2.6) */ > +static notmuch_status_t > +get_gpg_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) > { > - GMimeCryptoContext *gpgctx; > + if (ctx == NULL || crypto == NULL) > + return NOTMUCH_STATUS_NULL_POINTER; > > if (crypto->gpgctx) { > - return crypto->gpgctx; > + *ctx = crypto->gpgctx; > + return NOTMUCH_STATUS_SUCCESS; > } > > /* TODO: GMimePasswordRequestFunc */ > - gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); > - if (! gpgctx) { > - fprintf (stderr, "Failed to construct gpg context.\n"); > - return NULL; > + crypto->gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); > + if (! crypto->gpgctx) { > + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; > } > - crypto->gpgctx = gpgctx; > > - g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, TRUE); > - g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, FALSE); > + g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) crypto->gpgctx, TRUE); > + g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) crypto->gpgctx, FALSE); > > - return crypto->gpgctx; > + *ctx = crypto->gpgctx; > + return NOTMUCH_STATUS_SUCCESS; > } > > -/* Create a PKCS7 context (GMime 2.6) */ > -static GMimeCryptoContext* > -create_pkcs7_context (_notmuch_crypto_t *crypto) > +/* Create or pass on a PKCS7 context (GMime 2.6) */ > +static notmuch_status_t > +get_pkcs7_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) > { > - GMimeCryptoContext *pkcs7ctx; > + if (ctx == NULL || crypto == NULL) > + return NOTMUCH_STATUS_NULL_POINTER; > > - if (crypto->pkcs7ctx) > - return crypto->pkcs7ctx; > + if (crypto->pkcs7ctx) { > + *ctx = crypto->pkcs7ctx; > + return NOTMUCH_STATUS_SUCCESS; > + } > > /* TODO: GMimePasswordRequestFunc */ > - pkcs7ctx = g_mime_pkcs7_context_new (NULL); > - if (! pkcs7ctx) { > - fprintf (stderr, "Failed to construct pkcs7 context.\n"); > - return NULL; > + crypto->pkcs7ctx = g_mime_pkcs7_context_new (NULL); > + if (! crypto->pkcs7ctx) { > + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; > } > - crypto->pkcs7ctx = pkcs7ctx; > > - g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx, > + g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) crypto->pkcs7ctx, > FALSE); > > - return crypto->pkcs7ctx; > + *ctx = crypto->pkcs7ctx; > + return NOTMUCH_STATUS_SUCCESS; > } > static const struct { > const char *protocol; > - GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); > + notmuch_status_t (*get_context) (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx); > } protocols[] = { > { > .protocol = "application/pgp-signature", > - .get_context = create_gpg_context, > + .get_context = get_gpg_context, > }, > { > .protocol = "application/pgp-encrypted", > - .get_context = create_gpg_context, > + .get_context = get_gpg_context, > }, > { > .protocol = "application/pkcs7-signature", > - .get_context = create_pkcs7_context, > + .get_context = get_pkcs7_context, > }, > { > .protocol = "application/x-pkcs7-signature", > - .get_context = create_pkcs7_context, > + .get_context = get_pkcs7_context, > }, > }; > > /* for the specified protocol return the context pointer (initializing > * if needed) */ > -GMimeCryptoContext * > -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) > +notmuch_status_t > +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, > + const char *protocol, > + GMimeCryptoContext **ctx) > { > - GMimeCryptoContext *cryptoctx = NULL; > - size_t i; > - > - if (! protocol) { > - fprintf (stderr, "Cryptographic protocol is empty.\n"); > - return cryptoctx; > - } > + if (! protocol) > + return NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL; > > /* As per RFC 1847 section 2.1: "the [protocol] value token is > * comprised of the type and sub-type tokens of the Content-Type". > @@ -114,15 +114,12 @@ _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protoc > * parameter names as defined in this document are > * case-insensitive." Thus, we use strcasecmp for the protocol. > */ > - for (i = 0; i < ARRAY_SIZE (protocols); i++) { > + for (size_t i = 0; i < ARRAY_SIZE (protocols); i++) { > if (strcasecmp (protocol, protocols[i].protocol) == 0) > - return protocols[i].get_context (crypto); > + return protocols[i].get_context (crypto, ctx); > } > > - fprintf (stderr, "Unknown or unsupported cryptographic protocol %s.\n", > - protocol); > - > - return NULL; > + return NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL; > } > > void > diff --git a/util/crypto.h b/util/crypto.h > index 6d15a6ae..d653ffb4 100644 > --- a/util/crypto.h > +++ b/util/crypto.h > @@ -21,8 +21,10 @@ typedef struct _notmuch_crypto { > > > #if (GMIME_MAJOR_VERSION < 3) > -GMimeCryptoContext * > -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); > +notmuch_status_t > +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, > + const char *protocol, > + GMimeCryptoContext **ctx); > #endif > > void > -- > 2.14.1 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v2 02/10] crypto: make shared crypto code behave library-like 2017-09-23 15:36 ` Jani Nikula @ 2017-10-10 3:33 ` Daniel Kahn Gillmor 0 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 3:33 UTC (permalink / raw) To: Jani Nikula, Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 904 bytes --] On Sat 2017-09-23 18:36:18 +0300, Jani Nikula wrote: >> if (!cryptoctx) >> return NULL; > > I guess this will work because we initialize cryptoctx to NULL, but if > we return the status, I think we should trust status == success means > cryptoctx is fine, and otherwise we shouldn't touch or look at > cryptoctx. this function (_mime_node_create) is *not* returning the status -- it's returning the mime node. a non-successful status might or might not mean that the cryptoctx is set up. so i could move this check within the "if (status)" if you prefer, but if we're going to do the check here, i don't see why we wouldn't just do it regardless of status. at any rate, this is all within the gmime 2.6 variant, and we'll be able to clean it out once we move to gmime 3.0. so i've left this part as-is, while having addressed the rest of your feedback. thanks for the review! --dkg [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 03/10] tests: prepare for more crypto tests (using add_gnupg_home) 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 01/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 02/10] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-23 15:38 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor ` (7 subsequent siblings) 10 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail Move add_gnupg_home to test-lib.sh to prepare it for reuse. --- test/T350-crypto.sh | 17 ----------------- test/test-lib.sh | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh index 1d408af7..e1b8fd83 100755 --- a/test/T350-crypto.sh +++ b/test/T350-crypto.sh @@ -7,23 +7,6 @@ test_description='PGP/MIME signature verification and decryption' . ./test-lib.sh || exit 1 -add_gnupg_home () -{ - local output - [ -d ${GNUPGHOME} ] && return - _gnupg_exit () { gpgconf --kill all 2>/dev/null || true; } - at_exit_function _gnupg_exit - mkdir -m 0700 "$GNUPGHOME" - gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 - test_debug "cat $GNUPGHOME/import.log" - if (gpg --quick-random --version >/dev/null 2>&1) ; then - echo quick-random >> "$GNUPGHOME"/gpg.conf - elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then - echo debug-quick-random >> "$GNUPGHOME"/gpg.conf - fi - echo no-emit-version >> "$GNUPGHOME"/gpg.conf -} - ################################################## add_gnupg_home diff --git a/test/test-lib.sh b/test/test-lib.sh index 35024649..b8427d97 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -93,6 +93,21 @@ unset GREP_OPTIONS # For emacsclient unset ALTERNATE_EDITOR +add_gnupg_home () +{ + local output + [ -d ${GNUPGHOME} ] && return + mkdir -m 0700 "$GNUPGHOME" + gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 + test_debug "cat $GNUPGHOME/import.log" + if (gpg --quick-random --version >/dev/null 2>&1) ; then + echo quick-random >> "$GNUPGHOME"/gpg.conf + elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then + echo debug-quick-random >> "$GNUPGHOME"/gpg.conf + fi + echo no-emit-version >> "$GNUPGHOME"/gpg.conf +} + # Each test should start with something like this, after copyright notices: # # test_description='Description of this test... -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v2 03/10] tests: prepare for more crypto tests (using add_gnupg_home) 2017-09-15 5:53 ` [PATCH v2 03/10] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor @ 2017-09-23 15:38 ` Jani Nikula 0 siblings, 0 replies; 67+ messages in thread From: Jani Nikula @ 2017-09-23 15:38 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Fri, 15 Sep 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > Move add_gnupg_home to test-lib.sh to prepare it for reuse. > --- > test/T350-crypto.sh | 17 ----------------- > test/test-lib.sh | 15 +++++++++++++++ > 2 files changed, 15 insertions(+), 17 deletions(-) > > diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh > index 1d408af7..e1b8fd83 100755 > --- a/test/T350-crypto.sh > +++ b/test/T350-crypto.sh > @@ -7,23 +7,6 @@ > test_description='PGP/MIME signature verification and decryption' > . ./test-lib.sh || exit 1 > > -add_gnupg_home () > -{ > - local output > - [ -d ${GNUPGHOME} ] && return > - _gnupg_exit () { gpgconf --kill all 2>/dev/null || true; } > - at_exit_function _gnupg_exit The above lines get dropped. Rebase fail? BR, Jani. > - mkdir -m 0700 "$GNUPGHOME" > - gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 > - test_debug "cat $GNUPGHOME/import.log" > - if (gpg --quick-random --version >/dev/null 2>&1) ; then > - echo quick-random >> "$GNUPGHOME"/gpg.conf > - elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then > - echo debug-quick-random >> "$GNUPGHOME"/gpg.conf > - fi > - echo no-emit-version >> "$GNUPGHOME"/gpg.conf > -} > - > ################################################## > > add_gnupg_home > diff --git a/test/test-lib.sh b/test/test-lib.sh > index 35024649..b8427d97 100644 > --- a/test/test-lib.sh > +++ b/test/test-lib.sh > @@ -93,6 +93,21 @@ unset GREP_OPTIONS > # For emacsclient > unset ALTERNATE_EDITOR > > +add_gnupg_home () > +{ > + local output > + [ -d ${GNUPGHOME} ] && return > + mkdir -m 0700 "$GNUPGHOME" > + gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 > + test_debug "cat $GNUPGHOME/import.log" > + if (gpg --quick-random --version >/dev/null 2>&1) ; then > + echo quick-random >> "$GNUPGHOME"/gpg.conf > + elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then > + echo debug-quick-random >> "$GNUPGHOME"/gpg.conf > + fi > + echo no-emit-version >> "$GNUPGHOME"/gpg.conf > +} > + > # Each test should start with something like this, after copyright notices: > # > # test_description='Description of this test... > -- > 2.14.1 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (2 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 03/10] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-23 16:10 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 05/10] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor ` (6 subsequent siblings) 10 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail This is currently mostly a wrapper around _notmuch_crypto_t that keeps its internals private and doesn't expose any of the GMime API. However, non-crypto indexing options might also be added later (e.g. filters or other transformations). --- lib/add-message.cc | 9 ++++++++- lib/indexopts.c | 22 ++++++++++++++++++++-- lib/notmuch-private.h | 7 +++++++ lib/notmuch.h | 19 +++++++++++++++++++ 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index 73bde7fa..1fd91c14 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_status_t notmuch_database_index_file (notmuch_database_t *notmuch, const char *filename, - notmuch_indexopts_t unused (*indexopts), + notmuch_indexopts_t *indexopts, notmuch_message_t **message_ret) { notmuch_message_file_t *message_file; @@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2; notmuch_private_status_t private_status; notmuch_bool_t is_ghost = FALSE, is_new = FALSE; + notmuch_indexopts_t *def_indexopts = NULL; const char *date; const char *from, *to, *subject; @@ -540,6 +541,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, if (is_new || is_ghost) _notmuch_message_set_header_values (message, date, from, subject); + if (!indexopts) + indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); + ret = _notmuch_message_index_file (message, message_file); if (ret) goto DONE; @@ -557,6 +561,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, } DONE: + if (def_indexopts) + notmuch_indexopts_destroy (def_indexopts); + if (message) { if ((ret == NOTMUCH_STATUS_SUCCESS || ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret) diff --git a/lib/indexopts.c b/lib/indexopts.c index 2f9b841b..1162900c 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -21,9 +21,27 @@ #include "notmuch-private.h" notmuch_indexopts_t * -notmuch_database_get_default_indexopts (notmuch_database_t unused (*db)) +notmuch_database_get_default_indexopts (notmuch_database_t *db) { - return NULL; + return talloc_zero (db, notmuch_indexopts_t); +} + +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + notmuch_bool_t try_decrypt) +{ + if (!indexopts) + return NOTMUCH_STATUS_NULL_POINTER; + indexopts->crypto.decrypt = try_decrypt; + return NOTMUCH_STATUS_SUCCESS; +} + +notmuch_bool_t +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts) +{ + if (!indexopts) + return FALSE; + return indexopts->crypto.decrypt; } void diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index b187a80f..3168cf3c 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -51,6 +51,7 @@ NOTMUCH_BEGIN_DECLS #include "xutil.h" #include "error_util.h" #include "string-util.h" +#include "crypto.h" #ifdef DEBUG # define DEBUG_DATABASE_SANITY 1 @@ -632,6 +633,12 @@ _notmuch_thread_create (void *ctx, notmuch_exclude_t omit_exclude, notmuch_sort_t sort); +/* param.c */ + +typedef struct _notmuch_indexopts { + _notmuch_crypto_t crypto; +} notmuch_indexopts_t; + NOTMUCH_END_DECLS #ifdef __cplusplus diff --git a/lib/notmuch.h b/lib/notmuch.h index 6c76fb40..8baa76ab 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -2214,6 +2214,25 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list); notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db); +/** + * Specify whether to decrypt encrypted parts while indexing. + * + * Be aware that the index is likely sufficient to reconstruct the + * cleartext of the message itself, so please ensure that the notmuch + * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE + * without considering the security of your index. + */ +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + notmuch_bool_t try_decrypt); + +/** + * Return whether to decrypt encrypted parts while indexing. + * see notmuch_indexopts_set_try_decrypt. + */ +notmuch_bool_t +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts); + /** * Destroy a notmuch_indexopts_t object. * -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt 2017-09-15 5:53 ` [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor @ 2017-09-23 16:10 ` Jani Nikula 2017-10-10 3:45 ` Daniel Kahn Gillmor 0 siblings, 1 reply; 67+ messages in thread From: Jani Nikula @ 2017-09-23 16:10 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Fri, 15 Sep 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > This is currently mostly a wrapper around _notmuch_crypto_t that keeps > its internals private and doesn't expose any of the GMime API. > However, non-crypto indexing options might also be added later > (e.g. filters or other transformations). > --- > lib/add-message.cc | 9 ++++++++- > lib/indexopts.c | 22 ++++++++++++++++++++-- > lib/notmuch-private.h | 7 +++++++ > lib/notmuch.h | 19 +++++++++++++++++++ > 4 files changed, 54 insertions(+), 3 deletions(-) > > diff --git a/lib/add-message.cc b/lib/add-message.cc > index 73bde7fa..1fd91c14 100644 > --- a/lib/add-message.cc > +++ b/lib/add-message.cc > @@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch, > notmuch_status_t > notmuch_database_index_file (notmuch_database_t *notmuch, > const char *filename, > - notmuch_indexopts_t unused (*indexopts), > + notmuch_indexopts_t *indexopts, > notmuch_message_t **message_ret) > { > notmuch_message_file_t *message_file; > @@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, > notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2; > notmuch_private_status_t private_status; > notmuch_bool_t is_ghost = FALSE, is_new = FALSE; > + notmuch_indexopts_t *def_indexopts = NULL; > > const char *date; > const char *from, *to, *subject; > @@ -540,6 +541,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, > if (is_new || is_ghost) > _notmuch_message_set_header_values (message, date, from, subject); > > + if (!indexopts) > + indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); > + > ret = _notmuch_message_index_file (message, message_file); > if (ret) > goto DONE; > @@ -557,6 +561,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, > } > > DONE: > + if (def_indexopts) > + notmuch_indexopts_destroy (def_indexopts); > + > if (message) { > if ((ret == NOTMUCH_STATUS_SUCCESS || > ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret) > diff --git a/lib/indexopts.c b/lib/indexopts.c > index 2f9b841b..1162900c 100644 > --- a/lib/indexopts.c > +++ b/lib/indexopts.c > @@ -21,9 +21,27 @@ > #include "notmuch-private.h" > > notmuch_indexopts_t * > -notmuch_database_get_default_indexopts (notmuch_database_t unused (*db)) > +notmuch_database_get_default_indexopts (notmuch_database_t *db) > { > - return NULL; > + return talloc_zero (db, notmuch_indexopts_t); I wonder about the lifetime of indexopts. Should default indexopts be part of the db object, so that your caller above doesn't have to alloc/destroy it for every file? Our library interface has a leaky abstraction of the talloc hierarchical refcounting. We don't talk about it in any of the docs, some of it is implied, most of it is completely surprising if the library interface user assumes a traditional C memory allocation model without refcounting. > +} > + > +notmuch_status_t > +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, > + notmuch_bool_t try_decrypt) > +{ > + if (!indexopts) > + return NOTMUCH_STATUS_NULL_POINTER; > + indexopts->crypto.decrypt = try_decrypt; > + return NOTMUCH_STATUS_SUCCESS; > +} > + > +notmuch_bool_t > +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts) > +{ > + if (!indexopts) > + return FALSE; > + return indexopts->crypto.decrypt; > } > > void > diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h > index b187a80f..3168cf3c 100644 > --- a/lib/notmuch-private.h > +++ b/lib/notmuch-private.h > @@ -51,6 +51,7 @@ NOTMUCH_BEGIN_DECLS > #include "xutil.h" > #include "error_util.h" > #include "string-util.h" > +#include "crypto.h" > > #ifdef DEBUG > # define DEBUG_DATABASE_SANITY 1 > @@ -632,6 +633,12 @@ _notmuch_thread_create (void *ctx, > notmuch_exclude_t omit_exclude, > notmuch_sort_t sort); > > +/* param.c */ > + > +typedef struct _notmuch_indexopts { > + _notmuch_crypto_t crypto; > +} notmuch_indexopts_t; > + > NOTMUCH_END_DECLS > > #ifdef __cplusplus > diff --git a/lib/notmuch.h b/lib/notmuch.h > index 6c76fb40..8baa76ab 100644 > --- a/lib/notmuch.h > +++ b/lib/notmuch.h > @@ -2214,6 +2214,25 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list); > notmuch_indexopts_t * > notmuch_database_get_default_indexopts (notmuch_database_t *db); > > +/** > + * Specify whether to decrypt encrypted parts while indexing. > + * > + * Be aware that the index is likely sufficient to reconstruct the > + * cleartext of the message itself, so please ensure that the notmuch > + * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE > + * without considering the security of your index. > + */ > +notmuch_status_t > +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, > + notmuch_bool_t try_decrypt); > + > +/** > + * Return whether to decrypt encrypted parts while indexing. > + * see notmuch_indexopts_set_try_decrypt. > + */ > +notmuch_bool_t > +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts); > + > /** > * Destroy a notmuch_indexopts_t object. > * > -- > 2.14.1 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt 2017-09-23 16:10 ` Jani Nikula @ 2017-10-10 3:45 ` Daniel Kahn Gillmor 2017-10-14 12:40 ` Jani Nikula 0 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 3:45 UTC (permalink / raw) To: Jani Nikula, Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 2168 bytes --] On Sat 2017-09-23 19:10:18 +0300, Jani Nikula wrote: >> --- a/lib/indexopts.c >> +++ b/lib/indexopts.c >> @@ -21,9 +21,27 @@ >> #include "notmuch-private.h" >> >> notmuch_indexopts_t * >> -notmuch_database_get_default_indexopts (notmuch_database_t unused (*db)) >> +notmuch_database_get_default_indexopts (notmuch_database_t *db) >> { >> - return NULL; >> + return talloc_zero (db, notmuch_indexopts_t); > > I wonder about the lifetime of indexopts. Should default indexopts be > part of the db object, so that your caller above doesn't have to > alloc/destroy it for every file? The caller doesn't have to alloc/destroy it for every file, they can alloc it once and pass it in for every file. I'd rather not have the indexopts be part of the db object itself explicitly, because i can imagine a longer-running program wanting to create two indexopts objects, configuring them differently, and re-using one or the other without wanting to modify the database itself. > Our library interface has a leaky abstraction of the talloc hierarchical > refcounting. We don't talk about it in any of the docs, some of it is > implied, most of it is completely surprising if the library interface > user assumes a traditional C memory allocation model without > refcounting. right, probably the most surprising thing would be if the user got a default indexopts from one database object, and then tried to apply it to another database object, after having deleted the first database object. However, we *do* talk about it in the docs now, so i think we've given fair warning: ------------- /** * get the current default indexing options for a given database. * * This object will survive until the database itself is destroyed, * but the caller may also release it earlier with * notmuch_indexopts_destroy. * * This object represents a set of options on how a message can be * added to the index. At the moment it is a featureless stub. * * @since libnotmuch 5.1 (notmuch 0.26) */ notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db); ------------- --dkg [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt 2017-10-10 3:45 ` Daniel Kahn Gillmor @ 2017-10-14 12:40 ` Jani Nikula 0 siblings, 0 replies; 67+ messages in thread From: Jani Nikula @ 2017-10-14 12:40 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Mon, 09 Oct 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > On Sat 2017-09-23 19:10:18 +0300, Jani Nikula wrote: >>> --- a/lib/indexopts.c >>> +++ b/lib/indexopts.c >>> @@ -21,9 +21,27 @@ >>> #include "notmuch-private.h" >>> >>> notmuch_indexopts_t * >>> -notmuch_database_get_default_indexopts (notmuch_database_t unused (*db)) >>> +notmuch_database_get_default_indexopts (notmuch_database_t *db) >>> { >>> - return NULL; >>> + return talloc_zero (db, notmuch_indexopts_t); >> >> I wonder about the lifetime of indexopts. Should default indexopts be >> part of the db object, so that your caller above doesn't have to >> alloc/destroy it for every file? > > The caller doesn't have to alloc/destroy it for every file, they can > alloc it once and pass it in for every file. My point was, if the caller doesn't do it, it'll get alloced and destroyed per file. BR, Jani. > > I'd rather not have the indexopts be part of the db object itself > explicitly, because i can imagine a longer-running program wanting to > create two indexopts objects, configuring them differently, and re-using > one or the other without wanting to modify the database itself. > >> Our library interface has a leaky abstraction of the talloc hierarchical >> refcounting. We don't talk about it in any of the docs, some of it is >> implied, most of it is completely surprising if the library interface >> user assumes a traditional C memory allocation model without >> refcounting. > > right, probably the most surprising thing would be if the user got a > default indexopts from one database object, and then tried to apply it > to another database object, after having deleted the first database > object. > > However, we *do* talk about it in the docs now, so i think we've given > fair warning: > > ------------- > /** > * get the current default indexing options for a given database. > * > * This object will survive until the database itself is destroyed, > * but the caller may also release it earlier with > * notmuch_indexopts_destroy. > * > * This object represents a set of options on how a message can be > * added to the index. At the moment it is a featureless stub. > * > * @since libnotmuch 5.1 (notmuch 0.26) > */ > notmuch_indexopts_t * > notmuch_database_get_default_indexopts (notmuch_database_t *db); > ------------- > > --dkg ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 05/10] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (3 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-23 16:05 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 06/10] config: indexing defaults will be stored in the database Daniel Kahn Gillmor ` (5 subsequent siblings) 10 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail If we see index options that ask us to decrypt when indexing a message, and we encounter an encrypted part, we'll try to descend into it. If we can decrypt, we add the property index-decryption=success. If we can't decrypt (or recognize the encrypted type of mail), we add the property index-decryption=failure. Note that a single message may have both values of the "index-decryption" property: "success" and "failure". For example, consider a message that includes multiple layers of encryption. If we manage to decrypt the outer layer ("index-decryption=success"), but fail on the inner layer ("index-decryption=failure"). Before re-indexing, we wipe this automatically-added property, so that it will subsequently correspond to the actual semantics of the stored index. --- lib/add-message.cc | 2 +- lib/index.cc | 103 +++++++++++++++++++++++++++++++++++++++++++++----- lib/message.cc | 14 ++++++- lib/notmuch-private.h | 1 + 4 files changed, 107 insertions(+), 13 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index 1fd91c14..66eb0a1f 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -544,7 +544,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, if (!indexopts) indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); - ret = _notmuch_message_index_file (message, message_file); + ret = _notmuch_message_index_file (message, indexopts, message_file); if (ret) goto DONE; diff --git a/lib/index.cc b/lib/index.cc index ceb444df..285928f7 100644 --- a/lib/index.cc +++ b/lib/index.cc @@ -364,9 +364,17 @@ _index_content_type (notmuch_message_t *message, GMimeObject *part) } } +static void +_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts, +#if (GMIME_MAJOR_VERSION < 3) + GMimeContentType *content_type, +#endif + GMimeMultipartEncrypted *part); + /* Callback to generate terms for each mime part of a message. */ static void _index_mime_part (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, GMimeObject *part) { GMimeStream *stream, *filter; @@ -409,17 +417,26 @@ _index_mime_part (notmuch_message_t *message, } } if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) { - /* Don't index encrypted parts, but index their content type. */ - _index_content_type (message, - g_mime_multipart_get_part (multipart, i)); - if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) && - (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) { - _notmuch_database_log (_notmuch_message_database (message), - "Warning: Unexpected extra parts of multipart/encrypted.\n"); + if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) { + _index_encrypted_mime_part(message, indexopts, +#if (GMIME_MAJOR_VERSION < 3) + content_type, +#endif + GMIME_MULTIPART_ENCRYPTED (part)); + } else { + /* Don't index the non-content parts of an + * encrypted message, but do index their content + * type. */ + _index_content_type (message, + g_mime_multipart_get_part (multipart, i)); + if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) { + _notmuch_database_log (_notmuch_message_database (message), + "Warning: Unexpected extra parts of multipart/encrypted.\n"); + } } continue; } - _index_mime_part (message, + _index_mime_part (message, indexopts, g_mime_multipart_get_part (multipart, i)); } return; @@ -430,7 +447,7 @@ _index_mime_part (notmuch_message_t *message, mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part)); - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); return; } @@ -502,8 +519,74 @@ _index_mime_part (notmuch_message_t *message, } } +/* descend (if desired) into the cleartext part of an encrypted MIME + * part while indexing. */ +static void +_index_encrypted_mime_part (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, +#if (GMIME_MAJOR_VERSION < 3) + GMimeContentType *content_type, +#endif + GMimeMultipartEncrypted *encrypted_data) +{ + notmuch_status_t status; +#if (GMIME_MAJOR_VERSION < 3) + GMimeCryptoContext* crypto_ctx = NULL; + const char *protocol = NULL; +#endif + GError *err = NULL; + notmuch_database_t * notmuch = NULL; + GMimeObject *clear = NULL; + + if (!indexopts || !notmuch_indexopts_get_try_decrypt (indexopts)) + return; + + notmuch = _notmuch_message_database (message); + +#if (GMIME_MAJOR_VERSION < 3) + protocol = g_mime_content_type_get_parameter (content_type, "protocol"); + status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto), + protocol, &crypto_ctx); + if (status) { + _notmuch_database_log (notmuch, "Warning: setup failed for decrypting " + "during indexing. (%d)\n", status); + status = notmuch_message_add_property (message, "index-decryption", "failure"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + return; + } +#endif + /* we don't need the GMimeDecryptResult, because we're not looking + * at validating signatures, and we don't care about indexing who + * the message was ostensibly encrypted to. + */ + clear = g_mime_multipart_encrypted_decrypt(encrypted_data, crypto_ctx, + NULL, &err); + if (err) { + _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n", + err->domain, err->code, err->message); + g_error_free(err); + /* Indicate that we failed to decrypt during indexing */ + status = notmuch_message_add_property (message, "index-decryption", "failure"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + return; + } + _index_mime_part (message, indexopts, clear); + g_object_unref (clear); + + status = notmuch_message_add_property (message, "index-decryption", "success"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + +} + notmuch_status_t _notmuch_message_index_file (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, notmuch_message_file_t *message_file) { GMimeMessage *mime_message; @@ -531,7 +614,7 @@ _notmuch_message_index_file (notmuch_message_t *message, subject = g_mime_message_get_subject (mime_message); _notmuch_message_gen_terms (message, "subject", subject); - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); return NOTMUCH_STATUS_SUCCESS; } diff --git a/lib/message.cc b/lib/message.cc index 0e3b5a4f..2ffa25a3 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -1961,7 +1961,7 @@ _notmuch_message_frozen (notmuch_message_t *message) notmuch_status_t notmuch_message_reindex (notmuch_message_t *message, - notmuch_indexopts_t unused (*indexopts)) + notmuch_indexopts_t *indexopts) { notmuch_database_t *notmuch = NULL; notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; @@ -1969,6 +1969,7 @@ notmuch_message_reindex (notmuch_message_t *message, notmuch_filenames_t *orig_filenames = NULL; const char *orig_thread_id = NULL; notmuch_message_file_t *message_file = NULL; + const char *autoproperties[] = { "index-decryption" }; int found = 0; @@ -1999,6 +2000,15 @@ notmuch_message_reindex (notmuch_message_t *message, goto DONE; } + /* all automatically-added properties should be removed before re-indexing */ + for (size_t i = 0; i < ARRAY_SIZE (autoproperties); i++) { + ret = notmuch_message_remove_all_properties (message, autoproperties[i]); + if (ret) { + INTERNAL_ERROR ("failed to remove automatically-added property '%s'", autoproperties[i]); + goto DONE; + } + } + /* re-add the filenames with the associated indexopts */ for (; notmuch_filenames_valid (orig_filenames); notmuch_filenames_move_to_next (orig_filenames)) { @@ -2038,7 +2048,7 @@ notmuch_message_reindex (notmuch_message_t *message, if (found == 0) _notmuch_message_set_header_values (message, date, from, subject); - ret = _notmuch_message_index_file (message, message_file); + ret = _notmuch_message_index_file (message, indexopts, message_file); if (ret == NOTMUCH_STATUS_FILE_ERROR) continue; diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index 3168cf3c..362106c8 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -447,6 +447,7 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch, notmuch_status_t _notmuch_message_index_file (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, notmuch_message_file_t *message_file); /* messages.c */ -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v2 05/10] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-09-15 5:53 ` [PATCH v2 05/10] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor @ 2017-09-23 16:05 ` Jani Nikula 2017-10-10 4:27 ` Daniel Kahn Gillmor 0 siblings, 1 reply; 67+ messages in thread From: Jani Nikula @ 2017-09-23 16:05 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Fri, 15 Sep 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > If we see index options that ask us to decrypt when indexing a > message, and we encounter an encrypted part, we'll try to descend into > it. > > If we can decrypt, we add the property index-decryption=success. > > If we can't decrypt (or recognize the encrypted type of mail), we add > the property index-decryption=failure. > > Note that a single message may have both values of the > "index-decryption" property: "success" and "failure". For example, > consider a message that includes multiple layers of encryption. If we > manage to decrypt the outer layer ("index-decryption=success"), but > fail on the inner layer ("index-decryption=failure"). > > Before re-indexing, we wipe this automatically-added property, so that > it will subsequently correspond to the actual semantics of the stored > index. > --- > lib/add-message.cc | 2 +- > lib/index.cc | 103 +++++++++++++++++++++++++++++++++++++++++++++----- > lib/message.cc | 14 ++++++- > lib/notmuch-private.h | 1 + > 4 files changed, 107 insertions(+), 13 deletions(-) > > diff --git a/lib/add-message.cc b/lib/add-message.cc > index 1fd91c14..66eb0a1f 100644 > --- a/lib/add-message.cc > +++ b/lib/add-message.cc > @@ -544,7 +544,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, > if (!indexopts) > indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); > > - ret = _notmuch_message_index_file (message, message_file); > + ret = _notmuch_message_index_file (message, indexopts, message_file); > if (ret) > goto DONE; > > diff --git a/lib/index.cc b/lib/index.cc > index ceb444df..285928f7 100644 > --- a/lib/index.cc > +++ b/lib/index.cc > @@ -364,9 +364,17 @@ _index_content_type (notmuch_message_t *message, GMimeObject *part) > } > } > > +static void > +_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts, > +#if (GMIME_MAJOR_VERSION < 3) > + GMimeContentType *content_type, > +#endif I don't think adding this within ifdefs servers a useful purpose. It just makes the code harder to read all over the place. Please just pass it in unconditionally. > + GMimeMultipartEncrypted *part); > + > /* Callback to generate terms for each mime part of a message. */ > static void > _index_mime_part (notmuch_message_t *message, > + notmuch_indexopts_t *indexopts, > GMimeObject *part) > { > GMimeStream *stream, *filter; > @@ -409,17 +417,26 @@ _index_mime_part (notmuch_message_t *message, > } > } > if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) { > - /* Don't index encrypted parts, but index their content type. */ > - _index_content_type (message, > - g_mime_multipart_get_part (multipart, i)); > - if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) && > - (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) { > - _notmuch_database_log (_notmuch_message_database (message), > - "Warning: Unexpected extra parts of multipart/encrypted.\n"); > + if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) { > + _index_encrypted_mime_part(message, indexopts, > +#if (GMIME_MAJOR_VERSION < 3) > + content_type, > +#endif > + GMIME_MULTIPART_ENCRYPTED (part)); > + } else { > + /* Don't index the non-content parts of an > + * encrypted message, but do index their content > + * type. */ > + _index_content_type (message, > + g_mime_multipart_get_part (multipart, i)); > + if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) { > + _notmuch_database_log (_notmuch_message_database (message), > + "Warning: Unexpected extra parts of multipart/encrypted.\n"); > + } > } > continue; > } > - _index_mime_part (message, > + _index_mime_part (message, indexopts, > g_mime_multipart_get_part (multipart, i)); > } > return; > @@ -430,7 +447,7 @@ _index_mime_part (notmuch_message_t *message, > > mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part)); > > - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); > + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); > > return; > } > @@ -502,8 +519,74 @@ _index_mime_part (notmuch_message_t *message, > } > } > > +/* descend (if desired) into the cleartext part of an encrypted MIME > + * part while indexing. */ > +static void > +_index_encrypted_mime_part (notmuch_message_t *message, > + notmuch_indexopts_t *indexopts, > +#if (GMIME_MAJOR_VERSION < 3) > + GMimeContentType *content_type, > +#endif > + GMimeMultipartEncrypted *encrypted_data) > +{ > + notmuch_status_t status; > +#if (GMIME_MAJOR_VERSION < 3) > + GMimeCryptoContext* crypto_ctx = NULL; > + const char *protocol = NULL; > +#endif > + GError *err = NULL; > + notmuch_database_t * notmuch = NULL; > + GMimeObject *clear = NULL; > + > + if (!indexopts || !notmuch_indexopts_get_try_decrypt (indexopts)) > + return; > + > + notmuch = _notmuch_message_database (message); > + > +#if (GMIME_MAJOR_VERSION < 3) > + protocol = g_mime_content_type_get_parameter (content_type, "protocol"); > + status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto), > + protocol, &crypto_ctx); > + if (status) { > + _notmuch_database_log (notmuch, "Warning: setup failed for decrypting " > + "during indexing. (%d)\n", status); > + status = notmuch_message_add_property (message, "index-decryption", "failure"); > + if (status) > + _notmuch_database_log (notmuch, "failed to add index-decryption " > + "property (%d)\n", status); > + return; > + } > +#endif I'd like this #if block to be abstracted to separate functions for gmime 2 vs. 3. Adding conditional compilation within functions is ugly and hard to track for both branches. > + /* we don't need the GMimeDecryptResult, because we're not looking > + * at validating signatures, and we don't care about indexing who > + * the message was ostensibly encrypted to. > + */ > + clear = g_mime_multipart_encrypted_decrypt(encrypted_data, crypto_ctx, > + NULL, &err); Speaking of which, where is crypto_ctx declared for gmime 3? > + if (err) { > + _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n", > + err->domain, err->code, err->message); > + g_error_free(err); > + /* Indicate that we failed to decrypt during indexing */ > + status = notmuch_message_add_property (message, "index-decryption", "failure"); > + if (status) > + _notmuch_database_log (notmuch, "failed to add index-decryption " > + "property (%d)\n", status); > + return; > + } > + _index_mime_part (message, indexopts, clear); > + g_object_unref (clear); > + > + status = notmuch_message_add_property (message, "index-decryption", "success"); > + if (status) > + _notmuch_database_log (notmuch, "failed to add index-decryption " > + "property (%d)\n", status); > + > +} > + > notmuch_status_t > _notmuch_message_index_file (notmuch_message_t *message, > + notmuch_indexopts_t *indexopts, > notmuch_message_file_t *message_file) > { > GMimeMessage *mime_message; > @@ -531,7 +614,7 @@ _notmuch_message_index_file (notmuch_message_t *message, > subject = g_mime_message_get_subject (mime_message); > _notmuch_message_gen_terms (message, "subject", subject); > > - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); > + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); > > return NOTMUCH_STATUS_SUCCESS; > } > diff --git a/lib/message.cc b/lib/message.cc > index 0e3b5a4f..2ffa25a3 100644 > --- a/lib/message.cc > +++ b/lib/message.cc > @@ -1961,7 +1961,7 @@ _notmuch_message_frozen (notmuch_message_t *message) > > notmuch_status_t > notmuch_message_reindex (notmuch_message_t *message, > - notmuch_indexopts_t unused (*indexopts)) > + notmuch_indexopts_t *indexopts) > { > notmuch_database_t *notmuch = NULL; > notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; > @@ -1969,6 +1969,7 @@ notmuch_message_reindex (notmuch_message_t *message, > notmuch_filenames_t *orig_filenames = NULL; > const char *orig_thread_id = NULL; > notmuch_message_file_t *message_file = NULL; > + const char *autoproperties[] = { "index-decryption" }; > > int found = 0; > > @@ -1999,6 +2000,15 @@ notmuch_message_reindex (notmuch_message_t *message, > goto DONE; > } > > + /* all automatically-added properties should be removed before re-indexing */ > + for (size_t i = 0; i < ARRAY_SIZE (autoproperties); i++) { > + ret = notmuch_message_remove_all_properties (message, autoproperties[i]); > + if (ret) { > + INTERNAL_ERROR ("failed to remove automatically-added property '%s'", autoproperties[i]); > + goto DONE; > + } > + } > + > /* re-add the filenames with the associated indexopts */ > for (; notmuch_filenames_valid (orig_filenames); > notmuch_filenames_move_to_next (orig_filenames)) { > @@ -2038,7 +2048,7 @@ notmuch_message_reindex (notmuch_message_t *message, > if (found == 0) > _notmuch_message_set_header_values (message, date, from, subject); > > - ret = _notmuch_message_index_file (message, message_file); > + ret = _notmuch_message_index_file (message, indexopts, message_file); > > if (ret == NOTMUCH_STATUS_FILE_ERROR) > continue; > diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h > index 3168cf3c..362106c8 100644 > --- a/lib/notmuch-private.h > +++ b/lib/notmuch-private.h > @@ -447,6 +447,7 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch, > > notmuch_status_t > _notmuch_message_index_file (notmuch_message_t *message, > + notmuch_indexopts_t *indexopts, > notmuch_message_file_t *message_file); > > /* messages.c */ > -- > 2.14.1 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v2 05/10] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-09-23 16:05 ` Jani Nikula @ 2017-10-10 4:27 ` Daniel Kahn Gillmor 0 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 4:27 UTC (permalink / raw) To: Jani Nikula, Notmuch Mail On Sat 2017-09-23 19:05:40 +0300, Jani Nikula wrote: > I'd like this #if block to be abstracted to separate functions for gmime > 2 vs. 3. Adding conditional compilation within functions is ugly and > hard to track for both branches. That abstraction has already happened -- it's in util/crypto.c; there's simply a chunk of code that gmime 2.6 needs that gmime 3.0 does not. :/ i welcome suggestions for how to rewrite this if you really think it's a problem, but i hope the latest revision of this series (which groups all changes into a single, cleaned-up #if block) will be acceptable. --dkg ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 06/10] config: indexing defaults will be stored in the database. 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (4 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 05/10] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 07/10] config: define new option index.try_decrypt Daniel Kahn Gillmor ` (4 subsequent siblings) 10 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail At indexing time, the database needs to know its internal defaults. It shouldn't be contingent on an external config file (since that can't be retrieved from the database object itself). This behaves the same as the query.* configurations, which are also stored in the database itself, so we're not introducing any new dependencies. --- notmuch-config.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/notmuch-config.c b/notmuch-config.c index cb9529b9..06c54544 100644 --- a/notmuch-config.c +++ b/notmuch-config.c @@ -809,6 +809,7 @@ _item_split (char *item, char **group, char **key) #define BUILT_WITH_PREFIX "built_with." #define QUERY_PREFIX "query." +#define INDEX_PREFIX "index." static int _print_db_config(notmuch_config_t *config, const char *name) @@ -859,6 +860,8 @@ notmuch_config_command_get (notmuch_config_t *config, char *item) notmuch_built_with (item + strlen (BUILT_WITH_PREFIX)) ? "true" : "false"); } else if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) { return _print_db_config (config, item); + } else if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) { + return _print_db_config (config, item); } else { char **value; size_t i, length; @@ -931,6 +934,9 @@ notmuch_config_command_set (notmuch_config_t *config, char *item, int argc, char if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) { return _set_db_config (config, item, argc, argv); } + if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) { + return _set_db_config (config, item, argc, argv); + } if (_item_split (item, &group, &key)) return 1; -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v2 07/10] config: define new option index.try_decrypt 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (5 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 06/10] config: indexing defaults will be stored in the database Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-23 16:17 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 08/10] cli/new: add --try-decrypt=(true|false) Daniel Kahn Gillmor ` (3 subsequent siblings) 10 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail By default, notmuch won't try to decrypt on indexing. With this patch, we make it possible to indicate a per-database preference using the config variable "index.try_decrypt", which by default will be false. --- doc/man1/notmuch-config.rst | 12 ++++++++++++ lib/indexopts.c | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index 6a51e64f..6f35d127 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -134,6 +134,18 @@ The available configuration items are described below. Default: ``gpg``. + **index.try_decrypt** + + When indexing an encrypted e-mail message, if this variable is + set to true, notmuch will try to decrypt the message and index + the cleartext. Be aware that the index is likely sufficient + to reconstruct the cleartext of the message itself, so please + ensure that the notmuch message index is adequately protected. + DO NOT USE ``index.try_decrypt=true`` without considering the + security of your index. + + Default: ``false``. + **built_with.<name>** Compile time feature <name>. Current possibilities include diff --git a/lib/indexopts.c b/lib/indexopts.c index 1162900c..5bd396ff 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -23,7 +23,21 @@ notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db) { - return talloc_zero (db, notmuch_indexopts_t); + notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t); + if (ret) { + char * try_decrypt; + notmuch_status_t err; + if (!(err = notmuch_database_get_config (db, "index.try_decrypt", &try_decrypt))) { + if (try_decrypt && + ((!(strcasecmp(try_decrypt, "true"))) || + (!(strcasecmp(try_decrypt, "yes"))) || + (!(strcasecmp(try_decrypt, "1"))))) + notmuch_indexopts_set_try_decrypt (ret, TRUE); + + free (try_decrypt); + } + } + return ret; } notmuch_status_t -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v2 07/10] config: define new option index.try_decrypt 2017-09-15 5:53 ` [PATCH v2 07/10] config: define new option index.try_decrypt Daniel Kahn Gillmor @ 2017-09-23 16:17 ` Jani Nikula 0 siblings, 0 replies; 67+ messages in thread From: Jani Nikula @ 2017-09-23 16:17 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Fri, 15 Sep 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > By default, notmuch won't try to decrypt on indexing. With this > patch, we make it possible to indicate a per-database preference using > the config variable "index.try_decrypt", which by default will be > false. > --- > doc/man1/notmuch-config.rst | 12 ++++++++++++ > lib/indexopts.c | 16 +++++++++++++++- > 2 files changed, 27 insertions(+), 1 deletion(-) > > diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst > index 6a51e64f..6f35d127 100644 > --- a/doc/man1/notmuch-config.rst > +++ b/doc/man1/notmuch-config.rst > @@ -134,6 +134,18 @@ The available configuration items are described below. > > Default: ``gpg``. > > + **index.try_decrypt** > + > + When indexing an encrypted e-mail message, if this variable is > + set to true, notmuch will try to decrypt the message and index > + the cleartext. Be aware that the index is likely sufficient > + to reconstruct the cleartext of the message itself, so please > + ensure that the notmuch message index is adequately protected. > + DO NOT USE ``index.try_decrypt=true`` without considering the > + security of your index. > + > + Default: ``false``. > + > **built_with.<name>** > > Compile time feature <name>. Current possibilities include > diff --git a/lib/indexopts.c b/lib/indexopts.c > index 1162900c..5bd396ff 100644 > --- a/lib/indexopts.c > +++ b/lib/indexopts.c > @@ -23,7 +23,21 @@ > notmuch_indexopts_t * > notmuch_database_get_default_indexopts (notmuch_database_t *db) > { > - return talloc_zero (db, notmuch_indexopts_t); > + notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t); > + if (ret) { > + char * try_decrypt; > + notmuch_status_t err; > + if (!(err = notmuch_database_get_config (db, "index.try_decrypt", &try_decrypt))) { I like the style of always separating assigment and conditional. I wonder if this function would look cleaner by doing early returns every step of the way instead of nested ifs. if (!ret) return ret; err = notmuch_database_get_config(); if (err) return ret; and so on. > + if (try_decrypt && > + ((!(strcasecmp(try_decrypt, "true"))) || > + (!(strcasecmp(try_decrypt, "yes"))) || > + (!(strcasecmp(try_decrypt, "1"))))) > + notmuch_indexopts_set_try_decrypt (ret, TRUE); > + > + free (try_decrypt); > + } > + } > + return ret; > } > > notmuch_status_t > -- > 2.14.1 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 08/10] cli/new: add --try-decrypt=(true|false) 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (6 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 07/10] config: define new option index.try_decrypt Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-23 16:46 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 09/10] cli/insert: " Daniel Kahn Gillmor ` (2 subsequent siblings) 10 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail Try to decrypt any encrypted parts of newly-discovered messages while indexing them. The cleartext of any successfully-decrypted messages will be indexed, with tags applied in the same form as from notmuch insert --try-decrypt=true. Note: if the deprecated crypto.gpg_path configuration option is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). We also add a new test making use of this functionality. This requires a bit of reorganization, because we need to allow passing --long-arguments to "notmuch new" via emacs_fcc_message --- completion/notmuch-completion.bash | 13 ++++++++-- doc/man1/notmuch-new.rst | 12 +++++++++ notmuch-new.c | 29 +++++++++++++++++++++- test/T357-index-decryption.sh | 51 ++++++++++++++++++++++++++++++++++++++ test/test-lib.sh | 11 +++++++- 5 files changed, 112 insertions(+), 4 deletions(-) create mode 100755 test/T357-index-decryption.sh diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 5201be63..17be6b8f 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -311,11 +311,20 @@ _notmuch_insert() _notmuch_new() { local cur prev words cword split - _init_completion || return + _init_completion -s || return + + $split && + case "${prev}" in + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; + esac + ! $split && case "${cur}" in -*) - local options="--no-hooks --quiet ${_notmuch_shared_options}" + local options="--no-hooks --try-decrypt= --quiet ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "${options}" -- ${cur}) ) ;; diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst index 6acfa112..c255f980 100644 --- a/doc/man1/notmuch-new.rst +++ b/doc/man1/notmuch-new.rst @@ -43,6 +43,18 @@ Supported options for **new** include ``--quiet`` Do not print progress or results. + ``--try-decrypt=(true|false)`` + + If true, when encountering an encrypted message, try to + decrypt it while indexing. If decryption is successful, index + the cleartext itself. Be aware that the index is likely + sufficient to reconstruct the cleartext of the message itself, + so please ensure that the notmuch message index is adequately + protected. DO NOT USE ``--try-decrypt=true`` without + considering the security of your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + EXIT STATUS =========== diff --git a/notmuch-new.c b/notmuch-new.c index faeb8f0a..cffcd8bc 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -49,6 +49,7 @@ typedef struct { size_t new_tags_length; const char **new_ignore; size_t new_ignore_length; + notmuch_indexopts_t *indexopts; int total_files; int processed_files; @@ -267,7 +268,7 @@ add_file (notmuch_database_t *notmuch, const char *filename, if (status) goto DONE; - status = notmuch_database_index_file (notmuch, filename, NULL, &message); + status = notmuch_database_index_file (notmuch, filename, state->indexopts, &message); switch (status) { /* Success. */ case NOTMUCH_STATUS_SUCCESS: @@ -949,6 +950,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) unsigned int i; notmuch_bool_t timer_is_active = FALSE; notmuch_bool_t no_hooks = FALSE; + int try_decrypt = -1; notmuch_bool_t quiet = FALSE, verbose = FALSE; notmuch_status_t status; @@ -957,6 +959,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) { NOTMUCH_OPT_BOOLEAN, &verbose, "verbose", 'v', 0 }, { NOTMUCH_OPT_BOOLEAN, &add_files_state.debug, "debug", 'd', 0 }, { NOTMUCH_OPT_BOOLEAN, &no_hooks, "no-hooks", 'n', 0 }, + { NOTMUCH_OPT_BOOLEAN, &try_decrypt, "try-decrypt", 0, 0 }, { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -1074,6 +1077,30 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) if (notmuch == NULL) return EXIT_FAILURE; + add_files_state.indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!add_files_state.indexopts) { + fprintf (stderr, "Error: could not create index options.\n"); + return EXIT_FAILURE; + } + if (try_decrypt == TRUE || try_decrypt == FALSE) { + status = notmuch_indexopts_set_try_decrypt (add_files_state.indexopts, try_decrypt); + if (status != NOTMUCH_STATUS_SUCCESS) { + fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n", + try_decrypt ? "True" : "False", notmuch_status_to_string (status)); + notmuch_indexopts_destroy (add_files_state.indexopts); + return EXIT_FAILURE; + } + } +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (add_files_state.indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif + + /* Set up our handler for SIGINT. We do this after having * potentially done a database upgrade we this interrupt handler * won't support. */ diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh new file mode 100755 index 00000000..7bbd81f6 --- /dev/null +++ b/test/T357-index-decryption.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# TODO: test index-decryption-failed + +test_description='indexing decrypted mail' +. ./test-lib.sh || exit 1 + +################################################## + +add_gnupg_home +# get key fingerprint +FINGERPRINT=$(gpg --no-tty --list-secret-keys --with-colons --fingerprint | grep '^fpr:' | cut -d: -f10) + +# create a test encrypted message +test_begin_subtest 'emacs delivery of encrypted message' +test_expect_success \ +'emacs_fcc_message \ + "test encrypted message for cleartext index 001" \ + "This is a test encrypted message with a wumpus.\n" \ + "(mml-secure-message-encrypt)"' + +test_begin_subtest "search for unindexed cleartext" +output=$(notmuch search wumpus) +expected='' +test_expect_equal \ + "$output" \ + "$expected" + +# create a test encrypted message that is indexed in the clear +test_begin_subtest 'emacs delivery of encrypted message' +test_expect_success \ +'emacs_fcc_message --try-decrypt=true \ + "test encrypted message for cleartext index 002" \ + "This is a test encrypted message with a wumpus.\n" \ + "(mml-secure-message-encrypt)"' + +test_begin_subtest "emacs delivery of encrypted message, indexed cleartext" +output=$(notmuch search wumpus) +expected='thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property for one message" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + +test_done diff --git a/test/test-lib.sh b/test/test-lib.sh index b8427d97..6a47354f 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -338,8 +338,17 @@ emacs_deliver_message () # Accepts arbitrary extra emacs/elisp functions to modify the message # before sending, which is useful to doing things like attaching files # to the message and encrypting/signing. +# +# If any GNU-style long-arguments (like --quiet or --try-decrypt=true) are +# at the head of the argument list, they are sent directly to "notmuch +# new" after message delivery emacs_fcc_message () { + local nmn_args='' + while [[ "$1" =~ ^-- ]]; do + nmn_args="$nmn_args $1" + shift + done local subject="$1" local body="$2" shift 2 @@ -358,7 +367,7 @@ emacs_fcc_message () (insert \"${body}\") $@ (notmuch-mua-send-and-exit))" || return 1 - notmuch new >/dev/null + notmuch new $nmn_args >/dev/null } # Add an existing, fixed corpus of email to the database. -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v2 08/10] cli/new: add --try-decrypt=(true|false) 2017-09-15 5:53 ` [PATCH v2 08/10] cli/new: add --try-decrypt=(true|false) Daniel Kahn Gillmor @ 2017-09-23 16:46 ` Jani Nikula 0 siblings, 0 replies; 67+ messages in thread From: Jani Nikula @ 2017-09-23 16:46 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Fri, 15 Sep 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > Try to decrypt any encrypted parts of newly-discovered messages while > indexing them. The cleartext of any successfully-decrypted messages > will be indexed, with tags applied in the same form as from notmuch > insert --try-decrypt=true. > > Note: if the deprecated crypto.gpg_path configuration option is set to > anything other than "gpg", we ignore it (and print a warning on > stderr, if built against gmime < 3.0). > > We also add a new test making use of this functionality. This > requires a bit of reorganization, because we need to allow passing > --long-arguments to "notmuch new" via emacs_fcc_message > --- > completion/notmuch-completion.bash | 13 ++++++++-- > doc/man1/notmuch-new.rst | 12 +++++++++ > notmuch-new.c | 29 +++++++++++++++++++++- > test/T357-index-decryption.sh | 51 ++++++++++++++++++++++++++++++++++++++ > test/test-lib.sh | 11 +++++++- > 5 files changed, 112 insertions(+), 4 deletions(-) > create mode 100755 test/T357-index-decryption.sh > > diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash > index 5201be63..17be6b8f 100644 > --- a/completion/notmuch-completion.bash > +++ b/completion/notmuch-completion.bash > @@ -311,11 +311,20 @@ _notmuch_insert() > _notmuch_new() > { > local cur prev words cword split > - _init_completion || return > + _init_completion -s || return > + > + $split && > + case "${prev}" in > + --try-decrypt) > + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) > + return > + ;; > + esac > > + ! $split && > case "${cur}" in > -*) > - local options="--no-hooks --quiet ${_notmuch_shared_options}" > + local options="--no-hooks --try-decrypt= --quiet ${_notmuch_shared_options}" > compopt -o nospace > COMPREPLY=( $(compgen -W "${options}" -- ${cur}) ) > ;; > diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst > index 6acfa112..c255f980 100644 > --- a/doc/man1/notmuch-new.rst > +++ b/doc/man1/notmuch-new.rst > @@ -43,6 +43,18 @@ Supported options for **new** include > ``--quiet`` > Do not print progress or results. > > + ``--try-decrypt=(true|false)`` > + > + If true, when encountering an encrypted message, try to > + decrypt it while indexing. If decryption is successful, index > + the cleartext itself. Be aware that the index is likely > + sufficient to reconstruct the cleartext of the message itself, > + so please ensure that the notmuch message index is adequately > + protected. DO NOT USE ``--try-decrypt=true`` without > + considering the security of your index. > + > + See also ``index.try_decrypt`` in **notmuch-config(1)**. > + > EXIT STATUS > =========== > > diff --git a/notmuch-new.c b/notmuch-new.c > index faeb8f0a..cffcd8bc 100644 > --- a/notmuch-new.c > +++ b/notmuch-new.c > @@ -49,6 +49,7 @@ typedef struct { > size_t new_tags_length; > const char **new_ignore; > size_t new_ignore_length; > + notmuch_indexopts_t *indexopts; > > int total_files; > int processed_files; > @@ -267,7 +268,7 @@ add_file (notmuch_database_t *notmuch, const char *filename, > if (status) > goto DONE; > > - status = notmuch_database_index_file (notmuch, filename, NULL, &message); > + status = notmuch_database_index_file (notmuch, filename, state->indexopts, &message); > switch (status) { > /* Success. */ > case NOTMUCH_STATUS_SUCCESS: > @@ -949,6 +950,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) > unsigned int i; > notmuch_bool_t timer_is_active = FALSE; > notmuch_bool_t no_hooks = FALSE; > + int try_decrypt = -1; > notmuch_bool_t quiet = FALSE, verbose = FALSE; > notmuch_status_t status; > > @@ -957,6 +959,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) > { NOTMUCH_OPT_BOOLEAN, &verbose, "verbose", 'v', 0 }, > { NOTMUCH_OPT_BOOLEAN, &add_files_state.debug, "debug", 'd', 0 }, > { NOTMUCH_OPT_BOOLEAN, &no_hooks, "no-hooks", 'n', 0 }, > + { NOTMUCH_OPT_BOOLEAN, &try_decrypt, "try-decrypt", 0, 0 }, > { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, > { 0, 0, 0, 0, 0 } > }; > @@ -1074,6 +1077,30 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) > if (notmuch == NULL) > return EXIT_FAILURE; > > + add_files_state.indexopts = notmuch_database_get_default_indexopts (notmuch); > + if (!add_files_state.indexopts) { > + fprintf (stderr, "Error: could not create index options.\n"); > + return EXIT_FAILURE; > + } > + if (try_decrypt == TRUE || try_decrypt == FALSE) { As discussed in the argument parsing thread, I'm not really fond of these implicit "tristate booleans". It's surprising to look at a boolean option and check if it's either true or false... But looks like we need to involve other folks here, and figure out what we want to do wrt stdbools and my --no-arg idea for --arg=false, etc. > + status = notmuch_indexopts_set_try_decrypt (add_files_state.indexopts, try_decrypt); > + if (status != NOTMUCH_STATUS_SUCCESS) { > + fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n", > + try_decrypt ? "True" : "False", notmuch_status_to_string (status)); > + notmuch_indexopts_destroy (add_files_state.indexopts); > + return EXIT_FAILURE; > + } > + } > +#if (GMIME_MAJOR_VERSION < 3) > + if (notmuch_indexopts_get_try_decrypt (add_files_state.indexopts)) { > + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); > + if (gpg_path && strcmp(gpg_path, "gpg")) > + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" > + "\tbut ignoring (use $PATH instead)\n", gpg_path); > + } > +#endif > + Seems like there's quite a bit of duplication with the other commands here. BR, Jani. > + > /* Set up our handler for SIGINT. We do this after having > * potentially done a database upgrade we this interrupt handler > * won't support. */ > diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh > new file mode 100755 > index 00000000..7bbd81f6 > --- /dev/null > +++ b/test/T357-index-decryption.sh > @@ -0,0 +1,51 @@ > +#!/usr/bin/env bash > + > +# TODO: test index-decryption-failed > + > +test_description='indexing decrypted mail' > +. ./test-lib.sh || exit 1 > + > +################################################## > + > +add_gnupg_home > +# get key fingerprint > +FINGERPRINT=$(gpg --no-tty --list-secret-keys --with-colons --fingerprint | grep '^fpr:' | cut -d: -f10) > + > +# create a test encrypted message > +test_begin_subtest 'emacs delivery of encrypted message' > +test_expect_success \ > +'emacs_fcc_message \ > + "test encrypted message for cleartext index 001" \ > + "This is a test encrypted message with a wumpus.\n" \ > + "(mml-secure-message-encrypt)"' > + > +test_begin_subtest "search for unindexed cleartext" > +output=$(notmuch search wumpus) > +expected='' > +test_expect_equal \ > + "$output" \ > + "$expected" > + > +# create a test encrypted message that is indexed in the clear > +test_begin_subtest 'emacs delivery of encrypted message' > +test_expect_success \ > +'emacs_fcc_message --try-decrypt=true \ > + "test encrypted message for cleartext index 002" \ > + "This is a test encrypted message with a wumpus.\n" \ > + "(mml-secure-message-encrypt)"' > + > +test_begin_subtest "emacs delivery of encrypted message, indexed cleartext" > +output=$(notmuch search wumpus) > +expected='thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox)' > +test_expect_equal \ > + "$output" \ > + "$expected" > + > +# and the same search, but by property ($expected is untouched): > +test_begin_subtest "emacs search by property for one message" > +output=$(notmuch search property:index-decryption=success) > +test_expect_equal \ > + "$output" \ > + "$expected" > + > +test_done > diff --git a/test/test-lib.sh b/test/test-lib.sh > index b8427d97..6a47354f 100644 > --- a/test/test-lib.sh > +++ b/test/test-lib.sh > @@ -338,8 +338,17 @@ emacs_deliver_message () > # Accepts arbitrary extra emacs/elisp functions to modify the message > # before sending, which is useful to doing things like attaching files > # to the message and encrypting/signing. > +# > +# If any GNU-style long-arguments (like --quiet or --try-decrypt=true) are > +# at the head of the argument list, they are sent directly to "notmuch > +# new" after message delivery > emacs_fcc_message () > { > + local nmn_args='' > + while [[ "$1" =~ ^-- ]]; do > + nmn_args="$nmn_args $1" > + shift > + done > local subject="$1" > local body="$2" > shift 2 > @@ -358,7 +367,7 @@ emacs_fcc_message () > (insert \"${body}\") > $@ > (notmuch-mua-send-and-exit))" || return 1 > - notmuch new >/dev/null > + notmuch new $nmn_args >/dev/null > } > > # Add an existing, fixed corpus of email to the database. > -- > 2.14.1 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v2 09/10] cli/insert: add --try-decrypt=(true|false) 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (7 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 08/10] cli/new: add --try-decrypt=(true|false) Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 10/10] cli/reindex: " Daniel Kahn Gillmor 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor 10 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail Allow an incoming message to be delivered while indexing the cleartext, on a per-message basis. This requires the secret keys for the message to be available. For the moment, the most functional approach is to ensure that gpg-agent is running and knows about any secret keys that might be useful to decrypt incoming mail. Any additional recommendations for how to phrase the caveat for this option are welcome. Note: if the deprecated crypto.gpg_path is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). --- completion/notmuch-completion.bash | 6 +++++- doc/man1/notmuch-insert.rst | 14 ++++++++++++++ notmuch-insert.c | 32 +++++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 17be6b8f..72a75a94 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -287,12 +287,16 @@ _notmuch_insert() sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) ) return ;; + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; esac ! $split && case "${cur}" in --*) - local options="--create-folder --folder= --keep --no-hooks ${_notmuch_shared_options}" + local options="--create-folder --folder= --keep --no-hooks --try-decrypt= ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) return diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst index f79600d6..647dac06 100644 --- a/doc/man1/notmuch-insert.rst +++ b/doc/man1/notmuch-insert.rst @@ -50,6 +50,20 @@ Supported options for **insert** include ``--no-hooks`` Prevent hooks from being run. + ``--try-decrypt=(true|false)`` + + If true and the message is encrypted, try to decrypt the + message while indexing. If decryption is successful, index + the cleartext itself. Either way, the message is always + stored to disk in its original form (ciphertext). Be aware + that the index is likely sufficient to reconstruct the + cleartext of the message itself, so please ensure that the + notmuch message index is adequately protected. DO NOT USE + ``--try-decrypt=true`` without considering the security of + your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + EXIT STATUS =========== diff --git a/notmuch-insert.c b/notmuch-insert.c index 648bd944..c46a3278 100644 --- a/notmuch-insert.c +++ b/notmuch-insert.c @@ -379,12 +379,13 @@ FAIL: */ static notmuch_status_t add_file (notmuch_database_t *notmuch, const char *path, tag_op_list_t *tag_ops, - notmuch_bool_t synchronize_flags, notmuch_bool_t keep) + notmuch_bool_t synchronize_flags, notmuch_bool_t keep, + notmuch_indexopts_t *indexopts) { notmuch_message_t *message; notmuch_status_t status; - status = notmuch_database_index_file (notmuch, path, NULL, &message); + status = notmuch_database_index_file (notmuch, path, indexopts, &message); if (status == NOTMUCH_STATUS_SUCCESS) { status = tag_op_list_apply (message, tag_ops, 0); if (status) { @@ -456,17 +457,20 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_bool_t create_folder = FALSE; notmuch_bool_t keep = FALSE; notmuch_bool_t no_hooks = FALSE; + int try_decrypt = -1; notmuch_bool_t synchronize_flags; const char *maildir; char *newpath; int opt_index; unsigned int i; + notmuch_indexopts_t *indexopts; notmuch_opt_desc_t options[] = { { NOTMUCH_OPT_STRING, &folder, "folder", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &create_folder, "create-folder", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &keep, "keep", 0, 0 }, { NOTMUCH_OPT_BOOLEAN, &no_hooks, "no-hooks", 'n', 0 }, + { NOTMUCH_OPT_BOOLEAN, &try_decrypt, "try-decrypt", 0, 0 }, { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, { NOTMUCH_OPT_END, 0, 0, 0, 0 } }; @@ -547,9 +551,31 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_exit_if_unmatched_db_uuid (notmuch); + indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!indexopts) { + fprintf (stderr, "Error: could not create index options.\n"); + return EXIT_FAILURE; + } + if (try_decrypt == TRUE || try_decrypt == FALSE) { + status = notmuch_indexopts_set_try_decrypt (indexopts, try_decrypt); + if (status != NOTMUCH_STATUS_SUCCESS) { + fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n", + try_decrypt ? "True" : "False", notmuch_status_to_string (status)); + notmuch_indexopts_destroy (indexopts); + return EXIT_FAILURE; + } + } +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif /* Index the message. */ - status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep); + status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, indexopts); /* Commit changes. */ close_status = notmuch_database_destroy (notmuch); -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v2 10/10] cli/reindex: add --try-decrypt=(true|false) 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (8 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 09/10] cli/insert: " Daniel Kahn Gillmor @ 2017-09-15 5:53 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor 10 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-09-15 5:53 UTC (permalink / raw) To: Notmuch Mail Try to decrypt any encrypted parts of newly-discovered messages while re-indexing them. The cleartext of any successfully-decrypted messages will be indexed, with tags applied in the same form as from notmuch insert --try-decrypt=true. Note: if the deprecated crypto.gpg_path configuration option is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). --- completion/notmuch-completion.bash | 10 +++++- doc/man1/notmuch-reindex.rst | 14 ++++++++ notmuch-reindex.c | 23 ++++++++++++++ test/T357-index-decryption.sh | 65 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 72a75a94..ef79affe 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -435,10 +435,18 @@ _notmuch_reindex() local cur prev words cword split _init_completion -s || return + $split && + case "${prev}" in + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; + esac + ! $split && case "${cur}" in -*) - local options="${_notmuch_shared_options}" + local options="--try-decrypt= ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) ;; diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst index e39cc4ee..60a060a7 100644 --- a/doc/man1/notmuch-reindex.rst +++ b/doc/man1/notmuch-reindex.rst @@ -19,6 +19,20 @@ The **reindex** command searches for all messages matching the supplied search terms, and re-creates the full-text index on these messages using the supplied options. +Supported options for **reindex** include + + ``--try-decrypt=(true|false)`` + + If true, when encountering an encrypted message, try to + decrypt it while reindexing. If decryption is successful, + index the cleartext itself. Be aware that the index is likely + sufficient to reconstruct the cleartext of the message itself, + so please ensure that the notmuch message index is adequately + protected. DO NOT USE ``--try-decrypt=true`` without + considering the security of your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + SEE ALSO ======== diff --git a/notmuch-reindex.c b/notmuch-reindex.c index bceac722..83cd0a57 100644 --- a/notmuch-reindex.c +++ b/notmuch-reindex.c @@ -90,6 +90,8 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) int opt_index; int ret; notmuch_indexopts_t *indexopts = NULL; + int try_decrypt = -1; + notmuch_status_t status; /* Set up our handler for SIGINT */ memset (&action, 0, sizeof (struct sigaction)); @@ -99,6 +101,7 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) sigaction (SIGINT, &action, NULL); notmuch_opt_desc_t options[] = { + { NOTMUCH_OPT_BOOLEAN, &try_decrypt, "try-decrypt", 0, 0 }, { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -115,6 +118,26 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_exit_if_unmatched_db_uuid (notmuch); + indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!indexopts) + return EXIT_FAILURE; + + if (try_decrypt == TRUE || try_decrypt == FALSE) { + status = notmuch_indexopts_set_try_decrypt (indexopts, try_decrypt); + if (status) + fprintf (stderr, "Warning: failed to set --try-decrypt to %d (%s)\n", + try_decrypt, notmuch_status_to_string (status)); + } + +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif + query_string = query_string_from_args (config, argc-opt_index, argv+opt_index); if (query_string == NULL) { fprintf (stderr, "Out of memory\n"); diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh index 7bbd81f6..3d18f5af 100755 --- a/test/T357-index-decryption.sh +++ b/test/T357-index-decryption.sh @@ -48,4 +48,69 @@ test_expect_equal \ "$output" \ "$expected" +# add a tag to all messages to ensure that it stays after reindexing +test_begin_subtest 'tagging all messages' +test_expect_success 'notmuch tag +blarney "encrypted message"' +test_begin_subtest "verify that tags have not changed" +output=$(notmuch search tag:blarney) +expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + +# see if first message shows up after reindexing with --try-decrypt=true (same $expected, untouched): +test_begin_subtest 'reindex old messages' +test_expect_success 'notmuch reindex --try-decrypt=true tag:encrypted and not property:index-decryption=success' +test_begin_subtest "reindexed encrypted message, including cleartext" +output=$(notmuch search wumpus) +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property for both messages" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + + +# try to remove cleartext indexing +test_begin_subtest 'reindex without cleartext' +test_expect_success 'notmuch reindex tag:encrypted and property:index-decryption=success' +test_begin_subtest "reindexed encrypted messages, without cleartext" +output=$(notmuch search wumpus) +expected='' +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property with both messages unindexed" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + +# ensure that the tags remain even when we are dropping the cleartext. +test_begin_subtest "verify that tags remain without cleartext" +output=$(notmuch search tag:blarney) +expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + + +# TODO: test removal of a message from the message store between +# indexing and reindexing. + +# TODO: insert the same message into the message store twice, index, +# remove one of them from the message store, and then reindex. +# reindexing should return a failure but the message should still be +# present? -- or what should the semantics be if you ask to reindex a +# message whose underlying files have been renamed or moved or +# removed? + test_done -- 2.14.1 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* cleartext indexing, round 3 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor ` (9 preceding siblings ...) 2017-09-15 5:53 ` [PATCH v2 10/10] cli/reindex: " Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 01/15] crypto: rename notmuch_crypto_t to _notmuch_crypto_t Daniel Kahn Gillmor ` (16 more replies) 10 siblings, 17 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail What follows is the third round of the latest revision of cleartext indexing patches. This series is rebased against current master, with the improved option-handling code, and takes into consideration the review that Jani did on round 2 of this series. I welcome additional review and feedback. Many of these patches may be familiar the third time around and will hopefully be uncontroversial. Please don't hesitate to review and ack the easy ones :) I've also pushed this series to the "cleartext-indexing" branch at https://gitlab.com/dkg/notmuch for those who prefer a direct git pull. it is currently git commit 6f7f6847141db2f031b29c68d966fa13c3be2da5. Regards, --dkg ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v3 01/15] crypto: rename notmuch_crypto_t to _notmuch_crypto_t 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 02/15] crypto: drop pretense of notmuch_crypto_context_t Daniel Kahn Gillmor ` (15 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail The notmuch_crypto_t struct isn't used externally, and we have no plans to explicitly export it. Prefix its name (and associated functions) with _ to make that intent clear. --- crypto.c | 12 ++++++------ mime-node.c | 6 +++--- notmuch-client.h | 12 ++++++------ notmuch-reply.c | 2 +- notmuch-show.c | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/crypto.c b/crypto.c index 9c557d6e..9ae303d4 100644 --- a/crypto.c +++ b/crypto.c @@ -22,7 +22,7 @@ #if (GMIME_MAJOR_VERSION < 3) /* Create a GPG context (GMime 2.6) */ static notmuch_crypto_context_t * -create_gpg_context (notmuch_crypto_t *crypto) +create_gpg_context (_notmuch_crypto_t *crypto) { notmuch_crypto_context_t *gpgctx; @@ -45,7 +45,7 @@ create_gpg_context (notmuch_crypto_t *crypto) /* Create a PKCS7 context (GMime 2.6) */ static notmuch_crypto_context_t * -create_pkcs7_context (notmuch_crypto_t *crypto) +create_pkcs7_context (_notmuch_crypto_t *crypto) { notmuch_crypto_context_t *pkcs7ctx; @@ -67,7 +67,7 @@ create_pkcs7_context (notmuch_crypto_t *crypto) } static const struct { const char *protocol; - notmuch_crypto_context_t *(*get_context) (notmuch_crypto_t *crypto); + notmuch_crypto_context_t *(*get_context) (_notmuch_crypto_t *crypto); } protocols[] = { { .protocol = "application/pgp-signature", @@ -90,7 +90,7 @@ static const struct { /* for the specified protocol return the context pointer (initializing * if needed) */ notmuch_crypto_context_t * -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) +_notmuch_crypto_get_context (_notmuch_crypto_t *crypto, const char *protocol) { notmuch_crypto_context_t *cryptoctx = NULL; size_t i; @@ -118,7 +118,7 @@ notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol) } int -notmuch_crypto_cleanup (notmuch_crypto_t *crypto) +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto) { if (crypto->gpgctx) { g_object_unref (crypto->gpgctx); @@ -133,7 +133,7 @@ notmuch_crypto_cleanup (notmuch_crypto_t *crypto) return 0; } #else -int notmuch_crypto_cleanup (unused(notmuch_crypto_t *crypto)) +int _notmuch_crypto_cleanup (unused(_notmuch_crypto_t *crypto)) { return 0; } diff --git a/mime-node.c b/mime-node.c index 8b767d78..56187746 100644 --- a/mime-node.c +++ b/mime-node.c @@ -33,7 +33,7 @@ typedef struct mime_node_context { GMimeMessage *mime_message; /* Context provided by the caller. */ - notmuch_crypto_t *crypto; + _notmuch_crypto_t *crypto; } mime_node_context_t; static int @@ -56,7 +56,7 @@ _mime_node_context_free (mime_node_context_t *res) notmuch_status_t mime_node_open (const void *ctx, notmuch_message_t *message, - notmuch_crypto_t *crypto, mime_node_t **root_out) + _notmuch_crypto_t *crypto, mime_node_t **root_out) { const char *filename = notmuch_message_get_filename (message); mime_node_context_t *mctx; @@ -265,7 +265,7 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { GMimeContentType *content_type = g_mime_object_get_content_type (part); const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); - cryptoctx = notmuch_crypto_get_context (node->ctx->crypto, protocol); + cryptoctx = _notmuch_crypto_get_context (node->ctx->crypto, protocol); if (!cryptoctx) return NULL; } diff --git a/notmuch-client.h b/notmuch-client.h index 0365baae..224912c5 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -72,7 +72,7 @@ typedef struct notmuch_show_format { const struct notmuch_show_params *params); } notmuch_show_format_t; -typedef struct notmuch_crypto { +typedef struct _notmuch_crypto { bool verify; bool decrypt; #if (GMIME_MAJOR_VERSION < 3) @@ -80,14 +80,14 @@ typedef struct notmuch_crypto { notmuch_crypto_context_t* pkcs7ctx; const char *gpgpath; #endif -} notmuch_crypto_t; +} _notmuch_crypto_t; typedef struct notmuch_show_params { bool entire_thread; bool omit_excluded; bool output_body; int part; - notmuch_crypto_t crypto; + _notmuch_crypto_t crypto; bool include_html; GMimeStream *out_stream; } notmuch_show_params_t; @@ -183,11 +183,11 @@ notmuch_exit_if_unsupported_format (void); #if (GMIME_MAJOR_VERSION <3) notmuch_crypto_context_t * -notmuch_crypto_get_context (notmuch_crypto_t *crypto, const char *protocol); +_notmuch_crypto_get_context (_notmuch_crypto_t *crypto, const char *protocol); #endif int -notmuch_crypto_cleanup (notmuch_crypto_t *crypto); +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto); int notmuch_count_command (notmuch_config_t *config, int argc, char *argv[]); @@ -449,7 +449,7 @@ struct mime_node { */ notmuch_status_t mime_node_open (const void *ctx, notmuch_message_t *message, - notmuch_crypto_t *crypto, mime_node_t **node_out); + _notmuch_crypto_t *crypto, mime_node_t **node_out); /* Return a new MIME node for the requested child part of parent. * parent will be used as the talloc context for the returned child diff --git a/notmuch-reply.c b/notmuch-reply.c index 3e5a1443..2c7cc4eb 100644 --- a/notmuch-reply.c +++ b/notmuch-reply.c @@ -759,7 +759,7 @@ notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[]) if (do_reply (config, query, ¶ms, format, reply_all) != 0) return EXIT_FAILURE; - notmuch_crypto_cleanup (¶ms.crypto); + _notmuch_crypto_cleanup (¶ms.crypto); notmuch_query_destroy (query); notmuch_database_destroy (notmuch); diff --git a/notmuch-show.c b/notmuch-show.c index 1cfc7465..7afd3947 100644 --- a/notmuch-show.c +++ b/notmuch-show.c @@ -1234,7 +1234,7 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[]) g_mime_stream_flush (params.out_stream); g_object_unref (params.out_stream); - notmuch_crypto_cleanup (¶ms.crypto); + _notmuch_crypto_cleanup (¶ms.crypto); notmuch_query_destroy (query); notmuch_database_destroy (notmuch); -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 02/15] crypto: drop pretense of notmuch_crypto_context_t 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 01/15] crypto: rename notmuch_crypto_t to _notmuch_crypto_t Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 03/15] crypto: _notmuch_crypto_cleanup should return void Daniel Kahn Gillmor ` (14 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail notmuch_crypto_context_t was introduced (i think) as some sort of abstraction layer to make notmuch somewhat independent of GMime. But it isn't even useful for GMime 3.0 or later -- we can drop the pretense that it's some sort of abstraction in this case, and just call it what it is, GMimeCryptoContext, which is useful for building against older versions of GMime. This also renames _notmuch_crypto_get_context() to _notmuch_crypto_get_gmime_context(). --- crypto.c | 16 ++++++++-------- mime-node.c | 8 ++++---- notmuch-client.h | 9 ++++----- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crypto.c b/crypto.c index 9ae303d4..fc7c5d1d 100644 --- a/crypto.c +++ b/crypto.c @@ -21,10 +21,10 @@ #include "notmuch-client.h" #if (GMIME_MAJOR_VERSION < 3) /* Create a GPG context (GMime 2.6) */ -static notmuch_crypto_context_t * +static GMimeCryptoContext * create_gpg_context (_notmuch_crypto_t *crypto) { - notmuch_crypto_context_t *gpgctx; + GMimeCryptoContext *gpgctx; if (crypto->gpgctx) return crypto->gpgctx; @@ -44,10 +44,10 @@ create_gpg_context (_notmuch_crypto_t *crypto) } /* Create a PKCS7 context (GMime 2.6) */ -static notmuch_crypto_context_t * +static GMimeCryptoContext * create_pkcs7_context (_notmuch_crypto_t *crypto) { - notmuch_crypto_context_t *pkcs7ctx; + GMimeCryptoContext *pkcs7ctx; if (crypto->pkcs7ctx) return crypto->pkcs7ctx; @@ -67,7 +67,7 @@ create_pkcs7_context (_notmuch_crypto_t *crypto) } static const struct { const char *protocol; - notmuch_crypto_context_t *(*get_context) (_notmuch_crypto_t *crypto); + GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); } protocols[] = { { .protocol = "application/pgp-signature", @@ -89,10 +89,10 @@ static const struct { /* for the specified protocol return the context pointer (initializing * if needed) */ -notmuch_crypto_context_t * -_notmuch_crypto_get_context (_notmuch_crypto_t *crypto, const char *protocol) +GMimeCryptoContext * +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) { - notmuch_crypto_context_t *cryptoctx = NULL; + GMimeCryptoContext *cryptoctx = NULL; size_t i; if (! protocol) { diff --git a/mime-node.c b/mime-node.c index 56187746..02e3df48 100644 --- a/mime-node.c +++ b/mime-node.c @@ -171,7 +171,7 @@ set_signature_list_destructor (mime_node_t *node) /* Verify a signed mime node (GMime 2.6) */ static void node_verify (mime_node_t *node, GMimeObject *part, - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) { GError *err = NULL; @@ -192,7 +192,7 @@ node_verify (mime_node_t *node, GMimeObject *part, /* Decrypt and optionally verify an encrypted mime node (GMime 2.6) */ static void node_decrypt_and_verify (mime_node_t *node, GMimeObject *part, - g_mime_3_unused(notmuch_crypto_context_t *cryptoctx)) + g_mime_3_unused(GMimeCryptoContext *cryptoctx)) { GError *err = NULL; GMimeDecryptResult *decrypt_result = NULL; @@ -227,7 +227,7 @@ static mime_node_t * _mime_node_create (mime_node_t *parent, GMimeObject *part) { mime_node_t *node = talloc_zero (parent, mime_node_t); - notmuch_crypto_context_t *cryptoctx = NULL; + GMimeCryptoContext *cryptoctx = NULL; /* Set basic node properties */ node->part = part; @@ -265,7 +265,7 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { GMimeContentType *content_type = g_mime_object_get_content_type (part); const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); - cryptoctx = _notmuch_crypto_get_context (node->ctx->crypto, protocol); + cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); if (!cryptoctx) return NULL; } diff --git a/notmuch-client.h b/notmuch-client.h index 224912c5..62acc70e 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -32,7 +32,6 @@ #include "gmime-extra.h" -typedef GMimeCryptoContext notmuch_crypto_context_t; /* This is automatically included only since gmime 2.6.10 */ #include <gmime/gmime-pkcs7-context.h> @@ -76,8 +75,8 @@ typedef struct _notmuch_crypto { bool verify; bool decrypt; #if (GMIME_MAJOR_VERSION < 3) - notmuch_crypto_context_t* gpgctx; - notmuch_crypto_context_t* pkcs7ctx; + GMimeCryptoContext* gpgctx; + GMimeCryptoContext* pkcs7ctx; const char *gpgpath; #endif } _notmuch_crypto_t; @@ -182,8 +181,8 @@ void notmuch_exit_if_unsupported_format (void); #if (GMIME_MAJOR_VERSION <3) -notmuch_crypto_context_t * -_notmuch_crypto_get_context (_notmuch_crypto_t *crypto, const char *protocol); +GMimeCryptoContext * +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); #endif int -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 03/15] crypto: _notmuch_crypto_cleanup should return void 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 01/15] crypto: rename notmuch_crypto_t to _notmuch_crypto_t Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 02/15] crypto: drop pretense of notmuch_crypto_context_t Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 04/15] crypto: move into libutil Daniel Kahn Gillmor ` (13 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail There's no chance that _notmuch_crypto_cleanup() will ever return anything other than 0, and no one ever checks its return value anyway. So make it return void instead of int. --- crypto.c | 7 ++----- notmuch-client.h | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/crypto.c b/crypto.c index fc7c5d1d..4c1b7eec 100644 --- a/crypto.c +++ b/crypto.c @@ -117,7 +117,7 @@ _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protoc return NULL; } -int +void _notmuch_crypto_cleanup (_notmuch_crypto_t *crypto) { if (crypto->gpgctx) { @@ -129,12 +129,9 @@ _notmuch_crypto_cleanup (_notmuch_crypto_t *crypto) g_object_unref (crypto->pkcs7ctx); crypto->pkcs7ctx = NULL; } - - return 0; } #else -int _notmuch_crypto_cleanup (unused(_notmuch_crypto_t *crypto)) +void _notmuch_crypto_cleanup (unused(_notmuch_crypto_t *crypto)) { - return 0; } #endif diff --git a/notmuch-client.h b/notmuch-client.h index 62acc70e..bdcfd893 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -185,7 +185,7 @@ GMimeCryptoContext * _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); #endif -int +void _notmuch_crypto_cleanup (_notmuch_crypto_t *crypto); int -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 04/15] crypto: move into libutil 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (2 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 03/15] crypto: _notmuch_crypto_cleanup should return void Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-12 10:54 ` David Bremner 2017-10-10 5:49 ` [PATCH v3 05/15] gmime-extra: remove duplicate GMimeAddressType typedef Daniel Kahn Gillmor ` (12 subsequent siblings) 16 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail This prepares us for using the crypto object in both the library and the client. --- Makefile.local | 1 - notmuch-client.h | 22 +--------------------- util/Makefile.local | 2 +- crypto.c => util/crypto.c | 6 +++++- util/crypto.h | 28 ++++++++++++++++++++++++++++ 5 files changed, 35 insertions(+), 24 deletions(-) rename crypto.c => util/crypto.c (97%) create mode 100644 util/crypto.h diff --git a/Makefile.local b/Makefile.local index 9d9c52c2..9505b7fe 100644 --- a/Makefile.local +++ b/Makefile.local @@ -246,7 +246,6 @@ notmuch_client_srcs = \ sprinter-text.c \ query-string.c \ mime-node.c \ - crypto.c \ tag-util.c notmuch_client_modules = $(notmuch_client_srcs:.c=.o) diff --git a/notmuch-client.h b/notmuch-client.h index bdcfd893..d17cdf01 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -32,9 +32,6 @@ #include "gmime-extra.h" -/* This is automatically included only since gmime 2.6.10 */ -#include <gmime/gmime-pkcs7-context.h> - #include "notmuch.h" /* This is separate from notmuch-private.h because we're trying to @@ -54,6 +51,7 @@ #include <ctype.h> #include "talloc-extra.h" +#include "crypto.h" #define unused(x) x __attribute__ ((unused)) @@ -71,16 +69,6 @@ typedef struct notmuch_show_format { const struct notmuch_show_params *params); } notmuch_show_format_t; -typedef struct _notmuch_crypto { - bool verify; - bool decrypt; -#if (GMIME_MAJOR_VERSION < 3) - GMimeCryptoContext* gpgctx; - GMimeCryptoContext* pkcs7ctx; - const char *gpgpath; -#endif -} _notmuch_crypto_t; - typedef struct notmuch_show_params { bool entire_thread; bool omit_excluded; @@ -180,14 +168,6 @@ typedef struct _notmuch_config notmuch_config_t; void notmuch_exit_if_unsupported_format (void); -#if (GMIME_MAJOR_VERSION <3) -GMimeCryptoContext * -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); -#endif - -void -_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto); - int notmuch_count_command (notmuch_config_t *config, int argc, char *argv[]); diff --git a/util/Makefile.local b/util/Makefile.local index 3027880b..ba03230e 100644 --- a/util/Makefile.local +++ b/util/Makefile.local @@ -5,7 +5,7 @@ extra_cflags += -I$(srcdir)/$(dir) libnotmuch_util_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \ $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \ - $(dir)/util.c $(dir)/gmime-extra.c + $(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c libnotmuch_util_modules := $(libnotmuch_util_c_srcs:.c=.o) diff --git a/crypto.c b/util/crypto.c similarity index 97% rename from crypto.c rename to util/crypto.c index 4c1b7eec..39954ca0 100644 --- a/crypto.c +++ b/util/crypto.c @@ -18,7 +18,11 @@ * Authors: Jameson Rollins <jrollins@finestructure.net> */ -#include "notmuch-client.h" +#include "crypto.h" +#include "lib/notmuch-private.h" + +#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) + #if (GMIME_MAJOR_VERSION < 3) /* Create a GPG context (GMime 2.6) */ static GMimeCryptoContext * diff --git a/util/crypto.h b/util/crypto.h new file mode 100644 index 00000000..80628dc5 --- /dev/null +++ b/util/crypto.h @@ -0,0 +1,28 @@ +#ifndef _CRYPTO_H +#define _CRYPTO_H + +#include <stdbool.h> +#if (GMIME_MAJOR_VERSION < 3) +#include "gmime-extra.h" +#endif + +typedef struct _notmuch_crypto { + bool verify; + bool decrypt; +#if (GMIME_MAJOR_VERSION < 3) + GMimeCryptoContext* gpgctx; + GMimeCryptoContext* pkcs7ctx; + const char *gpgpath; +#endif +} _notmuch_crypto_t; + + +#if (GMIME_MAJOR_VERSION < 3) +GMimeCryptoContext * +_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); +#endif + +void +_notmuch_crypto_cleanup (_notmuch_crypto_t *crypto); + +#endif -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v3 04/15] crypto: move into libutil 2017-10-10 5:49 ` [PATCH v3 04/15] crypto: move into libutil Daniel Kahn Gillmor @ 2017-10-12 10:54 ` David Bremner 2017-10-12 14:07 ` Daniel Kahn Gillmor 0 siblings, 1 reply; 67+ messages in thread From: David Bremner @ 2017-10-12 10:54 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > This prepares us for using the crypto object in both the library and > the client. I think we could be more precise about names here, to help outsiders get up to speed on the code. libutil was renamed to libnotmuch_util, and "the library" should probably be "libnotmuch". > diff --git a/crypto.c b/util/crypto.c > similarity index 97% > rename from crypto.c > rename to util/crypto.c > index 4c1b7eec..39954ca0 100644 > --- a/crypto.c > +++ b/util/crypto.c > @@ -18,7 +18,11 @@ > * Authors: Jameson Rollins <jrollins@finestructure.net> > */ > > -#include "notmuch-client.h" > +#include "crypto.h" > +#include "lib/notmuch-private.h" > + This seems like a kind of layering violation. What's in notmuch-private.h that needs to be exported to things outside of lib/ ? ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v3 04/15] crypto: move into libutil 2017-10-12 10:54 ` David Bremner @ 2017-10-12 14:07 ` Daniel Kahn Gillmor 2017-10-12 21:07 ` David Bremner 0 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-12 14:07 UTC (permalink / raw) To: David Bremner, Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 1555 bytes --] Hi Bremner-- Thanks for the review! On Thu 2017-10-12 07:54:33 -0300, David Bremner wrote: > Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > >> This prepares us for using the crypto object in both the library and >> the client. > > I think we could be more precise about names here, to help outsiders get > up to speed on the code. libutil was renamed to libnotmuch_util, and > "the library" should probably be "libnotmuch". Are you asking for improved comment text in the git commit, or for something else? I'm happy to provide improved comment text, if that's what you're asking for. >> diff --git a/crypto.c b/util/crypto.c >> similarity index 97% >> rename from crypto.c >> rename to util/crypto.c >> index 4c1b7eec..39954ca0 100644 >> --- a/crypto.c >> +++ b/util/crypto.c >> @@ -18,7 +18,11 @@ >> * Authors: Jameson Rollins <jrollins@finestructure.net> >> */ >> >> -#include "notmuch-client.h" >> +#include "crypto.h" >> +#include "lib/notmuch-private.h" >> + > > This seems like a kind of layering violation. What's in > notmuch-private.h that needs to be exported to things outside of lib/ ? when building against gmime-3.0, this #include supplies: #define unused(x) x __attribute__ ((unused)) When building against gmime-2.6, it provides: #include <strings.h> I could replace it with these two things explicitly (and i could put them inside the GMIME_MAJOR_VERSION tests) if that would be preferable to #including lib/notmuch-private.h. Any preference? --dkg [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v3 04/15] crypto: move into libutil 2017-10-12 14:07 ` Daniel Kahn Gillmor @ 2017-10-12 21:07 ` David Bremner 0 siblings, 0 replies; 67+ messages in thread From: David Bremner @ 2017-10-12 21:07 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > Hi Bremner-- > > Thanks for the review! > > On Thu 2017-10-12 07:54:33 -0300, David Bremner wrote: >> Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: >> >>> This prepares us for using the crypto object in both the library and >>> the client. >> >> I think we could be more precise about names here, to help outsiders get >> up to speed on the code. libutil was renamed to libnotmuch_util, and >> "the library" should probably be "libnotmuch". > > Are you asking for improved comment text in the git commit, or for > something else? I'm happy to provide improved comment text, if that's > what you're asking for. > yes. >>> diff --git a/crypto.c b/util/crypto.c >>> similarity index 97% >>> rename from crypto.c >>> rename to util/crypto.c >>> index 4c1b7eec..39954ca0 100644 >>> --- a/crypto.c >>> +++ b/util/crypto.c >>> @@ -18,7 +18,11 @@ >>> * Authors: Jameson Rollins <jrollins@finestructure.net> >>> */ >>> >>> -#include "notmuch-client.h" >>> +#include "crypto.h" >>> +#include "lib/notmuch-private.h" >>> + >> >> This seems like a kind of layering violation. What's in >> notmuch-private.h that needs to be exported to things outside of lib/ ? > > when building against gmime-3.0, this #include supplies: > > #define unused(x) x __attribute__ ((unused)) > > When building against gmime-2.6, it provides: > > #include <strings.h> > > > I could replace it with these two things explicitly (and i could put > them inside the GMIME_MAJOR_VERSION tests) if that would be preferable > to #including lib/notmuch-private.h. Any preference? I guess the most correct thing would be to have a header file in util/ which has the 'unused' definition in it and is included where it is needed. Or failing that to repeat the definition as we already do. d ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v3 05/15] gmime-extra: remove duplicate GMimeAddressType typedef 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (3 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 04/15] crypto: move into libutil Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 06/15] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor ` (11 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail "typedef GMimeAddressType GMimeRecipientType" is already present further down in the compatibility wrapper (with other typedefs). We don't need it twice. --- util/gmime-extra.h | 1 - 1 file changed, 1 deletion(-) diff --git a/util/gmime-extra.h b/util/gmime-extra.h index de275bc1..e060bcc2 100644 --- a/util/gmime-extra.h +++ b/util/gmime-extra.h @@ -18,7 +18,6 @@ GMimeStream *g_mime_stream_stdout_new(void); #define g_mime_certificate_get_fpr16(cert) g_mime_certificate_get_key_id (cert) #define g_mime_certificate_get_uid(cert) g_mime_certificate_get_name (cert); #else /* GMime >= 3.0 */ -typedef GMimeAddressType GMimeRecipientType; #define GMIME_ENABLE_RFC_2047_WORKAROUNDS 0xdeadbeef #define g_mime_certificate_get_uid(cert) g_mime_certificate_get_key_id (cert); -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 06/15] crypto: make shared crypto code behave library-like 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (4 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 05/15] gmime-extra: remove duplicate GMimeAddressType typedef Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 07/15] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor ` (10 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail If we're going to reuse the crypto code across both the library and the client, then it needs to report error states properly and not write to stderr. --- lib/database.cc | 6 ++++ lib/notmuch.h | 17 +++++++++++ mime-node.c | 7 ++++- util/crypto.c | 92 ++++++++++++++++++++++++++++----------------------------- util/crypto.h | 7 +++-- 5 files changed, 79 insertions(+), 50 deletions(-) diff --git a/lib/database.cc b/lib/database.cc index 35c66939..02444e09 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -413,6 +413,12 @@ notmuch_status_to_string (notmuch_status_t status) return "Operation requires a database upgrade"; case NOTMUCH_STATUS_PATH_ERROR: return "Path supplied is illegal for this function"; + case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL: + return "Crypto protocol missing, malformed, or unintelligible"; + case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION: + return "Crypto engine initialization failure"; + case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL: + return "Unknown crypto protocol"; default: case NOTMUCH_STATUS_LAST_STATUS: return "Unknown error status value"; diff --git a/lib/notmuch.h b/lib/notmuch.h index e8e0cc12..669e01b1 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -191,6 +191,23 @@ typedef enum _notmuch_status { * function, in a way not covered by a more specific argument. */ NOTMUCH_STATUS_ILLEGAL_ARGUMENT, + /** + * A MIME object claimed to have cryptographic protection which + * notmuch tried to handle, but the protocol was not specified in + * an intelligible way. + */ + NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL, + /** + * Notmuch attempted to do crypto processing, but could not + * initialize the engine needed to do so. + */ + NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION, + /** + * A MIME object claimed to have cryptographic protection, and + * notmuch attempted to process it, but the specific protocol was + * something that notmuch doesn't know how to handle. + */ + NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL, /** * Not an actual status value. Just a way to find out how many * valid status values there are. diff --git a/mime-node.c b/mime-node.c index 02e3df48..07620f43 100644 --- a/mime-node.c +++ b/mime-node.c @@ -265,7 +265,12 @@ _mime_node_create (mime_node_t *parent, GMimeObject *part) || (GMIME_IS_MULTIPART_SIGNED (part) && node->ctx->crypto->verify)) { GMimeContentType *content_type = g_mime_object_get_content_type (part); const char *protocol = g_mime_content_type_get_parameter (content_type, "protocol"); - cryptoctx = _notmuch_crypto_get_gmime_context (node->ctx->crypto, protocol); + notmuch_status_t status; + status = _notmuch_crypto_get_gmime_ctx_for_protocol (node->ctx->crypto, + protocol, &cryptoctx); + if (status) /* this is a warning, not an error */ + fprintf (stderr, "Warning: %s (%s).\n", notmuch_status_to_string (status), + protocol ? protocol : "NULL"); if (!cryptoctx) return NULL; } diff --git a/util/crypto.c b/util/crypto.c index 39954ca0..908ba290 100644 --- a/util/crypto.c +++ b/util/crypto.c @@ -24,85 +24,86 @@ #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) #if (GMIME_MAJOR_VERSION < 3) -/* Create a GPG context (GMime 2.6) */ -static GMimeCryptoContext * -create_gpg_context (_notmuch_crypto_t *crypto) +/* Create or pass on a GPG context (GMime 2.6) */ +static notmuch_status_t +get_gpg_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) { - GMimeCryptoContext *gpgctx; + if (ctx == NULL || crypto == NULL) + return NOTMUCH_STATUS_NULL_POINTER; - if (crypto->gpgctx) - return crypto->gpgctx; + if (crypto->gpgctx) { + *ctx = crypto->gpgctx; + return NOTMUCH_STATUS_SUCCESS; + } /* TODO: GMimePasswordRequestFunc */ - gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); - if (! gpgctx) { - fprintf (stderr, "Failed to construct gpg context.\n"); - return NULL; + crypto->gpgctx = g_mime_gpg_context_new (NULL, crypto->gpgpath ? crypto->gpgpath : "gpg"); + if (! crypto->gpgctx) { + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; } - crypto->gpgctx = gpgctx; - g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) gpgctx, true); - g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) gpgctx, false); + g_mime_gpg_context_set_use_agent ((GMimeGpgContext *) crypto->gpgctx, true); + g_mime_gpg_context_set_always_trust ((GMimeGpgContext *) crypto->gpgctx, false); - return gpgctx; + *ctx = crypto->gpgctx; + return NOTMUCH_STATUS_SUCCESS; } -/* Create a PKCS7 context (GMime 2.6) */ -static GMimeCryptoContext * -create_pkcs7_context (_notmuch_crypto_t *crypto) +/* Create or pass on a PKCS7 context (GMime 2.6) */ +static notmuch_status_t +get_pkcs7_context (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx) { - GMimeCryptoContext *pkcs7ctx; + if (ctx == NULL || crypto == NULL) + return NOTMUCH_STATUS_NULL_POINTER; - if (crypto->pkcs7ctx) - return crypto->pkcs7ctx; + if (crypto->pkcs7ctx) { + *ctx = crypto->pkcs7ctx; + return NOTMUCH_STATUS_SUCCESS; + } /* TODO: GMimePasswordRequestFunc */ - pkcs7ctx = g_mime_pkcs7_context_new (NULL); - if (! pkcs7ctx) { - fprintf (stderr, "Failed to construct pkcs7 context.\n"); - return NULL; + crypto->pkcs7ctx = g_mime_pkcs7_context_new (NULL); + if (! crypto->pkcs7ctx) { + return NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION; } - crypto->pkcs7ctx = pkcs7ctx; - g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) pkcs7ctx, + g_mime_pkcs7_context_set_always_trust ((GMimePkcs7Context *) crypto->pkcs7ctx, false); - return pkcs7ctx; + *ctx = crypto->pkcs7ctx; + return NOTMUCH_STATUS_SUCCESS; } static const struct { const char *protocol; - GMimeCryptoContext *(*get_context) (_notmuch_crypto_t *crypto); + notmuch_status_t (*get_context) (_notmuch_crypto_t *crypto, GMimeCryptoContext **ctx); } protocols[] = { { .protocol = "application/pgp-signature", - .get_context = create_gpg_context, + .get_context = get_gpg_context, }, { .protocol = "application/pgp-encrypted", - .get_context = create_gpg_context, + .get_context = get_gpg_context, }, { .protocol = "application/pkcs7-signature", - .get_context = create_pkcs7_context, + .get_context = get_pkcs7_context, }, { .protocol = "application/x-pkcs7-signature", - .get_context = create_pkcs7_context, + .get_context = get_pkcs7_context, }, }; /* for the specified protocol return the context pointer (initializing * if needed) */ -GMimeCryptoContext * -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol) +notmuch_status_t +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, + const char *protocol, + GMimeCryptoContext **ctx) { - GMimeCryptoContext *cryptoctx = NULL; - size_t i; - - if (! protocol) { - fprintf (stderr, "Cryptographic protocol is empty.\n"); - return cryptoctx; - } + if (! protocol) + return NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL; /* As per RFC 1847 section 2.1: "the [protocol] value token is * comprised of the type and sub-type tokens of the Content-Type". @@ -110,15 +111,12 @@ _notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protoc * parameter names as defined in this document are * case-insensitive." Thus, we use strcasecmp for the protocol. */ - for (i = 0; i < ARRAY_SIZE (protocols); i++) { + for (size_t i = 0; i < ARRAY_SIZE (protocols); i++) { if (strcasecmp (protocol, protocols[i].protocol) == 0) - return protocols[i].get_context (crypto); + return protocols[i].get_context (crypto, ctx); } - fprintf (stderr, "Unknown or unsupported cryptographic protocol %s.\n", - protocol); - - return NULL; + return NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL; } void diff --git a/util/crypto.h b/util/crypto.h index 80628dc5..1ff0297d 100644 --- a/util/crypto.h +++ b/util/crypto.h @@ -4,6 +4,7 @@ #include <stdbool.h> #if (GMIME_MAJOR_VERSION < 3) #include "gmime-extra.h" +#include "notmuch.h" #endif typedef struct _notmuch_crypto { @@ -18,8 +19,10 @@ typedef struct _notmuch_crypto { #if (GMIME_MAJOR_VERSION < 3) -GMimeCryptoContext * -_notmuch_crypto_get_gmime_context (_notmuch_crypto_t *crypto, const char *protocol); +notmuch_status_t +_notmuch_crypto_get_gmime_ctx_for_protocol (_notmuch_crypto_t *crypto, + const char *protocol, + GMimeCryptoContext **ctx); #endif void -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 07/15] tests: prepare for more crypto tests (using add_gnupg_home) 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (5 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 06/15] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor ` (9 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail Move add_gnupg_home to test-lib.sh to prepare it for reuse. --- test/T350-crypto.sh | 17 ----------------- test/test-lib.sh | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh index 53bf4113..401a22c9 100755 --- a/test/T350-crypto.sh +++ b/test/T350-crypto.sh @@ -7,23 +7,6 @@ test_description='PGP/MIME signature verification and decryption' . ./test-lib.sh || exit 1 -add_gnupg_home () -{ - local output - [ -d ${GNUPGHOME} ] && return - _gnupg_exit () { gpgconf --kill all 2>/dev/null || true; } - at_exit_function _gnupg_exit - mkdir -m 0700 "$GNUPGHOME" - gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 - test_debug "cat $GNUPGHOME/import.log" - if (gpg --quick-random --version >/dev/null 2>&1) ; then - echo quick-random >> "$GNUPGHOME"/gpg.conf - elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then - echo debug-quick-random >> "$GNUPGHOME"/gpg.conf - fi - echo no-emit-version >> "$GNUPGHOME"/gpg.conf -} - ################################################## add_gnupg_home diff --git a/test/test-lib.sh b/test/test-lib.sh index 35024649..84051bc9 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -93,6 +93,23 @@ unset GREP_OPTIONS # For emacsclient unset ALTERNATE_EDITOR +add_gnupg_home () +{ + local output + [ -d ${GNUPGHOME} ] && return + _gnupg_exit () { gpgconf --kill all 2>/dev/null || true; } + at_exit_function _gnupg_exit + mkdir -m 0700 "$GNUPGHOME" + gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 + test_debug "cat $GNUPGHOME/import.log" + if (gpg --quick-random --version >/dev/null 2>&1) ; then + echo quick-random >> "$GNUPGHOME"/gpg.conf + elif (gpg --debug-quick-random --version >/dev/null 2>&1) ; then + echo debug-quick-random >> "$GNUPGHOME"/gpg.conf + fi + echo no-emit-version >> "$GNUPGHOME"/gpg.conf +} + # Each test should start with something like this, after copyright notices: # # test_description='Description of this test... -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 08/15] index: implement notmuch_indexopts_t with try_decrypt 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (6 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 07/15] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-11 0:29 ` avoid double typedef Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 09/15] gmime-extra: drop compat layer for g_mime_multipart_encrypted_decrypt Daniel Kahn Gillmor ` (8 subsequent siblings) 16 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail This is currently mostly a wrapper around _notmuch_crypto_t that keeps its internals private and doesn't expose any of the GMime API. However, non-crypto indexing options might also be added later (e.g. filters or other transformations). --- lib/add-message.cc | 9 ++++++++- lib/indexopts.c | 22 ++++++++++++++++++++-- lib/notmuch-private.h | 7 +++++++ lib/notmuch.h | 20 ++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index bce10a0f..bdbbeed7 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_status_t notmuch_database_index_file (notmuch_database_t *notmuch, const char *filename, - notmuch_indexopts_t unused (*indexopts), + notmuch_indexopts_t *indexopts, notmuch_message_t **message_ret) { notmuch_message_file_t *message_file; @@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2; notmuch_private_status_t private_status; bool is_ghost = false, is_new = false; + notmuch_indexopts_t *def_indexopts = NULL; const char *date; const char *from, *to, *subject; @@ -540,6 +541,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, if (is_new || is_ghost) _notmuch_message_set_header_values (message, date, from, subject); + if (!indexopts) + indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); + ret = _notmuch_message_index_file (message, message_file); if (ret) goto DONE; @@ -557,6 +561,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, } DONE: + if (def_indexopts) + notmuch_indexopts_destroy (def_indexopts); + if (message) { if ((ret == NOTMUCH_STATUS_SUCCESS || ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret) diff --git a/lib/indexopts.c b/lib/indexopts.c index 2f9b841b..cc1d6422 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -21,9 +21,27 @@ #include "notmuch-private.h" notmuch_indexopts_t * -notmuch_database_get_default_indexopts (notmuch_database_t unused (*db)) +notmuch_database_get_default_indexopts (notmuch_database_t *db) { - return NULL; + return talloc_zero (db, notmuch_indexopts_t); +} + +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + bool try_decrypt) +{ + if (!indexopts) + return NOTMUCH_STATUS_NULL_POINTER; + indexopts->crypto.decrypt = try_decrypt; + return NOTMUCH_STATUS_SUCCESS; +} + +bool +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts) +{ + if (!indexopts) + return false; + return indexopts->crypto.decrypt; } void diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index e86f4582..d6008bd0 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -52,6 +52,7 @@ NOTMUCH_BEGIN_DECLS #include "xutil.h" #include "error_util.h" #include "string-util.h" +#include "crypto.h" #ifdef DEBUG # define DEBUG_DATABASE_SANITY 1 @@ -633,6 +634,12 @@ _notmuch_thread_create (void *ctx, notmuch_exclude_t omit_exclude, notmuch_sort_t sort); +/* param.c */ + +typedef struct _notmuch_indexopts { + _notmuch_crypto_t crypto; +} notmuch_indexopts_t; + NOTMUCH_END_DECLS #ifdef __cplusplus diff --git a/lib/notmuch.h b/lib/notmuch.h index 669e01b1..0b2e8305 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -42,6 +42,7 @@ NOTMUCH_BEGIN_DECLS #include <time.h> +#include <stdbool.h> #pragma GCC visibility push(default) @@ -2214,6 +2215,25 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list); notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db); +/** + * Specify whether to decrypt encrypted parts while indexing. + * + * Be aware that the index is likely sufficient to reconstruct the + * cleartext of the message itself, so please ensure that the notmuch + * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE + * without considering the security of your index. + */ +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + bool try_decrypt); + +/** + * Return whether to decrypt encrypted parts while indexing. + * see notmuch_indexopts_set_try_decrypt. + */ +bool +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts); + /** * Destroy a notmuch_indexopts_t object. * -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* avoid double typedef 2017-10-10 5:49 ` [PATCH v3 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor @ 2017-10-11 0:29 ` Daniel Kahn Gillmor 2017-10-11 0:30 ` [PATCH v4 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor 2017-10-11 6:22 ` avoid double typedef Tomi Ollila 0 siblings, 2 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-11 0:29 UTC (permalink / raw) To: Notmuch Mail On IRC, Domo pointed out that older gcc complains when a typedef gets repeated. So i'm updating patch 8 in this series to avoid double typedefs. The rest of the series should be unchanged, so i'm avoiding re-flooding the lst with them, but i'm happy to send along a full round of v4 if folks would prefer that. the "cleartext-index" branch of https://gitlab.com/dkg/notmuch has been updated to include this fix, and is now at 9be9d7a6bf083968458ad2cecf4af7f0d99f1cb3 Regards, --dkg ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v4 08/15] index: implement notmuch_indexopts_t with try_decrypt 2017-10-11 0:29 ` avoid double typedef Daniel Kahn Gillmor @ 2017-10-11 0:30 ` Daniel Kahn Gillmor 2017-10-12 11:18 ` David Bremner 2017-10-11 6:22 ` avoid double typedef Tomi Ollila 1 sibling, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-11 0:30 UTC (permalink / raw) To: Notmuch Mail This is currently mostly a wrapper around _notmuch_crypto_t that keeps its internals private and doesn't expose any of the GMime API. However, non-crypto indexing options might also be added later (e.g. filters or other transformations). --- lib/add-message.cc | 9 ++++++++- lib/indexopts.c | 22 ++++++++++++++++++++-- lib/notmuch-private.h | 7 +++++++ lib/notmuch.h | 20 ++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index bce10a0f..bdbbeed7 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -460,7 +460,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_status_t notmuch_database_index_file (notmuch_database_t *notmuch, const char *filename, - notmuch_indexopts_t unused (*indexopts), + notmuch_indexopts_t *indexopts, notmuch_message_t **message_ret) { notmuch_message_file_t *message_file; @@ -468,6 +468,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2; notmuch_private_status_t private_status; bool is_ghost = false, is_new = false; + notmuch_indexopts_t *def_indexopts = NULL; const char *date; const char *from, *to, *subject; @@ -540,6 +541,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, if (is_new || is_ghost) _notmuch_message_set_header_values (message, date, from, subject); + if (!indexopts) + indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); + ret = _notmuch_message_index_file (message, message_file); if (ret) goto DONE; @@ -557,6 +561,9 @@ notmuch_database_index_file (notmuch_database_t *notmuch, } DONE: + if (def_indexopts) + notmuch_indexopts_destroy (def_indexopts); + if (message) { if ((ret == NOTMUCH_STATUS_SUCCESS || ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret) diff --git a/lib/indexopts.c b/lib/indexopts.c index 2f9b841b..cc1d6422 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -21,9 +21,27 @@ #include "notmuch-private.h" notmuch_indexopts_t * -notmuch_database_get_default_indexopts (notmuch_database_t unused (*db)) +notmuch_database_get_default_indexopts (notmuch_database_t *db) { - return NULL; + return talloc_zero (db, notmuch_indexopts_t); +} + +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + bool try_decrypt) +{ + if (!indexopts) + return NOTMUCH_STATUS_NULL_POINTER; + indexopts->crypto.decrypt = try_decrypt; + return NOTMUCH_STATUS_SUCCESS; +} + +bool +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts) +{ + if (!indexopts) + return false; + return indexopts->crypto.decrypt; } void diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index e86f4582..da6a8c38 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -52,6 +52,7 @@ NOTMUCH_BEGIN_DECLS #include "xutil.h" #include "error_util.h" #include "string-util.h" +#include "crypto.h" #ifdef DEBUG # define DEBUG_DATABASE_SANITY 1 @@ -633,6 +634,12 @@ _notmuch_thread_create (void *ctx, notmuch_exclude_t omit_exclude, notmuch_sort_t sort); +/* param.c */ + +struct _notmuch_indexopts { + _notmuch_crypto_t crypto; +}; + NOTMUCH_END_DECLS #ifdef __cplusplus diff --git a/lib/notmuch.h b/lib/notmuch.h index 669e01b1..0b2e8305 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -42,6 +42,7 @@ NOTMUCH_BEGIN_DECLS #include <time.h> +#include <stdbool.h> #pragma GCC visibility push(default) @@ -2214,6 +2215,25 @@ notmuch_config_list_destroy (notmuch_config_list_t *config_list); notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db); +/** + * Specify whether to decrypt encrypted parts while indexing. + * + * Be aware that the index is likely sufficient to reconstruct the + * cleartext of the message itself, so please ensure that the notmuch + * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE + * without considering the security of your index. + */ +notmuch_status_t +notmuch_indexopts_set_try_decrypt (notmuch_indexopts_t *indexopts, + bool try_decrypt); + +/** + * Return whether to decrypt encrypted parts while indexing. + * see notmuch_indexopts_set_try_decrypt. + */ +bool +notmuch_indexopts_get_try_decrypt (const notmuch_indexopts_t *indexopts); + /** * Destroy a notmuch_indexopts_t object. * -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v4 08/15] index: implement notmuch_indexopts_t with try_decrypt 2017-10-11 0:30 ` [PATCH v4 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor @ 2017-10-12 11:18 ` David Bremner 2017-10-12 14:30 ` Daniel Kahn Gillmor 0 siblings, 1 reply; 67+ messages in thread From: David Bremner @ 2017-10-12 11:18 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > > + if (!indexopts) > + indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); > + I think I'd like parens here indexopts = (def_indexopts = notmuch_database_get_default_indexopts (notmuch)); or even split into two assignments. Others might disagree though. ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v4 08/15] index: implement notmuch_indexopts_t with try_decrypt 2017-10-12 11:18 ` David Bremner @ 2017-10-12 14:30 ` Daniel Kahn Gillmor 0 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-12 14:30 UTC (permalink / raw) To: David Bremner, Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 785 bytes --] On Thu 2017-10-12 08:18:15 -0300, David Bremner wrote: > Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > >> >> + if (!indexopts) >> + indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); >> + > > I think I'd like parens here > > indexopts = (def_indexopts = notmuch_database_get_default_indexopts (notmuch)); > > or even split into two assignments. Others might disagree though. I've gone with the simpler variant of splitting into two assignments. My updated branch is still cleartext-index on https://gitlab.com/dkg/notmuch, with commit ID 77e8c5079e854d1934e47637c15f06bbc14a4819. Shall i push this to the mailing list as v5 of this series, or are there more reviews coming that i should try to integrate? --dkg [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: avoid double typedef 2017-10-11 0:29 ` avoid double typedef Daniel Kahn Gillmor 2017-10-11 0:30 ` [PATCH v4 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor @ 2017-10-11 6:22 ` Tomi Ollila 1 sibling, 0 replies; 67+ messages in thread From: Tomi Ollila @ 2017-10-11 6:22 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail On Tue, Oct 10 2017, Daniel Kahn Gillmor wrote: > On IRC, Domo pointed out that older gcc complains when a typedef gets > repeated. So i'm updating patch 8 in this series to avoid double > typedefs. The rest of the series should be unchanged, so i'm avoiding > re-flooding the lst with them, but i'm happy to send along a full > round of v4 if folks would prefer that. This version fixed the (c99 related ;) gcc compilation error on that particular test system (actually this system where this email is sent). The me the series looks good (based on ~30 min code look-through session) and the tests pass (as good as w/o these changes). Tomi > > the "cleartext-index" branch of https://gitlab.com/dkg/notmuch has > been updated to include this fix, and is now at > 9be9d7a6bf083968458ad2cecf4af7f0d99f1cb3 > > Regards, > > --dkg > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > https://notmuchmail.org/mailman/listinfo/notmuch ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v3 09/15] gmime-extra: drop compat layer for g_mime_multipart_encrypted_decrypt 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (7 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-14 14:02 ` David Bremner 2017-10-10 5:49 ` [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor ` (7 subsequent siblings) 16 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail In practice, we're going to see this function invoked differently depending on which gmime we build against. The compatibility layer forces our code into the lowest-common-denominator -- unable to make use of new features even when built against a newer version. Dropping the compatibility layer paves the way for clearer use of features from GMime 3.0 in future commits. --- mime-node.c | 4 ++++ util/gmime-extra.h | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mime-node.c b/mime-node.c index 07620f43..c3d5cb9b 100644 --- a/mime-node.c +++ b/mime-node.c @@ -200,7 +200,11 @@ node_decrypt_and_verify (mime_node_t *node, GMimeObject *part, node->decrypt_attempted = true; node->decrypted_child = g_mime_multipart_encrypted_decrypt +#if (GMIME_MAJOR_VERSION < 3) (encrypteddata, cryptoctx, &decrypt_result, &err); +#else + (encrypteddata, GMIME_DECRYPT_NONE, NULL, &decrypt_result, &err); +#endif if (! node->decrypted_child) { fprintf (stderr, "Failed to decrypt part: %s\n", err ? err->message : "no error explanation given"); diff --git a/util/gmime-extra.h b/util/gmime-extra.h index e060bcc2..40bf1454 100644 --- a/util/gmime-extra.h +++ b/util/gmime-extra.h @@ -29,7 +29,6 @@ GMimeStream *g_mime_stream_stdout_new(void); #define g_mime_init(flags) g_mime_init() #define g_mime_message_add_recipient(m,t,n,a) g_mime_message_add_mailbox (m,t,n,a) #define g_mime_message_set_subject(m,s) g_mime_message_set_subject(m,s,NULL) -#define g_mime_multipart_encrypted_decrypt(mpe,ctx,out,err) g_mime_multipart_encrypted_decrypt(mpe, GMIME_DECRYPT_NONE, NULL, out, err) #define g_mime_multipart_signed_verify(mps,ctx,err) g_mime_multipart_signed_verify(mps, GMIME_ENCRYPT_NONE, err) #define g_mime_object_write_to_stream(o,s) g_mime_object_write_to_stream (o,NULL,s) #define g_mime_object_set_header(o,h,v) g_mime_object_set_header (o,h,v,NULL) -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v3 09/15] gmime-extra: drop compat layer for g_mime_multipart_encrypted_decrypt 2017-10-10 5:49 ` [PATCH v3 09/15] gmime-extra: drop compat layer for g_mime_multipart_encrypted_decrypt Daniel Kahn Gillmor @ 2017-10-14 14:02 ` David Bremner 0 siblings, 0 replies; 67+ messages in thread From: David Bremner @ 2017-10-14 14:02 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > In practice, we're going to see this function invoked differently > depending on which gmime we build against. The compatibility layer > forces our code into the lowest-common-denominator -- unable to make > use of new features even when built against a newer version. > > Dropping the compatibility layer paves the way for clearer use of > features from GMime 3.0 in future commits. pushed patches 7 and 9. d ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (8 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 09/15] gmime-extra: drop compat layer for g_mime_multipart_encrypted_decrypt Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-13 1:08 ` David Bremner 2017-10-10 5:49 ` [PATCH v3 11/15] config: indexing defaults will be stored in the database Daniel Kahn Gillmor ` (6 subsequent siblings) 16 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail If we see index options that ask us to decrypt when indexing a message, and we encounter an encrypted part, we'll try to descend into it. If we can decrypt, we add the property index-decryption=success. If we can't decrypt (or recognize the encrypted type of mail), we add the property index-decryption=failure. Note that a single message may have both values of the "index-decryption" property: "success" and "failure". For example, consider a message that includes multiple layers of encryption. If we manage to decrypt the outer layer ("index-decryption=success"), but fail on the inner layer ("index-decryption=failure"). Before re-indexing, we wipe this automatically-added property, so that it will subsequently correspond to the actual semantics of the stored index. --- lib/add-message.cc | 2 +- lib/index.cc | 91 ++++++++++++++++++++++++++++++++++++++++++++++----- lib/message.cc | 14 ++++++-- lib/notmuch-private.h | 1 + 4 files changed, 96 insertions(+), 12 deletions(-) diff --git a/lib/add-message.cc b/lib/add-message.cc index bdbbeed7..abd50057 100644 --- a/lib/add-message.cc +++ b/lib/add-message.cc @@ -544,7 +544,7 @@ notmuch_database_index_file (notmuch_database_t *notmuch, if (!indexopts) indexopts = def_indexopts = notmuch_database_get_default_indexopts (notmuch); - ret = _notmuch_message_index_file (message, message_file); + ret = _notmuch_message_index_file (message, indexopts, message_file); if (ret) goto DONE; diff --git a/lib/index.cc b/lib/index.cc index e5ae2ba7..4b8b2344 100644 --- a/lib/index.cc +++ b/lib/index.cc @@ -364,9 +364,15 @@ _index_content_type (notmuch_message_t *message, GMimeObject *part) } } +static void +_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts, + GMimeContentType *content_type, + GMimeMultipartEncrypted *part); + /* Callback to generate terms for each mime part of a message. */ static void _index_mime_part (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, GMimeObject *part) { GMimeStream *stream, *filter; @@ -385,6 +391,7 @@ _index_mime_part (notmuch_message_t *message, } _index_content_type (message, part); + content_type = g_mime_object_get_content_type (part); if (GMIME_IS_MULTIPART (part)) { GMimeMultipart *multipart = GMIME_MULTIPART (part); @@ -409,17 +416,21 @@ _index_mime_part (notmuch_message_t *message, } } if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) { - /* Don't index encrypted parts, but index their content type. */ _index_content_type (message, g_mime_multipart_get_part (multipart, i)); - if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) && - (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) { - _notmuch_database_log (_notmuch_message_database (message), - "Warning: Unexpected extra parts of multipart/encrypted.\n"); + if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) { + _index_encrypted_mime_part(message, indexopts, + content_type, + GMIME_MULTIPART_ENCRYPTED (part)); + } else { + if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) { + _notmuch_database_log (_notmuch_message_database (message), + "Warning: Unexpected extra parts of multipart/encrypted.\n"); + } } continue; } - _index_mime_part (message, + _index_mime_part (message, indexopts, g_mime_multipart_get_part (multipart, i)); } return; @@ -430,7 +441,7 @@ _index_mime_part (notmuch_message_t *message, mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part)); - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); return; } @@ -464,7 +475,6 @@ _index_mime_part (notmuch_message_t *message, filter = g_mime_stream_filter_new (stream); - content_type = g_mime_object_get_content_type (part); discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type); g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter), @@ -502,8 +512,71 @@ _index_mime_part (notmuch_message_t *message, } } +/* descend (if desired) into the cleartext part of an encrypted MIME + * part while indexing. */ +static void +_index_encrypted_mime_part (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, + g_mime_3_unused(GMimeContentType *content_type), + GMimeMultipartEncrypted *encrypted_data) +{ + notmuch_status_t status; + GError *err = NULL; + notmuch_database_t * notmuch = NULL; + GMimeObject *clear = NULL; + + if (!indexopts || !notmuch_indexopts_get_try_decrypt (indexopts)) + return; + + notmuch = _notmuch_message_database (message); + +#if (GMIME_MAJOR_VERSION < 3) + { + GMimeCryptoContext* crypto_ctx = NULL; + const char *protocol = NULL; + protocol = g_mime_content_type_get_parameter (content_type, "protocol"); + status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto), + protocol, &crypto_ctx); + if (status) { + _notmuch_database_log (notmuch, "Warning: setup failed for decrypting " + "during indexing. (%d)\n", status); + status = notmuch_message_add_property (message, "index-decryption", "failure"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + return; + } + clear = g_mime_multipart_encrypted_decrypt(encrypted_data, crypto_ctx, + NULL, &err); + } +#else + clear = g_mime_multipart_encrypted_decrypt(encrypted_data, GMIME_DECRYPT_NONE, NULL, + NULL, &err); +#endif + if (err) { + _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n", + err->domain, err->code, err->message); + g_error_free(err); + /* Indicate that we failed to decrypt during indexing */ + status = notmuch_message_add_property (message, "index-decryption", "failure"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + return; + } + _index_mime_part (message, indexopts, clear); + g_object_unref (clear); + + status = notmuch_message_add_property (message, "index-decryption", "success"); + if (status) + _notmuch_database_log (notmuch, "failed to add index-decryption " + "property (%d)\n", status); + +} + notmuch_status_t _notmuch_message_index_file (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, notmuch_message_file_t *message_file) { GMimeMessage *mime_message; @@ -531,7 +604,7 @@ _notmuch_message_index_file (notmuch_message_t *message, subject = g_mime_message_get_subject (mime_message); _notmuch_message_gen_terms (message, "subject", subject); - _index_mime_part (message, g_mime_message_get_mime_part (mime_message)); + _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message)); return NOTMUCH_STATUS_SUCCESS; } diff --git a/lib/message.cc b/lib/message.cc index 4ab0ed26..0102580b 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -1961,7 +1961,7 @@ _notmuch_message_frozen (notmuch_message_t *message) notmuch_status_t notmuch_message_reindex (notmuch_message_t *message, - notmuch_indexopts_t unused (*indexopts)) + notmuch_indexopts_t *indexopts) { notmuch_database_t *notmuch = NULL; notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; @@ -1969,6 +1969,7 @@ notmuch_message_reindex (notmuch_message_t *message, notmuch_filenames_t *orig_filenames = NULL; const char *orig_thread_id = NULL; notmuch_message_file_t *message_file = NULL; + const char *autoproperties[] = { "index-decryption" }; int found = 0; @@ -1999,6 +2000,15 @@ notmuch_message_reindex (notmuch_message_t *message, goto DONE; } + /* all automatically-added properties should be removed before re-indexing */ + for (size_t i = 0; i < ARRAY_SIZE (autoproperties); i++) { + ret = notmuch_message_remove_all_properties (message, autoproperties[i]); + if (ret) { + INTERNAL_ERROR ("failed to remove automatically-added property '%s'", autoproperties[i]); + goto DONE; + } + } + /* re-add the filenames with the associated indexopts */ for (; notmuch_filenames_valid (orig_filenames); notmuch_filenames_move_to_next (orig_filenames)) { @@ -2038,7 +2048,7 @@ notmuch_message_reindex (notmuch_message_t *message, if (found == 0) _notmuch_message_set_header_values (message, date, from, subject); - ret = _notmuch_message_index_file (message, message_file); + ret = _notmuch_message_index_file (message, indexopts, message_file); if (ret == NOTMUCH_STATUS_FILE_ERROR) continue; diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index d6008bd0..1f4e411f 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -448,6 +448,7 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch, notmuch_status_t _notmuch_message_index_file (notmuch_message_t *message, + notmuch_indexopts_t *indexopts, notmuch_message_file_t *message_file); /* messages.c */ -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-10-10 5:49 ` [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor @ 2017-10-13 1:08 ` David Bremner 2017-10-13 14:35 ` Daniel Kahn Gillmor 0 siblings, 1 reply; 67+ messages in thread From: David Bremner @ 2017-10-13 1:08 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > + if (status) { > + _notmuch_database_log (notmuch, "Warning: setup failed for decrypting " > + "during indexing. (%d)\n", status); > + status = notmuch_message_add_property (message, "index-decryption", "failure"); > + if (status) > + _notmuch_database_log (notmuch, "failed to add index-decryption " > + "property (%d)\n", status); > + return; > + } The second _notmuch_database_log will override the first here. You can use _notmuch_database_log_append if you don't want to clear the existing log (e.g. at least for the second _log here). > + const char *autoproperties[] = { "index-decryption" }; I'm always a bit nervous when I see the same string hard coded into two different places. What about using some prefix naming scheme like "index.auto.decryption" with the idea that all properties with the prefix "index.auto." could be blown away. If we settle on a prefix based naming scheme now, we could do the minor tweak to the properties API later so that it's only a single call. ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-10-13 1:08 ` David Bremner @ 2017-10-13 14:35 ` Daniel Kahn Gillmor 2017-10-13 15:19 ` David Bremner 2017-10-14 11:15 ` David Bremner 0 siblings, 2 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-13 14:35 UTC (permalink / raw) To: David Bremner, Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 1902 bytes --] On Thu 2017-10-12 22:08:26 -0300, David Bremner wrote: > The second _notmuch_database_log will override the first > here. You can use _notmuch_database_log_append if you don't want to > clear the existing log (e.g. at least for the second _log > here). good catch, thanks! I'll include that in my next revision. It's already fixed in my gitlab branch. >> + const char *autoproperties[] = { "index-decryption" }; > > I'm always a bit nervous when I see the same string hard coded into two > different places. What about using some prefix naming scheme like > "index.auto.decryption" with the idea that all properties with the > prefix "index.auto." could be blown away. If we settle on a prefix based > naming scheme now, we could do the minor tweak to the properties API > later so that it's only a single call. Hm, i see what you're saying, but i'm a bit worried that stuffing this "property of a property" into the name can cause two different forms of confusion. first confusion: "index.auto.decryption" has at least two different meanings at first blush. For example, it might mean "this message was automatically decrypted", which isn't necessarily the case. second confusion: Even if we used a less ambiguous term like "flush-on-reindex", it would seem weird to need to say "notmuch search property:index.flush-on-reindex.decryption=failure", since the property we're interested in is whether there was a failure in decrypting during index time, not whether we expect the thing to be flushed on reindex. We could maybe avoid both of these confusions by dropping "auto" altogether and defining the following rule: * properties prefixed with "index." are set during indexing and will be cleared (and possibly re-set) upon reindexing wdyt? If you're ok with that, i could try to update the series to work that way (and include documentation along those lines). --dkg [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-10-13 14:35 ` Daniel Kahn Gillmor @ 2017-10-13 15:19 ` David Bremner 2017-10-14 11:15 ` David Bremner 1 sibling, 0 replies; 67+ messages in thread From: David Bremner @ 2017-10-13 15:19 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > > * properties prefixed with "index." are set during indexing and will be > cleared (and possibly re-set) upon reindexing > Fine. I think the model is that "applications" should take ownership of ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set. 2017-10-13 14:35 ` Daniel Kahn Gillmor 2017-10-13 15:19 ` David Bremner @ 2017-10-14 11:15 ` David Bremner 1 sibling, 0 replies; 67+ messages in thread From: David Bremner @ 2017-10-14 11:15 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > > "index.auto.decryption" has at least two different meanings at first > blush. For example, it might mean "this message was automatically > decrypted", which isn't necessarily the case. I think in either case you need to document what the properties mean. I don't think "index-decryption" by itself is unambiguous either. > second confusion: > > Even if we used a less ambiguous term like "flush-on-reindex", it would > seem weird to need to say "notmuch search > property:index.flush-on-reindex.decryption=failure", Yes, you seem to envision interactive searching for these properities. I'll have to take your word for it. I tend to think about properties as more like APIs than as end user facing. > We could maybe avoid both of these confusions by dropping "auto" > altogether and defining the following rule: > > * properties prefixed with "index." are set during indexing and will be > cleared (and possibly re-set) upon reindexing > > wdyt? > > If you're ok with that, i could try to update the series to work that > way (and include documentation along those lines). I don't object to this. ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v3 11/15] config: indexing defaults will be stored in the database. 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (9 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-14 18:08 ` David Bremner 2017-10-10 5:49 ` [PATCH v3 12/15] config: define new option index.try_decrypt Daniel Kahn Gillmor ` (5 subsequent siblings) 16 siblings, 1 reply; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail At indexing time, the database needs to know its internal defaults. It shouldn't be contingent on an external config file (since that can't be retrieved from the database object itself). This behaves the same as the query.* configurations, which are also stored in the database itself, so we're not introducing any new dependencies. --- notmuch-config.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/notmuch-config.c b/notmuch-config.c index 8fb59f96..e82f7dd7 100644 --- a/notmuch-config.c +++ b/notmuch-config.c @@ -809,6 +809,7 @@ _item_split (char *item, char **group, char **key) #define BUILT_WITH_PREFIX "built_with." #define QUERY_PREFIX "query." +#define INDEX_PREFIX "index." static int _print_db_config(notmuch_config_t *config, const char *name) @@ -859,6 +860,8 @@ notmuch_config_command_get (notmuch_config_t *config, char *item) notmuch_built_with (item + strlen (BUILT_WITH_PREFIX)) ? "true" : "false"); } else if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) { return _print_db_config (config, item); + } else if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) { + return _print_db_config (config, item); } else { char **value; size_t i, length; @@ -931,6 +934,9 @@ notmuch_config_command_set (notmuch_config_t *config, char *item, int argc, char if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) { return _set_db_config (config, item, argc, argv); } + if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) { + return _set_db_config (config, item, argc, argv); + } if (_item_split (item, &group, &key)) return 1; -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: [PATCH v3 11/15] config: indexing defaults will be stored in the database. 2017-10-10 5:49 ` [PATCH v3 11/15] config: indexing defaults will be stored in the database Daniel Kahn Gillmor @ 2017-10-14 18:08 ` David Bremner 2017-10-15 6:28 ` Daniel Kahn Gillmor 0 siblings, 1 reply; 67+ messages in thread From: David Bremner @ 2017-10-14 18:08 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > static int > _print_db_config(notmuch_config_t *config, const char *name) > @@ -859,6 +860,8 @@ notmuch_config_command_get (notmuch_config_t *config, char *item) > notmuch_built_with (item + strlen (BUILT_WITH_PREFIX)) ? "true" : "false"); > } else if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) { > return _print_db_config (config, item); > + } else if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) { > + return _print_db_config (config, item); > } else { > char **value; I wonder if we should sanity check the value of 'item' more here. With 'query.', it makes sense to get or set anything, since it's just the name of a stored query. With 'index.', presumably only certain parameters make sense. As a motivating example, consider someone who sets $ notmuch config index.try_decrypt true then changes their mind $ notmuch config index.try-decrypt false They _think_ they are safe, but notmuch is silently going to continue decrypting their mail ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: [PATCH v3 11/15] config: indexing defaults will be stored in the database. 2017-10-14 18:08 ` David Bremner @ 2017-10-15 6:28 ` Daniel Kahn Gillmor 0 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-15 6:28 UTC (permalink / raw) To: David Bremner, Notmuch Mail On Sat 2017-10-14 15:08:51 -0300, David Bremner wrote: > Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > >> static int >> _print_db_config(notmuch_config_t *config, const char *name) >> @@ -859,6 +860,8 @@ notmuch_config_command_get (notmuch_config_t *config, char *item) >> notmuch_built_with (item + strlen (BUILT_WITH_PREFIX)) ? "true" : "false"); >> } else if (STRNCMP_LITERAL (item, QUERY_PREFIX) == 0) { >> return _print_db_config (config, item); >> + } else if (STRNCMP_LITERAL (item, INDEX_PREFIX) == 0) { >> + return _print_db_config (config, item); >> } else { >> char **value; > > I wonder if we should sanity check the value of 'item' more here. With > 'query.', it makes sense to get or set anything, since it's just the > name of a stored query. With 'index.', presumably only certain > parameters make sense. As a motivating example, consider someone who > sets > > $ notmuch config index.try_decrypt true > > then changes their mind > > $ notmuch config index.try-decrypt false > > They _think_ they are safe, but notmuch is silently going to continue > decrypting their mail Do you have any suggestions about how we should do this check? Some sort of registry of known options? If we have such a registry, how would an addon or a frontend use notmuch config without having to modify that registry? --dkg ^ permalink raw reply [flat|nested] 67+ messages in thread
* [PATCH v3 12/15] config: define new option index.try_decrypt 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (10 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 11/15] config: indexing defaults will be stored in the database Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 13/15] cli/new: add --try-decrypt=(true|false) Daniel Kahn Gillmor ` (4 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail By default, notmuch won't try to decrypt on indexing. With this patch, we make it possible to indicate a per-database preference using the config variable "index.try_decrypt", which by default will be false. --- doc/man1/notmuch-config.rst | 12 ++++++++++++ lib/indexopts.c | 18 +++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst index 6a51e64f..6f35d127 100644 --- a/doc/man1/notmuch-config.rst +++ b/doc/man1/notmuch-config.rst @@ -134,6 +134,18 @@ The available configuration items are described below. Default: ``gpg``. + **index.try_decrypt** + + When indexing an encrypted e-mail message, if this variable is + set to true, notmuch will try to decrypt the message and index + the cleartext. Be aware that the index is likely sufficient + to reconstruct the cleartext of the message itself, so please + ensure that the notmuch message index is adequately protected. + DO NOT USE ``index.try_decrypt=true`` without considering the + security of your index. + + Default: ``false``. + **built_with.<name>** Compile time feature <name>. Current possibilities include diff --git a/lib/indexopts.c b/lib/indexopts.c index cc1d6422..987d8952 100644 --- a/lib/indexopts.c +++ b/lib/indexopts.c @@ -23,7 +23,23 @@ notmuch_indexopts_t * notmuch_database_get_default_indexopts (notmuch_database_t *db) { - return talloc_zero (db, notmuch_indexopts_t); + notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t); + if (!ret) + return ret; + + char * try_decrypt; + notmuch_status_t err = notmuch_database_get_config (db, "index.try_decrypt", &try_decrypt); + if (err) + return ret; + + if (try_decrypt && + ((!(strcasecmp(try_decrypt, "true"))) || + (!(strcasecmp(try_decrypt, "yes"))) || + (!(strcasecmp(try_decrypt, "1"))))) + notmuch_indexopts_set_try_decrypt (ret, true); + + free (try_decrypt); + return ret; } notmuch_status_t -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 13/15] cli/new: add --try-decrypt=(true|false) 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (11 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 12/15] config: define new option index.try_decrypt Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 14/15] cli/insert: " Daniel Kahn Gillmor ` (3 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail Try to decrypt any encrypted parts of newly-discovered messages while indexing them. The cleartext of any successfully-decrypted messages will be indexed, with tags applied in the same form as from notmuch insert --try-decrypt=true. Note: if the deprecated crypto.gpg_path configuration option is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). We also add a new test making use of this functionality. This requires a bit of reorganization, because we need to allow passing --long-arguments to "notmuch new" via emacs_fcc_message --- completion/notmuch-completion.bash | 13 ++++++++-- doc/man1/notmuch-new.rst | 12 +++++++++ notmuch-new.c | 29 +++++++++++++++++++++- test/T357-index-decryption.sh | 51 ++++++++++++++++++++++++++++++++++++++ test/test-lib.sh | 11 +++++++- 5 files changed, 112 insertions(+), 4 deletions(-) create mode 100755 test/T357-index-decryption.sh diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 5201be63..17be6b8f 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -311,11 +311,20 @@ _notmuch_insert() _notmuch_new() { local cur prev words cword split - _init_completion || return + _init_completion -s || return + + $split && + case "${prev}" in + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; + esac + ! $split && case "${cur}" in -*) - local options="--no-hooks --quiet ${_notmuch_shared_options}" + local options="--no-hooks --try-decrypt= --quiet ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "${options}" -- ${cur}) ) ;; diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst index 6acfa112..c255f980 100644 --- a/doc/man1/notmuch-new.rst +++ b/doc/man1/notmuch-new.rst @@ -43,6 +43,18 @@ Supported options for **new** include ``--quiet`` Do not print progress or results. + ``--try-decrypt=(true|false)`` + + If true, when encountering an encrypted message, try to + decrypt it while indexing. If decryption is successful, index + the cleartext itself. Be aware that the index is likely + sufficient to reconstruct the cleartext of the message itself, + so please ensure that the notmuch message index is adequately + protected. DO NOT USE ``--try-decrypt=true`` without + considering the security of your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + EXIT STATUS =========== diff --git a/notmuch-new.c b/notmuch-new.c index 0f50457e..93046602 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -49,6 +49,7 @@ typedef struct { size_t new_tags_length; const char **new_ignore; size_t new_ignore_length; + notmuch_indexopts_t *indexopts; int total_files; int processed_files; @@ -267,7 +268,7 @@ add_file (notmuch_database_t *notmuch, const char *filename, if (status) goto DONE; - status = notmuch_database_index_file (notmuch, filename, NULL, &message); + status = notmuch_database_index_file (notmuch, filename, state->indexopts, &message); switch (status) { /* Success. */ case NOTMUCH_STATUS_SUCCESS: @@ -956,6 +957,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) bool timer_is_active = false; bool no_hooks = false; bool quiet = false, verbose = false; + bool try_decrypt_set = false, try_decrypt = false; notmuch_status_t status; notmuch_opt_desc_t options[] = { @@ -963,6 +965,8 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) { .opt_bool = &verbose, .name = "verbose" }, { .opt_bool = &add_files_state.debug, .name = "debug" }, { .opt_bool = &no_hooks, .name = "no-hooks" }, + { .opt_bool = &try_decrypt, .name = "try-decrypt", + .present = &try_decrypt_set }, { .opt_inherit = notmuch_shared_options }, { } }; @@ -1080,6 +1084,29 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) if (notmuch == NULL) return EXIT_FAILURE; + add_files_state.indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!add_files_state.indexopts) { + fprintf (stderr, "Error: could not create index options.\n"); + return EXIT_FAILURE; + } + if (try_decrypt_set) { + status = notmuch_indexopts_set_try_decrypt (add_files_state.indexopts, try_decrypt); + if (status != NOTMUCH_STATUS_SUCCESS) { + fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n", + try_decrypt ? "True" : "False", notmuch_status_to_string (status)); + notmuch_indexopts_destroy (add_files_state.indexopts); + return EXIT_FAILURE; + } + } +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (add_files_state.indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif + /* Set up our handler for SIGINT. We do this after having * potentially done a database upgrade we this interrupt handler * won't support. */ diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh new file mode 100755 index 00000000..7bbd81f6 --- /dev/null +++ b/test/T357-index-decryption.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# TODO: test index-decryption-failed + +test_description='indexing decrypted mail' +. ./test-lib.sh || exit 1 + +################################################## + +add_gnupg_home +# get key fingerprint +FINGERPRINT=$(gpg --no-tty --list-secret-keys --with-colons --fingerprint | grep '^fpr:' | cut -d: -f10) + +# create a test encrypted message +test_begin_subtest 'emacs delivery of encrypted message' +test_expect_success \ +'emacs_fcc_message \ + "test encrypted message for cleartext index 001" \ + "This is a test encrypted message with a wumpus.\n" \ + "(mml-secure-message-encrypt)"' + +test_begin_subtest "search for unindexed cleartext" +output=$(notmuch search wumpus) +expected='' +test_expect_equal \ + "$output" \ + "$expected" + +# create a test encrypted message that is indexed in the clear +test_begin_subtest 'emacs delivery of encrypted message' +test_expect_success \ +'emacs_fcc_message --try-decrypt=true \ + "test encrypted message for cleartext index 002" \ + "This is a test encrypted message with a wumpus.\n" \ + "(mml-secure-message-encrypt)"' + +test_begin_subtest "emacs delivery of encrypted message, indexed cleartext" +output=$(notmuch search wumpus) +expected='thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property for one message" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + +test_done diff --git a/test/test-lib.sh b/test/test-lib.sh index 84051bc9..9c336662 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -340,8 +340,17 @@ emacs_deliver_message () # Accepts arbitrary extra emacs/elisp functions to modify the message # before sending, which is useful to doing things like attaching files # to the message and encrypting/signing. +# +# If any GNU-style long-arguments (like --quiet or --try-decrypt=true) are +# at the head of the argument list, they are sent directly to "notmuch +# new" after message delivery emacs_fcc_message () { + local nmn_args='' + while [[ "$1" =~ ^-- ]]; do + nmn_args="$nmn_args $1" + shift + done local subject="$1" local body="$2" shift 2 @@ -360,7 +369,7 @@ emacs_fcc_message () (insert \"${body}\") $@ (notmuch-mua-send-and-exit))" || return 1 - notmuch new >/dev/null + notmuch new $nmn_args >/dev/null } # Add an existing, fixed corpus of email to the database. -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 14/15] cli/insert: add --try-decrypt=(true|false) 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (12 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 13/15] cli/new: add --try-decrypt=(true|false) Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 15/15] cli/reindex: " Daniel Kahn Gillmor ` (2 subsequent siblings) 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail Allow an incoming message to be delivered while indexing the cleartext, on a per-message basis. This requires the secret keys for the message to be available. For the moment, the most functional approach is to ensure that gpg-agent is running and knows about any secret keys that might be useful to decrypt incoming mail. Any additional recommendations for how to phrase the caveat for this option are welcome. Note: if the deprecated crypto.gpg_path is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). --- completion/notmuch-completion.bash | 6 +++++- doc/man1/notmuch-insert.rst | 14 ++++++++++++++ notmuch-insert.c | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 17be6b8f..72a75a94 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -287,12 +287,16 @@ _notmuch_insert() sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) ) return ;; + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; esac ! $split && case "${cur}" in --*) - local options="--create-folder --folder= --keep --no-hooks ${_notmuch_shared_options}" + local options="--create-folder --folder= --keep --no-hooks --try-decrypt= ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) return diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst index f79600d6..647dac06 100644 --- a/doc/man1/notmuch-insert.rst +++ b/doc/man1/notmuch-insert.rst @@ -50,6 +50,20 @@ Supported options for **insert** include ``--no-hooks`` Prevent hooks from being run. + ``--try-decrypt=(true|false)`` + + If true and the message is encrypted, try to decrypt the + message while indexing. If decryption is successful, index + the cleartext itself. Either way, the message is always + stored to disk in its original form (ciphertext). Be aware + that the index is likely sufficient to reconstruct the + cleartext of the message itself, so please ensure that the + notmuch message index is adequately protected. DO NOT USE + ``--try-decrypt=true`` without considering the security of + your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + EXIT STATUS =========== diff --git a/notmuch-insert.c b/notmuch-insert.c index 32be7419..79f9cb7d 100644 --- a/notmuch-insert.c +++ b/notmuch-insert.c @@ -379,12 +379,13 @@ FAIL: */ static notmuch_status_t add_file (notmuch_database_t *notmuch, const char *path, tag_op_list_t *tag_ops, - bool synchronize_flags, bool keep) + bool synchronize_flags, bool keep, + notmuch_indexopts_t *indexopts) { notmuch_message_t *message; notmuch_status_t status; - status = notmuch_database_index_file (notmuch, path, NULL, &message); + status = notmuch_database_index_file (notmuch, path, indexopts, &message); if (status == NOTMUCH_STATUS_SUCCESS) { status = tag_op_list_apply (message, tag_ops, 0); if (status) { @@ -456,17 +457,21 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) bool create_folder = false; bool keep = false; bool no_hooks = false; + bool try_decrypt = false, try_decrypt_set = false; bool synchronize_flags; char *maildir; char *newpath; int opt_index; unsigned int i; + notmuch_indexopts_t *indexopts; notmuch_opt_desc_t options[] = { { .opt_string = &folder, .name = "folder" }, { .opt_bool = &create_folder, .name = "create-folder" }, { .opt_bool = &keep, .name = "keep" }, { .opt_bool = &no_hooks, .name = "no-hooks" }, + { .opt_bool = &try_decrypt, .name = "try-decrypt", + .present = &try_decrypt_set }, { .opt_inherit = notmuch_shared_options }, { } }; @@ -545,9 +550,31 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_exit_if_unmatched_db_uuid (notmuch); + indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!indexopts) { + fprintf (stderr, "Error: could not create index options.\n"); + return EXIT_FAILURE; + } + if (try_decrypt_set) { + status = notmuch_indexopts_set_try_decrypt (indexopts, try_decrypt); + if (status != NOTMUCH_STATUS_SUCCESS) { + fprintf (stderr, "Error: Failed to set try_decrypt to %s. (%s)\n", + try_decrypt ? "True" : "False", notmuch_status_to_string (status)); + notmuch_indexopts_destroy (indexopts); + return EXIT_FAILURE; + } + } +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif /* Index the message. */ - status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep); + status = add_file (notmuch, newpath, tag_ops, synchronize_flags, keep, indexopts); /* Commit changes. */ close_status = notmuch_database_destroy (notmuch); -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* [PATCH v3 15/15] cli/reindex: add --try-decrypt=(true|false) 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (13 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 14/15] cli/insert: " Daniel Kahn Gillmor @ 2017-10-10 5:49 ` Daniel Kahn Gillmor 2017-10-10 15:50 ` cleartext indexing, round 3 Jameson Graef Rollins 2017-10-13 1:28 ` David Bremner 16 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 5:49 UTC (permalink / raw) To: Notmuch Mail Try to decrypt any encrypted parts of newly-discovered messages while re-indexing them. The cleartext of any successfully-decrypted messages will be indexed, with tags applied in the same form as from notmuch insert --try-decrypt=true. Note: if the deprecated crypto.gpg_path configuration option is set to anything other than "gpg", we ignore it (and print a warning on stderr, if built against gmime < 3.0). --- completion/notmuch-completion.bash | 10 +++++- doc/man1/notmuch-reindex.rst | 14 ++++++++ notmuch-reindex.c | 24 ++++++++++++++ test/T357-index-decryption.sh | 65 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash index 72a75a94..7aae4297 100644 --- a/completion/notmuch-completion.bash +++ b/completion/notmuch-completion.bash @@ -435,10 +435,18 @@ _notmuch_reindex() local cur prev words cword split _init_completion -s || return + $split && + case "${prev}" in + --try-decrypt) + COMPREPLY=( $( compgen -W "true false" -- "${cur}" ) ) + return + ;; + esac + ! $split && case "${cur}" in -*) - local options="${_notmuch_shared_options}" + local options="--try-decrypt= ${_notmuch_shared_options}" compopt -o nospace COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) ;; diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst index e39cc4ee..60a060a7 100644 --- a/doc/man1/notmuch-reindex.rst +++ b/doc/man1/notmuch-reindex.rst @@ -19,6 +19,20 @@ The **reindex** command searches for all messages matching the supplied search terms, and re-creates the full-text index on these messages using the supplied options. +Supported options for **reindex** include + + ``--try-decrypt=(true|false)`` + + If true, when encountering an encrypted message, try to + decrypt it while reindexing. If decryption is successful, + index the cleartext itself. Be aware that the index is likely + sufficient to reconstruct the cleartext of the message itself, + so please ensure that the notmuch message index is adequately + protected. DO NOT USE ``--try-decrypt=true`` without + considering the security of your index. + + See also ``index.try_decrypt`` in **notmuch-config(1)**. + SEE ALSO ======== diff --git a/notmuch-reindex.c b/notmuch-reindex.c index 57ff5904..f4ad31c5 100644 --- a/notmuch-reindex.c +++ b/notmuch-reindex.c @@ -90,6 +90,8 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) int opt_index; int ret; notmuch_indexopts_t *indexopts = NULL; + bool try_decrypt = false, try_decrypt_set = false; + notmuch_status_t status; /* Set up our handler for SIGINT */ memset (&action, 0, sizeof (struct sigaction)); @@ -100,6 +102,8 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_opt_desc_t options[] = { { .opt_inherit = notmuch_shared_options }, + { .opt_bool = &try_decrypt, .name = "try-decrypt", + .present = &try_decrypt_set }, { } }; @@ -115,6 +119,26 @@ notmuch_reindex_command (notmuch_config_t *config, int argc, char *argv[]) notmuch_exit_if_unmatched_db_uuid (notmuch); + indexopts = notmuch_database_get_default_indexopts (notmuch); + if (!indexopts) + return EXIT_FAILURE; + + if (try_decrypt_set) { + status = notmuch_indexopts_set_try_decrypt (indexopts, try_decrypt); + if (status) + fprintf (stderr, "Warning: failed to set --try-decrypt to %d (%s)\n", + try_decrypt, notmuch_status_to_string (status)); + } + +#if (GMIME_MAJOR_VERSION < 3) + if (notmuch_indexopts_get_try_decrypt (indexopts)) { + const char* gpg_path = notmuch_config_get_crypto_gpg_path (config); + if (gpg_path && strcmp(gpg_path, "gpg")) + fprintf (stderr, "Warning: deprecated crypto.gpg_path is set to '%s'\n" + "\tbut ignoring (use $PATH instead)\n", gpg_path); + } +#endif + query_string = query_string_from_args (config, argc-opt_index, argv+opt_index); if (query_string == NULL) { fprintf (stderr, "Out of memory\n"); diff --git a/test/T357-index-decryption.sh b/test/T357-index-decryption.sh index 7bbd81f6..3d18f5af 100755 --- a/test/T357-index-decryption.sh +++ b/test/T357-index-decryption.sh @@ -48,4 +48,69 @@ test_expect_equal \ "$output" \ "$expected" +# add a tag to all messages to ensure that it stays after reindexing +test_begin_subtest 'tagging all messages' +test_expect_success 'notmuch tag +blarney "encrypted message"' +test_begin_subtest "verify that tags have not changed" +output=$(notmuch search tag:blarney) +expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + +# see if first message shows up after reindexing with --try-decrypt=true (same $expected, untouched): +test_begin_subtest 'reindex old messages' +test_expect_success 'notmuch reindex --try-decrypt=true tag:encrypted and not property:index-decryption=success' +test_begin_subtest "reindexed encrypted message, including cleartext" +output=$(notmuch search wumpus) +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property for both messages" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + + +# try to remove cleartext indexing +test_begin_subtest 'reindex without cleartext' +test_expect_success 'notmuch reindex tag:encrypted and property:index-decryption=success' +test_begin_subtest "reindexed encrypted messages, without cleartext" +output=$(notmuch search wumpus) +expected='' +test_expect_equal \ + "$output" \ + "$expected" + +# and the same search, but by property ($expected is untouched): +test_begin_subtest "emacs search by property with both messages unindexed" +output=$(notmuch search property:index-decryption=success) +test_expect_equal \ + "$output" \ + "$expected" + +# ensure that the tags remain even when we are dropping the cleartext. +test_begin_subtest "verify that tags remain without cleartext" +output=$(notmuch search tag:blarney) +expected='thread:0000000000000001 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 001 (blarney encrypted inbox) +thread:0000000000000002 2000-01-01 [1/1] Notmuch Test Suite; test encrypted message for cleartext index 002 (blarney encrypted inbox)' +test_expect_equal \ + "$output" \ + "$expected" + + +# TODO: test removal of a message from the message store between +# indexing and reindexing. + +# TODO: insert the same message into the message store twice, index, +# remove one of them from the message store, and then reindex. +# reindexing should return a failure but the message should still be +# present? -- or what should the semantics be if you ask to reindex a +# message whose underlying files have been renamed or moved or +# removed? + test_done -- 2.14.2 ^ permalink raw reply related [flat|nested] 67+ messages in thread
* Re: cleartext indexing, round 3 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (14 preceding siblings ...) 2017-10-10 5:49 ` [PATCH v3 15/15] cli/reindex: " Daniel Kahn Gillmor @ 2017-10-10 15:50 ` Jameson Graef Rollins 2017-10-10 16:47 ` Daniel Kahn Gillmor 2017-10-13 1:28 ` David Bremner 16 siblings, 1 reply; 67+ messages in thread From: Jameson Graef Rollins @ 2017-10-10 15:50 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 422 bytes --] On Tue, Oct 10 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: > I've also pushed this series to the "cleartext-indexing" branch at > https://gitlab.com/dkg/notmuch for those who prefer a direct git pull. > it is currently git commit 6f7f6847141db2f031b29c68d966fa13c3be2da5. Hey, Daniel, I think you mean the branch named "cleartext-index". That's the branch in your repo that corresponds to that hash. jamie. [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: cleartext indexing, round 3 2017-10-10 15:50 ` cleartext indexing, round 3 Jameson Graef Rollins @ 2017-10-10 16:47 ` Daniel Kahn Gillmor 0 siblings, 0 replies; 67+ messages in thread From: Daniel Kahn Gillmor @ 2017-10-10 16:47 UTC (permalink / raw) To: Jameson Graef Rollins, Notmuch Mail [-- Attachment #1: Type: text/plain, Size: 627 bytes --] On Tue 2017-10-10 08:50:17 -0700, Jameson Graef Rollins wrote: > On Tue, Oct 10 2017, Daniel Kahn Gillmor <dkg@fifthhorseman.net> wrote: >> I've also pushed this series to the "cleartext-indexing" branch at >> https://gitlab.com/dkg/notmuch for those who prefer a direct git pull. >> it is currently git commit 6f7f6847141db2f031b29c68d966fa13c3be2da5. > > Hey, Daniel, I think you mean the branch named "cleartext-index". > That's the branch in your repo that corresponds to that hash. yep, that's the correct name for the branch in question. sorry for the accidental extra gerunding in my e-mail message. --dkg [-- Attachment #2: signature.asc --] [-- Type: application/pgp-signature, Size: 832 bytes --] ^ permalink raw reply [flat|nested] 67+ messages in thread
* Re: cleartext indexing, round 3 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor ` (15 preceding siblings ...) 2017-10-10 15:50 ` cleartext indexing, round 3 Jameson Graef Rollins @ 2017-10-13 1:28 ` David Bremner 16 siblings, 0 replies; 67+ messages in thread From: David Bremner @ 2017-10-13 1:28 UTC (permalink / raw) To: Daniel Kahn Gillmor, Notmuch Mail Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes: > What follows is the third round of the latest revision of cleartext > indexing patches. pushed patches 1,2,3, and 5 to master. d ^ permalink raw reply [flat|nested] 67+ messages in thread
end of thread, other threads:[~2017-10-15 6:48 UTC | newest] Thread overview: 67+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2017-09-12 23:01 [PATCH 01/10] reorganize indexing of multipart/signed and multipart/encrypted Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 02/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 03/10] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 04/10] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 05/10] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 06/10] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 07/10] Define new config option index.try_decrypt Daniel Kahn Gillmor 2017-09-12 23:29 ` Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 08/10] add --try-decrypt=(true|false) to notmuch new Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 09/10] add --try-decrypt=(true|false) to notmuch insert Daniel Kahn Gillmor 2017-09-12 23:01 ` [PATCH 10/10] add --try-decrypt=(true|false) to notmuch reindex Daniel Kahn Gillmor 2017-09-15 5:53 ` cleartext-indexing Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 01/10] crypto: Move crypto.c into libutil Daniel Kahn Gillmor 2017-09-23 15:23 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 02/10] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor 2017-09-23 15:36 ` Jani Nikula 2017-10-10 3:33 ` Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 03/10] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor 2017-09-23 15:38 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 04/10] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor 2017-09-23 16:10 ` Jani Nikula 2017-10-10 3:45 ` Daniel Kahn Gillmor 2017-10-14 12:40 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 05/10] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor 2017-09-23 16:05 ` Jani Nikula 2017-10-10 4:27 ` Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 06/10] config: indexing defaults will be stored in the database Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 07/10] config: define new option index.try_decrypt Daniel Kahn Gillmor 2017-09-23 16:17 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 08/10] cli/new: add --try-decrypt=(true|false) Daniel Kahn Gillmor 2017-09-23 16:46 ` Jani Nikula 2017-09-15 5:53 ` [PATCH v2 09/10] cli/insert: " Daniel Kahn Gillmor 2017-09-15 5:53 ` [PATCH v2 10/10] cli/reindex: " Daniel Kahn Gillmor 2017-10-10 5:49 ` cleartext indexing, round 3 Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 01/15] crypto: rename notmuch_crypto_t to _notmuch_crypto_t Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 02/15] crypto: drop pretense of notmuch_crypto_context_t Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 03/15] crypto: _notmuch_crypto_cleanup should return void Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 04/15] crypto: move into libutil Daniel Kahn Gillmor 2017-10-12 10:54 ` David Bremner 2017-10-12 14:07 ` Daniel Kahn Gillmor 2017-10-12 21:07 ` David Bremner 2017-10-10 5:49 ` [PATCH v3 05/15] gmime-extra: remove duplicate GMimeAddressType typedef Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 06/15] crypto: make shared crypto code behave library-like Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 07/15] tests: prepare for more crypto tests (using add_gnupg_home) Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor 2017-10-11 0:29 ` avoid double typedef Daniel Kahn Gillmor 2017-10-11 0:30 ` [PATCH v4 08/15] index: implement notmuch_indexopts_t with try_decrypt Daniel Kahn Gillmor 2017-10-12 11:18 ` David Bremner 2017-10-12 14:30 ` Daniel Kahn Gillmor 2017-10-11 6:22 ` avoid double typedef Tomi Ollila 2017-10-10 5:49 ` [PATCH v3 09/15] gmime-extra: drop compat layer for g_mime_multipart_encrypted_decrypt Daniel Kahn Gillmor 2017-10-14 14:02 ` David Bremner 2017-10-10 5:49 ` [PATCH v3 10/15] crypto: index encrypted parts when indexopts try_decrypt is set Daniel Kahn Gillmor 2017-10-13 1:08 ` David Bremner 2017-10-13 14:35 ` Daniel Kahn Gillmor 2017-10-13 15:19 ` David Bremner 2017-10-14 11:15 ` David Bremner 2017-10-10 5:49 ` [PATCH v3 11/15] config: indexing defaults will be stored in the database Daniel Kahn Gillmor 2017-10-14 18:08 ` David Bremner 2017-10-15 6:28 ` Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 12/15] config: define new option index.try_decrypt Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 13/15] cli/new: add --try-decrypt=(true|false) Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 14/15] cli/insert: " Daniel Kahn Gillmor 2017-10-10 5:49 ` [PATCH v3 15/15] cli/reindex: " Daniel Kahn Gillmor 2017-10-10 15:50 ` cleartext indexing, round 3 Jameson Graef Rollins 2017-10-10 16:47 ` Daniel Kahn Gillmor 2017-10-13 1:28 ` David Bremner
Code repositories for project(s) associated with this public inbox https://yhetil.org/notmuch.git/ This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).