v3 of message properties patches

v3 of message properties patches

[David Bremner]

This obsoletes

     id:1470184228-12517-1-git-send-email-david@tethera.net

In particular it fixes the compile errors (boo), and adds dkg's
has: queries, with minimal doc and tests.

interdiff follows:

diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index 075f88c..86d02ee 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -58,6 +58,8 @@ indicate user-supplied values):
 
 -  query:<name>
 
+-  has:<key>=<value>
+
 The **from:** prefix is used to match the name or address of the sender
 of an email message.
 
@@ -139,6 +141,11 @@ queries added with **notmuch-config(1)**. Named queries are only
 available if notmuch is built with **Xapian Field Processors** (see
 below).
 
+The **has:** prefix searches for messages with a particular
+<key>=<value> property pair. Properties are used internally by notmuch
+(and extensions) to add metadata to messages. A given key can be
+present on a given message with several different values.
+
 Operators
 ---------
 
@@ -217,7 +224,7 @@ exact matches like "tag:inbox"  or **probabilistic**, supporting a more flexible
 
 
 Boolean
-   **tag:**, **id:**, **thread:**, **folder:**, **path:**
+   **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has**
 Probabilistic
    **from:**, **to:**, **subject:**, **attachment:**, **mimetype:**
 
diff --git a/lib/database.cc b/lib/database.cc
index 3a741f0..3bdbd07 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -97,6 +97,9 @@ typedef struct {
  *		        STRING is the name of a file within that
  *		        directory for this mail message.
  *
+ *      has:       Has a property with key=value
+ *                 FIXME: if no = is present, should match on any value
+ *
  *    A mail document also has four values:
  *
  *	TIMESTAMP:	The time_t value corresponding to the message's
@@ -260,6 +263,7 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "is",			"K" },
     { "id",			"Q" },
     { "path",			"P" },
+    { "has",			"XPROPERTY" },
     /*
      * Without the ":", since this is a multi-letter prefix, Xapian
      * will add a colon itself if the first letter of the path is
diff --git a/notmuch-restore.c b/notmuch-restore.c
index 3cd8a40..d6429ef 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -83,7 +83,7 @@ process_properties_line (notmuch_database_t *notmuch, const char* line)
 	goto DONE;
 
     if (print_status_database ("notmuch restore", notmuch,
-			       notmuch_message_remove_all_properties (message)))
+			       notmuch_message_remove_all_properties (message, NULL)))
 	goto DONE;
 
     tok = id_p + id_len;
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index 8952eb7..ee7910b 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -91,7 +91,7 @@ test_expect_equal_file EXPECTED OUTPUT
 
 test_begin_subtest "notmuch_message_remove_all_properties"
 cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
-EXPECT0(notmuch_message_remove_all_properties (message));
+EXPECT0(notmuch_message_remove_all_properties (message, NULL));
 print_properties (message, "", FALSE);
 EOF
 cat <<'EOF' >EXPECTED
@@ -237,4 +237,15 @@ notmuch restore < BEFORE2
 notmuch dump | grep '^#=' > OUTPUT
 test_expect_equal_file PROPERTIES OUTPUT
 
+test_begin_subtest "test 'has:' queries: empty"
+notmuch search has:testkey1=charles > OUTPUT
+test_expect_equal_file /dev/null OUTPUT
+
+test_begin_subtest "test 'has:' queries: single message"
+notmuch search --output=messages has:testkey1=alice > OUTPUT
+cat <<EOF >EXPECTED
+id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done

[PATCH 1/9] lib: read "property" terms from messages.

[David Bremner]

This is a first step towards providing an API to attach
arbitrary (key,value) pairs to messages and retrieve all of the values
for a given key.
---
 lib/database.cc       |  1 +
 lib/message.cc        | 29 ++++++++++++++++++++++++++++-
 lib/notmuch-private.h |  3 +++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/lib/database.cc b/lib/database.cc
index 66ee267..3a741f0 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -251,6 +251,7 @@ static prefix_t BOOLEAN_PREFIX_INTERNAL[] = {
     { "directory",		"XDIRECTORY" },
     { "file-direntry",		"XFDIRENTRY" },
     { "directory-direntry",	"XDDIRENTRY" },
+    { "property",               "XPROPERTY"  },
 };
 
 static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
diff --git a/lib/message.cc b/lib/message.cc
index 24e698a..63a8da5 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -37,6 +37,8 @@ struct visible _notmuch_message {
     notmuch_string_list_t *filename_list;
     char *author;
     notmuch_message_file_t *message_file;
+    notmuch_string_list_t *property_term_list;
+    notmuch_string_map_t *property_map;
     notmuch_message_list_t *replies;
     unsigned long flags;
     /* For flags that are initialized on-demand, lazy_flags indicates
@@ -116,6 +118,8 @@ _notmuch_message_create_for_document (const void *talloc_owner,
     message->filename_list = NULL;
     message->message_file = NULL;
     message->author = NULL;
+    message->property_term_list = NULL;
+    message->property_map = NULL;
 
     message->replies = _notmuch_message_list_create (message);
     if (unlikely (message->replies == NULL)) {
@@ -314,6 +318,7 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
 	*id_prefix = _find_prefix ("id"),
 	*type_prefix = _find_prefix ("type"),
 	*filename_prefix = _find_prefix ("file-direntry"),
+	*property_prefix = _find_prefix ("property"),
 	*replyto_prefix = _find_prefix ("replyto");
 
     /* We do this all in a single pass because Xapian decompresses the
@@ -369,11 +374,21 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
 	    _notmuch_database_get_terms_with_prefix (message, i, end,
 						     filename_prefix);
 
+
+    /* Get property terms. Mimic the setup with filenames above */
+    assert (strcmp (filename_prefix, property_prefix) < 0);
+    if (!message->property_map && !message->property_term_list)
+	message->property_term_list =
+	    _notmuch_database_get_terms_with_prefix (message, i, end,
+						     property_prefix);
+
     /* Get reply to */
-    assert (strcmp (filename_prefix, replyto_prefix) < 0);
+    assert (strcmp (property_prefix, replyto_prefix) < 0);
     if (!message->in_reply_to)
 	message->in_reply_to =
 	    _notmuch_message_get_term (message, i, end, replyto_prefix);
+
+
     /* It's perfectly valid for a message to have no In-Reply-To
      * header. For these cases, we return an empty string. */
     if (!message->in_reply_to)
@@ -405,6 +420,18 @@ _notmuch_message_invalidate_metadata (notmuch_message_t *message,
 	message->filename_term_list = message->filename_list = NULL;
     }
 
+    if (strcmp ("property", prefix_name) == 0) {
+
+	if (message->property_term_list)
+	    talloc_free (message->property_term_list);
+	message->property_term_list = NULL;
+
+	if (message->property_map)
+	    talloc_unlink (message, message->property_map);
+
+	message->property_map = NULL;
+    }
+
     if (strcmp ("replyto", prefix_name) == 0) {
 	talloc_free (message->in_reply_to);
 	message->in_reply_to = NULL;
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 3721431..df63905 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -537,6 +537,9 @@ _notmuch_string_list_append (notmuch_string_list_t *list,
 void
 _notmuch_string_list_sort (notmuch_string_list_t *list);
 
+/* string-map.c */
+typedef struct _notmuch_string_map  notmuch_string_map_t;
+
 /* tags.c */
 
 notmuch_tags_t *
-- 
2.8.1

[PATCH 2/9] lib: private string map (associative array) API

[David Bremner]

The choice of array implementation is deliberate, for future iterator support
---
 lib/Makefile.local    |   2 +
 lib/notmuch-private.h |  11 ++++
 lib/string-map.c      | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 166 insertions(+)
 create mode 100644 lib/string-map.c

diff --git a/lib/Makefile.local b/lib/Makefile.local
index beb9635..c012ed1 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -40,6 +40,7 @@ libnotmuch_c_srcs =		\
 	$(dir)/messages.c	\
 	$(dir)/sha1.c		\
 	$(dir)/built-with.c	\
+	$(dir)/string-map.c    \
 	$(dir)/tags.c
 
 libnotmuch_cxx_srcs =		\
@@ -48,6 +49,7 @@ libnotmuch_cxx_srcs =		\
 	$(dir)/directory.cc	\
 	$(dir)/index.cc		\
 	$(dir)/message.cc	\
+	$(dir)/message-property.cc \
 	$(dir)/query.cc		\
 	$(dir)/query-fp.cc      \
 	$(dir)/config.cc	\
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index df63905..5abde88 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -540,6 +540,17 @@ _notmuch_string_list_sort (notmuch_string_list_t *list);
 /* string-map.c */
 typedef struct _notmuch_string_map  notmuch_string_map_t;
 
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx);
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value);
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
+
 /* tags.c */
 
 notmuch_tags_t *
diff --git a/lib/string-map.c b/lib/string-map.c
new file mode 100644
index 0000000..0491a10
--- /dev/null
+++ b/lib/string-map.c
@@ -0,0 +1,153 @@
+/* string-map.c - associative arrays of strings
+ *
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+
+/* Create a new notmuch_string_map_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+
+typedef struct _notmuch_string_pair_t {
+    char *key;
+    char *value;
+} notmuch_string_pair_t;
+
+struct _notmuch_string_map {
+    notmuch_bool_t sorted;
+    size_t length;
+    notmuch_string_pair_t *pairs;
+};
+
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx)
+{
+    notmuch_string_map_t *map;
+
+    map = talloc (ctx, notmuch_string_map_t);
+    if (unlikely (map == NULL))
+	return NULL;
+
+    map->length = 0;
+    map->pairs = NULL;
+    map->sorted = TRUE;
+
+    return map;
+}
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value)
+{
+
+    map->length++;
+    map->sorted = FALSE;
+
+    if (map->pairs)
+	map->pairs = talloc_realloc (map, map->pairs, notmuch_string_pair_t, map->length + 1);
+    else
+	map->pairs = talloc_array (map, notmuch_string_pair_t, map->length + 1);
+
+    map->pairs[map->length - 1].key = talloc_strdup (map, key);
+    map->pairs[map->length - 1].value = talloc_strdup (map, value);
+
+    /* Add sentinel */
+    map->pairs[map->length].key = NULL;
+    map->pairs[map->length].value = NULL;
+
+}
+
+static int
+cmppair (const void *pa, const void *pb)
+{
+    notmuch_string_pair_t *a = (notmuch_string_pair_t *) pa;
+    notmuch_string_pair_t *b = (notmuch_string_pair_t *) pb;
+
+    return strcmp (a->key, b->key);
+}
+
+static void
+_notmuch_string_map_sort (notmuch_string_map_t *map)
+{
+    if (map->length == 0)
+	return;
+
+    if (map->sorted)
+	return;
+
+    qsort (map->pairs, map->length, sizeof (notmuch_string_pair_t), cmppair);
+
+    map->sorted = TRUE;
+}
+
+static notmuch_bool_t
+string_cmp (const char *a, const char *b, notmuch_bool_t exact)
+{
+    if (exact)
+	return (strcmp (a, b));
+    else
+	return (strncmp (a, b, strlen (a)));
+}
+
+static notmuch_string_pair_t *
+bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, notmuch_bool_t exact)
+{
+    size_t first = 0;
+    size_t last = len - 1;
+    size_t mid;
+
+    if (len <= 0)
+	return NULL;
+
+    while (last > first) {
+	mid = (first + last) / 2;
+	int sign = string_cmp (key, array[mid].key, exact);
+
+	if (sign <= 0)
+	    last = mid;
+	else
+	    first = mid + 1;
+    }
+
+
+    if (string_cmp (key, array[first].key, exact) == 0)
+	return array + first;
+    else
+	return NULL;
+
+}
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
+{
+    notmuch_string_pair_t *pair;
+
+    /* this means that calling append invalidates iterators */
+    _notmuch_string_map_sort (map);
+
+    pair = bsearch_first (map->pairs, map->length, key, TRUE);
+    if (! pair)
+	return NULL;
+
+    return pair->value;
+}
-- 
2.8.1

[PATCH 3/9] lib: basic message-property API

[David Bremner]

Initially, support get, set and removal of single key/value pair, as
well as removing all properties.
---
 lib/message-private.h         |  16 +++++++
 lib/message-property.cc       | 108 ++++++++++++++++++++++++++++++++++++++++++
 lib/message.cc                |  52 +++++++++++++++++++-
 lib/notmuch.h                 |  72 ++++++++++++++++++++++++++++
 test/T610-message-property.sh |  84 ++++++++++++++++++++++++++++++++
 5 files changed, 330 insertions(+), 2 deletions(-)
 create mode 100644 lib/message-private.h
 create mode 100644 lib/message-property.cc
 create mode 100755 test/T610-message-property.sh

diff --git a/lib/message-private.h b/lib/message-private.h
new file mode 100644
index 0000000..7419925
--- /dev/null
+++ b/lib/message-private.h
@@ -0,0 +1,16 @@
+#ifndef MESSAGE_PRIVATE_H
+#define MESSAGE_PRIVATE_H
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message);
+
+notmuch_bool_t
+_notmuch_message_frozen (notmuch_message_t *message);
+
+void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix);
+
+void
+_notmuch_message_invalidate_metadata (notmuch_message_t *message,  const char *prefix_name);
+
+#endif
diff --git a/lib/message-property.cc b/lib/message-property.cc
new file mode 100644
index 0000000..1f04a20
--- /dev/null
+++ b/lib/message-property.cc
@@ -0,0 +1,108 @@
+/* message-property.cc - Properties are like tags, but (key,value) pairs.
+ * keys are allowed to repeat.
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "message-private.h"
+
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value)
+{
+    if (! value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    *value = _notmuch_string_map_get (_notmuch_message_property_map (message), key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_notmuch_message_modify_property (notmuch_message_t *message, const char *key, const char *value,
+				  notmuch_bool_t delete_it)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+    char *term = NULL;
+
+    status = _notmuch_database_ensure_writable (_notmuch_message_database (message));
+    if (status)
+	return status;
+
+    if (key == NULL || value == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (index (key, '='))
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+
+    term = talloc_asprintf (message, "%s=%s", key, value);
+
+    if (delete_it)
+	private_status = _notmuch_message_remove_term (message, "property", term);
+    else
+	private_status = _notmuch_message_add_term (message, "property", term);
+
+    if (private_status)
+	return COERCE_STATUS (private_status,
+			      "Unhandled error modifying message property");
+    if (! _notmuch_message_frozen (message))
+	_notmuch_message_sync (message);
+
+    if (term)
+	talloc_free (term);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, FALSE);
+}
+
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, TRUE);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+    notmuch_status_t status;
+    const char * term_prefix;
+
+    status = _notmuch_database_ensure_writable (_notmuch_message_database (message));
+    if (status)
+	return status;
+
+    _notmuch_message_invalidate_metadata (message, "property");
+    if (key)
+	term_prefix = talloc_asprintf (message, "%s%s=", _find_prefix ("property"), key);
+    else
+	term_prefix = _find_prefix ("property");
+
+    /* XXX better error reporting ? */
+    _notmuch_message_remove_terms (message, term_prefix);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/lib/message.cc b/lib/message.cc
index 63a8da5..9d3e807 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -20,6 +20,7 @@
 
 #include "notmuch-private.h"
 #include "database-private.h"
+#include "message-private.h"
 
 #include <stdint.h>
 
@@ -395,7 +396,7 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
 	message->in_reply_to = talloc_strdup (message, "");
 }
 
-static void
+void
 _notmuch_message_invalidate_metadata (notmuch_message_t *message,
 				      const char *prefix_name)
 {
@@ -552,7 +553,7 @@ notmuch_message_get_replies (notmuch_message_t *message)
     return _notmuch_messages_create (message->replies);
 }
 
-static void
+void
 _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 {
     Xapian::TermIterator i;
@@ -1799,3 +1800,50 @@ _notmuch_message_database (notmuch_message_t *message)
 {
     return message->notmuch;
 }
+
+void
+_notmuch_message_ensure_property_map (notmuch_message_t *message)
+{
+    notmuch_string_node_t *node;
+
+    if (message->property_map)
+	return;
+
+    if (!message->property_term_list)
+	_notmuch_message_ensure_metadata (message);
+
+    message->property_map = _notmuch_string_map_create (message);
+
+    for (node = message->property_term_list->head; node; node = node->next) {
+	const char *key;
+	char *value;
+
+	value = index(node->string, '=');
+	if (!value)
+	    INTERNAL_ERROR ("malformed property term");
+
+	*value = '\0';
+	value++;
+	key = node->string;
+
+	_notmuch_string_map_append (message->property_map, key, value);
+
+    }
+
+    talloc_free (message->property_term_list);
+    message->property_term_list = NULL;
+}
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_property_map (message);
+
+    return message->property_map;
+}
+
+notmuch_bool_t
+_notmuch_message_frozen (notmuch_message_t *message)
+{
+    return message->frozen;
+}
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 2faa146..b304dca 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -180,6 +180,11 @@ typedef enum _notmuch_status {
      */
     NOTMUCH_STATUS_PATH_ERROR,
     /**
+     * One of the arguments violates the preconditions for the
+     * function, in a way not covered by a more specific argument.
+     */
+    NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+    /**
      * Not an actual status value. Just a way to find out how many
      * valid status values there are.
      */
@@ -1654,6 +1659,73 @@ void
 notmuch_message_destroy (notmuch_message_t *message);
 
 /**
+ * @name Message Properties
+ *
+ * This interface provides the ability to attach arbitrary (key,value)
+ * string pairs to a message, to remove such pairs, and to iterate
+ * over them.  The caller should take some care as to what keys they
+ * add or delete values for, as other subsystems or extensions may
+ * depend on these properties.
+ *
+ */
+/**@{*/
+/**
+ * Retrieve the value for a single property key
+ *
+ * *value* is set to a string owned by the message or NULL if there is
+ * no such key. In the case of multiple values for the given key, the
+ * first one is retrieved.
+ *
+ * @returns
+ * - NOTMUCH_STATUS_NULL_POINTER: *value* may not be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+
+ */
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value);
+
+/**
+ * Add a (key,value) pair to a message
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ */
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove a (key,value) pair from a message.
+ *
+ * It is not an error to remove a non-existant (key,value) pair
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ */
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove all (key,value) pairs from the given message.
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     key      key to delete properties for. If NULL, delete
+ *			   properties for all keys
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ *
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
+
+/**@}*/
+
+/**
  * Is the given 'tags' iterator pointing at a valid tag.
  *
  * When this function returns TRUE, notmuch_tags_get will return a
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
new file mode 100755
index 0000000..0217950
--- /dev/null
+++ b/test/T610-message-property.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+test_description="message property API"
+
+. ./test-lib.sh || exit 1
+
+add_email_corpus
+
+cat <<EOF > c_head
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <notmuch-test.h>
+
+int main (int argc, char** argv)
+{
+   notmuch_database_t *db;
+   notmuch_message_t *message = NULL;
+   const char *val;
+   notmuch_status_t stat;
+
+   EXPECT0(notmuch_database_open (argv[1], NOTMUCH_DATABASE_MODE_READ_WRITE, &db));
+   EXPECT0(notmuch_database_find_message(db, "4EFC743A.3060609@april.org", &message));
+   if (message == NULL) {
+	fprintf (stderr, "unable to find message");
+	exit (1);
+   }
+EOF
+
+cat <<EOF > c_tail
+   EXPECT0(notmuch_database_destroy(db));
+}
+EOF
+
+test_begin_subtest "notmuch_message_{add,get,remove}_property"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+   EXPECT0(notmuch_message_add_property (message, "testkey1", "testvalue1"));
+   EXPECT0(notmuch_message_get_property (message, "testkey1", &val));
+   printf("testkey1[1] = %s\n", val);
+   EXPECT0(notmuch_message_add_property (message, "testkey2", "this value has spaces and = sign"));
+   EXPECT0(notmuch_message_get_property (message, "testkey1", &val));
+   printf("testkey1[2] = %s\n", val);
+   EXPECT0(notmuch_message_get_property (message, "testkey1", &val));
+
+   EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+   printf("testkey2 = %s\n", val);
+
+   /* Add second value for key */
+   EXPECT0(notmuch_message_add_property (message, "testkey2", "zztestvalue3"));
+   EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+   printf("testkey2 = %s\n", val);
+
+   /* remove first value for key */
+   EXPECT0(notmuch_message_remove_property (message, "testkey2", "this value has spaces and = sign"));
+   EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+   printf("testkey2 = %s\n", val);
+
+   /* remove non-existant value for key */
+   EXPECT0(notmuch_message_remove_property (message, "testkey2", "this value has spaces and = sign"));
+   EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+   printf("testkey2 = %s\n", val);
+
+   /* remove only value for key */
+   EXPECT0(notmuch_message_remove_property (message, "testkey2", "zztestvalue3"));
+   EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+   printf("testkey2 = %s\n", val == NULL ? "NULL" : val);
+}
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+testkey1[1] = testvalue1
+testkey1[2] = testvalue1
+testkey2 = this value has spaces and = sign
+testkey2 = this value has spaces and = sign
+testkey2 = zztestvalue3
+testkey2 = zztestvalue3
+testkey2 = NULL
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+
+
+test_done
-- 
2.8.1

[PATCH 4/9] lib: extend private string map API with iterators

[David Bremner]

Support for prefix based iterators is perhaps overengineering, but I
wanted to mimic the existing database_config API.
---
 lib/notmuch-private.h | 21 ++++++++++++++-
 lib/string-map.c      | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 5abde88..65f7ead 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -539,7 +539,7 @@ _notmuch_string_list_sort (notmuch_string_list_t *list);
 
 /* string-map.c */
 typedef struct _notmuch_string_map  notmuch_string_map_t;
-
+typedef struct _notmuch_string_map_iterator notmuch_string_map_iterator_t;
 notmuch_string_map_t *
 _notmuch_string_map_create (const void *ctx);
 
@@ -551,6 +551,25 @@ _notmuch_string_map_append (notmuch_string_map_t *map,
 const char *
 _notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
 
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     notmuch_bool_t exact);
+
+notmuch_bool_t
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iter);
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator);
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+
 /* tags.c */
 
 notmuch_tags_t *
diff --git a/lib/string-map.c b/lib/string-map.c
index 0491a10..591ff6d 100644
--- a/lib/string-map.c
+++ b/lib/string-map.c
@@ -38,6 +38,12 @@ struct _notmuch_string_map {
     notmuch_string_pair_t *pairs;
 };
 
+struct _notmuch_string_map_iterator {
+    notmuch_string_pair_t *current;
+    notmuch_bool_t exact;
+    const char *key;
+};
+
 notmuch_string_map_t *
 _notmuch_string_map_create (const void *ctx)
 {
@@ -151,3 +157,69 @@ _notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
 
     return pair->value;
 }
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     notmuch_bool_t exact)
+{
+    notmuch_string_map_iterator_t *iter;
+
+    _notmuch_string_map_sort (map);
+
+    iter = talloc (map, notmuch_string_map_iterator_t);
+    if (unlikely (iter == NULL))
+	return NULL;
+
+    iter->key = talloc_strdup (iter, key);
+    iter->exact = exact;
+    iter->current = bsearch_first (map->pairs, map->length, key, exact);
+    return iter;
+}
+
+notmuch_bool_t
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iterator)
+{
+    if (iterator->current == NULL)
+	return FALSE;
+
+    /* sentinel */
+    if (iterator->current->key == NULL)
+	return FALSE;
+
+    return (0 == string_cmp (iterator->key, iterator->current->key, iterator->exact));
+
+}
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iterator)
+{
+
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return;
+
+    (iterator->current)++;
+}
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->key;
+}
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->value;
+}
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator)
+{
+    talloc_free (iterator);
+}
-- 
2.8.1

[PATCH 5/9] lib: iterator API for message properties

[David Bremner]

This is a thin wrapper around the string map iterator API just introduced.
---
 lib/message-property.cc       |  38 +++++++++++++++
 lib/notmuch.h                 |  95 +++++++++++++++++++++++++++++++++++++
 test/T610-message-property.sh | 107 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 240 insertions(+)

diff --git a/lib/message-property.cc b/lib/message-property.cc
index 1f04a20..0b13cac 100644
--- a/lib/message-property.cc
+++ b/lib/message-property.cc
@@ -106,3 +106,41 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
 
     return NOTMUCH_STATUS_SUCCESS;
 }
+
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
+{
+    notmuch_string_map_t *map;
+    map = _notmuch_message_property_map (message);
+    return _notmuch_string_map_iterator_create (map, key, exact);
+}
+
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_valid (properties);
+}
+
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_move_to_next (properties);
+}
+
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_key (properties);
+}
+
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_value (properties);
+}
+
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties)
+{
+    _notmuch_string_map_iterator_destroy (properties);
+}
diff --git a/lib/notmuch.h b/lib/notmuch.h
index b304dca..e03a05d 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1723,6 +1723,101 @@ notmuch_message_remove_property (notmuch_message_t *message, const char *key, co
 notmuch_status_t
 notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
 
+/**
+ * Opaque message property iterator
+ */
+typedef struct _notmuch_string_map_iterator notmuch_message_properties_t;
+
+/**
+ * Get the properties for *message*, returning a
+ * notmuch_message_properties_t object which can be used to iterate
+ * over all properties.
+ *
+ * The notmuch_message_properties_t object is owned by the message and
+ * as such, will only be valid for as long as the message is valid,
+ * (which is until the query from which it derived is destroyed).
+ *
+ * @param[in] message  The message to examine
+ * @param[in] key      key or key prefix
+ * @param[in] exact    if TRUE, require exact match with key. Otherwise
+ *		       treat as prefix.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_message_properties_t *list;
+ *
+ *     for (list = notmuch_message_get_properties (message, "testkey1", TRUE);
+ *          notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+ *        printf("%s\n", notmuch_message_properties_value(list));
+ *     }
+ *
+ *     notmuch_message_properties_destroy (list);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_message_properties_t object. (For consistency, we do
+ * provide a notmuch_message_properities_destroy function, but there's
+ * no good reason to call it if the message is about to be destroyed).
+ */
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact);
+
+/**
+ * Is the given *properties* iterator pointing at a valid (key,value)
+ * pair.
+ *
+ * When this function returns TRUE,
+ * notmuch_message_properties_{key,value} will return a valid string,
+ * and notmuch_message_properties_move_to_next will do what it
+ * says. Whereas when this function returns FALSE, calling any of
+ * these functions results in undefined behaviour.
+ *
+ * See the documentation of notmuch_message_properties_get for example
+ * code showing how to iterate over a notmuch_message_properties_t
+ * object.
+ */
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties);
+
+/**
+ * Move the *properties* iterator to the next (key,value) pair
+ *
+ * If *properties* is already pointing at the last pair then the iterator
+ * will be moved to a point just beyond that last pair, (where
+ * notmuch_message_properties_valid will return FALSE).
+ *
+ * See the documentation of notmuch_message_get_properties for example
+ * code showing how to iterate over a notmuch_message_properties_t object.
+ */
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * this could be useful if iterating for a prefix
+ */
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * This could be useful if iterating for a prefix.
+ */
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties);
+
+
+/**
+ * Destroy a notmuch_message_properties_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_message_properties_t object will be reclaimed when the
+ * containing message object is destroyed.
+ */
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties);
+
 /**@}*/
 
 /**
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index 0217950..b5ddb7a 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -9,8 +9,18 @@ cat <<EOF > c_head
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
+#include <talloc.h>
 #include <notmuch-test.h>
 
+void print_properties (notmuch_message_t *message, const char *prefix, notmuch_bool_t exact) {
+    notmuch_message_properties_t *list;
+    for (list = notmuch_message_get_properties (message, prefix, exact);
+         notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+       printf("%s\n", notmuch_message_properties_value(list));
+    }
+    notmuch_message_properties_destroy (list);
+}
+
 int main (int argc, char** argv)
 {
    notmuch_database_t *db;
@@ -79,6 +89,103 @@ testkey2 = NULL
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "notmuch_message_get_properties: empty list"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+   notmuch_message_properties_t *list;
+   list = notmuch_message_get_properties (message, "nonexistent", TRUE);
+   printf("valid = %d\n", notmuch_message_properties_valid (list));
+   notmuch_message_properties_destroy (list);
+}
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+valid = 0
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "notmuch_message_properties: one value"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+print_properties (message, "testkey1", TRUE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+testvalue1
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "notmuch_message_properties: multiple values"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "testkey1", "bob"));
+EXPECT0(notmuch_message_add_property (message, "testkey1", "testvalue2"));
+EXPECT0(notmuch_message_add_property (message, "testkey1", "alice"));
+print_properties (message, "testkey1", TRUE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+alice
+bob
+testvalue1
+testvalue2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "notmuch_message_properties: prefix"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "testkey3", "bob3"));
+EXPECT0(notmuch_message_add_property (message, "testkey3", "testvalue3"));
+EXPECT0(notmuch_message_add_property (message, "testkey3", "alice3"));
+print_properties (message, "testkey", FALSE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+alice
+bob
+testvalue1
+testvalue2
+alice3
+bob3
+testvalue3
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "notmuch_message_properties: modify during iteration"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+    const char *keys[1000] = {NULL};
+    const char *vals[1000] = {NULL};
+    notmuch_message_properties_t *properties;
+    int i;
+
+    for (properties = notmuch_message_get_properties (message, "", FALSE), i=0;
+	 notmuch_message_properties_valid (properties);
+	 notmuch_message_properties_move_to_next (properties), i++)
+    {
+	const char *key, *value;
+
+	keys[i]=talloc_strdup(message,
+		    notmuch_message_properties_key (properties));
+        vals[i]=talloc_strdup(message,
+		    notmuch_message_properties_value (properties));
+
+	EXPECT0(notmuch_message_remove_property (message, keys[i], vals[i]));
+    }
+
+    print_properties (message, "", FALSE);
+
+    for (i = 0; keys[i] && vals[i]; i++) {
+        EXPECT0(notmuch_message_add_property (message, keys[i], vals[i]));
+    }
+}
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
 
 test_done
-- 
2.8.1

[PATCH 6/9] CLI: refactor dumping of tags.

[David Bremner]

This is mainly code movement, to make room in the loop over messages for
dumping properties.
---
 notmuch-dump.c | 127 +++++++++++++++++++++++++++++++--------------------------
 1 file changed, 69 insertions(+), 58 deletions(-)

diff --git a/notmuch-dump.c b/notmuch-dump.c
index cae1db8..d80ed8b8 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -78,13 +78,78 @@ print_dump_header (gzFile output, int output_format, int include)
 }
 
 static int
+dump_tags_message (void *ctx,
+		   notmuch_message_t *message, int output_format,
+		   gzFile output,
+		   char **buffer_p, size_t *size_p)
+{
+    int first = 1;
+    const char *message_id;
+
+    message_id = notmuch_message_get_message_id (message);
+
+    if (output_format == DUMP_FORMAT_BATCH_TAG &&
+	strchr (message_id, '\n')) {
+	/* This will produce a line break in the output, which
+	 * would be difficult to handle in tools.  However, it's
+	 * also impossible to produce an email containing a line
+	 * break in a message ID because of unfolding, so we can
+	 * safely disallow it. */
+	fprintf (stderr, "Warning: skipping message id containing line break: \"%s\"\n", message_id);
+	return EXIT_SUCCESS;
+    }
+
+    if (output_format == DUMP_FORMAT_SUP) {
+	gzprintf (output, "%s (", message_id);
+    }
+
+    for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags)) {
+	const char *tag_str = notmuch_tags_get (tags);
+
+	if (! first)
+	    gzputs (output, " ");
+
+	first = 0;
+
+	if (output_format == DUMP_FORMAT_SUP) {
+	    gzputs (output, tag_str);
+	} else {
+	    if (hex_encode (ctx, tag_str,
+			    buffer_p, size_p) != HEX_SUCCESS) {
+		fprintf (stderr, "Error: failed to hex-encode tag %s\n",
+			 tag_str);
+		return EXIT_FAILURE;
+	    }
+	    gzprintf (output, "+%s", *buffer_p);
+	}
+    }
+
+    if (output_format == DUMP_FORMAT_SUP) {
+	gzputs (output, ")\n");
+    } else {
+	if (make_boolean_term (ctx, "id", message_id,
+			       buffer_p, size_p)) {
+	    fprintf (stderr, "Error quoting message id %s: %s\n",
+		     message_id, strerror (errno));
+	    return EXIT_FAILURE;
+	}
+	gzprintf (output, " -- %s\n", *buffer_p);
+    }
+    return EXIT_SUCCESS;
+}
+
+static int
 database_dump_file (notmuch_database_t *notmuch, gzFile output,
 		    const char *query_str, int output_format, int include)
 {
     notmuch_query_t *query;
     notmuch_messages_t *messages;
     notmuch_message_t *message;
-    notmuch_tags_t *tags;
+    notmuch_status_t status;
+    char *buffer = NULL;
+    size_t buffer_size = 0;
 
     print_dump_header (output, output_format, include);
 
@@ -110,10 +175,6 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
      */
     notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
 
-    char *buffer = NULL;
-    size_t buffer_size = 0;
-    notmuch_status_t status;
-
     status = notmuch_query_search_messages_st (query, &messages);
     if (print_status_query ("notmuch dump", query, status))
 	return EXIT_FAILURE;
@@ -121,62 +182,12 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
     for (;
 	 notmuch_messages_valid (messages);
 	 notmuch_messages_move_to_next (messages)) {
-	int first = 1;
-	const char *message_id;
 
 	message = notmuch_messages_get (messages);
-	message_id = notmuch_message_get_message_id (message);
-
-	if (output_format == DUMP_FORMAT_BATCH_TAG &&
-	    strchr (message_id, '\n')) {
-	    /* This will produce a line break in the output, which
-	     * would be difficult to handle in tools.  However, it's
-	     * also impossible to produce an email containing a line
-	     * break in a message ID because of unfolding, so we can
-	     * safely disallow it. */
-	    fprintf (stderr, "Warning: skipping message id containing line break: \"%s\"\n", message_id);
-	    notmuch_message_destroy (message);
-	    continue;
-	}
 
-	if (output_format == DUMP_FORMAT_SUP) {
-	    gzprintf (output, "%s (", message_id);
-	}
-
-	for (tags = notmuch_message_get_tags (message);
-	     notmuch_tags_valid (tags);
-	     notmuch_tags_move_to_next (tags)) {
-	    const char *tag_str = notmuch_tags_get (tags);
-
-	    if (! first)
-		gzputs (output, " ");
-
-	    first = 0;
-
-	    if (output_format == DUMP_FORMAT_SUP) {
-		gzputs (output, tag_str);
-	    } else {
-		if (hex_encode (notmuch, tag_str,
-				&buffer, &buffer_size) != HEX_SUCCESS) {
-		    fprintf (stderr, "Error: failed to hex-encode tag %s\n",
-			     tag_str);
-		    return EXIT_FAILURE;
-		}
-		gzprintf (output, "+%s", buffer);
-	    }
-	}
-
-	if (output_format == DUMP_FORMAT_SUP) {
-	    gzputs (output, ")\n");
-	} else {
-	    if (make_boolean_term (notmuch, "id", message_id,
-				   &buffer, &buffer_size)) {
-		    fprintf (stderr, "Error quoting message id %s: %s\n",
-			     message_id, strerror (errno));
-		    return EXIT_FAILURE;
-	    }
-	    gzprintf (output, " -- %s\n", buffer);
-	}
+	if (dump_tags_message (notmuch, message, output_format, output,
+			       &buffer, &buffer_size))
+	    return EXIT_FAILURE;
 
 	notmuch_message_destroy (message);
     }
-- 
2.8.1

[PATCH 7/9] CLI: add properties to dump output

[David Bremner]

Part of providing extensibility via properties is to make sure that user
data is not lost. Thus we need to be able to dump and restore
properties.
---
 doc/man1/notmuch-dump.rst     | 18 ++++++---
 notmuch-client.h              |  3 ++
 notmuch-dump.c                | 85 +++++++++++++++++++++++++++++++++++++++----
 notmuch-new.c                 |  2 +-
 test/T610-message-property.sh | 21 +++++++++++
 5 files changed, 116 insertions(+), 13 deletions(-)

diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index 94986a8..751850b 100644
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -71,7 +71,7 @@ Supported options for **dump** include
             characters. Note also that tags with spaces will not be
             correctly restored with this format.
 
-    ``--include=(config|tags)``
+    ``--include=(config|properties|tags)``
 
     Control what kind of metadata is included in the output.
 
@@ -81,14 +81,22 @@ Supported options for **dump** include
 	starts with "#@ ", followed by a space seperated key-value
 	pair.  Both key and value are hex encoded if needed.
 
+      **properties**
+
+	Output per-message (key,value) metadata.  Each line starts
+	with "#= ", followed by a message id, and a space seperated
+	list of key=value pairs.  pair.  Ids, keys and values are hex
+	encoded if needed.
+
       **tags**
 
-	Output per-message metadata, namely tags. See *format* above
+	Output per-message boolean metadata, namely tags. See *format* above
 	for description of the output.
 
-      The default is to include both tags and configuration
-      information. As of version 2 of the dump format, there is a
-      header line of the following form
+      The default is to include all available types of data.  The
+      option can be specified multiple times to select some subset. As
+      of version 2 of the dump format, there is a header line of the
+      following form
 
       |
       |  #notmuch-dump <*format*>:<*version*> <*included*>
diff --git a/notmuch-client.h b/notmuch-client.h
index ebc092b..9ce2aef 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -449,8 +449,11 @@ typedef enum dump_formats {
 typedef enum dump_includes {
     DUMP_INCLUDE_TAGS = 1,
     DUMP_INCLUDE_CONFIG = 2,
+    DUMP_INCLUDE_PROPERTIES = 4
 } dump_include_t;
 
+#define DUMP_INCLUDE_DEFAULT (DUMP_INCLUDE_TAGS | DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_PROPERTIES)
+
 #define NOTMUCH_DUMP_VERSION 2
 
 int
diff --git a/notmuch-dump.c b/notmuch-dump.c
index d80ed8b8..e7965ce 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -69,12 +69,77 @@ database_dump_config (notmuch_database_t *notmuch, gzFile output)
 static void
 print_dump_header (gzFile output, int output_format, int include)
 {
-    gzprintf (output, "#notmuch-dump %s:%d %s%s%s\n",
+    const char *sep = "";
+
+    gzprintf (output, "#notmuch-dump %s:%d ",
 	      (output_format == DUMP_FORMAT_SUP) ? "sup" : "batch-tag",
-	      NOTMUCH_DUMP_VERSION,
-	      (include & DUMP_INCLUDE_CONFIG) ? "config" : "",
-	      (include & DUMP_INCLUDE_TAGS) && (include & DUMP_INCLUDE_CONFIG) ? "," : "",
-	      (include & DUMP_INCLUDE_TAGS) ? "tags" : "");
+	      NOTMUCH_DUMP_VERSION);
+
+    if (include & DUMP_INCLUDE_CONFIG) {
+	gzputs (output, "config");
+	sep = ",";
+    }
+    if (include & DUMP_INCLUDE_PROPERTIES) {
+	gzprintf (output, "%sproperties", sep);
+	sep = ",";
+    }
+    if (include & DUMP_INCLUDE_TAGS) {
+	gzprintf (output, "%sproperties", sep);
+    }
+    gzputs (output, "\n");
+}
+
+static int
+dump_properties_message (void *ctx,
+			 notmuch_message_t *message,
+			 gzFile output,
+			 char **buffer_p, size_t *size_p)
+{
+    const char *message_id;
+    notmuch_message_properties_t *list;
+    notmuch_bool_t first = TRUE;
+
+    message_id = notmuch_message_get_message_id (message);
+
+    if (strchr (message_id, '\n')) {
+	fprintf (stderr, "Warning: skipping message id containing line break: \"%s\"\n", message_id);
+	return 0;
+    }
+
+    for (list = notmuch_message_get_properties (message, "", FALSE);
+	 notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+	const char *key, *val;
+
+	if (first) {
+	    if (hex_encode (ctx, message_id, buffer_p, size_p) != HEX_SUCCESS) {
+		fprintf (stderr, "Error: failed to hex-encode message-id %s\n", message_id);
+		return 1;
+	    }
+	    gzprintf (output, "#= %s", *buffer_p);
+	    first = FALSE;
+	}
+
+	key = notmuch_message_properties_key (list);
+	val = notmuch_message_properties_value (list);
+
+	if (hex_encode (ctx, key, buffer_p, size_p) != HEX_SUCCESS) {
+	    fprintf (stderr, "Error: failed to hex-encode key %s\n", key);
+	    return 1;
+	}
+	gzprintf (output, " %s", *buffer_p);
+
+	if (hex_encode (ctx, val, buffer_p, size_p) != HEX_SUCCESS) {
+	    fprintf (stderr, "Error: failed to hex-encode value %s\n", val);
+	    return 1;
+	}
+	gzprintf (output, "=%s", *buffer_p);
+    }
+    notmuch_message_properties_destroy (list);
+
+    if (! first)
+	gzprintf (output, "\n", *buffer_p);
+
+    return 0;
 }
 
 static int
@@ -159,7 +224,7 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
 	    return EXIT_FAILURE;
     }
 
-    if (! (include & DUMP_INCLUDE_TAGS))
+    if (! (include & (DUMP_INCLUDE_TAGS | DUMP_INCLUDE_PROPERTIES)))
 	return EXIT_SUCCESS;
 
     if (! query_str)
@@ -189,6 +254,11 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
 			       &buffer, &buffer_size))
 	    return EXIT_FAILURE;
 
+	if ((include & DUMP_INCLUDE_PROPERTIES) &&
+	    dump_properties_message (notmuch, message, output,
+				     &buffer, &buffer_size))
+	    return EXIT_FAILURE;
+
 	notmuch_message_destroy (message);
     }
 
@@ -312,6 +382,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
 				  { 0, 0 } } },
 	{ NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I',
 	  (notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG },
+				  { "properties", DUMP_INCLUDE_PROPERTIES },
 				  { "tags", DUMP_INCLUDE_TAGS} } },
 	{ NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },
 	{ NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
@@ -326,7 +397,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
     notmuch_process_shared_options (argv[0]);
 
     if (include == 0)
-	include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS;
+	include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS | DUMP_INCLUDE_PROPERTIES;
 
     if (opt_index < argc) {
 	query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);
diff --git a/notmuch-new.c b/notmuch-new.c
index 799fec2..c55dea7 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -1042,7 +1042,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
 	    }
 
 	    if (notmuch_database_dump (notmuch, backup_name, "",
-				       DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS, TRUE)) {
+				       DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_DEFAULT, TRUE)) {
 		fprintf (stderr, "Backup failed. Aborting upgrade.");
 		return EXIT_FAILURE;
 	    }
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index b5ddb7a..a9b76de 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -89,6 +89,17 @@ testkey2 = NULL
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "notmuch_message_remove_all_properties"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_remove_all_properties (message, NULL));
+print_properties (message, "", FALSE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest "notmuch_message_get_properties: empty list"
 cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
 {
@@ -188,4 +199,14 @@ cat <<'EOF' >EXPECTED
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "dump message properties"
+cat <<EOF > PROPERTIES
+#= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue1 testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3
+EOF
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "fancy key with áccènts", "import value with ="));
+EOF
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
 test_done
-- 
2.8.1

[PATCH 8/9] cli: optionally restore message properties from dump file

[David Bremner]

This somewhat mimics the config line parsing, except there can be
arbitrarily many key value pairs, so one more level of looping is
required.
---
 doc/man1/notmuch-restore.rst  | 13 +++++--
 notmuch-restore.c             | 85 +++++++++++++++++++++++++++++++++++++++++--
 test/T610-message-property.sh | 28 ++++++++++++++
 3 files changed, 120 insertions(+), 6 deletions(-)

diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst
index 87fa22e..a0c1b3c 100644
--- a/doc/man1/notmuch-restore.rst
+++ b/doc/man1/notmuch-restore.rst
@@ -50,7 +50,7 @@ Supported options for **restore** include
             format, this heuristic, based the fact that batch-tag format
             contains no parentheses, should be accurate.
 
-    ``--include=(config|tags)``
+    ``--include=(config|properties|tags)``
 
       Control what kind of metadata is restored.
 
@@ -60,13 +60,20 @@ Supported options for **restore** include
 	  with "#@ ", followed by a space seperated key-value pair.
 	  Both key and value are hex encoded if needed.
 
+	**properties**
+
+	  Output per-message (key,value) metadata.  Each line starts
+	  with "#= ", followed by a message id, and a space seperated
+	  list of key=value pairs.  pair.  Ids, keys and values are
+	  hex encoded if needed.
+
 	**tags**
 
 	  Output per-message metadata, namely tags. See *format* above
 	  for more details.
 
-      The default is to restore both tags and configuration
-      information
+      The default is to restore all available types of data.  The
+      option can be specified multiple times to select some subset.
 
     ``--input=``\ <filename>
         Read input from given file instead of stdin.
diff --git a/notmuch-restore.c b/notmuch-restore.c
index 371237c..d6429ef 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -57,6 +57,72 @@ process_config_line (notmuch_database_t *notmuch, const char* line)
     return ret;
 }
 
+static int
+process_properties_line (notmuch_database_t *notmuch, const char* line)
+
+{
+    const char *id_p, *tok;
+    size_t id_len = 0, tok_len = 0;
+    char *id;
+
+    notmuch_message_t *message = NULL;
+    const char *delim = " \t\n";
+    int ret = EXIT_FAILURE;
+
+    void *local = talloc_new (NULL);
+
+    id_p = strtok_len_c (line, delim, &id_len);
+    id = talloc_strndup (local, id_p, id_len);
+    if (hex_decode_inplace (id) != HEX_SUCCESS) {
+	fprintf (stderr, "hex decoding failure on line %s\n", line);
+	goto DONE;
+    }
+
+    if (print_status_database ("notmuch restore", notmuch,
+			       notmuch_database_find_message (notmuch, id, &message)))
+	goto DONE;
+
+    if (print_status_database ("notmuch restore", notmuch,
+			       notmuch_message_remove_all_properties (message, NULL)))
+	goto DONE;
+
+    tok = id_p + id_len;
+
+    while ((tok = strtok_len_c (tok + tok_len, delim, &tok_len)) != NULL) {
+	char *key, *value;
+	size_t off = strcspn (tok, "=");
+	if (off > tok_len) {
+	    fprintf (stderr, "unparsable token %s\n", tok);
+	    goto DONE;
+	}
+
+	key = talloc_strndup (local, tok, off);
+	value = talloc_strndup (local, tok + off + 1, tok_len - off - 1);
+
+	if (hex_decode_inplace (key) != HEX_SUCCESS) {
+	    fprintf (stderr, "hex decoding failure on key %s\n", key);
+	    goto DONE;
+	}
+
+	if (hex_decode_inplace (value) != HEX_SUCCESS) {
+	    fprintf (stderr, "hex decoding failure on value %s\n", value);
+	    goto DONE;
+	}
+
+	if (print_status_database ("notmuch restore", notmuch,
+				   notmuch_message_add_property (message, key, value)))
+	    goto DONE;
+
+    }
+
+    ret = EXIT_SUCCESS;
+
+  DONE:
+    talloc_free (local);
+    return ret;
+}
+
+
 static regex_t regex;
 
 /* Non-zero return indicates an error in retrieving the message,
@@ -188,6 +254,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
 				  { 0, 0 } } },
 	{ NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I',
 	  (notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG },
+				  { "properties", DUMP_INCLUDE_PROPERTIES },
 				  { "tags", DUMP_INCLUDE_TAGS} } },
 
 	{ NOTMUCH_OPT_STRING, &input_file_name, "input", 'i', 0 },
@@ -206,7 +273,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
     notmuch_exit_if_unmatched_db_uuid (notmuch);
 
     if (include == 0) {
-	include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS;
+	include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_PROPERTIES | DUMP_INCLUDE_TAGS;
     }
 
     name_for_error = input_file_name ? input_file_name : "stdin";
@@ -273,13 +340,18 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
 	    if (ret)
 		goto DONE;
 	}
+	if ((include & DUMP_INCLUDE_PROPERTIES) && line_len >= 2 && line[0] == '#' && line[1] == '=') {
+	    ret = process_properties_line (notmuch, line + 2);
+	    if (ret)
+		goto DONE;
+	}
 
     } while ((line_len == 0) ||
 	     (line[0] == '#') ||
 	     /* the cast is safe because we checked about for line_len < 0 */
 	     (strspn (line, " \t\n") == (unsigned)line_len));
 
-    if (! (include & DUMP_INCLUDE_TAGS)) {
+    if (! ((include & DUMP_INCLUDE_TAGS) || (include & DUMP_INCLUDE_PROPERTIES))) {
 	ret = EXIT_SUCCESS;
 	goto DONE;
     }
@@ -306,6 +378,13 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
 	    talloc_free (line_ctx);
 
 	line_ctx = talloc_new (config);
+
+	if ((include & DUMP_INCLUDE_PROPERTIES) && line_len >= 2 && line[0] == '#' && line[1] == '=') {
+	    ret = process_properties_line (notmuch, line + 2);
+	    if (ret)
+		goto DONE;
+	}
+
 	if (input_format == DUMP_FORMAT_SUP) {
 	    ret = parse_sup_line (line_ctx, line, &query_string, tag_ops);
 	} else {
@@ -344,7 +423,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
 	    break;
 
     }  while (! (ret = gz_getline (line_ctx, &line, &line_len, input)));
-    
+
 
     /* EOF is normal loop termination condition, UTIL_SUCCESS is
      * impossible here */
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index a9b76de..c92b99b 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -209,4 +209,32 @@ EOF
 notmuch dump | grep '^#=' > OUTPUT
 test_expect_equal_file PROPERTIES OUTPUT
 
+
+test_begin_subtest "restore missing message property (single line)"
+notmuch dump | grep '^#=' > BEFORE1
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob"));
+EOF
+notmuch restore < BEFORE1
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
+
+test_begin_subtest "restore missing message property (full dump)"
+notmuch dump > BEFORE2
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob"));
+EOF
+notmuch restore < BEFORE2
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
+test_begin_subtest "restore clear extra message property"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "testkey1", "charles"));
+EOF
+notmuch restore < BEFORE2
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
 test_done
-- 
2.8.1

[PATCH 9/9] add has: query prefix to search for specific properties

[David Bremner]

From: Daniel Kahn Gillmor <dkg@fifthhorseman.net>

We want to be able to query the properties directly, like:

   notmuch count has:foo=bar

which should return a count of messages where the property with key
"foo" has value equal to "bar".
---
 doc/man7/notmuch-search-terms.rst |  9 ++++++++-
 lib/database.cc                   |  4 ++++
 test/T610-message-property.sh     | 11 +++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index 075f88c..86d02ee 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -58,6 +58,8 @@ indicate user-supplied values):
 
 -  query:<name>
 
+-  has:<key>=<value>
+
 The **from:** prefix is used to match the name or address of the sender
 of an email message.
 
@@ -139,6 +141,11 @@ queries added with **notmuch-config(1)**. Named queries are only
 available if notmuch is built with **Xapian Field Processors** (see
 below).
 
+The **has:** prefix searches for messages with a particular
+<key>=<value> property pair. Properties are used internally by notmuch
+(and extensions) to add metadata to messages. A given key can be
+present on a given message with several different values.
+
 Operators
 ---------
 
@@ -217,7 +224,7 @@ exact matches like "tag:inbox"  or **probabilistic**, supporting a more flexible
 
 
 Boolean
-   **tag:**, **id:**, **thread:**, **folder:**, **path:**
+   **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has**
 Probabilistic
    **from:**, **to:**, **subject:**, **attachment:**, **mimetype:**
 
diff --git a/lib/database.cc b/lib/database.cc
index 3a741f0..3bdbd07 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -97,6 +97,9 @@ typedef struct {
  *		        STRING is the name of a file within that
  *		        directory for this mail message.
  *
+ *      has:       Has a property with key=value
+ *                 FIXME: if no = is present, should match on any value
+ *
  *    A mail document also has four values:
  *
  *	TIMESTAMP:	The time_t value corresponding to the message's
@@ -260,6 +263,7 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "is",			"K" },
     { "id",			"Q" },
     { "path",			"P" },
+    { "has",			"XPROPERTY" },
     /*
      * Without the ":", since this is a multi-letter prefix, Xapian
      * will add a colon itself if the first letter of the path is
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index c92b99b..ee7910b 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -237,4 +237,15 @@ notmuch restore < BEFORE2
 notmuch dump | grep '^#=' > OUTPUT
 test_expect_equal_file PROPERTIES OUTPUT
 
+test_begin_subtest "test 'has:' queries: empty"
+notmuch search has:testkey1=charles > OUTPUT
+test_expect_equal_file /dev/null OUTPUT
+
+test_begin_subtest "test 'has:' queries: single message"
+notmuch search --output=messages has:testkey1=alice > OUTPUT
+cat <<EOF >EXPECTED
+id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done
-- 
2.8.1

Re: [PATCH 9/9] add has: query prefix to search for specific properties

[David Bremner]

David Bremner <david@tethera.net> writes:

>  
> +The **has:** prefix searches for messages with a particular
> +<key>=<value> property pair. Properties are used internally by notmuch
> +(and extensions) to add metadata to messages. A given key can be
> +present on a given message with several different values.
> +

I probably should have added to the commit message that this
documentation is intentionally minimal, because it's not intended for
people creating properties; that should go in API documentation, I think.


> +    { "has",			"XPROPERTY" },
>      /*
>       * Without the ":", since this is a multi-letter prefix, Xapian
>       * will add a colon itself if the first letter of the path

I don't really understand the comment on the next line (since it seems
that no : is added to XPROPERTY by Xapian), but I guess we should
probably be consistent. Maybe Jani can explain the comment?

d

Re: [PATCH 9/9] add has: query prefix to search for specific properties

[Jani Nikula]

On Sat, 06 Aug 2016, David Bremner <david@tethera.net> wrote:
> David Bremner <david@tethera.net> writes:
>
>>  
>> +The **has:** prefix searches for messages with a particular
>> +<key>=<value> property pair. Properties are used internally by notmuch
>> +(and extensions) to add metadata to messages. A given key can be
>> +present on a given message with several different values.
>> +
>
> I probably should have added to the commit message that this
> documentation is intentionally minimal, because it's not intended for
> people creating properties; that should go in API documentation, I think.
>
>
>> +    { "has",			"XPROPERTY" },
>>      /*
>>       * Without the ":", since this is a multi-letter prefix, Xapian
>>       * will add a colon itself if the first letter of the path
>
> I don't really understand the comment on the next line (since it seems
> that no : is added to XPROPERTY by Xapian), but I guess we should
> probably be consistent. Maybe Jani can explain the comment?

My recollection is vague, but it has to do with user-defined prefixes
(XSOMETHING) and terms beginning with uppercase letters. I think it
would possibly be useful (mandatory?) to append ":" at the end of
XPROPERTY, if it's possible we'll have properties beginning with
uppercase letters.

Quoting https://xapian.org/docs/omega/termprefixes.html

"""
X starts a multi-capital letter user-defined prefix. If you want a
prefix for something without a standard prefix, you create your own
starting with an X (e.g. XSHOESIZE). The prefix ends with the first
non-capital. If the term you're prefixing starts with a capital, add a
":" between prefix and term to resolve ambiguity about where the prefix
ends and the term begins.
"""

BR,
Jani.

Re: [PATCH 9/9] add has: query prefix to search for specific properties

[David Bremner]

Jani Nikula <jani@nikula.org> writes:

>> I don't really understand the comment on the next line (since it seems
>> that no : is added to XPROPERTY by Xapian), but I guess we should
>> probably be consistent. Maybe Jani can explain the comment?
>
> My recollection is vague, but it has to do with user-defined prefixes
> (XSOMETHING) and terms beginning with uppercase letters. I think it
> would possibly be useful (mandatory?) to append ":" at the end of
> XPROPERTY, if it's possible we'll have properties beginning with
> uppercase letters.
>
> Quoting https://xapian.org/docs/omega/termprefixes.html
>
> """
> X starts a multi-capital letter user-defined prefix. If you want a
> prefix for something without a standard prefix, you create your own
> starting with an X (e.g. XSHOESIZE). The prefix ends with the first
> non-capital. If the term you're prefixing starts with a capital, add a
> ":" between prefix and term to resolve ambiguity about where the prefix
> ends and the term begins.
> """

Well, it seems harmless (modulo the 1 extra byte) to add it.  I was
wondering if the comment (and requirement) dated back to when we added
XFOLDER: terms with some Xapian term generator, but I think I wasn't
able to verify that poking around in the history.

d

Re: [PATCH 9/9] add has: query prefix to search for specific properties

[Jameson Graef Rollins]

[-- Attachment #1: Type: text/plain, Size: 583 bytes --]

On Sat, Aug 06 2016, David Bremner <david@tethera.net> wrote:
> @@ -217,7 +224,7 @@ exact matches like "tag:inbox"  or **probabilistic**, supporting a more flexible
>  
>  
>  Boolean
> -   **tag:**, **id:**, **thread:**, **folder:**, **path:**
> +   **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has**
>  Probabilistic
>     **from:**, **to:**, **subject:**, **attachment:**, **mimetype:**

Super minor annoying comment, but I wonder if there's a missing colon in
the above addition:

> +   **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has:**

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]

Re: [PATCH 9/9] add has: query prefix to search for specific properties

[David Bremner]

Jameson Graef Rollins <jrollins@finestructure.net> writes:

> Super minor annoying comment, but I wonder if there's a missing colon in
> the above addition:
>
>> +   **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has:**

annoying, but correct ;).

d

[PATCH] lib: reword comment about XFOLDER: prefix

[David Bremner]

I believe the current one is misleading, because in my experiments
Xapian did not add : when prefix and term were both upper case. Indeed,
it's hard to see how it could, because prefixes are added at a layer
above Xapian in our code. See _notmuch_message_add_term for an example.

Also try to explain why this is a good idea.
---
 lib/database.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index 57a98c9..30ee303 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -260,10 +260,10 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "id",			"Q" },
     { "path",			"P" },
     /*
-     * Without the ":", since this is a multi-letter prefix, Xapian
-     * will add a colon itself if the first letter of the path is
-     * upper-case ASCII. Including the ":" forces there to always be a
-     * colon, which keeps our own logic simpler.
+     * Unconditionally add ':' to reduce potential ambiguity with
+     * overlapping prefixes and/or terms that start with capital
+     * letters. See xapian document termprefixes.html for related
+     * discussion.
      */
     { "folder",			"XFOLDER:" },
 };
-- 
2.8.1

[PATCH] lib: reword comment about XFOLDER: prefix

[David Bremner]

I believe the current one is misleading, because in my experiments
Xapian did not add : when prefix and term were both upper case. Indeed,
it's hard to see how it could, because prefixes are added at a layer
above Xapian in our code. See _notmuch_message_add_term for an example.

Also try to explain why this is a good idea.  As far as I can ascertain,
this is more of an issue for a system trying to work with an unknown set
of prefixes. Since notmuch has a fixed set of prefixes, and we can
hopefully be trusted not to add XGOLD and XGOLDEN as prefixes, it is
harder for problems to arise.
---
 lib/database.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index 57a98c9..5577aaf 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -260,10 +260,10 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "id",			"Q" },
     { "path",			"P" },
     /*
-     * Without the ":", since this is a multi-letter prefix, Xapian
-     * will add a colon itself if the first letter of the path is
-     * upper-case ASCII. Including the ":" forces there to always be a
-     * colon, which keeps our own logic simpler.
+     * Unconditionally add ':' to reduce potential ambiguity with
+     * overlapping prefixes and/or terms that start with capital
+     * letters. See Xapian document termprefixes.html for related
+     * discussion.
      */
     { "folder",			"XFOLDER:" },
 };
-- 
2.8.1

Re: [PATCH] lib: reword comment about XFOLDER: prefix

[David Bremner]

David Bremner <david@tethera.net> writes:

> I believe the current one is misleading, because in my experiments
> Xapian did not add : when prefix and term were both upper case. Indeed,
> it's hard to see how it could, because prefixes are added at a layer
> above Xapian in our code. See _notmuch_message_add_term for an example.
>
> Also try to explain why this is a good idea.  As far as I can ascertain,
> this is more of an issue for a system trying to work with an unknown set
> of prefixes. Since notmuch has a fixed set of prefixes, and we can
> hopefully be trusted not to add XGOLD and XGOLDEN as prefixes, it is
> harder for problems to arise.

pushed,

d

Re: v3 of message properties patches

[Daniel Kahn Gillmor]

[-- Attachment #1: Type: text/plain, Size: 981 bytes --]

On Sat 2016-08-06 09:52:30 -0400, David Bremner wrote:
> This obsoletes
>
>      id:1470184228-12517-1-git-send-email-david@tethera.net
>
> In particular it fixes the compile errors (boo), and adds dkg's
> has: queries, with minimal doc and tests.

I've been running this series of patches since shortly after they were
released.  They work for me, and they're a critical feature to enable
the cleartext index series i've been trying to land.  I'd like to
nominate them for inclusion in 0.23 (if we can get them pushed to
master, i might even nominate my cleartext index series for 0.23 as
well).

As i've been maintaining them against the master branch, they've
diverged slightly from the series here.  I'd be happy to send the
updated revisions here if folks want them, but there are no substantive
changes.

You can see the current branch at proprties-v3 (commit ID
9d98707a90c0fcd0eb7b0671edb7d9e39aff551f) on
git://lair.fifthhorseman.net/~dkg/notmuch

Regards,

        --dkg

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 930 bytes --]

Re: v3 of message properties patches

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

> I've been running this series of patches since shortly after they were
> released.  They work for me, and they're a critical feature to enable
> the cleartext index series i've been trying to land.

OK, that's certainly more testing and API evaluation than most series
get. I'd still like someone other than me to sanity check the code
itself, and the design decisions that hard to change later, namely
changes to the database format and the dump-restore format. And of
course the main reason we want code review is for to find the issues
that I am not aware of. Tomi, I know you've read the code at some level,
are you happy with merging the series?

>
> As i've been maintaining them against the master branch, they've
> diverged slightly from the series here.  I'd be happy to send the
> updated revisions here if folks want them, but there are no substantive
> changes.

I can confirm that's just a rebase against master of what I have.

Re: v3 of message properties patches

[Tomi Ollila]

On Wed, Sep 21 2016, David Bremner <david@tethera.net> wrote:

> Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:
>
>> I've been running this series of patches since shortly after they were
>> released.  They work for me, and they're a critical feature to enable
>> the cleartext index series i've been trying to land.
>
> OK, that's certainly more testing and API evaluation than most series
> get. I'd still like someone other than me to sanity check the code
> itself, and the design decisions that hard to change later, namely
> changes to the database format and the dump-restore format. And of
> course the main reason we want code review is for to find the issues
> that I am not aware of. Tomi, I know you've read the code at some level,
> are you happy with merging the series?

I am happy with merging the series, just amend the sepErates
(and possibly spcs->tab after $(dir)/string-map.c in -3-)

I looked through the code and did not see anything that should broke
anything. As Daniel has been using it I can slack off from testing it myself :D

Tomi

>

>>
>> As i've been maintaining them against the master branch, they've
>> diverged slightly from the series here.  I'd be happy to send the
>> updated revisions here if folks want them, but there are no substantive
>> changes.
>
> I can confirm that's just a rebase against master of what I have.

Re: v3 of message properties patches

[David Bremner]

Tomi Ollila <tomi.ollila@iki.fi> writes:
>
> I am happy with merging the series, just amend the sepErates
> (and possibly spcs->tab after $(dir)/string-map.c in -3-)
>
> I looked through the code and did not see anything that should broke
> anything. As Daniel has been using it I can slack off from testing it myself :D
>
> Tomi

I have merged this series, amended as follows:

diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index 29964da..5857027 100644
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -84,7 +84,7 @@ Supported options for **dump** include
       **properties**
 
 	Output per-message (key,value) metadata.  Each line starts
-	with "#= ", followed by a message id, and a space seperated
+	with "#= ", followed by a message id, and a space separated
 	list of key=value pairs.  pair.  Ids, keys and values are hex
 	encoded if needed.
 
diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst
index ebdb3ff..c681fa2 100644
--- a/doc/man1/notmuch-restore.rst
+++ b/doc/man1/notmuch-restore.rst
@@ -63,7 +63,7 @@ Supported options for **restore** include
 	**properties**
 
 	  Output per-message (key,value) metadata.  Each line starts
-	  with "#= ", followed by a message id, and a space seperated
+	  with "#= ", followed by a message id, and a space separated
 	  list of key=value pairs.  pair.  Ids, keys and values are
 	  hex encoded if needed.
 
diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index a85d94b..de93d73 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -58,7 +58,7 @@ indicate user-supplied values):
 
 -  query:<name>
 
--  has:<key>=<value>
+-  property:<key>=<value>
 
 The **from:** prefix is used to match the name or address of the sender
 of an email message.
@@ -141,7 +141,7 @@ queries added with **notmuch-config(1)**. Named queries are only
 available if notmuch is built with **Xapian Field Processors** (see
 below).
 
-The **has:** prefix searches for messages with a particular
+The **property:** prefix searches for messages with a particular
 <key>=<value> property pair. Properties are used internally by notmuch
 (and extensions) to add metadata to messages. A given key can be
 present on a given message with several different values.
@@ -224,7 +224,7 @@ exact matches like "tag:inbox"  or **probabilistic**, supporting a more flexible
 
 
 Boolean
-   **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has:**
+   **tag:**, **id:**, **thread:**, **folder:**, **path:**, **property:**
 Probabilistic
    **from:**, **to:**, **subject:**, **attachment:**, **mimetype:**
 
diff --git a/lib/Makefile.local b/lib/Makefile.local
index c012ed1..3d1030a 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -40,7 +40,7 @@ libnotmuch_c_srcs =		\
 	$(dir)/messages.c	\
 	$(dir)/sha1.c		\
 	$(dir)/built-with.c	\
-	$(dir)/string-map.c    \
+	$(dir)/string-map.c	\
 	$(dir)/tags.c
 
 libnotmuch_cxx_srcs =		\
diff --git a/lib/database.cc b/lib/database.cc
index d3e2e0e..392e8b2 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -97,7 +97,7 @@ typedef struct {
  *		        STRING is the name of a file within that
  *		        directory for this mail message.
  *
- *      has:       Has a property with key=value
+ *      property:       Has a property with key=value
  *                 FIXME: if no = is present, should match on any value
  *
  *    A mail document also has four values:
@@ -254,7 +254,6 @@ static prefix_t BOOLEAN_PREFIX_INTERNAL[] = {
     { "directory",		"XDIRECTORY" },
     { "file-direntry",		"XFDIRENTRY" },
     { "directory-direntry",	"XDDIRENTRY" },
-    { "property",               "XPROPERTY"  },
 };
 
 static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
@@ -263,7 +262,7 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "is",			"K" },
     { "id",			"Q" },
     { "path",			"P" },
-    { "has",			"XPROPERTY" },
+    { "property",		"XPROPERTY" },
     /*
      * Unconditionally add ':' to reduce potential ambiguity with
      * overlapping prefixes and/or terms that start with capital
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index ee7910b..65ff19d 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -237,12 +237,12 @@ notmuch restore < BEFORE2
 notmuch dump | grep '^#=' > OUTPUT
 test_expect_equal_file PROPERTIES OUTPUT
 
-test_begin_subtest "test 'has:' queries: empty"
-notmuch search has:testkey1=charles > OUTPUT
+test_begin_subtest "test 'property:' queries: empty"
+notmuch search property:testkey1=charles > OUTPUT
 test_expect_equal_file /dev/null OUTPUT
 
-test_begin_subtest "test 'has:' queries: single message"
-notmuch search --output=messages has:testkey1=alice > OUTPUT
+test_begin_subtest "test 'property:' queries: single message"
+notmuch search --output=messages property:testkey1=alice > OUTPUT
 cat <<EOF >EXPECTED
 id:4EFC743A.3060609@april.org
 EOF