* [PATCH 1/9] lib: read "property" terms from messages.
2016-08-06 13:52 v3 of message properties patches David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 2/9] lib: private string map (associative array) API David Bremner
` (8 subsequent siblings)
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
This is a first step towards providing an API to attach
arbitrary (key,value) pairs to messages and retrieve all of the values
for a given key.
---
lib/database.cc | 1 +
lib/message.cc | 29 ++++++++++++++++++++++++++++-
lib/notmuch-private.h | 3 +++
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/lib/database.cc b/lib/database.cc
index 66ee267..3a741f0 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -251,6 +251,7 @@ static prefix_t BOOLEAN_PREFIX_INTERNAL[] = {
{ "directory", "XDIRECTORY" },
{ "file-direntry", "XFDIRENTRY" },
{ "directory-direntry", "XDDIRENTRY" },
+ { "property", "XPROPERTY" },
};
static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
diff --git a/lib/message.cc b/lib/message.cc
index 24e698a..63a8da5 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -37,6 +37,8 @@ struct visible _notmuch_message {
notmuch_string_list_t *filename_list;
char *author;
notmuch_message_file_t *message_file;
+ notmuch_string_list_t *property_term_list;
+ notmuch_string_map_t *property_map;
notmuch_message_list_t *replies;
unsigned long flags;
/* For flags that are initialized on-demand, lazy_flags indicates
@@ -116,6 +118,8 @@ _notmuch_message_create_for_document (const void *talloc_owner,
message->filename_list = NULL;
message->message_file = NULL;
message->author = NULL;
+ message->property_term_list = NULL;
+ message->property_map = NULL;
message->replies = _notmuch_message_list_create (message);
if (unlikely (message->replies == NULL)) {
@@ -314,6 +318,7 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
*id_prefix = _find_prefix ("id"),
*type_prefix = _find_prefix ("type"),
*filename_prefix = _find_prefix ("file-direntry"),
+ *property_prefix = _find_prefix ("property"),
*replyto_prefix = _find_prefix ("replyto");
/* We do this all in a single pass because Xapian decompresses the
@@ -369,11 +374,21 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
_notmuch_database_get_terms_with_prefix (message, i, end,
filename_prefix);
+
+ /* Get property terms. Mimic the setup with filenames above */
+ assert (strcmp (filename_prefix, property_prefix) < 0);
+ if (!message->property_map && !message->property_term_list)
+ message->property_term_list =
+ _notmuch_database_get_terms_with_prefix (message, i, end,
+ property_prefix);
+
/* Get reply to */
- assert (strcmp (filename_prefix, replyto_prefix) < 0);
+ assert (strcmp (property_prefix, replyto_prefix) < 0);
if (!message->in_reply_to)
message->in_reply_to =
_notmuch_message_get_term (message, i, end, replyto_prefix);
+
+
/* It's perfectly valid for a message to have no In-Reply-To
* header. For these cases, we return an empty string. */
if (!message->in_reply_to)
@@ -405,6 +420,18 @@ _notmuch_message_invalidate_metadata (notmuch_message_t *message,
message->filename_term_list = message->filename_list = NULL;
}
+ if (strcmp ("property", prefix_name) == 0) {
+
+ if (message->property_term_list)
+ talloc_free (message->property_term_list);
+ message->property_term_list = NULL;
+
+ if (message->property_map)
+ talloc_unlink (message, message->property_map);
+
+ message->property_map = NULL;
+ }
+
if (strcmp ("replyto", prefix_name) == 0) {
talloc_free (message->in_reply_to);
message->in_reply_to = NULL;
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 3721431..df63905 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -537,6 +537,9 @@ _notmuch_string_list_append (notmuch_string_list_t *list,
void
_notmuch_string_list_sort (notmuch_string_list_t *list);
+/* string-map.c */
+typedef struct _notmuch_string_map notmuch_string_map_t;
+
/* tags.c */
notmuch_tags_t *
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 2/9] lib: private string map (associative array) API
2016-08-06 13:52 v3 of message properties patches David Bremner
2016-08-06 13:52 ` [PATCH 1/9] lib: read "property" terms from messages David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 3/9] lib: basic message-property API David Bremner
` (7 subsequent siblings)
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
The choice of array implementation is deliberate, for future iterator support
---
lib/Makefile.local | 2 +
lib/notmuch-private.h | 11 ++++
lib/string-map.c | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 166 insertions(+)
create mode 100644 lib/string-map.c
diff --git a/lib/Makefile.local b/lib/Makefile.local
index beb9635..c012ed1 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -40,6 +40,7 @@ libnotmuch_c_srcs = \
$(dir)/messages.c \
$(dir)/sha1.c \
$(dir)/built-with.c \
+ $(dir)/string-map.c \
$(dir)/tags.c
libnotmuch_cxx_srcs = \
@@ -48,6 +49,7 @@ libnotmuch_cxx_srcs = \
$(dir)/directory.cc \
$(dir)/index.cc \
$(dir)/message.cc \
+ $(dir)/message-property.cc \
$(dir)/query.cc \
$(dir)/query-fp.cc \
$(dir)/config.cc \
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index df63905..5abde88 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -540,6 +540,17 @@ _notmuch_string_list_sort (notmuch_string_list_t *list);
/* string-map.c */
typedef struct _notmuch_string_map notmuch_string_map_t;
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx);
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+ const char *key,
+ const char *value);
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
+
/* tags.c */
notmuch_tags_t *
diff --git a/lib/string-map.c b/lib/string-map.c
new file mode 100644
index 0000000..0491a10
--- /dev/null
+++ b/lib/string-map.c
@@ -0,0 +1,153 @@
+/* string-map.c - associative arrays of strings
+ *
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+
+/* Create a new notmuch_string_map_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+
+typedef struct _notmuch_string_pair_t {
+ char *key;
+ char *value;
+} notmuch_string_pair_t;
+
+struct _notmuch_string_map {
+ notmuch_bool_t sorted;
+ size_t length;
+ notmuch_string_pair_t *pairs;
+};
+
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx)
+{
+ notmuch_string_map_t *map;
+
+ map = talloc (ctx, notmuch_string_map_t);
+ if (unlikely (map == NULL))
+ return NULL;
+
+ map->length = 0;
+ map->pairs = NULL;
+ map->sorted = TRUE;
+
+ return map;
+}
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+ const char *key,
+ const char *value)
+{
+
+ map->length++;
+ map->sorted = FALSE;
+
+ if (map->pairs)
+ map->pairs = talloc_realloc (map, map->pairs, notmuch_string_pair_t, map->length + 1);
+ else
+ map->pairs = talloc_array (map, notmuch_string_pair_t, map->length + 1);
+
+ map->pairs[map->length - 1].key = talloc_strdup (map, key);
+ map->pairs[map->length - 1].value = talloc_strdup (map, value);
+
+ /* Add sentinel */
+ map->pairs[map->length].key = NULL;
+ map->pairs[map->length].value = NULL;
+
+}
+
+static int
+cmppair (const void *pa, const void *pb)
+{
+ notmuch_string_pair_t *a = (notmuch_string_pair_t *) pa;
+ notmuch_string_pair_t *b = (notmuch_string_pair_t *) pb;
+
+ return strcmp (a->key, b->key);
+}
+
+static void
+_notmuch_string_map_sort (notmuch_string_map_t *map)
+{
+ if (map->length == 0)
+ return;
+
+ if (map->sorted)
+ return;
+
+ qsort (map->pairs, map->length, sizeof (notmuch_string_pair_t), cmppair);
+
+ map->sorted = TRUE;
+}
+
+static notmuch_bool_t
+string_cmp (const char *a, const char *b, notmuch_bool_t exact)
+{
+ if (exact)
+ return (strcmp (a, b));
+ else
+ return (strncmp (a, b, strlen (a)));
+}
+
+static notmuch_string_pair_t *
+bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, notmuch_bool_t exact)
+{
+ size_t first = 0;
+ size_t last = len - 1;
+ size_t mid;
+
+ if (len <= 0)
+ return NULL;
+
+ while (last > first) {
+ mid = (first + last) / 2;
+ int sign = string_cmp (key, array[mid].key, exact);
+
+ if (sign <= 0)
+ last = mid;
+ else
+ first = mid + 1;
+ }
+
+
+ if (string_cmp (key, array[first].key, exact) == 0)
+ return array + first;
+ else
+ return NULL;
+
+}
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
+{
+ notmuch_string_pair_t *pair;
+
+ /* this means that calling append invalidates iterators */
+ _notmuch_string_map_sort (map);
+
+ pair = bsearch_first (map->pairs, map->length, key, TRUE);
+ if (! pair)
+ return NULL;
+
+ return pair->value;
+}
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 3/9] lib: basic message-property API
2016-08-06 13:52 v3 of message properties patches David Bremner
2016-08-06 13:52 ` [PATCH 1/9] lib: read "property" terms from messages David Bremner
2016-08-06 13:52 ` [PATCH 2/9] lib: private string map (associative array) API David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 4/9] lib: extend private string map API with iterators David Bremner
` (6 subsequent siblings)
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
Initially, support get, set and removal of single key/value pair, as
well as removing all properties.
---
lib/message-private.h | 16 +++++++
lib/message-property.cc | 108 ++++++++++++++++++++++++++++++++++++++++++
lib/message.cc | 52 +++++++++++++++++++-
lib/notmuch.h | 72 ++++++++++++++++++++++++++++
test/T610-message-property.sh | 84 ++++++++++++++++++++++++++++++++
5 files changed, 330 insertions(+), 2 deletions(-)
create mode 100644 lib/message-private.h
create mode 100644 lib/message-property.cc
create mode 100755 test/T610-message-property.sh
diff --git a/lib/message-private.h b/lib/message-private.h
new file mode 100644
index 0000000..7419925
--- /dev/null
+++ b/lib/message-private.h
@@ -0,0 +1,16 @@
+#ifndef MESSAGE_PRIVATE_H
+#define MESSAGE_PRIVATE_H
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message);
+
+notmuch_bool_t
+_notmuch_message_frozen (notmuch_message_t *message);
+
+void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix);
+
+void
+_notmuch_message_invalidate_metadata (notmuch_message_t *message, const char *prefix_name);
+
+#endif
diff --git a/lib/message-property.cc b/lib/message-property.cc
new file mode 100644
index 0000000..1f04a20
--- /dev/null
+++ b/lib/message-property.cc
@@ -0,0 +1,108 @@
+/* message-property.cc - Properties are like tags, but (key,value) pairs.
+ * keys are allowed to repeat.
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "message-private.h"
+
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value)
+{
+ if (! value)
+ return NOTMUCH_STATUS_NULL_POINTER;
+
+ *value = _notmuch_string_map_get (_notmuch_message_property_map (message), key);
+
+ return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_notmuch_message_modify_property (notmuch_message_t *message, const char *key, const char *value,
+ notmuch_bool_t delete_it)
+{
+ notmuch_private_status_t private_status;
+ notmuch_status_t status;
+ char *term = NULL;
+
+ status = _notmuch_database_ensure_writable (_notmuch_message_database (message));
+ if (status)
+ return status;
+
+ if (key == NULL || value == NULL)
+ return NOTMUCH_STATUS_NULL_POINTER;
+
+ if (index (key, '='))
+ return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+
+ term = talloc_asprintf (message, "%s=%s", key, value);
+
+ if (delete_it)
+ private_status = _notmuch_message_remove_term (message, "property", term);
+ else
+ private_status = _notmuch_message_add_term (message, "property", term);
+
+ if (private_status)
+ return COERCE_STATUS (private_status,
+ "Unhandled error modifying message property");
+ if (! _notmuch_message_frozen (message))
+ _notmuch_message_sync (message);
+
+ if (term)
+ talloc_free (term);
+
+ return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value)
+{
+ return _notmuch_message_modify_property (message, key, value, FALSE);
+}
+
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value)
+{
+ return _notmuch_message_modify_property (message, key, value, TRUE);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+ notmuch_status_t status;
+ const char * term_prefix;
+
+ status = _notmuch_database_ensure_writable (_notmuch_message_database (message));
+ if (status)
+ return status;
+
+ _notmuch_message_invalidate_metadata (message, "property");
+ if (key)
+ term_prefix = talloc_asprintf (message, "%s%s=", _find_prefix ("property"), key);
+ else
+ term_prefix = _find_prefix ("property");
+
+ /* XXX better error reporting ? */
+ _notmuch_message_remove_terms (message, term_prefix);
+
+ return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/lib/message.cc b/lib/message.cc
index 63a8da5..9d3e807 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -20,6 +20,7 @@
#include "notmuch-private.h"
#include "database-private.h"
+#include "message-private.h"
#include <stdint.h>
@@ -395,7 +396,7 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
message->in_reply_to = talloc_strdup (message, "");
}
-static void
+void
_notmuch_message_invalidate_metadata (notmuch_message_t *message,
const char *prefix_name)
{
@@ -552,7 +553,7 @@ notmuch_message_get_replies (notmuch_message_t *message)
return _notmuch_messages_create (message->replies);
}
-static void
+void
_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
{
Xapian::TermIterator i;
@@ -1799,3 +1800,50 @@ _notmuch_message_database (notmuch_message_t *message)
{
return message->notmuch;
}
+
+void
+_notmuch_message_ensure_property_map (notmuch_message_t *message)
+{
+ notmuch_string_node_t *node;
+
+ if (message->property_map)
+ return;
+
+ if (!message->property_term_list)
+ _notmuch_message_ensure_metadata (message);
+
+ message->property_map = _notmuch_string_map_create (message);
+
+ for (node = message->property_term_list->head; node; node = node->next) {
+ const char *key;
+ char *value;
+
+ value = index(node->string, '=');
+ if (!value)
+ INTERNAL_ERROR ("malformed property term");
+
+ *value = '\0';
+ value++;
+ key = node->string;
+
+ _notmuch_string_map_append (message->property_map, key, value);
+
+ }
+
+ talloc_free (message->property_term_list);
+ message->property_term_list = NULL;
+}
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message)
+{
+ _notmuch_message_ensure_property_map (message);
+
+ return message->property_map;
+}
+
+notmuch_bool_t
+_notmuch_message_frozen (notmuch_message_t *message)
+{
+ return message->frozen;
+}
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 2faa146..b304dca 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -180,6 +180,11 @@ typedef enum _notmuch_status {
*/
NOTMUCH_STATUS_PATH_ERROR,
/**
+ * One of the arguments violates the preconditions for the
+ * function, in a way not covered by a more specific argument.
+ */
+ NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+ /**
* Not an actual status value. Just a way to find out how many
* valid status values there are.
*/
@@ -1654,6 +1659,73 @@ void
notmuch_message_destroy (notmuch_message_t *message);
/**
+ * @name Message Properties
+ *
+ * This interface provides the ability to attach arbitrary (key,value)
+ * string pairs to a message, to remove such pairs, and to iterate
+ * over them. The caller should take some care as to what keys they
+ * add or delete values for, as other subsystems or extensions may
+ * depend on these properties.
+ *
+ */
+/**@{*/
+/**
+ * Retrieve the value for a single property key
+ *
+ * *value* is set to a string owned by the message or NULL if there is
+ * no such key. In the case of multiple values for the given key, the
+ * first one is retrieved.
+ *
+ * @returns
+ * - NOTMUCH_STATUS_NULL_POINTER: *value* may not be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+
+ */
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value);
+
+/**
+ * Add a (key,value) pair to a message
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ */
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove a (key,value) pair from a message.
+ *
+ * It is not an error to remove a non-existant (key,value) pair
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ */
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove all (key,value) pairs from the given message.
+ *
+ * @param[in,out] message message to operate on.
+ * @param[in] key key to delete properties for. If NULL, delete
+ * properties for all keys
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ * read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ *
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
+
+/**@}*/
+
+/**
* Is the given 'tags' iterator pointing at a valid tag.
*
* When this function returns TRUE, notmuch_tags_get will return a
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
new file mode 100755
index 0000000..0217950
--- /dev/null
+++ b/test/T610-message-property.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+test_description="message property API"
+
+. ./test-lib.sh || exit 1
+
+add_email_corpus
+
+cat <<EOF > c_head
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <notmuch-test.h>
+
+int main (int argc, char** argv)
+{
+ notmuch_database_t *db;
+ notmuch_message_t *message = NULL;
+ const char *val;
+ notmuch_status_t stat;
+
+ EXPECT0(notmuch_database_open (argv[1], NOTMUCH_DATABASE_MODE_READ_WRITE, &db));
+ EXPECT0(notmuch_database_find_message(db, "4EFC743A.3060609@april.org", &message));
+ if (message == NULL) {
+ fprintf (stderr, "unable to find message");
+ exit (1);
+ }
+EOF
+
+cat <<EOF > c_tail
+ EXPECT0(notmuch_database_destroy(db));
+}
+EOF
+
+test_begin_subtest "notmuch_message_{add,get,remove}_property"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+ EXPECT0(notmuch_message_add_property (message, "testkey1", "testvalue1"));
+ EXPECT0(notmuch_message_get_property (message, "testkey1", &val));
+ printf("testkey1[1] = %s\n", val);
+ EXPECT0(notmuch_message_add_property (message, "testkey2", "this value has spaces and = sign"));
+ EXPECT0(notmuch_message_get_property (message, "testkey1", &val));
+ printf("testkey1[2] = %s\n", val);
+ EXPECT0(notmuch_message_get_property (message, "testkey1", &val));
+
+ EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+ printf("testkey2 = %s\n", val);
+
+ /* Add second value for key */
+ EXPECT0(notmuch_message_add_property (message, "testkey2", "zztestvalue3"));
+ EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+ printf("testkey2 = %s\n", val);
+
+ /* remove first value for key */
+ EXPECT0(notmuch_message_remove_property (message, "testkey2", "this value has spaces and = sign"));
+ EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+ printf("testkey2 = %s\n", val);
+
+ /* remove non-existant value for key */
+ EXPECT0(notmuch_message_remove_property (message, "testkey2", "this value has spaces and = sign"));
+ EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+ printf("testkey2 = %s\n", val);
+
+ /* remove only value for key */
+ EXPECT0(notmuch_message_remove_property (message, "testkey2", "zztestvalue3"));
+ EXPECT0(notmuch_message_get_property (message, "testkey2", &val));
+ printf("testkey2 = %s\n", val == NULL ? "NULL" : val);
+}
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+testkey1[1] = testvalue1
+testkey1[2] = testvalue1
+testkey2 = this value has spaces and = sign
+testkey2 = this value has spaces and = sign
+testkey2 = zztestvalue3
+testkey2 = zztestvalue3
+testkey2 = NULL
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+
+
+test_done
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 4/9] lib: extend private string map API with iterators
2016-08-06 13:52 v3 of message properties patches David Bremner
` (2 preceding siblings ...)
2016-08-06 13:52 ` [PATCH 3/9] lib: basic message-property API David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 5/9] lib: iterator API for message properties David Bremner
` (5 subsequent siblings)
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
Support for prefix based iterators is perhaps overengineering, but I
wanted to mimic the existing database_config API.
---
lib/notmuch-private.h | 21 ++++++++++++++-
lib/string-map.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 5abde88..65f7ead 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -539,7 +539,7 @@ _notmuch_string_list_sort (notmuch_string_list_t *list);
/* string-map.c */
typedef struct _notmuch_string_map notmuch_string_map_t;
-
+typedef struct _notmuch_string_map_iterator notmuch_string_map_iterator_t;
notmuch_string_map_t *
_notmuch_string_map_create (const void *ctx);
@@ -551,6 +551,25 @@ _notmuch_string_map_append (notmuch_string_map_t *map,
const char *
_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+ notmuch_bool_t exact);
+
+notmuch_bool_t
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iter);
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator);
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+
/* tags.c */
notmuch_tags_t *
diff --git a/lib/string-map.c b/lib/string-map.c
index 0491a10..591ff6d 100644
--- a/lib/string-map.c
+++ b/lib/string-map.c
@@ -38,6 +38,12 @@ struct _notmuch_string_map {
notmuch_string_pair_t *pairs;
};
+struct _notmuch_string_map_iterator {
+ notmuch_string_pair_t *current;
+ notmuch_bool_t exact;
+ const char *key;
+};
+
notmuch_string_map_t *
_notmuch_string_map_create (const void *ctx)
{
@@ -151,3 +157,69 @@ _notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
return pair->value;
}
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+ notmuch_bool_t exact)
+{
+ notmuch_string_map_iterator_t *iter;
+
+ _notmuch_string_map_sort (map);
+
+ iter = talloc (map, notmuch_string_map_iterator_t);
+ if (unlikely (iter == NULL))
+ return NULL;
+
+ iter->key = talloc_strdup (iter, key);
+ iter->exact = exact;
+ iter->current = bsearch_first (map->pairs, map->length, key, exact);
+ return iter;
+}
+
+notmuch_bool_t
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iterator)
+{
+ if (iterator->current == NULL)
+ return FALSE;
+
+ /* sentinel */
+ if (iterator->current->key == NULL)
+ return FALSE;
+
+ return (0 == string_cmp (iterator->key, iterator->current->key, iterator->exact));
+
+}
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iterator)
+{
+
+ if (! _notmuch_string_map_iterator_valid (iterator))
+ return;
+
+ (iterator->current)++;
+}
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator)
+{
+ if (! _notmuch_string_map_iterator_valid (iterator))
+ return NULL;
+
+ return iterator->current->key;
+}
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator)
+{
+ if (! _notmuch_string_map_iterator_valid (iterator))
+ return NULL;
+
+ return iterator->current->value;
+}
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator)
+{
+ talloc_free (iterator);
+}
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 5/9] lib: iterator API for message properties
2016-08-06 13:52 v3 of message properties patches David Bremner
` (3 preceding siblings ...)
2016-08-06 13:52 ` [PATCH 4/9] lib: extend private string map API with iterators David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 6/9] CLI: refactor dumping of tags David Bremner
` (4 subsequent siblings)
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
This is a thin wrapper around the string map iterator API just introduced.
---
lib/message-property.cc | 38 +++++++++++++++
lib/notmuch.h | 95 +++++++++++++++++++++++++++++++++++++
test/T610-message-property.sh | 107 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 240 insertions(+)
diff --git a/lib/message-property.cc b/lib/message-property.cc
index 1f04a20..0b13cac 100644
--- a/lib/message-property.cc
+++ b/lib/message-property.cc
@@ -106,3 +106,41 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
return NOTMUCH_STATUS_SUCCESS;
}
+
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
+{
+ notmuch_string_map_t *map;
+ map = _notmuch_message_property_map (message);
+ return _notmuch_string_map_iterator_create (map, key, exact);
+}
+
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties)
+{
+ return _notmuch_string_map_iterator_valid (properties);
+}
+
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties)
+{
+ return _notmuch_string_map_iterator_move_to_next (properties);
+}
+
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties)
+{
+ return _notmuch_string_map_iterator_key (properties);
+}
+
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties)
+{
+ return _notmuch_string_map_iterator_value (properties);
+}
+
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties)
+{
+ _notmuch_string_map_iterator_destroy (properties);
+}
diff --git a/lib/notmuch.h b/lib/notmuch.h
index b304dca..e03a05d 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1723,6 +1723,101 @@ notmuch_message_remove_property (notmuch_message_t *message, const char *key, co
notmuch_status_t
notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
+/**
+ * Opaque message property iterator
+ */
+typedef struct _notmuch_string_map_iterator notmuch_message_properties_t;
+
+/**
+ * Get the properties for *message*, returning a
+ * notmuch_message_properties_t object which can be used to iterate
+ * over all properties.
+ *
+ * The notmuch_message_properties_t object is owned by the message and
+ * as such, will only be valid for as long as the message is valid,
+ * (which is until the query from which it derived is destroyed).
+ *
+ * @param[in] message The message to examine
+ * @param[in] key key or key prefix
+ * @param[in] exact if TRUE, require exact match with key. Otherwise
+ * treat as prefix.
+ *
+ * Typical usage might be:
+ *
+ * notmuch_message_properties_t *list;
+ *
+ * for (list = notmuch_message_get_properties (message, "testkey1", TRUE);
+ * notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+ * printf("%s\n", notmuch_message_properties_value(list));
+ * }
+ *
+ * notmuch_message_properties_destroy (list);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_message_properties_t object. (For consistency, we do
+ * provide a notmuch_message_properities_destroy function, but there's
+ * no good reason to call it if the message is about to be destroyed).
+ */
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact);
+
+/**
+ * Is the given *properties* iterator pointing at a valid (key,value)
+ * pair.
+ *
+ * When this function returns TRUE,
+ * notmuch_message_properties_{key,value} will return a valid string,
+ * and notmuch_message_properties_move_to_next will do what it
+ * says. Whereas when this function returns FALSE, calling any of
+ * these functions results in undefined behaviour.
+ *
+ * See the documentation of notmuch_message_properties_get for example
+ * code showing how to iterate over a notmuch_message_properties_t
+ * object.
+ */
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties);
+
+/**
+ * Move the *properties* iterator to the next (key,value) pair
+ *
+ * If *properties* is already pointing at the last pair then the iterator
+ * will be moved to a point just beyond that last pair, (where
+ * notmuch_message_properties_valid will return FALSE).
+ *
+ * See the documentation of notmuch_message_get_properties for example
+ * code showing how to iterate over a notmuch_message_properties_t object.
+ */
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * this could be useful if iterating for a prefix
+ */
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * This could be useful if iterating for a prefix.
+ */
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties);
+
+
+/**
+ * Destroy a notmuch_message_properties_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_message_properties_t object will be reclaimed when the
+ * containing message object is destroyed.
+ */
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties);
+
/**@}*/
/**
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index 0217950..b5ddb7a 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -9,8 +9,18 @@ cat <<EOF > c_head
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
+#include <talloc.h>
#include <notmuch-test.h>
+void print_properties (notmuch_message_t *message, const char *prefix, notmuch_bool_t exact) {
+ notmuch_message_properties_t *list;
+ for (list = notmuch_message_get_properties (message, prefix, exact);
+ notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+ printf("%s\n", notmuch_message_properties_value(list));
+ }
+ notmuch_message_properties_destroy (list);
+}
+
int main (int argc, char** argv)
{
notmuch_database_t *db;
@@ -79,6 +89,103 @@ testkey2 = NULL
EOF
test_expect_equal_file EXPECTED OUTPUT
+test_begin_subtest "notmuch_message_get_properties: empty list"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+ notmuch_message_properties_t *list;
+ list = notmuch_message_get_properties (message, "nonexistent", TRUE);
+ printf("valid = %d\n", notmuch_message_properties_valid (list));
+ notmuch_message_properties_destroy (list);
+}
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+valid = 0
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+test_begin_subtest "notmuch_message_properties: one value"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+print_properties (message, "testkey1", TRUE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+testvalue1
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "notmuch_message_properties: multiple values"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "testkey1", "bob"));
+EXPECT0(notmuch_message_add_property (message, "testkey1", "testvalue2"));
+EXPECT0(notmuch_message_add_property (message, "testkey1", "alice"));
+print_properties (message, "testkey1", TRUE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+alice
+bob
+testvalue1
+testvalue2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "notmuch_message_properties: prefix"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "testkey3", "bob3"));
+EXPECT0(notmuch_message_add_property (message, "testkey3", "testvalue3"));
+EXPECT0(notmuch_message_add_property (message, "testkey3", "alice3"));
+print_properties (message, "testkey", FALSE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+alice
+bob
+testvalue1
+testvalue2
+alice3
+bob3
+testvalue3
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "notmuch_message_properties: modify during iteration"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+ const char *keys[1000] = {NULL};
+ const char *vals[1000] = {NULL};
+ notmuch_message_properties_t *properties;
+ int i;
+
+ for (properties = notmuch_message_get_properties (message, "", FALSE), i=0;
+ notmuch_message_properties_valid (properties);
+ notmuch_message_properties_move_to_next (properties), i++)
+ {
+ const char *key, *value;
+
+ keys[i]=talloc_strdup(message,
+ notmuch_message_properties_key (properties));
+ vals[i]=talloc_strdup(message,
+ notmuch_message_properties_value (properties));
+
+ EXPECT0(notmuch_message_remove_property (message, keys[i], vals[i]));
+ }
+
+ print_properties (message, "", FALSE);
+
+ for (i = 0; keys[i] && vals[i]; i++) {
+ EXPECT0(notmuch_message_add_property (message, keys[i], vals[i]));
+ }
+}
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
test_done
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 6/9] CLI: refactor dumping of tags.
2016-08-06 13:52 v3 of message properties patches David Bremner
` (4 preceding siblings ...)
2016-08-06 13:52 ` [PATCH 5/9] lib: iterator API for message properties David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 7/9] CLI: add properties to dump output David Bremner
` (3 subsequent siblings)
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
This is mainly code movement, to make room in the loop over messages for
dumping properties.
---
notmuch-dump.c | 127 +++++++++++++++++++++++++++++++--------------------------
1 file changed, 69 insertions(+), 58 deletions(-)
diff --git a/notmuch-dump.c b/notmuch-dump.c
index cae1db8..d80ed8b8 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -78,13 +78,78 @@ print_dump_header (gzFile output, int output_format, int include)
}
static int
+dump_tags_message (void *ctx,
+ notmuch_message_t *message, int output_format,
+ gzFile output,
+ char **buffer_p, size_t *size_p)
+{
+ int first = 1;
+ const char *message_id;
+
+ message_id = notmuch_message_get_message_id (message);
+
+ if (output_format == DUMP_FORMAT_BATCH_TAG &&
+ strchr (message_id, '\n')) {
+ /* This will produce a line break in the output, which
+ * would be difficult to handle in tools. However, it's
+ * also impossible to produce an email containing a line
+ * break in a message ID because of unfolding, so we can
+ * safely disallow it. */
+ fprintf (stderr, "Warning: skipping message id containing line break: \"%s\"\n", message_id);
+ return EXIT_SUCCESS;
+ }
+
+ if (output_format == DUMP_FORMAT_SUP) {
+ gzprintf (output, "%s (", message_id);
+ }
+
+ for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+ notmuch_tags_valid (tags);
+ notmuch_tags_move_to_next (tags)) {
+ const char *tag_str = notmuch_tags_get (tags);
+
+ if (! first)
+ gzputs (output, " ");
+
+ first = 0;
+
+ if (output_format == DUMP_FORMAT_SUP) {
+ gzputs (output, tag_str);
+ } else {
+ if (hex_encode (ctx, tag_str,
+ buffer_p, size_p) != HEX_SUCCESS) {
+ fprintf (stderr, "Error: failed to hex-encode tag %s\n",
+ tag_str);
+ return EXIT_FAILURE;
+ }
+ gzprintf (output, "+%s", *buffer_p);
+ }
+ }
+
+ if (output_format == DUMP_FORMAT_SUP) {
+ gzputs (output, ")\n");
+ } else {
+ if (make_boolean_term (ctx, "id", message_id,
+ buffer_p, size_p)) {
+ fprintf (stderr, "Error quoting message id %s: %s\n",
+ message_id, strerror (errno));
+ return EXIT_FAILURE;
+ }
+ gzprintf (output, " -- %s\n", *buffer_p);
+ }
+ return EXIT_SUCCESS;
+}
+
+static int
database_dump_file (notmuch_database_t *notmuch, gzFile output,
const char *query_str, int output_format, int include)
{
notmuch_query_t *query;
notmuch_messages_t *messages;
notmuch_message_t *message;
- notmuch_tags_t *tags;
+ notmuch_status_t status;
+ char *buffer = NULL;
+ size_t buffer_size = 0;
print_dump_header (output, output_format, include);
@@ -110,10 +175,6 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
*/
notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
- char *buffer = NULL;
- size_t buffer_size = 0;
- notmuch_status_t status;
-
status = notmuch_query_search_messages_st (query, &messages);
if (print_status_query ("notmuch dump", query, status))
return EXIT_FAILURE;
@@ -121,62 +182,12 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
for (;
notmuch_messages_valid (messages);
notmuch_messages_move_to_next (messages)) {
- int first = 1;
- const char *message_id;
message = notmuch_messages_get (messages);
- message_id = notmuch_message_get_message_id (message);
-
- if (output_format == DUMP_FORMAT_BATCH_TAG &&
- strchr (message_id, '\n')) {
- /* This will produce a line break in the output, which
- * would be difficult to handle in tools. However, it's
- * also impossible to produce an email containing a line
- * break in a message ID because of unfolding, so we can
- * safely disallow it. */
- fprintf (stderr, "Warning: skipping message id containing line break: \"%s\"\n", message_id);
- notmuch_message_destroy (message);
- continue;
- }
- if (output_format == DUMP_FORMAT_SUP) {
- gzprintf (output, "%s (", message_id);
- }
-
- for (tags = notmuch_message_get_tags (message);
- notmuch_tags_valid (tags);
- notmuch_tags_move_to_next (tags)) {
- const char *tag_str = notmuch_tags_get (tags);
-
- if (! first)
- gzputs (output, " ");
-
- first = 0;
-
- if (output_format == DUMP_FORMAT_SUP) {
- gzputs (output, tag_str);
- } else {
- if (hex_encode (notmuch, tag_str,
- &buffer, &buffer_size) != HEX_SUCCESS) {
- fprintf (stderr, "Error: failed to hex-encode tag %s\n",
- tag_str);
- return EXIT_FAILURE;
- }
- gzprintf (output, "+%s", buffer);
- }
- }
-
- if (output_format == DUMP_FORMAT_SUP) {
- gzputs (output, ")\n");
- } else {
- if (make_boolean_term (notmuch, "id", message_id,
- &buffer, &buffer_size)) {
- fprintf (stderr, "Error quoting message id %s: %s\n",
- message_id, strerror (errno));
- return EXIT_FAILURE;
- }
- gzprintf (output, " -- %s\n", buffer);
- }
+ if (dump_tags_message (notmuch, message, output_format, output,
+ &buffer, &buffer_size))
+ return EXIT_FAILURE;
notmuch_message_destroy (message);
}
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 7/9] CLI: add properties to dump output
2016-08-06 13:52 v3 of message properties patches David Bremner
` (5 preceding siblings ...)
2016-08-06 13:52 ` [PATCH 6/9] CLI: refactor dumping of tags David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 8/9] cli: optionally restore message properties from dump file David Bremner
` (2 subsequent siblings)
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
Part of providing extensibility via properties is to make sure that user
data is not lost. Thus we need to be able to dump and restore
properties.
---
doc/man1/notmuch-dump.rst | 18 ++++++---
notmuch-client.h | 3 ++
notmuch-dump.c | 85 +++++++++++++++++++++++++++++++++++++++----
notmuch-new.c | 2 +-
test/T610-message-property.sh | 21 +++++++++++
5 files changed, 116 insertions(+), 13 deletions(-)
diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index 94986a8..751850b 100644
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -71,7 +71,7 @@ Supported options for **dump** include
characters. Note also that tags with spaces will not be
correctly restored with this format.
- ``--include=(config|tags)``
+ ``--include=(config|properties|tags)``
Control what kind of metadata is included in the output.
@@ -81,14 +81,22 @@ Supported options for **dump** include
starts with "#@ ", followed by a space seperated key-value
pair. Both key and value are hex encoded if needed.
+ **properties**
+
+ Output per-message (key,value) metadata. Each line starts
+ with "#= ", followed by a message id, and a space seperated
+ list of key=value pairs. pair. Ids, keys and values are hex
+ encoded if needed.
+
**tags**
- Output per-message metadata, namely tags. See *format* above
+ Output per-message boolean metadata, namely tags. See *format* above
for description of the output.
- The default is to include both tags and configuration
- information. As of version 2 of the dump format, there is a
- header line of the following form
+ The default is to include all available types of data. The
+ option can be specified multiple times to select some subset. As
+ of version 2 of the dump format, there is a header line of the
+ following form
|
| #notmuch-dump <*format*>:<*version*> <*included*>
diff --git a/notmuch-client.h b/notmuch-client.h
index ebc092b..9ce2aef 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -449,8 +449,11 @@ typedef enum dump_formats {
typedef enum dump_includes {
DUMP_INCLUDE_TAGS = 1,
DUMP_INCLUDE_CONFIG = 2,
+ DUMP_INCLUDE_PROPERTIES = 4
} dump_include_t;
+#define DUMP_INCLUDE_DEFAULT (DUMP_INCLUDE_TAGS | DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_PROPERTIES)
+
#define NOTMUCH_DUMP_VERSION 2
int
diff --git a/notmuch-dump.c b/notmuch-dump.c
index d80ed8b8..e7965ce 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -69,12 +69,77 @@ database_dump_config (notmuch_database_t *notmuch, gzFile output)
static void
print_dump_header (gzFile output, int output_format, int include)
{
- gzprintf (output, "#notmuch-dump %s:%d %s%s%s\n",
+ const char *sep = "";
+
+ gzprintf (output, "#notmuch-dump %s:%d ",
(output_format == DUMP_FORMAT_SUP) ? "sup" : "batch-tag",
- NOTMUCH_DUMP_VERSION,
- (include & DUMP_INCLUDE_CONFIG) ? "config" : "",
- (include & DUMP_INCLUDE_TAGS) && (include & DUMP_INCLUDE_CONFIG) ? "," : "",
- (include & DUMP_INCLUDE_TAGS) ? "tags" : "");
+ NOTMUCH_DUMP_VERSION);
+
+ if (include & DUMP_INCLUDE_CONFIG) {
+ gzputs (output, "config");
+ sep = ",";
+ }
+ if (include & DUMP_INCLUDE_PROPERTIES) {
+ gzprintf (output, "%sproperties", sep);
+ sep = ",";
+ }
+ if (include & DUMP_INCLUDE_TAGS) {
+ gzprintf (output, "%sproperties", sep);
+ }
+ gzputs (output, "\n");
+}
+
+static int
+dump_properties_message (void *ctx,
+ notmuch_message_t *message,
+ gzFile output,
+ char **buffer_p, size_t *size_p)
+{
+ const char *message_id;
+ notmuch_message_properties_t *list;
+ notmuch_bool_t first = TRUE;
+
+ message_id = notmuch_message_get_message_id (message);
+
+ if (strchr (message_id, '\n')) {
+ fprintf (stderr, "Warning: skipping message id containing line break: \"%s\"\n", message_id);
+ return 0;
+ }
+
+ for (list = notmuch_message_get_properties (message, "", FALSE);
+ notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+ const char *key, *val;
+
+ if (first) {
+ if (hex_encode (ctx, message_id, buffer_p, size_p) != HEX_SUCCESS) {
+ fprintf (stderr, "Error: failed to hex-encode message-id %s\n", message_id);
+ return 1;
+ }
+ gzprintf (output, "#= %s", *buffer_p);
+ first = FALSE;
+ }
+
+ key = notmuch_message_properties_key (list);
+ val = notmuch_message_properties_value (list);
+
+ if (hex_encode (ctx, key, buffer_p, size_p) != HEX_SUCCESS) {
+ fprintf (stderr, "Error: failed to hex-encode key %s\n", key);
+ return 1;
+ }
+ gzprintf (output, " %s", *buffer_p);
+
+ if (hex_encode (ctx, val, buffer_p, size_p) != HEX_SUCCESS) {
+ fprintf (stderr, "Error: failed to hex-encode value %s\n", val);
+ return 1;
+ }
+ gzprintf (output, "=%s", *buffer_p);
+ }
+ notmuch_message_properties_destroy (list);
+
+ if (! first)
+ gzprintf (output, "\n", *buffer_p);
+
+ return 0;
}
static int
@@ -159,7 +224,7 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
return EXIT_FAILURE;
}
- if (! (include & DUMP_INCLUDE_TAGS))
+ if (! (include & (DUMP_INCLUDE_TAGS | DUMP_INCLUDE_PROPERTIES)))
return EXIT_SUCCESS;
if (! query_str)
@@ -189,6 +254,11 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output,
&buffer, &buffer_size))
return EXIT_FAILURE;
+ if ((include & DUMP_INCLUDE_PROPERTIES) &&
+ dump_properties_message (notmuch, message, output,
+ &buffer, &buffer_size))
+ return EXIT_FAILURE;
+
notmuch_message_destroy (message);
}
@@ -312,6 +382,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
{ 0, 0 } } },
{ NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I',
(notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG },
+ { "properties", DUMP_INCLUDE_PROPERTIES },
{ "tags", DUMP_INCLUDE_TAGS} } },
{ NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 },
{ NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
@@ -326,7 +397,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
notmuch_process_shared_options (argv[0]);
if (include == 0)
- include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS;
+ include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS | DUMP_INCLUDE_PROPERTIES;
if (opt_index < argc) {
query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);
diff --git a/notmuch-new.c b/notmuch-new.c
index 799fec2..c55dea7 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -1042,7 +1042,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
}
if (notmuch_database_dump (notmuch, backup_name, "",
- DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS, TRUE)) {
+ DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_DEFAULT, TRUE)) {
fprintf (stderr, "Backup failed. Aborting upgrade.");
return EXIT_FAILURE;
}
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index b5ddb7a..a9b76de 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -89,6 +89,17 @@ testkey2 = NULL
EOF
test_expect_equal_file EXPECTED OUTPUT
+test_begin_subtest "notmuch_message_remove_all_properties"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_remove_all_properties (message, NULL));
+print_properties (message, "", FALSE);
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
test_begin_subtest "notmuch_message_get_properties: empty list"
cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
{
@@ -188,4 +199,14 @@ cat <<'EOF' >EXPECTED
EOF
test_expect_equal_file EXPECTED OUTPUT
+test_begin_subtest "dump message properties"
+cat <<EOF > PROPERTIES
+#= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue1 testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3
+EOF
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "fancy key with áccènts", "import value with ="));
+EOF
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
test_done
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 8/9] cli: optionally restore message properties from dump file
2016-08-06 13:52 v3 of message properties patches David Bremner
` (6 preceding siblings ...)
2016-08-06 13:52 ` [PATCH 7/9] CLI: add properties to dump output David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 13:52 ` [PATCH 9/9] add has: query prefix to search for specific properties David Bremner
2016-09-20 21:56 ` v3 of message properties patches Daniel Kahn Gillmor
9 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch
This somewhat mimics the config line parsing, except there can be
arbitrarily many key value pairs, so one more level of looping is
required.
---
doc/man1/notmuch-restore.rst | 13 +++++--
notmuch-restore.c | 85 +++++++++++++++++++++++++++++++++++++++++--
test/T610-message-property.sh | 28 ++++++++++++++
3 files changed, 120 insertions(+), 6 deletions(-)
diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst
index 87fa22e..a0c1b3c 100644
--- a/doc/man1/notmuch-restore.rst
+++ b/doc/man1/notmuch-restore.rst
@@ -50,7 +50,7 @@ Supported options for **restore** include
format, this heuristic, based the fact that batch-tag format
contains no parentheses, should be accurate.
- ``--include=(config|tags)``
+ ``--include=(config|properties|tags)``
Control what kind of metadata is restored.
@@ -60,13 +60,20 @@ Supported options for **restore** include
with "#@ ", followed by a space seperated key-value pair.
Both key and value are hex encoded if needed.
+ **properties**
+
+ Output per-message (key,value) metadata. Each line starts
+ with "#= ", followed by a message id, and a space seperated
+ list of key=value pairs. pair. Ids, keys and values are
+ hex encoded if needed.
+
**tags**
Output per-message metadata, namely tags. See *format* above
for more details.
- The default is to restore both tags and configuration
- information
+ The default is to restore all available types of data. The
+ option can be specified multiple times to select some subset.
``--input=``\ <filename>
Read input from given file instead of stdin.
diff --git a/notmuch-restore.c b/notmuch-restore.c
index 371237c..d6429ef 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -57,6 +57,72 @@ process_config_line (notmuch_database_t *notmuch, const char* line)
return ret;
}
+static int
+process_properties_line (notmuch_database_t *notmuch, const char* line)
+
+{
+ const char *id_p, *tok;
+ size_t id_len = 0, tok_len = 0;
+ char *id;
+
+ notmuch_message_t *message = NULL;
+ const char *delim = " \t\n";
+ int ret = EXIT_FAILURE;
+
+ void *local = talloc_new (NULL);
+
+ id_p = strtok_len_c (line, delim, &id_len);
+ id = talloc_strndup (local, id_p, id_len);
+ if (hex_decode_inplace (id) != HEX_SUCCESS) {
+ fprintf (stderr, "hex decoding failure on line %s\n", line);
+ goto DONE;
+ }
+
+ if (print_status_database ("notmuch restore", notmuch,
+ notmuch_database_find_message (notmuch, id, &message)))
+ goto DONE;
+
+ if (print_status_database ("notmuch restore", notmuch,
+ notmuch_message_remove_all_properties (message, NULL)))
+ goto DONE;
+
+ tok = id_p + id_len;
+
+ while ((tok = strtok_len_c (tok + tok_len, delim, &tok_len)) != NULL) {
+ char *key, *value;
+ size_t off = strcspn (tok, "=");
+ if (off > tok_len) {
+ fprintf (stderr, "unparsable token %s\n", tok);
+ goto DONE;
+ }
+
+ key = talloc_strndup (local, tok, off);
+ value = talloc_strndup (local, tok + off + 1, tok_len - off - 1);
+
+ if (hex_decode_inplace (key) != HEX_SUCCESS) {
+ fprintf (stderr, "hex decoding failure on key %s\n", key);
+ goto DONE;
+ }
+
+ if (hex_decode_inplace (value) != HEX_SUCCESS) {
+ fprintf (stderr, "hex decoding failure on value %s\n", value);
+ goto DONE;
+ }
+
+ if (print_status_database ("notmuch restore", notmuch,
+ notmuch_message_add_property (message, key, value)))
+ goto DONE;
+
+ }
+
+ ret = EXIT_SUCCESS;
+
+ DONE:
+ talloc_free (local);
+ return ret;
+}
+
+
static regex_t regex;
/* Non-zero return indicates an error in retrieving the message,
@@ -188,6 +254,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
{ 0, 0 } } },
{ NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I',
(notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG },
+ { "properties", DUMP_INCLUDE_PROPERTIES },
{ "tags", DUMP_INCLUDE_TAGS} } },
{ NOTMUCH_OPT_STRING, &input_file_name, "input", 'i', 0 },
@@ -206,7 +273,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
notmuch_exit_if_unmatched_db_uuid (notmuch);
if (include == 0) {
- include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS;
+ include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_PROPERTIES | DUMP_INCLUDE_TAGS;
}
name_for_error = input_file_name ? input_file_name : "stdin";
@@ -273,13 +340,18 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
if (ret)
goto DONE;
}
+ if ((include & DUMP_INCLUDE_PROPERTIES) && line_len >= 2 && line[0] == '#' && line[1] == '=') {
+ ret = process_properties_line (notmuch, line + 2);
+ if (ret)
+ goto DONE;
+ }
} while ((line_len == 0) ||
(line[0] == '#') ||
/* the cast is safe because we checked about for line_len < 0 */
(strspn (line, " \t\n") == (unsigned)line_len));
- if (! (include & DUMP_INCLUDE_TAGS)) {
+ if (! ((include & DUMP_INCLUDE_TAGS) || (include & DUMP_INCLUDE_PROPERTIES))) {
ret = EXIT_SUCCESS;
goto DONE;
}
@@ -306,6 +378,13 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
talloc_free (line_ctx);
line_ctx = talloc_new (config);
+
+ if ((include & DUMP_INCLUDE_PROPERTIES) && line_len >= 2 && line[0] == '#' && line[1] == '=') {
+ ret = process_properties_line (notmuch, line + 2);
+ if (ret)
+ goto DONE;
+ }
+
if (input_format == DUMP_FORMAT_SUP) {
ret = parse_sup_line (line_ctx, line, &query_string, tag_ops);
} else {
@@ -344,7 +423,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
break;
} while (! (ret = gz_getline (line_ctx, &line, &line_len, input)));
-
+
/* EOF is normal loop termination condition, UTIL_SUCCESS is
* impossible here */
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index a9b76de..c92b99b 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -209,4 +209,32 @@ EOF
notmuch dump | grep '^#=' > OUTPUT
test_expect_equal_file PROPERTIES OUTPUT
+
+test_begin_subtest "restore missing message property (single line)"
+notmuch dump | grep '^#=' > BEFORE1
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob"));
+EOF
+notmuch restore < BEFORE1
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
+
+test_begin_subtest "restore missing message property (full dump)"
+notmuch dump > BEFORE2
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_remove_property (message, "testkey1", "bob"));
+EOF
+notmuch restore < BEFORE2
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
+test_begin_subtest "restore clear extra message property"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+EXPECT0(notmuch_message_add_property (message, "testkey1", "charles"));
+EOF
+notmuch restore < BEFORE2
+notmuch dump | grep '^#=' > OUTPUT
+test_expect_equal_file PROPERTIES OUTPUT
+
test_done
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH 9/9] add has: query prefix to search for specific properties
2016-08-06 13:52 v3 of message properties patches David Bremner
` (7 preceding siblings ...)
2016-08-06 13:52 ` [PATCH 8/9] cli: optionally restore message properties from dump file David Bremner
@ 2016-08-06 13:52 ` David Bremner
2016-08-06 14:18 ` David Bremner
2016-08-13 19:00 ` [PATCH 9/9] add has: query prefix to search for specific properties Jameson Graef Rollins
2016-09-20 21:56 ` v3 of message properties patches Daniel Kahn Gillmor
9 siblings, 2 replies; 22+ messages in thread
From: David Bremner @ 2016-08-06 13:52 UTC (permalink / raw)
To: notmuch; +Cc: Daniel Kahn Gillmor
From: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
We want to be able to query the properties directly, like:
notmuch count has:foo=bar
which should return a count of messages where the property with key
"foo" has value equal to "bar".
---
doc/man7/notmuch-search-terms.rst | 9 ++++++++-
lib/database.cc | 4 ++++
test/T610-message-property.sh | 11 +++++++++++
3 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index 075f88c..86d02ee 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -58,6 +58,8 @@ indicate user-supplied values):
- query:<name>
+- has:<key>=<value>
+
The **from:** prefix is used to match the name or address of the sender
of an email message.
@@ -139,6 +141,11 @@ queries added with **notmuch-config(1)**. Named queries are only
available if notmuch is built with **Xapian Field Processors** (see
below).
+The **has:** prefix searches for messages with a particular
+<key>=<value> property pair. Properties are used internally by notmuch
+(and extensions) to add metadata to messages. A given key can be
+present on a given message with several different values.
+
Operators
---------
@@ -217,7 +224,7 @@ exact matches like "tag:inbox" or **probabilistic**, supporting a more flexible
Boolean
- **tag:**, **id:**, **thread:**, **folder:**, **path:**
+ **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has**
Probabilistic
**from:**, **to:**, **subject:**, **attachment:**, **mimetype:**
diff --git a/lib/database.cc b/lib/database.cc
index 3a741f0..3bdbd07 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -97,6 +97,9 @@ typedef struct {
* STRING is the name of a file within that
* directory for this mail message.
*
+ * has: Has a property with key=value
+ * FIXME: if no = is present, should match on any value
+ *
* A mail document also has four values:
*
* TIMESTAMP: The time_t value corresponding to the message's
@@ -260,6 +263,7 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
{ "is", "K" },
{ "id", "Q" },
{ "path", "P" },
+ { "has", "XPROPERTY" },
/*
* Without the ":", since this is a multi-letter prefix, Xapian
* will add a colon itself if the first letter of the path is
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index c92b99b..ee7910b 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -237,4 +237,15 @@ notmuch restore < BEFORE2
notmuch dump | grep '^#=' > OUTPUT
test_expect_equal_file PROPERTIES OUTPUT
+test_begin_subtest "test 'has:' queries: empty"
+notmuch search has:testkey1=charles > OUTPUT
+test_expect_equal_file /dev/null OUTPUT
+
+test_begin_subtest "test 'has:' queries: single message"
+notmuch search --output=messages has:testkey1=alice > OUTPUT
+cat <<EOF >EXPECTED
+id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
test_done
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH 9/9] add has: query prefix to search for specific properties
2016-08-06 13:52 ` [PATCH 9/9] add has: query prefix to search for specific properties David Bremner
@ 2016-08-06 14:18 ` David Bremner
2016-08-13 9:38 ` Jani Nikula
2016-08-13 19:00 ` [PATCH 9/9] add has: query prefix to search for specific properties Jameson Graef Rollins
1 sibling, 1 reply; 22+ messages in thread
From: David Bremner @ 2016-08-06 14:18 UTC (permalink / raw)
To: notmuch; +Cc: Daniel Kahn Gillmor, Jani Nikula
David Bremner <david@tethera.net> writes:
>
> +The **has:** prefix searches for messages with a particular
> +<key>=<value> property pair. Properties are used internally by notmuch
> +(and extensions) to add metadata to messages. A given key can be
> +present on a given message with several different values.
> +
I probably should have added to the commit message that this
documentation is intentionally minimal, because it's not intended for
people creating properties; that should go in API documentation, I think.
> + { "has", "XPROPERTY" },
> /*
> * Without the ":", since this is a multi-letter prefix, Xapian
> * will add a colon itself if the first letter of the path
I don't really understand the comment on the next line (since it seems
that no : is added to XPROPERTY by Xapian), but I guess we should
probably be consistent. Maybe Jani can explain the comment?
d
^ permalink raw reply [flat|nested] 22+ messages in thread* Re: [PATCH 9/9] add has: query prefix to search for specific properties
2016-08-06 14:18 ` David Bremner
@ 2016-08-13 9:38 ` Jani Nikula
2016-08-13 11:31 ` David Bremner
2016-08-18 8:14 ` [PATCH] lib: reword comment about XFOLDER: prefix David Bremner
0 siblings, 2 replies; 22+ messages in thread
From: Jani Nikula @ 2016-08-13 9:38 UTC (permalink / raw)
To: David Bremner, notmuch; +Cc: Daniel Kahn Gillmor
On Sat, 06 Aug 2016, David Bremner <david@tethera.net> wrote:
> David Bremner <david@tethera.net> writes:
>
>>
>> +The **has:** prefix searches for messages with a particular
>> +<key>=<value> property pair. Properties are used internally by notmuch
>> +(and extensions) to add metadata to messages. A given key can be
>> +present on a given message with several different values.
>> +
>
> I probably should have added to the commit message that this
> documentation is intentionally minimal, because it's not intended for
> people creating properties; that should go in API documentation, I think.
>
>
>> + { "has", "XPROPERTY" },
>> /*
>> * Without the ":", since this is a multi-letter prefix, Xapian
>> * will add a colon itself if the first letter of the path
>
> I don't really understand the comment on the next line (since it seems
> that no : is added to XPROPERTY by Xapian), but I guess we should
> probably be consistent. Maybe Jani can explain the comment?
My recollection is vague, but it has to do with user-defined prefixes
(XSOMETHING) and terms beginning with uppercase letters. I think it
would possibly be useful (mandatory?) to append ":" at the end of
XPROPERTY, if it's possible we'll have properties beginning with
uppercase letters.
Quoting https://xapian.org/docs/omega/termprefixes.html
"""
X starts a multi-capital letter user-defined prefix. If you want a
prefix for something without a standard prefix, you create your own
starting with an X (e.g. XSHOESIZE). The prefix ends with the first
non-capital. If the term you're prefixing starts with a capital, add a
":" between prefix and term to resolve ambiguity about where the prefix
ends and the term begins.
"""
BR,
Jani.
^ permalink raw reply [flat|nested] 22+ messages in thread* Re: [PATCH 9/9] add has: query prefix to search for specific properties
2016-08-13 9:38 ` Jani Nikula
@ 2016-08-13 11:31 ` David Bremner
2016-08-18 8:14 ` [PATCH] lib: reword comment about XFOLDER: prefix David Bremner
1 sibling, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-13 11:31 UTC (permalink / raw)
To: Jani Nikula, notmuch; +Cc: Daniel Kahn Gillmor
Jani Nikula <jani@nikula.org> writes:
>> I don't really understand the comment on the next line (since it seems
>> that no : is added to XPROPERTY by Xapian), but I guess we should
>> probably be consistent. Maybe Jani can explain the comment?
>
> My recollection is vague, but it has to do with user-defined prefixes
> (XSOMETHING) and terms beginning with uppercase letters. I think it
> would possibly be useful (mandatory?) to append ":" at the end of
> XPROPERTY, if it's possible we'll have properties beginning with
> uppercase letters.
>
> Quoting https://xapian.org/docs/omega/termprefixes.html
>
> """
> X starts a multi-capital letter user-defined prefix. If you want a
> prefix for something without a standard prefix, you create your own
> starting with an X (e.g. XSHOESIZE). The prefix ends with the first
> non-capital. If the term you're prefixing starts with a capital, add a
> ":" between prefix and term to resolve ambiguity about where the prefix
> ends and the term begins.
> """
Well, it seems harmless (modulo the 1 extra byte) to add it. I was
wondering if the comment (and requirement) dated back to when we added
XFOLDER: terms with some Xapian term generator, but I think I wasn't
able to verify that poking around in the history.
d
^ permalink raw reply [flat|nested] 22+ messages in thread
* [PATCH] lib: reword comment about XFOLDER: prefix
2016-08-13 9:38 ` Jani Nikula
2016-08-13 11:31 ` David Bremner
@ 2016-08-18 8:14 ` David Bremner
2016-08-18 8:18 ` David Bremner
1 sibling, 1 reply; 22+ messages in thread
From: David Bremner @ 2016-08-18 8:14 UTC (permalink / raw)
To: Jani Nikula, David Bremner, notmuch
I believe the current one is misleading, because in my experiments
Xapian did not add : when prefix and term were both upper case. Indeed,
it's hard to see how it could, because prefixes are added at a layer
above Xapian in our code. See _notmuch_message_add_term for an example.
Also try to explain why this is a good idea.
---
lib/database.cc | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/database.cc b/lib/database.cc
index 57a98c9..30ee303 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -260,10 +260,10 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
{ "id", "Q" },
{ "path", "P" },
/*
- * Without the ":", since this is a multi-letter prefix, Xapian
- * will add a colon itself if the first letter of the path is
- * upper-case ASCII. Including the ":" forces there to always be a
- * colon, which keeps our own logic simpler.
+ * Unconditionally add ':' to reduce potential ambiguity with
+ * overlapping prefixes and/or terms that start with capital
+ * letters. See xapian document termprefixes.html for related
+ * discussion.
*/
{ "folder", "XFOLDER:" },
};
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH] lib: reword comment about XFOLDER: prefix
2016-08-18 8:14 ` [PATCH] lib: reword comment about XFOLDER: prefix David Bremner
@ 2016-08-18 8:18 ` David Bremner
2016-08-19 12:18 ` David Bremner
0 siblings, 1 reply; 22+ messages in thread
From: David Bremner @ 2016-08-18 8:18 UTC (permalink / raw)
To: David Bremner, Jani Nikula, notmuch
I believe the current one is misleading, because in my experiments
Xapian did not add : when prefix and term were both upper case. Indeed,
it's hard to see how it could, because prefixes are added at a layer
above Xapian in our code. See _notmuch_message_add_term for an example.
Also try to explain why this is a good idea. As far as I can ascertain,
this is more of an issue for a system trying to work with an unknown set
of prefixes. Since notmuch has a fixed set of prefixes, and we can
hopefully be trusted not to add XGOLD and XGOLDEN as prefixes, it is
harder for problems to arise.
---
lib/database.cc | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/database.cc b/lib/database.cc
index 57a98c9..5577aaf 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -260,10 +260,10 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
{ "id", "Q" },
{ "path", "P" },
/*
- * Without the ":", since this is a multi-letter prefix, Xapian
- * will add a colon itself if the first letter of the path is
- * upper-case ASCII. Including the ":" forces there to always be a
- * colon, which keeps our own logic simpler.
+ * Unconditionally add ':' to reduce potential ambiguity with
+ * overlapping prefixes and/or terms that start with capital
+ * letters. See Xapian document termprefixes.html for related
+ * discussion.
*/
{ "folder", "XFOLDER:" },
};
--
2.8.1
^ permalink raw reply related [flat|nested] 22+ messages in thread* Re: [PATCH] lib: reword comment about XFOLDER: prefix
2016-08-18 8:18 ` David Bremner
@ 2016-08-19 12:18 ` David Bremner
0 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-08-19 12:18 UTC (permalink / raw)
To: Jani Nikula, notmuch
David Bremner <david@tethera.net> writes:
> I believe the current one is misleading, because in my experiments
> Xapian did not add : when prefix and term were both upper case. Indeed,
> it's hard to see how it could, because prefixes are added at a layer
> above Xapian in our code. See _notmuch_message_add_term for an example.
>
> Also try to explain why this is a good idea. As far as I can ascertain,
> this is more of an issue for a system trying to work with an unknown set
> of prefixes. Since notmuch has a fixed set of prefixes, and we can
> hopefully be trusted not to add XGOLD and XGOLDEN as prefixes, it is
> harder for problems to arise.
pushed,
d
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: [PATCH 9/9] add has: query prefix to search for specific properties
2016-08-06 13:52 ` [PATCH 9/9] add has: query prefix to search for specific properties David Bremner
2016-08-06 14:18 ` David Bremner
@ 2016-08-13 19:00 ` Jameson Graef Rollins
2016-08-13 23:14 ` David Bremner
1 sibling, 1 reply; 22+ messages in thread
From: Jameson Graef Rollins @ 2016-08-13 19:00 UTC (permalink / raw)
To: David Bremner, notmuch; +Cc: Daniel Kahn Gillmor
[-- Attachment #1: Type: text/plain, Size: 583 bytes --]
On Sat, Aug 06 2016, David Bremner <david@tethera.net> wrote:
> @@ -217,7 +224,7 @@ exact matches like "tag:inbox" or **probabilistic**, supporting a more flexible
>
>
> Boolean
> - **tag:**, **id:**, **thread:**, **folder:**, **path:**
> + **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has**
> Probabilistic
> **from:**, **to:**, **subject:**, **attachment:**, **mimetype:**
Super minor annoying comment, but I wonder if there's a missing colon in
the above addition:
> + **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has:**
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: v3 of message properties patches
2016-08-06 13:52 v3 of message properties patches David Bremner
` (8 preceding siblings ...)
2016-08-06 13:52 ` [PATCH 9/9] add has: query prefix to search for specific properties David Bremner
@ 2016-09-20 21:56 ` Daniel Kahn Gillmor
2016-09-21 12:01 ` David Bremner
9 siblings, 1 reply; 22+ messages in thread
From: Daniel Kahn Gillmor @ 2016-09-20 21:56 UTC (permalink / raw)
To: David Bremner, notmuch
[-- Attachment #1: Type: text/plain, Size: 981 bytes --]
On Sat 2016-08-06 09:52:30 -0400, David Bremner wrote:
> This obsoletes
>
> id:1470184228-12517-1-git-send-email-david@tethera.net
>
> In particular it fixes the compile errors (boo), and adds dkg's
> has: queries, with minimal doc and tests.
I've been running this series of patches since shortly after they were
released. They work for me, and they're a critical feature to enable
the cleartext index series i've been trying to land. I'd like to
nominate them for inclusion in 0.23 (if we can get them pushed to
master, i might even nominate my cleartext index series for 0.23 as
well).
As i've been maintaining them against the master branch, they've
diverged slightly from the series here. I'd be happy to send the
updated revisions here if folks want them, but there are no substantive
changes.
You can see the current branch at proprties-v3 (commit ID
9d98707a90c0fcd0eb7b0671edb7d9e39aff551f) on
git://lair.fifthhorseman.net/~dkg/notmuch
Regards,
--dkg
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 930 bytes --]
^ permalink raw reply [flat|nested] 22+ messages in thread* Re: v3 of message properties patches
2016-09-20 21:56 ` v3 of message properties patches Daniel Kahn Gillmor
@ 2016-09-21 12:01 ` David Bremner
2016-09-21 13:04 ` Tomi Ollila
0 siblings, 1 reply; 22+ messages in thread
From: David Bremner @ 2016-09-21 12:01 UTC (permalink / raw)
To: Daniel Kahn Gillmor, notmuch; +Cc: Tomi Ollila
Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:
> I've been running this series of patches since shortly after they were
> released. They work for me, and they're a critical feature to enable
> the cleartext index series i've been trying to land.
OK, that's certainly more testing and API evaluation than most series
get. I'd still like someone other than me to sanity check the code
itself, and the design decisions that hard to change later, namely
changes to the database format and the dump-restore format. And of
course the main reason we want code review is for to find the issues
that I am not aware of. Tomi, I know you've read the code at some level,
are you happy with merging the series?
>
> As i've been maintaining them against the master branch, they've
> diverged slightly from the series here. I'd be happy to send the
> updated revisions here if folks want them, but there are no substantive
> changes.
I can confirm that's just a rebase against master of what I have.
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: v3 of message properties patches
2016-09-21 12:01 ` David Bremner
@ 2016-09-21 13:04 ` Tomi Ollila
2016-09-21 21:21 ` David Bremner
0 siblings, 1 reply; 22+ messages in thread
From: Tomi Ollila @ 2016-09-21 13:04 UTC (permalink / raw)
To: David Bremner, Daniel Kahn Gillmor, notmuch
On Wed, Sep 21 2016, David Bremner <david@tethera.net> wrote:
> Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:
>
>> I've been running this series of patches since shortly after they were
>> released. They work for me, and they're a critical feature to enable
>> the cleartext index series i've been trying to land.
>
> OK, that's certainly more testing and API evaluation than most series
> get. I'd still like someone other than me to sanity check the code
> itself, and the design decisions that hard to change later, namely
> changes to the database format and the dump-restore format. And of
> course the main reason we want code review is for to find the issues
> that I am not aware of. Tomi, I know you've read the code at some level,
> are you happy with merging the series?
I am happy with merging the series, just amend the sepErates
(and possibly spcs->tab after $(dir)/string-map.c in -3-)
I looked through the code and did not see anything that should broke
anything. As Daniel has been using it I can slack off from testing it myself :D
Tomi
>
>>
>> As i've been maintaining them against the master branch, they've
>> diverged slightly from the series here. I'd be happy to send the
>> updated revisions here if folks want them, but there are no substantive
>> changes.
>
> I can confirm that's just a rebase against master of what I have.
^ permalink raw reply [flat|nested] 22+ messages in thread
* Re: v3 of message properties patches
2016-09-21 13:04 ` Tomi Ollila
@ 2016-09-21 21:21 ` David Bremner
0 siblings, 0 replies; 22+ messages in thread
From: David Bremner @ 2016-09-21 21:21 UTC (permalink / raw)
To: Tomi Ollila, Daniel Kahn Gillmor, notmuch
Tomi Ollila <tomi.ollila@iki.fi> writes:
>
> I am happy with merging the series, just amend the sepErates
> (and possibly spcs->tab after $(dir)/string-map.c in -3-)
>
> I looked through the code and did not see anything that should broke
> anything. As Daniel has been using it I can slack off from testing it myself :D
>
> Tomi
I have merged this series, amended as follows:
diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index 29964da..5857027 100644
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -84,7 +84,7 @@ Supported options for **dump** include
**properties**
Output per-message (key,value) metadata. Each line starts
- with "#= ", followed by a message id, and a space seperated
+ with "#= ", followed by a message id, and a space separated
list of key=value pairs. pair. Ids, keys and values are hex
encoded if needed.
diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst
index ebdb3ff..c681fa2 100644
--- a/doc/man1/notmuch-restore.rst
+++ b/doc/man1/notmuch-restore.rst
@@ -63,7 +63,7 @@ Supported options for **restore** include
**properties**
Output per-message (key,value) metadata. Each line starts
- with "#= ", followed by a message id, and a space seperated
+ with "#= ", followed by a message id, and a space separated
list of key=value pairs. pair. Ids, keys and values are
hex encoded if needed.
diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index a85d94b..de93d73 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -58,7 +58,7 @@ indicate user-supplied values):
- query:<name>
-- has:<key>=<value>
+- property:<key>=<value>
The **from:** prefix is used to match the name or address of the sender
of an email message.
@@ -141,7 +141,7 @@ queries added with **notmuch-config(1)**. Named queries are only
available if notmuch is built with **Xapian Field Processors** (see
below).
-The **has:** prefix searches for messages with a particular
+The **property:** prefix searches for messages with a particular
<key>=<value> property pair. Properties are used internally by notmuch
(and extensions) to add metadata to messages. A given key can be
present on a given message with several different values.
@@ -224,7 +224,7 @@ exact matches like "tag:inbox" or **probabilistic**, supporting a more flexible
Boolean
- **tag:**, **id:**, **thread:**, **folder:**, **path:**, **has:**
+ **tag:**, **id:**, **thread:**, **folder:**, **path:**, **property:**
Probabilistic
**from:**, **to:**, **subject:**, **attachment:**, **mimetype:**
diff --git a/lib/Makefile.local b/lib/Makefile.local
index c012ed1..3d1030a 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -40,7 +40,7 @@ libnotmuch_c_srcs = \
$(dir)/messages.c \
$(dir)/sha1.c \
$(dir)/built-with.c \
- $(dir)/string-map.c \
+ $(dir)/string-map.c \
$(dir)/tags.c
libnotmuch_cxx_srcs = \
diff --git a/lib/database.cc b/lib/database.cc
index d3e2e0e..392e8b2 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -97,7 +97,7 @@ typedef struct {
* STRING is the name of a file within that
* directory for this mail message.
*
- * has: Has a property with key=value
+ * property: Has a property with key=value
* FIXME: if no = is present, should match on any value
*
* A mail document also has four values:
@@ -254,7 +254,6 @@ static prefix_t BOOLEAN_PREFIX_INTERNAL[] = {
{ "directory", "XDIRECTORY" },
{ "file-direntry", "XFDIRENTRY" },
{ "directory-direntry", "XDDIRENTRY" },
- { "property", "XPROPERTY" },
};
static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
@@ -263,7 +262,7 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
{ "is", "K" },
{ "id", "Q" },
{ "path", "P" },
- { "has", "XPROPERTY" },
+ { "property", "XPROPERTY" },
/*
* Unconditionally add ':' to reduce potential ambiguity with
* overlapping prefixes and/or terms that start with capital
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index ee7910b..65ff19d 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -237,12 +237,12 @@ notmuch restore < BEFORE2
notmuch dump | grep '^#=' > OUTPUT
test_expect_equal_file PROPERTIES OUTPUT
-test_begin_subtest "test 'has:' queries: empty"
-notmuch search has:testkey1=charles > OUTPUT
+test_begin_subtest "test 'property:' queries: empty"
+notmuch search property:testkey1=charles > OUTPUT
test_expect_equal_file /dev/null OUTPUT
-test_begin_subtest "test 'has:' queries: single message"
-notmuch search --output=messages has:testkey1=alice > OUTPUT
+test_begin_subtest "test 'property:' queries: single message"
+notmuch search --output=messages property:testkey1=alice > OUTPUT
cat <<EOF >EXPECTED
id:4EFC743A.3060609@april.org
EOF
^ permalink raw reply related [flat|nested] 22+ messages in thread