WIP: store message headers in document data area

WIP: store message headers in document data area

[David Bremner]

Every Xapian document (thing in the database) has a data area that
stores an arbitrary string. That string is not usable for searching
(unlike terms or values), but can be used e.g. for display. Currently
we don't use this part of the Xapian document at all.  While I was
working adding all of the subjects of multi-file messages for regex
search, it seemed like it might be useful to fix the default subject
displayed to the user independent of how search terms/values are
added. I also have the feeling this might be useful for encrypted
headers. Before I suggested that I wanted to get a clearer idea of how
hard that would be to do.

There's really not that much new code here, it's mainly tests, and
some code movement. Unlike message properties, these are not currently
backed up. I guess that could be done, although I'd like to have a
clearer idea of the use cases before I do that.

[PATCH 01/12] test: link test_C snippets to libnotmuch_util.a

[David Bremner]

This will allow testing (some) private APIs.  I couldn't understand
the trailing / on -L${NOTMUCH_BUILDDIR}/lib/, but it seems harmless so
I left it.
---
 test/test-lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test-lib.sh b/test/test-lib.sh
index fca5277d..e15cde45 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -1015,7 +1015,7 @@ test_C () {
     exec_file="test${test_count}"
     test_file="${exec_file}.c"
     cat > ${test_file}
-    ${TEST_CC} ${TEST_CFLAGS} -I${NOTMUCH_SRCDIR}/test -I${NOTMUCH_SRCDIR}/lib -o ${exec_file} ${test_file} -L${NOTMUCH_BUILDDIR}/lib/ -lnotmuch -ltalloc
+    ${TEST_CC} ${TEST_CFLAGS} -I${NOTMUCH_SRCDIR}/test -I${NOTMUCH_SRCDIR}/lib -I${NOTMUCH_SRCDIR}/util -o ${exec_file} ${test_file} -L${NOTMUCH_BUILDDIR}/lib/ -L${NOTMUCH_BUILDDIR}/util/ -lnotmuch -lnotmuch_util -ltalloc
     echo "== stdout ==" > OUTPUT.stdout
     echo "== stderr ==" > OUTPUT.stderr
     ./${exec_file} "$@" 1>>OUTPUT.stdout 2>>OUTPUT.stderr
-- 
2.17.1

[PATCH 02/12] lib: move string-map functions to libnotmuch_util

[David Bremner]

Although they are not yet needed for the CLI, this will facilitate
writing unit tests.
---
 lib/Makefile.local         |  1 -
 lib/notmuch-private.h      | 33 +--------------------------------
 util/Makefile.local        |  3 ++-
 {lib => util}/string-map.c |  0
 util/string-map.h          | 36 ++++++++++++++++++++++++++++++++++++
 5 files changed, 39 insertions(+), 34 deletions(-)
 rename {lib => util}/string-map.c (100%)
 create mode 100644 util/string-map.h

diff --git a/lib/Makefile.local b/lib/Makefile.local
index 5dc057c0..a9a310b4 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -42,7 +42,6 @@ libnotmuch_c_srcs =		\
 	$(dir)/messages.c	\
 	$(dir)/sha1.c		\
 	$(dir)/built-with.c	\
-	$(dir)/string-map.c	\
 	$(dir)/indexopts.c	\
 	$(dir)/tags.c
 
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 3764a6a9..063df5cd 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -580,38 +580,7 @@ _notmuch_string_list_append (notmuch_string_list_t *list,
 void
 _notmuch_string_list_sort (notmuch_string_list_t *list);
 
-/* string-map.c */
-typedef struct _notmuch_string_map  notmuch_string_map_t;
-typedef struct _notmuch_string_map_iterator notmuch_string_map_iterator_t;
-notmuch_string_map_t *
-_notmuch_string_map_create (const void *ctx);
-
-void
-_notmuch_string_map_append (notmuch_string_map_t *map,
-			    const char *key,
-			    const char *value);
-
-const char *
-_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
-
-notmuch_string_map_iterator_t *
-_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
-				     bool exact);
-
-bool
-_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
-
-void
-_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iter);
-
-const char *
-_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator);
-
-const char *
-_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
-
-void
-_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+#include "string-map.h"
 
 /* tags.c */
 
diff --git a/util/Makefile.local b/util/Makefile.local
index ba03230e..71a66158 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -5,7 +5,8 @@ extra_cflags += -I$(srcdir)/$(dir)
 
 libnotmuch_util_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \
 		  $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \
-		$(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c
+		$(dir)/util.c $(dir)/gmime-extra.c $(dir)/crypto.c \
+		$(dir)/string-map.c
 
 libnotmuch_util_modules := $(libnotmuch_util_c_srcs:.c=.o)
 
diff --git a/lib/string-map.c b/util/string-map.c
similarity index 100%
rename from lib/string-map.c
rename to util/string-map.c
diff --git a/util/string-map.h b/util/string-map.h
new file mode 100644
index 00000000..42d16da4
--- /dev/null
+++ b/util/string-map.h
@@ -0,0 +1,36 @@
+#ifndef STRING_MAP_H
+#define STRING_MAP_H
+
+#include <stdbool.h>
+typedef struct _notmuch_string_map  notmuch_string_map_t;
+typedef struct _notmuch_string_map_iterator notmuch_string_map_iterator_t;
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx);
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value);
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     bool exact);
+
+bool
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iter);
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator);
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+#endif
-- 
2.17.1

[PATCH 03/12] test: add initial tests for string-map

[David Bremner]

These test every non-destroy function, albeit lightly.
---
 test/T710-string-map.sh | 117 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100755 test/T710-string-map.sh

diff --git a/test/T710-string-map.sh b/test/T710-string-map.sh
new file mode 100755
index 00000000..b2f65381
--- /dev/null
+++ b/test/T710-string-map.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+test_description='string-map unit tests'
+. $(dirname "$0")/test-lib.sh || exit 1
+
+cat <<'EOF' > c_head
+#include <stdlib.h>
+#include <talloc.h>
+#include <string.h>
+#include "string-map.h"
+static void
+dump_map(notmuch_string_map_t *map)
+{
+    int count=0;
+    for (notmuch_string_map_iterator_t *i=_notmuch_string_map_iterator_create (map, "", false);
+         _notmuch_string_map_iterator_valid (i);
+         _notmuch_string_map_iterator_move_to_next (i), count++) {
+        printf("key[%d]=%s\nval[%d]=%s\n", count, _notmuch_string_map_iterator_key(i),
+                                     count, _notmuch_string_map_iterator_value(i));
+    }
+}
+int main (int argc, char** argv)
+{
+    void *ctx = talloc_new (NULL);
+EOF
+
+cat <<EOF > c_tail
+}
+EOF
+
+test_begin_subtest "empty map"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "single pair"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "two pairs"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey2
+val[1]=testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "duplicate key, sorting"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1", "testval3");
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "get first"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1", "testval3");
+    printf ("%s\n",_notmuch_string_map_get (map, "testkey1"));
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+testval1
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_done
-- 
2.17.1

[PATCH 04/12] util/string-map: add _notmuch_string_map_serialize

[David Bremner]

The anticipated usage of this is to serialize a string map into the
data area of a xapian document
---
 test/T710-string-map.sh | 312 ++++++++++++++++++++++++++++++++++++++++
 util/string-map.c       |  42 ++++++
 util/string-map.h       |   7 +
 3 files changed, 361 insertions(+)

diff --git a/test/T710-string-map.sh b/test/T710-string-map.sh
index b2f65381..9c5c1d8e 100755
--- a/test/T710-string-map.sh
+++ b/test/T710-string-map.sh
@@ -114,4 +114,316 @@ testval1
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "serialize empty"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    dump_map (map);
+    printf ("%s",_notmuch_string_map_serialize (ctx, map));
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "serialize"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1", "testval3");
+    dump_map (map);
+    printf ("%s",_notmuch_string_map_serialize (ctx, map));
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+testkey1
+testval1
+testkey1
+testval3
+testkey2
+testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "serialize, key with embedded newline"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1", "testval3");
+    _notmuch_string_map_append (map, "testkey2\nreallynot", "testval4");
+    dump_map (map);
+    printf ("%s",_notmuch_string_map_serialize (ctx, map));
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+key[3]=testkey2
+reallynot
+val[3]=testval4
+testkey1
+testval1
+testkey1
+testval3
+testkey2
+testval2
+testkey2\nreallynot
+testval4
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "serialize, key with embedded backslash"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1", "testval3");
+    _notmuch_string_map_append (map, "testkey2\\not", "testval4");
+    dump_map (map);
+    printf ("%s",_notmuch_string_map_serialize (ctx, map));
+}
+EOF
+cat<<'EOF' > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+key[3]=testkey2\not
+val[3]=testval4
+testkey1
+testval1
+testkey1
+testval3
+testkey2
+testval2
+testkey2\\not
+testval4
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "serialize, value with embedded newline"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1", "testval3");
+    _notmuch_string_map_append (map, "testkey2", "testval4\nvalue continues");
+    dump_map (map);
+    printf ("%s",_notmuch_string_map_serialize (ctx, map));
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+key[3]=testkey2
+val[3]=testval4
+value continues
+testkey1
+testval1
+testkey1
+testval3
+testkey2
+testval2
+testkey2
+testval4\nvalue continues
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "serialize, key and value with embedded newline"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1\nkey continues", "testval3\nvalue continues");
+    dump_map (map);
+    printf ("%s",_notmuch_string_map_serialize (ctx, map));
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+key continues
+val[1]=testval3
+value continues
+key[2]=testkey2
+val[2]=testval2
+testkey1
+testval1
+testkey1\nkey continues
+testval3\nvalue continues
+testkey2
+testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "serialize, key and value with embedded literal \n"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1\\nkey continues", "testval3\\nvalue continues");
+    dump_map (map);
+    printf ("%s",_notmuch_string_map_serialize (ctx, map));
+}
+EOF
+cat<<'EOF' > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1\nkey continues
+val[1]=testval3\nvalue continues
+key[2]=testkey2
+val[2]=testval2
+testkey1
+testval1
+testkey1\\nkey continues
+testval3\\nvalue continues
+testkey2
+testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "deserialize empty string"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_deserialize (ctx, "");
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "deserialize"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_deserialize (ctx,
+        "testkey1\n"
+        "testval1\n"
+        "testkey1\n"
+        "testval3\n"
+        "testkey2\n"
+        "testval2\n");
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "deserialize, key and value with embedded newline"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_deserialize (ctx,
+        "testkey1\n"
+        "testval1\n"
+        "testkey1\\nkey continues\n"
+        "testval3\\nvalue continues\n"
+        "testkey2\n"
+        "testval2\n");
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+key continues
+val[1]=testval3
+value continues
+key[2]=testkey2
+val[2]=testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "deserialize, keys and values with embedded backslashes"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    const char * str =
+         "testkey1\n"
+         "testval1\\b\n"
+         "testkey1\n"
+         "testval3\\\n"
+         "testkey2\\\\not\n"
+         "testval2\n"
+         "testkey2\\toto\n"
+         "testval4\n";
+
+    notmuch_string_map_t *map = _notmuch_string_map_deserialize (ctx, str);
+    fputs(str, stdout);
+    dump_map (map);
+}
+EOF
+cat<<'EOF' > EXPECTED
+== stdout ==
+testkey1
+testval1\b
+testkey1
+testval3\
+testkey2\\not
+testval2
+testkey2\toto
+testval4
+key[0]=testkey1
+val[0]=testval1\b
+key[1]=testkey1
+val[1]=testval3\
+key[2]=testkey2\not
+val[2]=testval2
+key[3]=testkey2\toto
+val[3]=testval4
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done
diff --git a/util/string-map.c b/util/string-map.c
index ad818207..ab0c42ab 100644
--- a/util/string-map.c
+++ b/util/string-map.c
@@ -226,3 +226,45 @@ _notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator)
 {
     talloc_free (iterator);
 }
+
+static const char *
+_append_escaped (void *ctx, const char *dest, const char *str) {
+    /* At worst we escape everything */
+    char *buf = talloc_zero_size (ctx, 2 * strlen(str) + 1);
+    char *ret;
+    int j = 0;
+    for (const char *cur = str; *cur; cur++){
+	if (*cur == '\n') {
+	    buf[j++] = '\\';
+	    buf[j++] = 'n';
+	} else if (*cur == '\\' && *(cur+1)== 'n') {
+	    buf[j++] = '\\';
+	    buf[j++] = '\\';
+	} else {
+	    buf[j++] = *cur;
+	}
+    }
+    /* this NUL is already there, but better safe than sorry */
+    buf[j] = '\0';
+    ret = talloc_asprintf (ctx, "%s%s", dest, buf);
+    talloc_free (buf);
+    return ret;
+}
+
+const char *
+_notmuch_string_map_serialize (void* ctx, notmuch_string_map_t *map)
+{
+    const char *ret;
+
+    _notmuch_string_map_sort (map);
+
+    ret=talloc_strdup(ctx, "");
+    for (size_t i=0; i < map->length; i++) {
+	ret = _append_escaped (ctx, ret, map->pairs[i].key);
+	ret = talloc_asprintf(ctx, "%s\n", ret);
+	ret = _append_escaped (ctx, ret, map->pairs[i].value);
+	ret = talloc_asprintf(ctx, "%s\n", ret);
+    }
+
+    return ret;
+}
diff --git a/util/string-map.h b/util/string-map.h
index 42d16da4..9baf3530 100644
--- a/util/string-map.h
+++ b/util/string-map.h
@@ -33,4 +33,11 @@ _notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
 
 void
 _notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+
+/*
+ * encode map as newline delimited string "key1\nval1\key2\nval2\n..."
+ * newlines will be escaped as "\n", and "\n" will be escaped as "\\n".
+ */
+const char *
+_notmuch_string_map_serialize (void *ctx, notmuch_string_map_t *map);
 #endif
-- 
2.17.1

[PATCH 05/12] util/string-map: add _notmuch_string_map_deserialize

[David Bremner]

The anticipated use case is loading the document data area from a
message Xapian document into a usable data structure.
---
 util/string-map.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++
 util/string-map.h |  8 ++++++++
 2 files changed, 57 insertions(+)

diff --git a/util/string-map.c b/util/string-map.c
index ab0c42ab..b29a9ba0 100644
--- a/util/string-map.c
+++ b/util/string-map.c
@@ -268,3 +268,52 @@ _notmuch_string_map_serialize (void* ctx, notmuch_string_map_t *map)
 
     return ret;
 }
+
+static char *
+unescape_newlines (void *ctx, const char *in, size_t len) {
+    size_t i,j;
+    /* removing escapes only makes things shorter */
+    char *out = talloc_zero_size (ctx, len+1);
+    for (i=0, j=0; i<len; i++) {
+	if (in[i] == '\\' && i < len - 1) {
+	    switch (in[i+1]) {
+	    case '\\':
+		i++;
+		out[j++] = '\\';
+		break;
+	    case 'n':
+		i++;
+		out[j++] = '\n';
+		break;
+	    default:
+		out[j++] = '\\';
+	    }
+	} else {
+	    out[j++] = in[i];
+	}
+    }
+    out[j]='\0';
+    return out;
+}
+
+notmuch_string_map_t *
+_notmuch_string_map_deserialize (void *ctx, const char *str)
+{
+    const char *tok = str;
+    const char *delim = "\n";
+    size_t tok_len = 0;
+    const char *pair [2];
+    size_t step = 0;
+
+    notmuch_string_map_t *map = _notmuch_string_map_create (ctx);
+
+    while ((tok = strtok_len_c (tok + tok_len, delim, &tok_len)) != NULL) {
+	pair[step] = unescape_newlines (ctx, tok, tok_len);
+	step++;
+	if (step == 2) {
+	    step = 0;
+	    _notmuch_string_map_append (map, pair[0], pair[1]);
+	}
+    }
+    return map;
+}
diff --git a/util/string-map.h b/util/string-map.h
index 9baf3530..22aa487c 100644
--- a/util/string-map.h
+++ b/util/string-map.h
@@ -40,4 +40,12 @@ _notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
  */
 const char *
 _notmuch_string_map_serialize (void *ctx, notmuch_string_map_t *map);
+
+/*
+ * decode newline delimited string "key1\nval1\key2\nval2\n..." into a string_map.
+ *
+ * \n decodes to newline and \\ decodes to \
+ */
+notmuch_string_map_t *
+_notmuch_string_map_deserialize (void *ctx, const char *str);
 #endif
-- 
2.17.1

[PATCH 06/12] util/string-map: add round trip test

[David Bremner]

In particular this tests that we are not escaping more than necessary.
---
 test/T710-string-map.sh | 58 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/test/T710-string-map.sh b/test/T710-string-map.sh
index 9c5c1d8e..8fd69a53 100755
--- a/test/T710-string-map.sh
+++ b/test/T710-string-map.sh
@@ -426,4 +426,62 @@ val[3]=testval4
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "Round trip from string"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    const char * str =
+         "testkey1\n"
+         "testval1\\b\n"
+         "testkey1\n"
+         "testval3\\\n"
+         "testkey1\\nkey continues\n"
+         "testval3\\nvalue continues\n"
+         "testkey2\n"
+         "testval2\n"
+         "testkey2\\\\not\n"
+         "testval2\n"
+         "testkey2\\toto\n"
+         "testval4\n";
+
+
+    notmuch_string_map_t *map = _notmuch_string_map_deserialize (ctx, str);
+    const char * str2 = _notmuch_string_map_serialize (ctx, map);
+    fputs (str, stdout);
+    fputs ("--------------------------------------------------\n", stdout);
+    fputs (str2, stdout);
+    printf ("%d\n", strcmp (str, str2));
+}
+EOF
+cat<<'EOF' > EXPECTED
+== stdout ==
+testkey1
+testval1\b
+testkey1
+testval3\
+testkey1\nkey continues
+testval3\nvalue continues
+testkey2
+testval2
+testkey2\\not
+testval2
+testkey2\toto
+testval4
+--------------------------------------------------
+testkey1
+testval1\b
+testkey1
+testval3\
+testkey1\nkey continues
+testval3\nvalue continues
+testkey2
+testval2
+testkey2\\not
+testval2
+testkey2\toto
+testval4
+0
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done
-- 
2.17.1

[PATCH 07/12] util/string-map: add _notmuch_string_map_set

[David Bremner]

In contrast to the existing _append, this is intended for interleaved
read and write operations.
---
 test/T710-string-map.sh | 30 ++++++++++++++++++++++++++++++
 util/string-map.c       | 19 +++++++++++++++++++
 util/string-map.h       |  5 +++++
 3 files changed, 54 insertions(+)

diff --git a/test/T710-string-map.sh b/test/T710-string-map.sh
index 8fd69a53..6f07f363 100755
--- a/test/T710-string-map.sh
+++ b/test/T710-string-map.sh
@@ -97,6 +97,36 @@ val[2]=testval2
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "set (replace)"
+cat c_head - c_tail <<'EOF' | test_C $
+{
+    notmuch_string_map_t *map = _notmuch_string_map_create(ctx);
+    _notmuch_string_map_append (map, "testkey1", "testval1");
+    _notmuch_string_map_append (map, "testkey2", "testval2");
+    _notmuch_string_map_append (map, "testkey1", "testval3");
+    dump_map (map);
+    _notmuch_string_map_set (map, "testkey1", "newval");
+    dump_map (map);
+}
+EOF
+cat<<EOF > EXPECTED
+== stdout ==
+key[0]=testkey1
+val[0]=testval1
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+key[0]=testkey1
+val[0]=newval
+key[1]=testkey1
+val[1]=testval3
+key[2]=testkey2
+val[2]=testval2
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest "get first"
 cat c_head - c_tail <<'EOF' | test_C $
 {
diff --git a/util/string-map.c b/util/string-map.c
index b29a9ba0..41a6881d 100644
--- a/util/string-map.c
+++ b/util/string-map.c
@@ -143,6 +143,25 @@ bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, bool e
 
 }
 
+void
+_notmuch_string_map_set (notmuch_string_map_t *map,
+			     const char *key,
+			     const char *value)
+{
+
+    notmuch_string_pair_t *pair;
+
+    /* this means that calling append invalidates iterators */
+    _notmuch_string_map_sort (map);
+
+    pair = bsearch_first (map->pairs, map->length, key, true);
+    if (! pair)
+	_notmuch_string_map_append (map, key, value);
+
+    pair->value = talloc_strdup (map, value);
+
+}
+
 const char *
 _notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
 {
diff --git a/util/string-map.h b/util/string-map.h
index 22aa487c..ff648b5c 100644
--- a/util/string-map.h
+++ b/util/string-map.h
@@ -12,6 +12,11 @@ _notmuch_string_map_append (notmuch_string_map_t *map,
 			    const char *key,
 			    const char *value);
 
+void
+_notmuch_string_map_set (notmuch_string_map_t *map,
+			 const char *key,
+			 const char *value);
+
 const char *
 _notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
 
-- 
2.17.1

[PATCH 08/12] lib/database: add NOTMUCH_FEATURE_MESSAGE_DATA

[David Bremner]

This feature is intended to mark the database as supporting a
key-value store in the document data area. The actual key-value store
is implemented in a subsequent commit.
---
 lib/database-private.h |  8 +++++++-
 lib/database.cc        | 12 ++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/database-private.h b/lib/database-private.h
index a499b259..3a15fd16 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -108,6 +108,12 @@ enum _notmuch_features {
      *
      * Introduced: version 3. */
     NOTMUCH_FEATURE_LAST_MOD = 1 << 6,
+
+    /* If set, messages store a serialized string-map in their data
+     * area
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_MESSAGE_DATA = 1 << 7,
 };
 
 /* In C++, a named enum is its own type, so define bitwise operators
@@ -233,7 +239,7 @@ struct _notmuch_database {
 #define NOTMUCH_FEATURES_CURRENT \
     (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_DIRECTORY_DOCS | \
      NOTMUCH_FEATURE_BOOL_FOLDER | NOTMUCH_FEATURE_GHOSTS | \
-     NOTMUCH_FEATURE_LAST_MOD)
+     NOTMUCH_FEATURE_LAST_MOD | NOTMUCH_FEATURE_MESSAGE_DATA )
 
 /* Return the list of terms from the given iterator matching a prefix.
  * The prefix will be stripped from the strings in the returned list.
diff --git a/lib/database.cc b/lib/database.cc
index 9cf8062c..47e903d5 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -383,6 +383,8 @@ static const struct {
       "indexed MIME types", "w"},
     { NOTMUCH_FEATURE_LAST_MOD,
       "modification tracking", "w"},
+    { NOTMUCH_FEATURE_MESSAGE_DATA,
+      "per message data", "rw"},
 };
 
 const char *
@@ -1342,6 +1344,16 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
     if (! notmuch_database_needs_upgrade (notmuch))
 	return NOTMUCH_STATUS_SUCCESS;
 
+    /*
+     * the upgrade for NOTMUCH_FEATURE_MESSAGE_DATA is actually done
+     * by NOTMUCH_FEATURE_FILE_TERMS
+     */
+    if ((new_features & NOTMUCH_FEATURE_MESSAGE_DATA) &&
+	!(target_features & NOTMUCH_FEATURE_FILE_TERMS)) {
+	_notmuch_database_log (notmuch, "inconsistent feature set");
+	return NOTMUCH_STATUS_UNSUPPORTED_OPERATION;
+    }
+
     if (progress_notify) {
 	/* Set up our handler for SIGALRM */
 	memset (&action, 0, sizeof (struct sigaction));
-- 
2.17.1

[PATCH 09/12] lib: add data_map field to message structs

[David Bremner]

This string-map stores the unserialized key-value map from the message
document data area. It is lazily read on the first read, and lazily
written by _notmuch_message_sync. Note that other than naming this is
independent from the other metadata, which is stored in document
terms (i.e. things you can search for).
---
 lib/message.cc | 53 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/lib/message.cc b/lib/message.cc
index 153e4bed..bf597bc5 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -41,6 +41,7 @@ struct _notmuch_message {
     notmuch_message_file_t *message_file;
     notmuch_string_list_t *property_term_list;
     notmuch_string_map_t *property_map;
+    notmuch_string_map_t *data_map;
     notmuch_message_list_t *replies;
     unsigned long flags;
     /* For flags that are initialized on-demand, lazy_flags indicates
@@ -129,6 +130,7 @@ _notmuch_message_create_for_document (const void *talloc_owner,
     message->author = NULL;
     message->property_term_list = NULL;
     message->property_map = NULL;
+    message->data_map = NULL;
 
     message->replies = _notmuch_message_list_create (message);
     if (unlikely (message->replies == NULL)) {
@@ -334,7 +336,6 @@ _notmuch_message_get_thread_id_only (notmuch_message_t *message)
     return message->thread_id;
 }
 
-
 static void
 _notmuch_message_ensure_metadata (notmuch_message_t *message, void *field)
 {
@@ -483,6 +484,19 @@ _notmuch_message_invalidate_metadata (notmuch_message_t *message,
     }
 }
 
+static void
+_notmuch_message_ensure_data_map (notmuch_message_t *message)
+{
+    if (message->data_map)
+	return;
+
+    char *blob = talloc_strdup(message, message->doc.get_data().c_str ());
+
+    message->data_map = _notmuch_string_map_deserialize (message, blob);
+
+    talloc_free (blob);
+}
+
 unsigned int
 _notmuch_message_get_doc_id (notmuch_message_t *message)
 {
@@ -1169,6 +1183,13 @@ _notmuch_message_sync (notmuch_message_t *message)
 				    _notmuch_database_new_revision (
 					message->notmuch)));
 
+    if (message->notmuch->features & NOTMUCH_FEATURE_MESSAGE_DATA
+	&& message->data_map) {
+	const char *blob = _notmuch_string_map_serialize (message,
+							  message->data_map);
+	message->doc.set_data (blob);
+    }
+
     db = static_cast <Xapian::WritableDatabase *> (message->notmuch->xapian_db);
     db->replace_document (message->doc_id, message->doc);
     message->modified = false;
@@ -1930,6 +1951,36 @@ notmuch_message_get_database (const notmuch_message_t *message)
     return message->notmuch;
 }
 
+notmuch_status_t
+_notmuch_message_data_get (notmuch_message_t *message, const char *key, const char **value)
+{
+    if (! message || !key || !value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    _notmuch_message_ensure_data_map (message);
+
+    *value = _notmuch_string_map_get (message->data_map, key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+_notmuch_message_data_set (notmuch_message_t *message, const char *key, const char *value)
+{
+    if (! message || !key || !value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    notmuch_status_t status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    _notmuch_message_ensure_data_map (message);
+
+    _notmuch_string_map_set (message->data_map, key, value);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 static void
 _notmuch_message_ensure_property_map (notmuch_message_t *message)
 {
-- 
2.17.1

[PATCH 10/12] lib: factor out xapian access from notmuch_message_get_header

[David Bremner]

In a later commit, we will introduce another layer of data structure
for headers. To keep the resulting function size down, factor out the
database access.
---
 lib/message.cc | 41 ++++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/lib/message.cc b/lib/message.cc
index bf597bc5..f59f0a5c 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -529,6 +529,29 @@ _notmuch_message_ensure_message_file (notmuch_message_t *message)
 	notmuch_message_get_database (message), message, filename);
 }
 
+static const char *
+_notmuch_message_get_header_from_xapian (notmuch_message_t *message,
+				 Xapian::valueno slot)
+{
+    try {
+	std::string value = message->doc.get_value (slot);
+
+	/* If we have NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES, then
+	 * empty values indicate empty headers.  If we don't, then
+	 * it could just mean we didn't record the header. */
+	if ((message->notmuch->features &
+	     NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES) ||
+	    ! value.empty())
+	    return talloc_strdup (message, value.c_str ());
+
+    } catch (Xapian::Error &error) {
+	_notmuch_database_log(notmuch_message_get_database (message), "A Xapian exception occurred when reading header: %s\n",
+			      error.get_msg().c_str());
+	message->notmuch->exception_reported = true;
+    }
+    return NULL;
+}
+
 const char *
 notmuch_message_get_header (notmuch_message_t *message, const char *header)
 {
@@ -544,23 +567,7 @@ notmuch_message_get_header (notmuch_message_t *message, const char *header)
 	slot = NOTMUCH_VALUE_MESSAGE_ID;
 
     if (slot != Xapian::BAD_VALUENO) {
-	try {
-	    std::string value = message->doc.get_value (slot);
-
-	    /* If we have NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES, then
-	     * empty values indicate empty headers.  If we don't, then
-	     * it could just mean we didn't record the header. */
-	    if ((message->notmuch->features &
-		 NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES) ||
-		! value.empty())
-		return talloc_strdup (message, value.c_str ());
-
-	} catch (Xapian::Error &error) {
-	    _notmuch_database_log(notmuch_message_get_database (message), "A Xapian exception occurred when reading header: %s\n",
-		     error.get_msg().c_str());
-	    message->notmuch->exception_reported = true;
-	    return NULL;
-	}
+	return _notmuch_message_get_header_from_xapian (message, slot);
     }
 
     /* Otherwise fall back to parsing the file */
-- 
2.17.1

[PATCH 11/12] lib/message: check message data_map for header

[David Bremner]

The will allow the indexing process to override the values in the
files. This is potentially useful for encrypted headers, or in
resolving duplicate files for the same message(-id).
---
 lib/message.cc | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/lib/message.cc b/lib/message.cc
index f59f0a5c..c3c71fd4 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -552,11 +552,30 @@ _notmuch_message_get_header_from_xapian (notmuch_message_t *message,
     return NULL;
 }
 
+static const char *
+_notmuch_message_get_header_from_data (notmuch_message_t *message, const char *header) {
+    _notmuch_message_ensure_data_map (message);
+
+    if (! (message->notmuch->features & NOTMUCH_FEATURE_MESSAGE_DATA))
+	return NULL;
+
+    const char *key = talloc_asprintf (message->data_map, "header.%s", header);
+
+    if (! key)
+	return NULL;
+
+    return _notmuch_string_map_get (message->data_map, key);
+}
+
 const char *
 notmuch_message_get_header (notmuch_message_t *message, const char *header)
 {
-    Xapian::valueno slot = Xapian::BAD_VALUENO;
+    /* Have we explicitly chosen / set this header */
+    const char *value = _notmuch_message_get_header_from_data (message, header);
+    if (value)
+	return value;
 
+    Xapian::valueno slot = Xapian::BAD_VALUENO;
     /* Fetch header from the appropriate xapian value field if
      * available */
     if (strcasecmp (header, "from") == 0)
-- 
2.17.1

[PATCH 12/12] lib/message: add notmuch_message_set_header

[David Bremner]

This saves a header value to the message document data area, where it
can be retrieved later by notmuch_message_get_header.
---
 lib/message.cc            | 24 +++++++++++++++
 lib/notmuch.h             |  6 +++-
 test/T720-message-data.sh | 64 +++++++++++++++++++++++++++++++++++++++
 util/string-map.c         |  4 +--
 4 files changed, 95 insertions(+), 3 deletions(-)
 create mode 100755 test/T720-message-data.sh

diff --git a/lib/message.cc b/lib/message.cc
index c3c71fd4..cfd51b2e 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -597,6 +597,30 @@ notmuch_message_get_header (notmuch_message_t *message, const char *header)
     return _notmuch_message_file_get_header (message->message_file, header);
 }
 
+notmuch_status_t
+notmuch_message_set_header (notmuch_message_t *message,
+			    const char *header,
+			    const char *value)
+{
+    /* We don't want to accept changes which will be silently lost on
+     * sync */
+    notmuch_status_t status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    const char *key = talloc_asprintf (message->data_map, "header.%s", header);
+
+    _notmuch_message_ensure_data_map (message);
+
+    _notmuch_string_map_set (message->data_map, key, value);
+    message->modified = true;
+
+    if (! message->frozen)
+	_notmuch_message_sync (message);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 /* Return the message ID from the In-Reply-To header of 'message'.
  *
  * Returns an empty string ("") if 'message' has no In-Reply-To
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 247f6ad7..ca809016 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -58,7 +58,7 @@ NOTMUCH_BEGIN_DECLS
  * version in Makefile.local.
  */
 #define LIBNOTMUCH_MAJOR_VERSION	5
-#define LIBNOTMUCH_MINOR_VERSION	2
+#define LIBNOTMUCH_MINOR_VERSION	3
 #define LIBNOTMUCH_MICRO_VERSION	0
 
 
@@ -1516,6 +1516,10 @@ notmuch_message_get_date  (notmuch_message_t *message);
 const char *
 notmuch_message_get_header (notmuch_message_t *message, const char *header);
 
+notmuch_status_t
+notmuch_message_set_header (notmuch_message_t *message,
+			    const char *header,
+			    const  char *value );
 /**
  * Get the tags for 'message', returning a notmuch_tags_t object which
  * can be used to iterate over all tags.
diff --git a/test/T720-message-data.sh b/test/T720-message-data.sh
new file mode 100755
index 00000000..3b74d0b8
--- /dev/null
+++ b/test/T720-message-data.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+test_description="message data API"
+
+. $(dirname "$0")/test-lib.sh || exit 1
+
+add_email_corpus
+
+cat <<EOF > c_head
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include <notmuch-test.h>
+
+int main (int argc, char** argv)
+{
+   notmuch_database_t *db;
+   char *val;
+   notmuch_status_t stat;
+
+   EXPECT0(notmuch_database_open (argv[1], NOTMUCH_DATABASE_MODE_READ_WRITE, &db));
+
+EOF
+
+cat <<EOF > c_tail
+   EXPECT0(notmuch_database_destroy(db));
+}
+EOF
+
+add_message '[subject]=initial' '[id]=subject-test-id'
+
+test_begin_subtest "find initial subject"
+notmuch search id:subject-test-id | notmuch_search_sanitize > OUTPUT
+cat <<EOF >EXPECTED
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; initial (inbox unread)
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "modify subject"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+   notmuch_message_t *message;
+   const char* subject;
+   EXPECT0 (notmuch_database_find_message (db, "subject-test-id", &message));
+   assert (message);
+   EXPECT0 (notmuch_message_set_header (message, "subject", "modified"));
+   subject = notmuch_message_get_header (message, "subject");
+   printf ("subject = %s\n", subject);
+}
+EOF
+cat <<'EOF' >EXPECTED
+== stdout ==
+subject = modified
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "modified subject persists"
+notmuch search id:subject-test-id | notmuch_search_sanitize > OUTPUT
+cat <<EOF >EXPECTED
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; modified (inbox unread)
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_done
diff --git a/util/string-map.c b/util/string-map.c
index 41a6881d..644803ac 100644
--- a/util/string-map.c
+++ b/util/string-map.c
@@ -157,8 +157,8 @@ _notmuch_string_map_set (notmuch_string_map_t *map,
     pair = bsearch_first (map->pairs, map->length, key, true);
     if (! pair)
 	_notmuch_string_map_append (map, key, value);
-
-    pair->value = talloc_strdup (map, value);
+    else
+	pair->value = talloc_strdup (map, value);
 
 }
 
-- 
2.17.1