Revised dump-restore patches

Revised dump-restore patches

[David Bremner]

I haven't written user docs yet. I'm considering whether I can improve
this small task by doing the larger task of splitting the man page and
having notmuch help call man.

The code is cleaned up quite a bit, the bug fix for 8bit characters is
rolled into the series, and following a suggestion of Jani, restore
now auto-detects what format the input is in.

I left the default dump format as "sup" (old style) in this round. I'm
leary of breaking people's scripts, so my current proposal is
deprecate the sup format (as default) for a release or two.

[PATCH Draft 2 1/9] hex-escape: (en|de)code strings to/from restricted character set

[David Bremner]

From: David Bremner <bremner@debian.org>

The character set is chosen to be suitable for pathnames, and the same
as that used by contrib/nmbug
---
 util/Makefile.local |    2 +-
 util/hex-escape.c   |  156 +++++++++++++++++++++++++++++++++++++++++++++++++++
 util/hex-escape.h   |   32 +++++++++++
 3 files changed, 189 insertions(+), 1 deletions(-)
 create mode 100644 util/hex-escape.c
 create mode 100644 util/hex-escape.h

diff --git a/util/Makefile.local b/util/Makefile.local
index 26e4c3f..2e63932 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -3,7 +3,7 @@
 dir := util
 extra_cflags += -I$(srcdir)/$(dir)
 
-libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c
+libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c
 
 libutil_modules := $(libutil_c_srcs:.c=.o)
 
diff --git a/util/hex-escape.c b/util/hex-escape.c
new file mode 100644
index 0000000..f58b9a2
--- /dev/null
+++ b/util/hex-escape.c
@@ -0,0 +1,156 @@
+/* pathname.c -  Manage encoding and decoding of byte strings into path names
+ *
+ * Copyright (c) 2011 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <talloc.h>
+#include "error_util.h"
+#include "hex-escape.h"
+
+static const size_t default_buf_size = 1024;
+
+static const char *output_charset =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.:,";
+
+static const int escape_char = '%';
+
+static int
+is_output (char c)
+{
+    return (strchr (output_charset, c) != NULL);
+}
+
+static int
+maybe_realloc (void *ctx, size_t needed, char **out, size_t *out_size)
+{
+    if (*out_size < needed) {
+
+	if (*out == NULL)
+	    *out = talloc_size (ctx, needed);
+	else
+	    *out = talloc_realloc (ctx, *out, char, needed);
+
+	if (*out == NULL)
+	    return 0;
+
+	*out_size = needed;
+    }
+    return 1;
+}
+
+hex_status_t
+hex_encode (void *ctx, const char *in, char **out, size_t *out_size)
+{
+
+    const unsigned char *p;
+    char *q;
+
+    size_t escape_count = 0;
+    size_t len = 0;
+    size_t needed;
+
+    assert (ctx); assert (in); assert (out); assert (out_size);
+
+    for (p = (unsigned char *) in; *p; p++) {
+	escape_count += (!is_output (*p));
+	len++;
+    }
+
+    needed = len + escape_count * 2 + 1;
+
+    if (*out == NULL)
+	*out_size = 0;
+
+    if (!maybe_realloc (ctx, needed, out, out_size))
+	return HEX_OUT_OF_MEMORY;
+
+    q = *out;
+    p = (unsigned char *) in;
+
+    while (*p) {
+	if (is_output (*p)) {
+	    *q++ = *p++;
+	} else {
+	    sprintf (q, "%%%02x", *p++);
+	    q += 3;
+	}
+    }
+
+    *q = '\0';
+    return HEX_SUCCESS;
+}
+
+hex_status_t
+hex_decode (void *ctx, const char *in, char **out, size_t * out_size)
+{
+
+    char buf[3];
+
+    const char *p;
+    unsigned char *q;
+
+    size_t escape_count = 0;
+    size_t needed = 0;
+
+    assert (ctx); assert (in); assert (out); assert (out_size);
+
+    size_t len = strlen (in);
+
+    for (p = in; *p; p++)
+	escape_count += (*p == escape_char);
+
+    needed = len - escape_count * 2 + 1;
+
+    if (!maybe_realloc (ctx, needed, out, out_size))
+	return HEX_OUT_OF_MEMORY;
+
+    p = in;
+    q = (unsigned char *) *out;
+    buf[2] = 0;
+
+    while (*p) {
+
+	if (*p == escape_char) {
+
+	    char *endp;
+
+	    if (len < 3)
+		return HEX_SYNTAX_ERROR;
+
+	    buf[0] = p[1];
+	    buf[1] = p[2];
+
+	    *q = strtol (buf, &endp, 16);
+
+	    if (endp != buf + 2)
+		return HEX_SYNTAX_ERROR;
+
+	    len -= 3;
+	    p += 3;
+	    q++;
+	} else {
+	    *q++ = *p++;
+	}
+    }
+
+    *q = '\0';
+
+    return HEX_SUCCESS;
+}
diff --git a/util/hex-escape.h b/util/hex-escape.h
new file mode 100644
index 0000000..e409626
--- /dev/null
+++ b/util/hex-escape.h
@@ -0,0 +1,32 @@
+#ifndef _HEX_ESCAPE_H
+#define _HEX_ESCAPE_H
+
+typedef enum hex_status {
+    HEX_SUCCESS = 0,
+    HEX_SYNTAX_ERROR,
+    HEX_OUT_OF_MEMORY
+} hex_status_t;
+
+/*
+ * The API is modelled on that for getline.
+ *
+ * If 'out' points to a NULL pointer a char array of the appropriate
+ * size is allocated using talloc, and out_size is updated.
+ *
+ * If 'out' points to a non-NULL pointer, it assumed to describe an
+ * existing char array, with the size given in *out_size.  This array
+ * may be resized by talloc_realloc if needed; in this case *out_size
+ * will also be updated.
+ *
+ * Note that it is an error to pass a NULL pointer for any parameter
+ * of these routines.
+ */
+
+hex_status_t
+hex_encode (void *talloc_ctx, const char *in, char **out,
+            size_t *out_size);
+
+hex_status_t
+hex_decode (void *talloc_ctx, const char *in, char **out,
+            size_t *out_size);
+#endif
-- 
1.7.7.3

[PATCH Draft 2 2/9] test/hex-xcode: new test binary

[David Bremner]

From: David Bremner <bremner@debian.org>

This program is used both as a test-bed/unit-tester for
../util/hex-escape.c, and also as a utility in future tests of dump
and restore.
---
 test/.gitignore     |    1 +
 test/Makefile.local |    6 +++-
 test/basic          |    2 +-
 test/hex-xcode.c    |   78 +++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 85 insertions(+), 2 deletions(-)
 create mode 100644 test/hex-xcode.c

diff --git a/test/.gitignore b/test/.gitignore
index e63c689..be7ab5e 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -3,4 +3,5 @@ corpus.mail
 smtp-dummy
 symbol-test
 arg-test
+hex-xcode
 tmp.*
diff --git a/test/Makefile.local b/test/Makefile.local
index fa2df73..ec507c4 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -13,6 +13,9 @@ smtp_dummy_modules = $(smtp_dummy_srcs:.c=.o)
 $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
 	$(call quiet,CC) -I. $^ -o $@
 
+$(dir)/hex-xcode: $(dir)/hex-xcode.o util/libutil.a
+	$(call quiet,CC) -I. $^ -o $@ -ltalloc
+
 $(dir)/smtp-dummy: $(smtp_dummy_modules)
 	$(call quiet,CC) $^ -o $@
 
@@ -21,7 +24,8 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
 
 .PHONY: test check
 
-test-binaries: $(dir)/arg-test $(dir)/smtp-dummy $(dir)/symbol-test
+test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
+	 $(dir)/smtp-dummy $(dir)/symbol-test
 
 test:	all test-binaries
 	@${dir}/notmuch-test $(OPTIONS)
diff --git a/test/basic b/test/basic
index d6aed24..af57026 100755
--- a/test/basic
+++ b/test/basic
@@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
 eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
 tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
 available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
-    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test)$/d" | \
+    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
     sort)
 test_expect_equal "$tests_in_suite" "$available"
 
diff --git a/test/hex-xcode.c b/test/hex-xcode.c
new file mode 100644
index 0000000..20c1603
--- /dev/null
+++ b/test/hex-xcode.c
@@ -0,0 +1,78 @@
+/* No, nothing to to with IDE from Apple Inc.
+   testbed for ../util/hex-escape.c.
+
+   usage: hex-xcode (e|d) < foo
+
+   e is for encode
+   d is for decode
+ */
+
+#include "notmuch-client.h"
+#include "hex-escape.h"
+#include <assert.h>
+
+static int
+xcode (void *ctx, char dir, char *in, char **buf_p, size_t *size_p)
+{
+    hex_status_t status;
+
+    if (dir == 'e')
+	status = hex_encode (ctx, in, buf_p, size_p);
+    else
+	status = hex_decode (ctx, in, buf_p, size_p);
+
+    if (status == HEX_SUCCESS)
+	puts (*buf_p);
+
+    return status;
+}
+
+
+int
+main (int argc, char **argv)
+{
+
+    assert (argc > 1 && argv[1]);
+
+    char dir = argv[1][0];
+
+    void *ctx = talloc_new (NULL);
+
+    char *line = NULL;
+    size_t line_size;
+    ssize_t line_len;
+
+    char *buffer = NULL;
+    size_t buf_size = 0;
+
+    int arg_index = 2;
+    notmuch_bool_t read_stdin = TRUE;
+
+    for (arg_index = 2; arg_index < argc; arg_index++) {
+
+	if (xcode (ctx, dir, argv[arg_index],
+		   &buffer, &buf_size) != HEX_SUCCESS)
+	    return 1;
+
+	read_stdin = FALSE;
+    }
+
+    if (!read_stdin)
+	return 0;
+
+    while ((line_len = getline (&line, &line_size, stdin)) != -1) {
+
+	chomp_newline (line);
+
+	if (xcode (ctx, dir, line, &buffer, &buf_size) != HEX_SUCCESS)
+	    return 1;
+
+    }
+
+    if (line)
+	free (line);
+
+    talloc_free (ctx);
+
+    return 0;
+}
-- 
1.7.7.3

[PATCH Draft 2 3/9] test/hex-escaping: new test for hex escaping routines

[David Bremner]

From: David Bremner <bremner@debian.org>

These are more like unit tests, to (try to) make sure the library
functionality is working before building more complicated things on
top of it.
---
 test/hex-escaping |   26 ++++++++++++++++++++++++++
 test/notmuch-test |    1 +
 2 files changed, 27 insertions(+), 0 deletions(-)
 create mode 100755 test/hex-escaping

diff --git a/test/hex-escaping b/test/hex-escaping
new file mode 100755
index 0000000..daa6446
--- /dev/null
+++ b/test/hex-escaping
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+test_description="hex encoding and decoding"
+. ./test-lib.sh
+
+test_begin_subtest "round trip"
+find $TEST_DIRECTORY/corpus -type f -print | sort | xargs cat > EXPECTED
+$TEST_DIRECTORY/hex-xcode e < EXPECTED | $TEST_DIRECTORY/hex-xcode d > OUTPUT
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "punctuation"
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+tag_enc1=$($TEST_DIRECTORY/hex-xcode e "$tag1")
+test_expect_equal "$tag_enc1" "comic_swear=%24%26%5e%25%24%5e%25%5c%5c%2f%2f-+%24%5e%25%24"
+
+test_begin_subtest "round trip newlines"
+printf 'this\n tag\t has\n spaces\n' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode e  < EXPECTED.$test_count |\
+	$TEST_DIRECTORY/hex-xcode d > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest "round trip 8bit chars"
+echo '%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode d  < EXPECTED.$test_count |\
+	$TEST_DIRECTORY/hex-xcode e > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index ded79e8..af72c73 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -50,6 +50,7 @@ TESTS="
   python
   hooks
   argument-parsing
+  hex-escaping
 "
 TESTS=${NOTMUCH_TESTS:=$TESTS}
 
-- 
1.7.7.3

[PATCH Draft 2 4/9] notmuch-dump: add --format=(notmuch|sup)

[David Bremner]

From: David Bremner <bremner@debian.org>

sup is the old format, and remains the default.

Each line of the notmuch format is "msg_id tag tag...tag" where each
space seperated token is 'hex-encoded' to remove troubling characters.
In particular this format won't have the same problem with e.g. spaces
in message-ids or tags; they will be round-trip-able.
---
 dump-restore-private.h |   12 ++++++++++++
 notmuch-dump.c         |   47 +++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 51 insertions(+), 8 deletions(-)
 create mode 100644 dump-restore-private.h

diff --git a/dump-restore-private.h b/dump-restore-private.h
new file mode 100644
index 0000000..34a5022
--- /dev/null
+++ b/dump-restore-private.h
@@ -0,0 +1,12 @@
+#ifndef DUMP_RESTORE_PRIVATE_H
+#define DUMP_RESTORE_PRIVATE_H
+
+#include "hex-escape.h"
+#include "command-line-arguments.h"
+
+typedef enum dump_formats {
+    DUMP_FORMAT_SUP,
+    DUMP_FORMAT_NOTMUCH
+} dump_format_t;
+
+#endif
diff --git a/notmuch-dump.c b/notmuch-dump.c
index a735875..0231db2 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -19,6 +19,7 @@
  */
 
 #include "notmuch-client.h"
+#include "dump-restore-private.h"
 
 int
 notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
@@ -44,9 +45,15 @@ notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
     char *output_file_name = NULL;
     int opt_index;
 
+    int output_format = DUMP_FORMAT_SUP;
+
     notmuch_opt_desc_t options[] = {
-	{ NOTMUCH_OPT_POSITION, &output_file_name, 0, 0, 0  },
-	{ 0, 0, 0, 0, 0 }
+	{ NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
+	  (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
+				  { "notmuch", DUMP_FORMAT_NOTMUCH },
+				  {0, 0} } },
+	{ NOTMUCH_OPT_POSITION, &output_file_name, 0, 0, 0 },
+	{ 0,		        0,	           0, 0, 0 }
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
@@ -85,29 +92,53 @@ notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
      */
     notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
 
+    char *buffer = NULL;
+    size_t buffer_size = 0;
+
     for (messages = notmuch_query_search_messages (query);
 	 notmuch_messages_valid (messages);
 	 notmuch_messages_move_to_next (messages))
     {
 	int first = 1;
-	message = notmuch_messages_get (messages);
+	const char *message_id;
 
-	fprintf (output,
-		 "%s (", notmuch_message_get_message_id (message));
+	message = notmuch_messages_get (messages);
+	message_id = notmuch_message_get_message_id (message);
+
+	if (output_format == DUMP_FORMAT_SUP) {
+	    fprintf (output, "%s (", message_id);
+	} else {
+	    if (hex_encode (notmuch, message_id,
+			    &buffer, &buffer_size) != HEX_SUCCESS)
+		return 1;
+	    fprintf (output, "%s ", buffer);
+	}
 
 	for (tags = notmuch_message_get_tags (message);
 	     notmuch_tags_valid (tags);
 	     notmuch_tags_move_to_next (tags))
 	{
+	    const char *tag_str = notmuch_tags_get (tags);
+
 	    if (! first)
-		fprintf (output, " ");
+		fputs (" ", output);
 
-	    fprintf (output, "%s", notmuch_tags_get (tags));
+	    if (output_format == DUMP_FORMAT_SUP) {
+		fputs (tag_str, output);
+	    } else {
+		if (hex_encode (notmuch, tag_str,
+				&buffer, &buffer_size) != HEX_SUCCESS)
+		    return 1;
 
+		fputs (buffer, output);
+	    }
 	    first = 0;
 	}
 
-	fprintf (output, ")\n");
+	if (output_format == DUMP_FORMAT_SUP)
+	    fputs (")\n", output);
+	else
+	    fputs ("\n", output);
 
 	notmuch_message_destroy (message);
     }
-- 
1.7.7.3

[PATCH Draft 2 5/9] test: add test for dump --format=notmuch

[David Bremner]

From: David Bremner <bremner@debian.org>

The first test is really to test our assumptions about the corpus,
namely that a certain set of message-id's is safe (i.e. doesn't change
under hex-escaping).  We then check dump output as best we can without
functionality-to-come in notmuch-restore.
---
 test/dump-restore |   12 ++++++++++++
 1 files changed, 12 insertions(+), 0 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index 439e998..48caf4e 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -82,4 +82,16 @@ test_begin_subtest "dump outfile -- from:cworth"
 notmuch dump dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
 
+test_begin_subtest "Check for a safe set of message-ids"
+notmuch search --output=messages from:cworth > EXPECTED.$test_count
+notmuch search --output=messages from:cworth |\
+	$TEST_DIRECTORY/hex-xcode e > OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
+# we have observed that cworth has sane message-ids, and hopefully sane tags.
+test_begin_subtest "dump --format=notmuch -- from:cworth"
+notmuch dump --format=sup -- from:cworth | tr -d \(\) > EXPECTED.$test_count
+notmuch dump --format=notmuch -- from:cworth > OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
 test_done
-- 
1.7.7.3

[PATCH Draft 2 6/9] notmuch-restore: add --format=notmuch support

[David Bremner]

From: David Bremner <bremner@debian.org>

This is format is whitespace separated tokens, encoded by
util/hex-escape.c
---
 notmuch-restore.c |   81 +++++++++++++++++++++++++++++++++++++++++------------
 1 files changed, 63 insertions(+), 18 deletions(-)

diff --git a/notmuch-restore.c b/notmuch-restore.c
index 87d9772..f618b47 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -19,6 +19,7 @@
  */
 
 #include "notmuch-client.h"
+#include "dump-restore-private.h"
 
 int
 notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
@@ -35,6 +36,7 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
     regex_t regex;
     int rerr;
     int opt_index;
+    int input_format = DUMP_FORMAT_SUP;
 
     config = notmuch_config_open (ctx, NULL, NULL);
     if (config == NULL)
@@ -48,6 +50,7 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
     synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config);
 
     notmuch_opt_desc_t options[] = {
+	FORMAT_DESC (input_format),
 	{ NOTMUCH_OPT_POSITION, &input_file_name, 0, 0, 0 },
 	{ NOTMUCH_OPT_BOOLEAN,  &accumulate, "accumulate", 'a', 0 },
 	{ 0, 0, 0, 0, 0 }
@@ -81,33 +84,63 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
      * non-space characters for the message-id, then one or more
      * spaces, then a list of space-separated tags as a sequence of
      * characters within literal '(' and ')'. */
-    if ( xregcomp (&regex,
-		   "^([^ ]+) \\(([^)]*)\\)$",
-		   REG_EXTENDED) )
-	INTERNAL_ERROR("compile time constant regex failed.");
+    if (input_format == DUMP_FORMAT_SUP) {
+	if ( xregcomp (&regex,
+		       "^([^ ]+) \\(([^)]*)\\)$",
+		       REG_EXTENDED) )
+	    INTERNAL_ERROR("compile time constant regex failed.");
+    }
+
+
+    /* These are out here to re-use the buffers with hex_decode */
+
+    char *message_id = NULL;
+    size_t message_id_size = 0;
+    char *tag = NULL;
+    size_t tag_size = 0;
 
     while ((line_len = getline (&line, &line_size, input)) != -1) {
 	regmatch_t match[3];
-	char *message_id, *file_tags, *tag, *next;
+	char  *file_tags, *next;
 	notmuch_message_t *message = NULL;
+
 	notmuch_status_t status;
 	notmuch_tags_t *db_tags;
 	char *db_tags_str;
 
 	chomp_newline (line);
 
-	rerr = xregexec (&regex, line, 3, match, 0);
-	if (rerr == REG_NOMATCH)
-	{
-	    fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
-		     line);
+	/* Silently ignore blank lines */
+
+	if (line[0] == '\0') {
 	    continue;
 	}
 
-	message_id = xstrndup (line + match[1].rm_so,
-			       match[1].rm_eo - match[1].rm_so);
-	file_tags = xstrndup (line + match[2].rm_so,
-			      match[2].rm_eo - match[2].rm_so);
+	if (input_format == DUMP_FORMAT_SUP) {
+	    rerr = xregexec (&regex, line, 3, match, 0);
+	    if (rerr == REG_NOMATCH)
+	    {
+		fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
+			 line);
+		continue;
+	    }
+	    message_id = talloc_strndup (notmuch, line + match[1].rm_so,
+					 match[1].rm_eo - match[1].rm_so);
+	    file_tags = talloc_strndup (notmuch, line + match[2].rm_so,
+					match[2].rm_eo - match[2].rm_so);
+	} else {
+	    char *p = line;
+	    char *raw_mid;
+
+	    raw_mid = strsep (&p, " \t");
+
+	    if (hex_decode (notmuch, raw_mid,
+			    &message_id, &message_id_size) != HEX_SUCCESS)
+		return 1;
+
+	    file_tags = xstrdup (p);
+	}
+
 
 	status = notmuch_database_find_message (notmuch, message_id, &message);
 	if (status || message == NULL) {
@@ -153,7 +186,16 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 
 	next = file_tags;
 	while (next) {
-	    tag = strsep (&next, " ");
+	    char *raw_tag = strsep (&next, " ");
+
+	    if (input_format == DUMP_FORMAT_NOTMUCH) {
+		if (hex_decode (notmuch, raw_tag,
+				&tag, &tag_size) != HEX_SUCCESS)
+		    return 1;
+	    } else {
+		tag = talloc_strdup (notmuch, raw_tag);
+	    }
+
 	    if (*tag == '\0')
 		continue;
 	    status = notmuch_message_add_tag (message, tag);
@@ -175,11 +217,14 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 	if (message)
 	    notmuch_message_destroy (message);
 	message = NULL;
-	free (message_id);
-	free (file_tags);
+	if (input_format == DUMP_FORMAT_SUP) {
+	    talloc_free (message_id);
+	    talloc_free (file_tags);
+	}
     }
 
-    regfree (&regex);
+    if (input_format == DUMP_FORMAT_SUP)
+	regfree (&regex);
 
     if (line)
 	free (line);
-- 
1.7.7.3

[PATCH Draft 2 7/9] test: second set of dump/restore --format=notmuch tests

[David Bremner]

From: David Bremner <bremner@debian.org>

These one need the completed functionality in notmuch-restore. Fairly
exotic tags are tested, but no weird message id's.
---
 test/dump-restore |   36 ++++++++++++++++++++++++++++++++++++
 1 files changed, 36 insertions(+), 0 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index 48caf4e..ef4a63e 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -94,4 +94,40 @@ notmuch dump --format=sup -- from:cworth | tr -d \(\) > EXPECTED.$test_count
 notmuch dump --format=notmuch -- from:cworth > OUTPUT.$test_count
 test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
 
+test_begin_subtest "format=notmuch, # round-trip"
+notmuch dump --format=sup | sort > EXPECTED.$test_count
+notmuch dump --format=notmuch | notmuch restore --format=notmuch
+notmuch dump --format=sup | sort > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+enc1=$($TEST_DIRECTORY/hex-xcode e "$tag1")
+
+tag2=$(printf 'this\n tag\t has\n spaces')
+enc2=$($TEST_DIRECTORY/hex-xcode e "$tag2")
+
+enc3='%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a'
+tag3=$($TEST_DIRECTORY/hex-xcode d $enc3)
+
+notmuch dump --format=notmuch > BACKUP
+
+notmuch tag +"$tag1" +"$tag2" +"$tag3" -inbox -unread "*"
+
+test_begin_subtest 'format=notmuch, round trip with strange tags'
+    notmuch dump --format=notmuch > EXPECTED.$test_count
+    notmuch dump --format=notmuch | notmuch restore --format=notmuch
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+
+test_begin_subtest 'format=notmuch, checking encoded output'
+    cp /dev/null EXPECTED.$test_count
+    notmuch dump --format=notmuch -- from:cworth |\
+	 awk "{ print \$1 \" $enc1 $enc2 $enc3\" }" > EXPECTED.$test_count
+
+    notmuch dump --format=notmuch -- from:cworth  > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+notmuch restore --format=notmuch < BACKUP
+
 test_done
-- 
1.7.7.3

[PATCH Draft 2 8/9] notmuch-restore: auto detect format of backup

[David Bremner]

From: David Bremner <bremner@debian.org>

The simple heuristic relies on the fact that '(' is not part of the
character set used by hex-escape. Since hex-escape is designed to be
OK for pathnames (and shells), this seems like a reasonable
assumption.

In principle the --format argument to notmuch-restore is notmuch
needed at this point, but it adds literally 5 lines of argument
description, so I left it.
---
 dump-restore-private.h |    5 +++--
 notmuch-restore.c      |   45 ++++++++++++++++++++++++++++++++-------------
 2 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/dump-restore-private.h b/dump-restore-private.h
index 34a5022..67795e5 100644
--- a/dump-restore-private.h
+++ b/dump-restore-private.h
@@ -5,8 +5,9 @@
 #include "command-line-arguments.h"
 
 typedef enum dump_formats {
-    DUMP_FORMAT_SUP,
-    DUMP_FORMAT_NOTMUCH
+    DUMP_FORMAT_AUTO,
+    DUMP_FORMAT_NOTMUCH,
+    DUMP_FORMAT_SUP
 } dump_format_t;
 
 #endif
diff --git a/notmuch-restore.c b/notmuch-restore.c
index f618b47..340f31f 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -36,7 +36,7 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
     regex_t regex;
     int rerr;
     int opt_index;
-    int input_format = DUMP_FORMAT_SUP;
+    int input_format = DUMP_FORMAT_AUTO;
 
     config = notmuch_config_open (ctx, NULL, NULL);
     if (config == NULL)
@@ -50,7 +50,11 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
     synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config);
 
     notmuch_opt_desc_t options[] = {
-	FORMAT_DESC (input_format),
+	{ NOTMUCH_OPT_KEYWORD, &input_format, "format", 'f',
+	  (notmuch_keyword_t []){ { "auto", DUMP_FORMAT_AUTO },
+				  { "notmuch", DUMP_FORMAT_NOTMUCH },
+				  { "sup", DUMP_FORMAT_SUP },
+				  {0, 0} } },
 	{ NOTMUCH_OPT_POSITION, &input_file_name, 0, 0, 0 },
 	{ NOTMUCH_OPT_BOOLEAN,  &accumulate, "accumulate", 'a', 0 },
 	{ 0, 0, 0, 0, 0 }
@@ -80,17 +84,6 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 	return 1;
     }
 
-    /* Dump output is one line per message. We match a sequence of
-     * non-space characters for the message-id, then one or more
-     * spaces, then a list of space-separated tags as a sequence of
-     * characters within literal '(' and ')'. */
-    if (input_format == DUMP_FORMAT_SUP) {
-	if ( xregcomp (&regex,
-		       "^([^ ]+) \\(([^)]*)\\)$",
-		       REG_EXTENDED) )
-	    INTERNAL_ERROR("compile time constant regex failed.");
-    }
-
 
     /* These are out here to re-use the buffers with hex_decode */
 
@@ -98,6 +91,7 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
     size_t message_id_size = 0;
     char *tag = NULL;
     size_t tag_size = 0;
+    notmuch_bool_t first_line = TRUE;
 
     while ((line_len = getline (&line, &line_size, input)) != -1) {
 	regmatch_t match[3];
@@ -109,6 +103,30 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 	char *db_tags_str;
 
 	chomp_newline (line);
+	if (first_line && input_format == DUMP_FORMAT_AUTO) {
+	    char *p;
+
+	    for (p = line; *p; p++) {
+		if (*p == '(')
+		    input_format = DUMP_FORMAT_SUP;
+	    }
+
+	    if (input_format == DUMP_FORMAT_AUTO)
+		input_format = DUMP_FORMAT_NOTMUCH;
+
+	}
+
+	/* sup dump output is one line per message. We match a
+	 * sequence of non-space characters for the message-id, then
+	 * one or more spaces, then a list of space-separated tags as
+	 * a sequence of characters within literal '(' and ')'. */
+	if (first_line && input_format == DUMP_FORMAT_SUP) {
+	    if ( xregcomp (&regex,
+			   "^([^ ]+) \\(([^)]*)\\)$",
+			   REG_EXTENDED) )
+		INTERNAL_ERROR("compile time constant regex failed.");
+	}
+
 
 	/* Silently ignore blank lines */
 
@@ -141,6 +159,7 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 	    file_tags = xstrdup (p);
 	}
 
+	first_line = FALSE;
 
 	status = notmuch_database_find_message (notmuch, message_id, &message);
 	if (status || message == NULL) {
-- 
1.7.7.3

[PATCH Draft 2 9/9] test: add tests for restore --auto

[David Bremner]

From: David Bremner <bremner@debian.org>

We start by restoring the original tags, and making sure that restore
worked. Next we test each possible input to autodetection, both
explicit (with --format=auto) and implicit (without --format).
---
 test/dump-restore |   34 ++++++++++++++++++++++++++++++++--
 1 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index ef4a63e..8400c7b 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -119,7 +119,6 @@ test_begin_subtest 'format=notmuch, round trip with strange tags'
     notmuch dump --format=notmuch > OUTPUT.$test_count
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
-
 test_begin_subtest 'format=notmuch, checking encoded output'
     cp /dev/null EXPECTED.$test_count
     notmuch dump --format=notmuch -- from:cworth |\
@@ -128,6 +127,37 @@ test_begin_subtest 'format=notmuch, checking encoded output'
     notmuch dump --format=notmuch -- from:cworth  > OUTPUT.$test_count
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
-notmuch restore --format=notmuch < BACKUP
+test_begin_subtest 'restoring sane tags'
+    notmuch restore --format=notmuch < BACKUP
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file BACKUP OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, restore=auto'
+    notmuch dump --format=notmuch > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore --format=auto < EXPECTED.$test_count
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=auto'
+    notmuch dump --format=sup > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore --format=auto < EXPECTED.$test_count
+    notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, restore=default'
+    notmuch dump --format=notmuch > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore < EXPECTED.$test_count
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=default'
+    notmuch dump --format=sup > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore < EXPECTED.$test_count
+    notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 test_done
-- 
1.7.7.3

Re: Revised dump-restore patches

[Jameson Graef Rollins]

[-- Attachment #1: Type: text/plain, Size: 691 bytes --]

On Sun, 18 Dec 2011 09:15:02 -0400, David Bremner <david@tethera.net> wrote:
> I left the default dump format as "sup" (old style) in this round. I'm
> leary of breaking people's scripts, so my current proposal is
> deprecate the sup format (as default) for a release or two.

It seems to me that as long as restore can detect what format the dump
file is in, it shouldn't matter what the default dump format is, right?
If someone makes a new dump in the new format, they'll presumably also
be restoring with the new code that supports that format as well.  I'm
having a hard time imagining a situation where moving to a new
fully-supported default dump format would break anything.

jamie.

[-- Attachment #2: Type: application/pgp-signature, Size: 835 bytes --]

Re: Revised dump-restore patches

[David Bremner]

On Sun, 18 Dec 2011 12:51:56 -0800, Jameson Graef Rollins <jrollins@finestructure.net> wrote:

> It seems to me that as long as restore can detect what format the dump
> file is in, it shouldn't matter what the default dump format is, right?
> If someone makes a new dump in the new format, they'll presumably also
> be restoring with the new code that supports that format as well.  I'm
> having a hard time imagining a situation where moving to a new
> fully-supported default dump format would break anything.

Well, nmbug would break, to take a random example ;). I expect other
people parse the dump output for their own neferious purposes.

d

New dump/restore format.

[David Bremner]

This version of this series add fairly extensive testing with strange
message ids full of spaces and punctuation, and some documentation.

[PATCH v3 01/10] hex-escape: (en|de)code strings to/from restricted character set

[David Bremner]

From: David Bremner <bremner@debian.org>

The character set is chosen to be suitable for pathnames, and the same
as that used by contrib/nmbug
---
 util/Makefile.local |    2 +-
 util/hex-escape.c   |  156 +++++++++++++++++++++++++++++++++++++++++++++++++++
 util/hex-escape.h   |   32 +++++++++++
 3 files changed, 189 insertions(+), 1 deletions(-)
 create mode 100644 util/hex-escape.c
 create mode 100644 util/hex-escape.h

diff --git a/util/Makefile.local b/util/Makefile.local
index 26e4c3f..2e63932 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -3,7 +3,7 @@
 dir := util
 extra_cflags += -I$(srcdir)/$(dir)
 
-libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c
+libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c
 
 libutil_modules := $(libutil_c_srcs:.c=.o)
 
diff --git a/util/hex-escape.c b/util/hex-escape.c
new file mode 100644
index 0000000..6c1260b
--- /dev/null
+++ b/util/hex-escape.c
@@ -0,0 +1,156 @@
+/* hex-escape.c -  Manage encoding and decoding of byte strings into path names
+ *
+ * Copyright (c) 2011 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <talloc.h>
+#include "error_util.h"
+#include "hex-escape.h"
+
+static const size_t default_buf_size = 1024;
+
+static const char *output_charset =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.:,";
+
+static const int escape_char = '%';
+
+static int
+is_output (char c)
+{
+    return (strchr (output_charset, c) != NULL);
+}
+
+static int
+maybe_realloc (void *ctx, size_t needed, char **out, size_t *out_size)
+{
+    if (*out_size < needed) {
+
+	if (*out == NULL)
+	    *out = talloc_size (ctx, needed);
+	else
+	    *out = talloc_realloc (ctx, *out, char, needed);
+
+	if (*out == NULL)
+	    return 0;
+
+	*out_size = needed;
+    }
+    return 1;
+}
+
+hex_status_t
+hex_encode (void *ctx, const char *in, char **out, size_t *out_size)
+{
+
+    const unsigned char *p;
+    char *q;
+
+    size_t escape_count = 0;
+    size_t len = 0;
+    size_t needed;
+
+    assert (ctx); assert (in); assert (out); assert (out_size);
+
+    for (p = (unsigned char *) in; *p; p++) {
+	escape_count += (!is_output (*p));
+	len++;
+    }
+
+    needed = len + escape_count * 2 + 1;
+
+    if (*out == NULL)
+	*out_size = 0;
+
+    if (!maybe_realloc (ctx, needed, out, out_size))
+	return HEX_OUT_OF_MEMORY;
+
+    q = *out;
+    p = (unsigned char *) in;
+
+    while (*p) {
+	if (is_output (*p)) {
+	    *q++ = *p++;
+	} else {
+	    sprintf (q, "%%%02x", *p++);
+	    q += 3;
+	}
+    }
+
+    *q = '\0';
+    return HEX_SUCCESS;
+}
+
+hex_status_t
+hex_decode (void *ctx, const char *in, char **out, size_t * out_size)
+{
+
+    char buf[3];
+
+    const char *p;
+    unsigned char *q;
+
+    size_t escape_count = 0;
+    size_t needed = 0;
+
+    assert (ctx); assert (in); assert (out); assert (out_size);
+
+    size_t len = strlen (in);
+
+    for (p = in; *p; p++)
+	escape_count += (*p == escape_char);
+
+    needed = len - escape_count * 2 + 1;
+
+    if (!maybe_realloc (ctx, needed, out, out_size))
+	return HEX_OUT_OF_MEMORY;
+
+    p = in;
+    q = (unsigned char *) *out;
+    buf[2] = 0;
+
+    while (*p) {
+
+	if (*p == escape_char) {
+
+	    char *endp;
+
+	    if (len < 3)
+		return HEX_SYNTAX_ERROR;
+
+	    buf[0] = p[1];
+	    buf[1] = p[2];
+
+	    *q = strtol (buf, &endp, 16);
+
+	    if (endp != buf + 2)
+		return HEX_SYNTAX_ERROR;
+
+	    len -= 3;
+	    p += 3;
+	    q++;
+	} else {
+	    *q++ = *p++;
+	}
+    }
+
+    *q = '\0';
+
+    return HEX_SUCCESS;
+}
diff --git a/util/hex-escape.h b/util/hex-escape.h
new file mode 100644
index 0000000..e409626
--- /dev/null
+++ b/util/hex-escape.h
@@ -0,0 +1,32 @@
+#ifndef _HEX_ESCAPE_H
+#define _HEX_ESCAPE_H
+
+typedef enum hex_status {
+    HEX_SUCCESS = 0,
+    HEX_SYNTAX_ERROR,
+    HEX_OUT_OF_MEMORY
+} hex_status_t;
+
+/*
+ * The API is modelled on that for getline.
+ *
+ * If 'out' points to a NULL pointer a char array of the appropriate
+ * size is allocated using talloc, and out_size is updated.
+ *
+ * If 'out' points to a non-NULL pointer, it assumed to describe an
+ * existing char array, with the size given in *out_size.  This array
+ * may be resized by talloc_realloc if needed; in this case *out_size
+ * will also be updated.
+ *
+ * Note that it is an error to pass a NULL pointer for any parameter
+ * of these routines.
+ */
+
+hex_status_t
+hex_encode (void *talloc_ctx, const char *in, char **out,
+            size_t *out_size);
+
+hex_status_t
+hex_decode (void *talloc_ctx, const char *in, char **out,
+            size_t *out_size);
+#endif
-- 
1.7.7.3

[PATCH v3 02/10] test/hex-xcode: new test binary

[David Bremner]

From: David Bremner <bremner@debian.org>

This program is used both as a test-bed/unit-tester for
../util/hex-escape.c, and also as a utility in future tests of dump
and restore.
---
 test/.gitignore     |    1 +
 test/Makefile.local |    6 ++-
 test/basic          |    2 +-
 test/hex-xcode.c    |  103 +++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 110 insertions(+), 2 deletions(-)
 create mode 100644 test/hex-xcode.c

diff --git a/test/.gitignore b/test/.gitignore
index e63c689..be7ab5e 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -3,4 +3,5 @@ corpus.mail
 smtp-dummy
 symbol-test
 arg-test
+hex-xcode
 tmp.*
diff --git a/test/Makefile.local b/test/Makefile.local
index fa2df73..ba697f4 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -13,6 +13,9 @@ smtp_dummy_modules = $(smtp_dummy_srcs:.c=.o)
 $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
 	$(call quiet,CC) -I. $^ -o $@
 
+$(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
+	$(call quiet,CC) -I. $^ -o $@ -ltalloc
+
 $(dir)/smtp-dummy: $(smtp_dummy_modules)
 	$(call quiet,CC) $^ -o $@
 
@@ -21,7 +24,8 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
 
 .PHONY: test check
 
-test-binaries: $(dir)/arg-test $(dir)/smtp-dummy $(dir)/symbol-test
+test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
+	 $(dir)/smtp-dummy $(dir)/symbol-test
 
 test:	all test-binaries
 	@${dir}/notmuch-test $(OPTIONS)
diff --git a/test/basic b/test/basic
index d6aed24..af57026 100755
--- a/test/basic
+++ b/test/basic
@@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
 eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
 tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
 available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
-    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test)$/d" | \
+    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
     sort)
 test_expect_equal "$tests_in_suite" "$available"
 
diff --git a/test/hex-xcode.c b/test/hex-xcode.c
new file mode 100644
index 0000000..eec6541
--- /dev/null
+++ b/test/hex-xcode.c
@@ -0,0 +1,103 @@
+/* No, nothing to to with IDE from Apple Inc.
+   testbed for ../util/hex-escape.c.
+
+   usage:
+   hex-xcode [--direction=(encode|decode)] [--omit-newline] < file
+   hex-xcode [--direction=(encode|decode)] [--omit-newline] arg1 arg2 arg3 ...
+
+ */
+
+#include "notmuch-client.h"
+#include "hex-escape.h"
+#include <assert.h>
+
+
+enum direction {
+    ENCODE,
+    DECODE
+};
+
+static int
+xcode (void *ctx, enum direction dir, char *in, char **buf_p, size_t *size_p)
+{
+    hex_status_t status;
+
+    if (dir == ENCODE)
+	status = hex_encode (ctx, in, buf_p, size_p);
+    else
+	status = hex_decode (ctx, in, buf_p, size_p);
+
+    if (status == HEX_SUCCESS)
+	fputs (*buf_p, stdout);
+
+    return status;
+}
+
+
+int
+main (int argc, char **argv)
+{
+
+
+    enum direction dir = DECODE;
+    int omit_newline = FALSE;
+
+    notmuch_opt_desc_t options[] = {
+	{ NOTMUCH_OPT_KEYWORD, &dir, "direction", 'd',
+	  (notmuch_keyword_t []){ { "encode", ENCODE },
+				  { "decode", DECODE },
+				  { 0, 0 } } },
+	{ NOTMUCH_OPT_BOOLEAN, &omit_newline, "omit-newline", 'n', 0 },
+	{ 0, 0, 0, 0, 0 }
+    };
+
+    int opt_index = parse_arguments (argc, argv, options, 1);
+
+    if (opt_index < 0)
+	exit (1);
+
+    void *ctx = talloc_new (NULL);
+
+    char *line = NULL;
+    size_t line_size;
+    ssize_t line_len;
+
+    char *buffer = NULL;
+    size_t buf_size = 0;
+
+    notmuch_bool_t read_stdin = TRUE;
+
+    for (; opt_index < argc; opt_index++) {
+
+	if (xcode (ctx, dir, argv[opt_index],
+		   &buffer, &buf_size) != HEX_SUCCESS)
+	    return 1;
+
+	if (!omit_newline)
+	    putchar ('\n');
+
+	read_stdin = FALSE;
+    }
+
+    if (!read_stdin)
+	return 0;
+
+    while ((line_len = getline (&line, &line_size, stdin)) != -1) {
+
+	chomp_newline (line);
+
+	if (xcode (ctx, dir, line, &buffer, &buf_size) != HEX_SUCCESS)
+	    return 1;
+
+	if (!omit_newline)
+	    putchar ('\n');
+
+    }
+
+    if (line)
+	free (line);
+
+    talloc_free (ctx);
+
+    return 0;
+}
-- 
1.7.7.3

[PATCH v3 03/10] test/hex-escaping: new test for hex escaping routines

[David Bremner]

From: David Bremner <bremner@debian.org>

These are more like unit tests, to (try to) make sure the library
functionality is working before building more complicated things on
top of it.
---
 test/hex-escaping |   26 ++++++++++++++++++++++++++
 test/notmuch-test |    1 +
 2 files changed, 27 insertions(+), 0 deletions(-)
 create mode 100755 test/hex-escaping

diff --git a/test/hex-escaping b/test/hex-escaping
new file mode 100755
index 0000000..f34cc8c
--- /dev/null
+++ b/test/hex-escaping
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+test_description="hex encoding and decoding"
+. ./test-lib.sh
+
+test_begin_subtest "round trip"
+find $TEST_DIRECTORY/corpus -type f -print | sort | xargs cat > EXPECTED
+$TEST_DIRECTORY/hex-xcode --direction=encode < EXPECTED | $TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "punctuation"
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+tag_enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag1")
+test_expect_equal "$tag_enc1" "comic_swear=%24%26%5e%25%24%5e%25%5c%5c%2f%2f-+%24%5e%25%24"
+
+test_begin_subtest "round trip newlines"
+printf 'this\n tag\t has\n spaces\n' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=encode  < EXPECTED.$test_count |\
+	$TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest "round trip 8bit chars"
+echo '%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=decode  < EXPECTED.$test_count |\
+	$TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index 6a99ae3..cbe2ef9 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -52,6 +52,7 @@ TESTS="
   python
   hooks
   argument-parsing
+  hex-escaping
 "
 TESTS=${NOTMUCH_TESTS:=$TESTS}
 
-- 
1.7.7.3

[PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup)

[David Bremner]

From: David Bremner <bremner@debian.org>

sup is the old format, and remains the default.

Each line of the notmuch format is "msg_id tag tag...tag" where each
space seperated token is 'hex-encoded' to remove troubling characters.
In particular this format won't have the same problem with e.g. spaces
in message-ids or tags; they will be round-trip-able.
---
 dump-restore-private.h |   12 ++++++++++++
 notmuch-dump.c         |   47 +++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 51 insertions(+), 8 deletions(-)
 create mode 100644 dump-restore-private.h

diff --git a/dump-restore-private.h b/dump-restore-private.h
new file mode 100644
index 0000000..34a5022
--- /dev/null
+++ b/dump-restore-private.h
@@ -0,0 +1,12 @@
+#ifndef DUMP_RESTORE_PRIVATE_H
+#define DUMP_RESTORE_PRIVATE_H
+
+#include "hex-escape.h"
+#include "command-line-arguments.h"
+
+typedef enum dump_formats {
+    DUMP_FORMAT_SUP,
+    DUMP_FORMAT_NOTMUCH
+} dump_format_t;
+
+#endif
diff --git a/notmuch-dump.c b/notmuch-dump.c
index a735875..0231db2 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -19,6 +19,7 @@
  */
 
 #include "notmuch-client.h"
+#include "dump-restore-private.h"
 
 int
 notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
@@ -44,9 +45,15 @@ notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
     char *output_file_name = NULL;
     int opt_index;
 
+    int output_format = DUMP_FORMAT_SUP;
+
     notmuch_opt_desc_t options[] = {
-	{ NOTMUCH_OPT_POSITION, &output_file_name, 0, 0, 0  },
-	{ 0, 0, 0, 0, 0 }
+	{ NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
+	  (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
+				  { "notmuch", DUMP_FORMAT_NOTMUCH },
+				  {0, 0} } },
+	{ NOTMUCH_OPT_POSITION, &output_file_name, 0, 0, 0 },
+	{ 0,		        0,	           0, 0, 0 }
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
@@ -85,29 +92,53 @@ notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
      */
     notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
 
+    char *buffer = NULL;
+    size_t buffer_size = 0;
+
     for (messages = notmuch_query_search_messages (query);
 	 notmuch_messages_valid (messages);
 	 notmuch_messages_move_to_next (messages))
     {
 	int first = 1;
-	message = notmuch_messages_get (messages);
+	const char *message_id;
 
-	fprintf (output,
-		 "%s (", notmuch_message_get_message_id (message));
+	message = notmuch_messages_get (messages);
+	message_id = notmuch_message_get_message_id (message);
+
+	if (output_format == DUMP_FORMAT_SUP) {
+	    fprintf (output, "%s (", message_id);
+	} else {
+	    if (hex_encode (notmuch, message_id,
+			    &buffer, &buffer_size) != HEX_SUCCESS)
+		return 1;
+	    fprintf (output, "%s ", buffer);
+	}
 
 	for (tags = notmuch_message_get_tags (message);
 	     notmuch_tags_valid (tags);
 	     notmuch_tags_move_to_next (tags))
 	{
+	    const char *tag_str = notmuch_tags_get (tags);
+
 	    if (! first)
-		fprintf (output, " ");
+		fputs (" ", output);
 
-	    fprintf (output, "%s", notmuch_tags_get (tags));
+	    if (output_format == DUMP_FORMAT_SUP) {
+		fputs (tag_str, output);
+	    } else {
+		if (hex_encode (notmuch, tag_str,
+				&buffer, &buffer_size) != HEX_SUCCESS)
+		    return 1;
 
+		fputs (buffer, output);
+	    }
 	    first = 0;
 	}
 
-	fprintf (output, ")\n");
+	if (output_format == DUMP_FORMAT_SUP)
+	    fputs (")\n", output);
+	else
+	    fputs ("\n", output);
 
 	notmuch_message_destroy (message);
     }
-- 
1.7.7.3

[PATCH v3 05/10] test: add test for dump --format=notmuch

[David Bremner]

From: David Bremner <bremner@debian.org>

The first test is really to test our assumptions about the corpus,
namely that a certain set of message-id's is safe (i.e. doesn't change
under hex-escaping).  We then check dump output as best we can without
functionality-to-come in notmuch-restore.
---
 test/dump-restore |   12 ++++++++++++
 1 files changed, 12 insertions(+), 0 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index 439e998..0e210d3 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -82,4 +82,16 @@ test_begin_subtest "dump outfile -- from:cworth"
 notmuch dump dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
 
+test_begin_subtest "Check for a safe set of message-ids"
+notmuch search --output=messages from:cworth > EXPECTED.$test_count
+notmuch search --output=messages from:cworth |\
+	$TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
+# we have observed that cworth has sane message-ids, and hopefully sane tags.
+test_begin_subtest "dump --format=notmuch -- from:cworth"
+notmuch dump --format=sup -- from:cworth | tr -d \(\) > EXPECTED.$test_count
+notmuch dump --format=notmuch -- from:cworth > OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
 test_done
-- 
1.7.7.3

[PATCH v3 06/10] notmuch-restore: add 'notmuch format' support, auto detect

[David Bremner]

From: David Bremner <bremner@debian.org>

This is format is whitespace separated tokens, encoded by
util/hex-escape.c

The format detection heuristic relies on the fact that '(' is not part
of the character set used by hex-escape. Since hex-escape is designed
to be OK for pathnames (and shells), this seems like a reasonable
assumption.

In principle the --format argument to notmuch-restore is notmuch
needed at this point, but it adds literally 5 lines of argument
description, so I left it.
---
 dump-restore-private.h |    5 +-
 notmuch-restore.c      |  111 ++++++++++++++++++++++++++++++++++++++---------
 2 files changed, 92 insertions(+), 24 deletions(-)

diff --git a/dump-restore-private.h b/dump-restore-private.h
index 34a5022..67795e5 100644
--- a/dump-restore-private.h
+++ b/dump-restore-private.h
@@ -5,8 +5,9 @@
 #include "command-line-arguments.h"
 
 typedef enum dump_formats {
-    DUMP_FORMAT_SUP,
-    DUMP_FORMAT_NOTMUCH
+    DUMP_FORMAT_AUTO,
+    DUMP_FORMAT_NOTMUCH,
+    DUMP_FORMAT_SUP
 } dump_format_t;
 
 #endif
diff --git a/notmuch-restore.c b/notmuch-restore.c
index 87d9772..3fdfecc 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -19,6 +19,7 @@
  */
 
 #include "notmuch-client.h"
+#include "dump-restore-private.h"
 
 int
 notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
@@ -35,6 +36,7 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
     regex_t regex;
     int rerr;
     int opt_index;
+    int input_format = DUMP_FORMAT_AUTO;
 
     config = notmuch_config_open (ctx, NULL, NULL);
     if (config == NULL)
@@ -48,6 +50,11 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
     synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config);
 
     notmuch_opt_desc_t options[] = {
+	{ NOTMUCH_OPT_KEYWORD, &input_format, "format", 'f',
+	  (notmuch_keyword_t []){ { "auto", DUMP_FORMAT_AUTO },
+				  { "notmuch", DUMP_FORMAT_NOTMUCH },
+				  { "sup", DUMP_FORMAT_SUP },
+				  {0, 0} } },
 	{ NOTMUCH_OPT_POSITION, &input_file_name, 0, 0, 0 },
 	{ NOTMUCH_OPT_BOOLEAN,  &accumulate, "accumulate", 'a', 0 },
 	{ 0, 0, 0, 0, 0 }
@@ -77,37 +84,85 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 	return 1;
     }
 
-    /* Dump output is one line per message. We match a sequence of
-     * non-space characters for the message-id, then one or more
-     * spaces, then a list of space-separated tags as a sequence of
-     * characters within literal '(' and ')'. */
-    if ( xregcomp (&regex,
-		   "^([^ ]+) \\(([^)]*)\\)$",
-		   REG_EXTENDED) )
-	INTERNAL_ERROR("compile time constant regex failed.");
+
+    /* These are out here to re-use the buffers with hex_decode */
+
+    char *message_id = NULL;
+    size_t message_id_size = 0;
+    char *tag = NULL;
+    size_t tag_size = 0;
+    notmuch_bool_t first_line = TRUE;
 
     while ((line_len = getline (&line, &line_size, input)) != -1) {
 	regmatch_t match[3];
-	char *message_id, *file_tags, *tag, *next;
+	char  *file_tags, *next;
 	notmuch_message_t *message = NULL;
+
 	notmuch_status_t status;
 	notmuch_tags_t *db_tags;
 	char *db_tags_str;
 
 	chomp_newline (line);
+	if (first_line && input_format == DUMP_FORMAT_AUTO) {
+	    char *p;
 
-	rerr = xregexec (&regex, line, 3, match, 0);
-	if (rerr == REG_NOMATCH)
-	{
-	    fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
-		     line);
+	    for (p = line; *p; p++) {
+		if (*p == '(')
+		    input_format = DUMP_FORMAT_SUP;
+	    }
+
+	    if (input_format == DUMP_FORMAT_AUTO)
+		input_format = DUMP_FORMAT_NOTMUCH;
+
+	}
+
+	/* sup dump output is one line per message. We match a
+	 * sequence of non-space characters for the message-id, then
+	 * one or more spaces, then a list of space-separated tags as
+	 * a sequence of characters within literal '(' and ')'. */
+	if (first_line && input_format == DUMP_FORMAT_SUP) {
+	    if ( xregcomp (&regex,
+			   "^([^ ]+) \\(([^)]*)\\)$",
+			   REG_EXTENDED) )
+		INTERNAL_ERROR("compile time constant regex failed.");
+	}
+
+
+	/* Silently ignore blank lines */
+
+	if (line[0] == '\0') {
 	    continue;
 	}
 
-	message_id = xstrndup (line + match[1].rm_so,
-			       match[1].rm_eo - match[1].rm_so);
-	file_tags = xstrndup (line + match[2].rm_so,
-			      match[2].rm_eo - match[2].rm_so);
+	if (input_format == DUMP_FORMAT_SUP) {
+	    rerr = xregexec (&regex, line, 3, match, 0);
+	    if (rerr == REG_NOMATCH)
+	    {
+		fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
+			 line);
+		continue;
+	    }
+	    message_id = talloc_strndup (notmuch, line + match[1].rm_so,
+					 match[1].rm_eo - match[1].rm_so);
+	    file_tags = talloc_strndup (notmuch, line + match[2].rm_so,
+					match[2].rm_eo - match[2].rm_so);
+	} else {
+	    char *p = line;
+	    char *raw_mid;
+
+	    raw_mid = strsep (&p, " \t");
+
+	    if (hex_decode (notmuch, raw_mid,
+			    &message_id, &message_id_size) != HEX_SUCCESS)
+		return 1;
+
+	    if (p)
+		file_tags = xstrdup (p);
+	    else
+		file_tags = NULL;
+	}
+
+	first_line = FALSE;
 
 	status = notmuch_database_find_message (notmuch, message_id, &message);
 	if (status || message == NULL) {
@@ -153,7 +208,16 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 
 	next = file_tags;
 	while (next) {
-	    tag = strsep (&next, " ");
+	    char *raw_tag = strsep (&next, " ");
+
+	    if (input_format == DUMP_FORMAT_NOTMUCH) {
+		if (hex_decode (notmuch, raw_tag,
+				&tag, &tag_size) != HEX_SUCCESS)
+		    return 1;
+	    } else {
+		tag = talloc_strdup (notmuch, raw_tag);
+	    }
+
 	    if (*tag == '\0')
 		continue;
 	    status = notmuch_message_add_tag (message, tag);
@@ -175,11 +239,14 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
 	if (message)
 	    notmuch_message_destroy (message);
 	message = NULL;
-	free (message_id);
-	free (file_tags);
+	if (input_format == DUMP_FORMAT_SUP) {
+	    talloc_free (message_id);
+	    talloc_free (file_tags);
+	}
     }
 
-    regfree (&regex);
+    if (input_format == DUMP_FORMAT_SUP)
+	regfree (&regex);
 
     if (line)
 	free (line);
-- 
1.7.7.3

[PATCH v3 07/10] test: second set of dump/restore --format=notmuch tests

[David Bremner]

From: David Bremner <bremner@debian.org>

These one need the completed functionality in notmuch-restore. Fairly
exotic tags are tested, but no weird message id's.

We test each possible input to autodetection, both explicit (with
--format=auto) and implicit (without --format).
---
 test/dump-restore |   66 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 66 insertions(+), 0 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index 0e210d3..60ae68e 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -94,4 +94,70 @@ notmuch dump --format=sup -- from:cworth | tr -d \(\) > EXPECTED.$test_count
 notmuch dump --format=notmuch -- from:cworth > OUTPUT.$test_count
 test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
 
+test_begin_subtest "format=notmuch, # round-trip"
+notmuch dump --format=sup | sort > EXPECTED.$test_count
+notmuch dump --format=notmuch | notmuch restore --format=notmuch
+notmuch dump --format=sup | sort > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag1")
+
+tag2=$(printf 'this\n tag\t has\n spaces')
+enc2=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag2")
+
+enc3='%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a'
+tag3=$($TEST_DIRECTORY/hex-xcode --direction=decode $enc3)
+
+notmuch dump --format=notmuch > BACKUP
+
+notmuch tag +"$tag1" +"$tag2" +"$tag3" -inbox -unread "*"
+
+test_begin_subtest 'format=notmuch, round trip with strange tags'
+    notmuch dump --format=notmuch > EXPECTED.$test_count
+    notmuch dump --format=notmuch | notmuch restore --format=notmuch
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, checking encoded output'
+    cp /dev/null EXPECTED.$test_count
+    notmuch dump --format=notmuch -- from:cworth |\
+	 awk "{ print \$1 \" $enc1 $enc2 $enc3\" }" > EXPECTED.$test_count
+
+    notmuch dump --format=notmuch -- from:cworth  > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'restoring sane tags'
+    notmuch restore --format=notmuch < BACKUP
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file BACKUP OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, restore=auto'
+    notmuch dump --format=notmuch > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore --format=auto < EXPECTED.$test_count
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=auto'
+    notmuch dump --format=sup > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore --format=auto < EXPECTED.$test_count
+    notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, restore=default'
+    notmuch dump --format=notmuch > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore < EXPECTED.$test_count
+    notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=default'
+    notmuch dump --format=sup > EXPECTED.$test_count
+    notmuch tag -inbox -unread "*"
+    notmuch restore < EXPECTED.$test_count
+    notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
 test_done
-- 
1.7.7.3

[PATCH v3 08/10] notmuch-{dump, restore}.1: document new format options

[David Bremner]

From: David Bremner <bremner@debian.org>

More or less arbitrarily, notmuch-dump.1 gets the more detailed
description of the format.
---
 man/man1/notmuch-dump.1    |   59 ++++++++++++++++++++++++++++++++++++++++++-
 man/man1/notmuch-restore.1 |   60 ++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 111 insertions(+), 8 deletions(-)

diff --git a/man/man1/notmuch-dump.1 b/man/man1/notmuch-dump.1
index 9ccf35d..f4f7964 100644
--- a/man/man1/notmuch-dump.1
+++ b/man/man1/notmuch-dump.1
@@ -5,7 +5,8 @@ notmuch-dump \- Creates a plain-text dump of the tags of each message.
 .SH SYNOPSIS
 
 .B "notmuch dump"
-.RI "[ <" filename "> ] [--]"
+.RI "[ <" filename "> ]"
+.RB  [ "\-\-format=(sup|notmuch)"  "] [--]"
 .RI "[ <" search-term ">...]"
 
 .SH DESCRIPTION
@@ -20,6 +21,62 @@ recreated from the messages themselves.  The output of notmuch dump is
 therefore the only critical thing to backup (and much more friendly to
 incremental backup than the native database files.)
 
+.TP 4
+.B \-\-format=(sup|notmuch)
+
+Notmuch restore supports two plain text dump formats, both with one message-id
+per line, followed by a list of tags.
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
+compatible with the format of files produced by sup-dump.
+So if you've previously been using sup for mail, then the
+.B "notmuch restore"
+command provides you a way to import all of your tags (or labels as
+sup calls them).
+Each line has the following form
+
+.RS 4
+.RI < message-id >
+.B (
+.RI < tag "> ..."
+.B )
+
+with zero or more tags are separated by spaces. Note that (malformed)
+message-ids may contained arbitrary non-null characters. Note also
+that tags with spaces will not be correctly restored with this format.
+
+.RE
+
+.RE
+.RS 4
+.TP 4
+.B notmuch
+
+The
+.B notmuch
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters.
+Each line has the form
+
+.RS 4
+.RI < encoded-message-id "> <" encoded-tag "> ..."
+
+where encoded means that every byte not matching the regex
+.B [A-Za-z0-9+-_@=.:,]
+is replace by
+.B %nn
+where nn is the two digit hex encoding.
+
+
+.RE
+
+
 With no search terms, a dump of all messages in the database will be
 generated.  A "--" argument instructs notmuch that the
 remaining arguments are search terms.
diff --git a/man/man1/notmuch-restore.1 b/man/man1/notmuch-restore.1
index 2191df0..3fb0e99 100644
--- a/man/man1/notmuch-restore.1
+++ b/man/man1/notmuch-restore.1
@@ -5,7 +5,7 @@ notmuch-restore \- Restores the tags from the given file (see notmuch dump).
 .SH SYNOPSIS
 
 .B "notmuch restore"
-.RB [ "--accumulate" ]
+.RI  [  options "...]"
 .RI "[ <" filename "> ]"
 
 .SH DESCRIPTION
@@ -15,21 +15,67 @@ Restores the tags from the given file (see
 
 The input is read from the given filename, if any, or from stdin.
 
-Note: The dump file format is specifically chosen to be
+
+Supported options for
+.B restore
+include
+.RS 4
+.TP 4
+.B \-\-accumulate
+
+The union of the existing and new tags is applied, instead of
+replacing each message's tags as they are read in from the dump file.
+
+.RE
+.RS 4
+.TP 4
+.B \-\-format=(sup|notmuch|auto)
+
+Notmuch restore supports two plain text dump formats, with one message-id
+per line, followed by a list of tags.
+For details of the actual formats, see \fBnotmuch-dump\fR(1).
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
 compatible with the format of files produced by sup-dump.
 So if you've previously been using sup for mail, then the
 .B "notmuch restore"
 command provides you a way to import all of your tags (or labels as
 sup calls them).
 
-The --accumulate switch causes the union of the existing and new tags to be
-applied, instead of replacing each message's tags as they are read in from the
-dump file.
+.RE
+.RS 4
+.TP 4
+.B notmuch
 
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for <search-terms>.
+The
+.B notmuch
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters.  This
+format hex-escapes all characters those outside of a small character
+set, intended to be suitable for e.g. pathnames in most UNIX-like
+systems.
 
 .RE
+
+.RS 4
+.TP 4
+.B auto
+
+This option (the default) tries to guess the format from the
+input. For correctly formed input in either supported format, this
+heuristic, based the fact that notmuch format contains no parentheses,
+should be accurate.
+
+.RE
+
+.RE
+
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.7.3

[PATCH v3 09/10] random-dump.c: new test-binary to generate dump files

[David Bremner]

From: David Bremner <bremner@debian.org>

This binary creates a "torture test" dump file for the new dump
format.
---
 test/Makefile.local |    4 ++
 test/basic          |    2 +-
 test/random-dump.c  |  144 +++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 149 insertions(+), 1 deletions(-)
 create mode 100644 test/random-dump.c

diff --git a/test/Makefile.local b/test/Makefile.local
index ba697f4..b59f837 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
 $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
 	$(call quiet,CC) -I. $^ -o $@ -ltalloc
 
+$(dir)/random-dump:  $(dir)/random-dump.o command-line-arguments.o util/libutil.a
+	$(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
+
 $(dir)/smtp-dummy: $(smtp_dummy_modules)
 	$(call quiet,CC) $^ -o $@
 
@@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
 .PHONY: test check
 
 test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
+	$(dir)/random-dump \
 	 $(dir)/smtp-dummy $(dir)/symbol-test
 
 test:	all test-binaries
diff --git a/test/basic b/test/basic
index af57026..e3a6cef 100755
--- a/test/basic
+++ b/test/basic
@@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
 eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
 tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
 available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
-    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
+    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d" | \
     sort)
 test_expect_equal "$tests_in_suite" "$available"
 
diff --git a/test/random-dump.c b/test/random-dump.c
new file mode 100644
index 0000000..1949425
--- /dev/null
+++ b/test/random-dump.c
@@ -0,0 +1,144 @@
+/*
+   Generate a random dump file in 'notmuch' format.
+   Generated message-id's and tags are intentionally nasty.
+
+   We restrict ourselves to 7 bit message-ids, because generating
+   random valid UTF-8 seems like work. And invalid UTF-8 can't be
+   round-tripped via Xapian.
+
+ */
+
+#include <stdlib.h>
+#include <assert.h>
+#include <talloc.h>
+#include <string.h>
+#include "math.h"
+#include "hex-escape.h"
+#include "command-line-arguments.h"
+
+static void
+hex_out (void *ctx, char *buf)
+{
+    static char *encoded_buf = NULL;
+    static size_t encoded_buf_size = 0;
+
+    if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) != HEX_SUCCESS) {
+	fprintf (stderr, "Hex encoding failed");
+	exit (1);
+    }
+
+    fputs (encoded_buf, stdout);
+}
+
+static void
+random_chars (char *buf, int from, int stop, int max_char,
+	      const char *blacklist)
+{
+    int i;
+
+    for (i = from; i < stop; i++) {
+	do {
+	    buf[i] = ' ' + (random () % (max_char - ' '));
+	} while (blacklist && strchr (blacklist, buf[i]));
+    }
+}
+
+static void
+random_tag (void *ctx, size_t len)
+{
+    static char *buf = NULL;
+    static size_t buf_len = 0;
+
+    int use = (random () % (len - 1)) + 1;
+
+    if (len > buf_len) {
+	buf = talloc_realloc (ctx, buf, char, len);
+	buf_len = len;
+    }
+
+    random_chars (buf, 0, use, 255, NULL);
+
+    buf[use] = '\0';
+
+    hex_out (ctx, buf);
+}
+
+static void
+random_message_id (void *ctx, size_t len)
+{
+    static char *buf = NULL;
+    static size_t buf_len = 0;
+
+    int lhs_len = (random () % (len / 2 - 1)) + 1;
+
+    int rhs_len = (random () % len / 2) + 1;
+
+    const char *blacklist = "\n\r@<>[]()";
+
+    if (len > buf_len) {
+	buf = talloc_realloc (ctx, buf, char, len);
+	buf_len = len;
+    }
+
+    random_chars (buf, 0, lhs_len, 127, blacklist);
+
+    buf[lhs_len] = '@';
+
+    random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127, blacklist);
+
+    hex_out (ctx, buf);
+}
+
+int
+main (int argc, char **argv)
+{
+
+    void *ctx = talloc_new (NULL);
+    int num_lines = 500;
+    int max_tags = 10;
+    int message_id_len = 100;
+    int tag_len = 50;
+    int seed = 734569;
+
+    int pad_tag = 0, pad_mid = 0;
+
+    notmuch_opt_desc_t options[] = {
+	{ NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
+	{ NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
+	{ NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
+	{ NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
+	{ NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
+	{ 0, 0, 0, 0, 0 }
+    };
+
+    int opt_index = parse_arguments (argc, argv, options, 1);
+
+    if (opt_index < 0)
+	exit (1);
+
+    pad_mid = ((int) log10 (num_lines) + 1);
+    pad_tag = ((int) log10 (max_tags)) + 1;
+
+    srandom (seed);
+
+    int line;
+    for (line = 0; line < num_lines; line++) {
+
+	printf ("%0*d-", pad_mid, line);
+
+	random_message_id (ctx, message_id_len);
+
+	int num_tags = random () % (max_tags + 1);
+
+	int j;
+	for (j = 0; j < num_tags; j++) {
+	    printf (" %0*d-", pad_tag, j);
+	    random_tag (ctx, tag_len);
+	}
+	putchar ('\n');
+    }
+
+    talloc_free (ctx);
+
+    return 0;
+}
-- 
1.7.7.3

[PATCH v3 10/10] test: new random message-id and tags dump/restore test.

[David Bremner]

From: David Bremner <bremner@debian.org>

The idea is to generate a random dump file, and then generate minimal
mail messages from that.

Currently this relies on the behaviour of the gmime message-id parser
that if it cannot find a leading '<' in the Message-Id field, it just
copies the string without attempting to parse it. The alternative is
to settle for some weaker notion of round-tripping in this test.
---
 test/dump-restore |   25 +++++++++++++++++++++++++
 1 files changed, 25 insertions(+), 0 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index 60ae68e..c354a44 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -160,4 +160,29 @@ test_begin_subtest 'format=sup, restore=default'
     notmuch dump --format=sup > OUTPUT.$test_count
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
+test_begin_subtest 'random message-ids and tags'
+
+     count=0
+
+    ${TEST_DIRECTORY}/random-dump sort > EXPECTED.$test_count
+
+    while read id tags ; do
+	file=${MAIL_DIR}/cur/dump-$count:2,
+	count=$((count + 1))
+	printf 'From: dump-test@example.com\n' > ${file}
+	printf 'To: example@example.net\n' > ${file}
+	printf 'Subject: StrangeMessageIDsAndTags\n' > ${file}
+	# the missing angle-braces are intentional, since they trigger
+	# gmime to pass the message-id through unparsed.
+	printf 'Message-Id: ' >> $file
+	${TEST_DIRECTORY}/hex-xcode --direction=decode $id >> $file
+    done < EXPECTED.$test_count
+
+    notmuch new
+    notmuch restore < EXPECTED.$test_count
+    notmuch dump --format=notmuch -- subject:StrangeMessageIDsAndTags | \
+	sed 's/ *$//' | sort > OUTPUT.$test_count
+
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
 test_done
-- 
1.7.7.3

Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files

[Jani Nikula]

[-- Attachment #1: Type: text/plain, Size: 5990 bytes --]

On Jan 15, 2012 3:41 AM, "David Bremner" <david@tethera.net> wrote:
>
> From: David Bremner <bremner@debian.org>
>
> This binary creates a "torture test" dump file for the new dump
> format.
> ---
>  test/Makefile.local |    4 ++
>  test/basic          |    2 +-
>  test/random-dump.c  |  144
+++++++++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 149 insertions(+), 1 deletions(-)
>  create mode 100644 test/random-dump.c
>
> diff --git a/test/Makefile.local b/test/Makefile.local
> index ba697f4..b59f837 100644
> --- a/test/Makefile.local
> +++ b/test/Makefile.local
> @@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o
command-line-arguments.o util/libutil.a
>  $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o
util/libutil.a
>        $(call quiet,CC) -I. $^ -o $@ -ltalloc
>
> +$(dir)/random-dump:  $(dir)/random-dump.o command-line-arguments.o
util/libutil.a
> +       $(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
> +
>  $(dir)/smtp-dummy: $(smtp_dummy_modules)
>        $(call quiet,CC) $^ -o $@
>
> @@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
>  .PHONY: test check
>
>  test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
> +       $(dir)/random-dump \
>         $(dir)/smtp-dummy $(dir)/symbol-test
>
>  test:  all test-binaries
> diff --git a/test/basic b/test/basic
> index af57026..e3a6cef 100755
> --- a/test/basic
> +++ b/test/basic
> @@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests
will be run by notmuch-test'
>  eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
>  tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
>  available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable
-printf '%f\n' | \
> -    sed -r -e
"/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d"
| \
> +    sed -r -e
"/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d"
| \
>     sort)
>  test_expect_equal "$tests_in_suite" "$available"
>
> diff --git a/test/random-dump.c b/test/random-dump.c
> new file mode 100644
> index 0000000..1949425
> --- /dev/null
> +++ b/test/random-dump.c
> @@ -0,0 +1,144 @@
> +/*
> +   Generate a random dump file in 'notmuch' format.
> +   Generated message-id's and tags are intentionally nasty.
> +
> +   We restrict ourselves to 7 bit message-ids, because generating
> +   random valid UTF-8 seems like work. And invalid UTF-8 can't be
> +   round-tripped via Xapian.
> +
> + */
> +
> +#include <stdlib.h>
> +#include <assert.h>
> +#include <talloc.h>
> +#include <string.h>
> +#include "math.h"
> +#include "hex-escape.h"
> +#include "command-line-arguments.h"
> +
> +static void
> +hex_out (void *ctx, char *buf)
> +{
> +    static char *encoded_buf = NULL;
> +    static size_t encoded_buf_size = 0;
> +
> +    if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) !=
HEX_SUCCESS) {
> +       fprintf (stderr, "Hex encoding failed");
> +       exit (1);
> +    }
> +
> +    fputs (encoded_buf, stdout);
> +}
> +
> +static void
> +random_chars (char *buf, int from, int stop, int max_char,
> +             const char *blacklist)
> +{
> +    int i;
> +
> +    for (i = from; i < stop; i++) {
> +       do {
> +           buf[i] = ' ' + (random () % (max_char - ' '));
> +       } while (blacklist && strchr (blacklist, buf[i]));
> +    }
> +}
> +
> +static void
> +random_tag (void *ctx, size_t len)
> +{
> +    static char *buf = NULL;
> +    static size_t buf_len = 0;
> +
> +    int use = (random () % (len - 1)) + 1;
> +
> +    if (len > buf_len) {
> +       buf = talloc_realloc (ctx, buf, char, len);
> +       buf_len = len;
> +    }
> +
> +    random_chars (buf, 0, use, 255, NULL);
> +
> +    buf[use] = '\0';
> +
> +    hex_out (ctx, buf);
> +}
> +
> +static void
> +random_message_id (void *ctx, size_t len)
> +{
> +    static char *buf = NULL;
> +    static size_t buf_len = 0;
> +
> +    int lhs_len = (random () % (len / 2 - 1)) + 1;
> +
> +    int rhs_len = (random () % len / 2) + 1;
> +
> +    const char *blacklist = "\n\r@<>[]()";
> +
> +    if (len > buf_len) {
> +       buf = talloc_realloc (ctx, buf, char, len);
> +       buf_len = len;
> +    }
> +
> +    random_chars (buf, 0, lhs_len, 127, blacklist);
> +
> +    buf[lhs_len] = '@';
> +
> +    random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127,
blacklist);
> +
> +    hex_out (ctx, buf);
> +}
> +
> +int
> +main (int argc, char **argv)
> +{
> +
> +    void *ctx = talloc_new (NULL);
> +    int num_lines = 500;
> +    int max_tags = 10;
> +    int message_id_len = 100;
> +    int tag_len = 50;
> +    int seed = 734569;
> +
> +    int pad_tag = 0, pad_mid = 0;
> +
> +    notmuch_opt_desc_t options[] = {
> +       { NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
> +       { NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
> +       { NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
> +       { NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
> +       { NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },

s/tag-len/seed/

> +       { 0, 0, 0, 0, 0 }
> +    };
> +
> +    int opt_index = parse_arguments (argc, argv, options, 1);
> +
> +    if (opt_index < 0)
> +       exit (1);
> +
> +    pad_mid = ((int) log10 (num_lines) + 1);
> +    pad_tag = ((int) log10 (max_tags)) + 1;
> +
> +    srandom (seed);
> +
> +    int line;
> +    for (line = 0; line < num_lines; line++) {
> +
> +       printf ("%0*d-", pad_mid, line);
> +
> +       random_message_id (ctx, message_id_len);
> +
> +       int num_tags = random () % (max_tags + 1);
> +
> +       int j;
> +       for (j = 0; j < num_tags; j++) {
> +           printf (" %0*d-", pad_tag, j);
> +           random_tag (ctx, tag_len);
> +       }
> +       putchar ('\n');
> +    }
> +
> +    talloc_free (ctx);
> +
> +    return 0;
> +}
> --
> 1.7.7.3
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> http://notmuchmail.org/mailman/listinfo/notmuch

[-- Attachment #2: Type: text/html, Size: 8488 bytes --]

Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files

[David Bremner]

On Sun, 15 Jan 2012 10:05:15 +0200, Jani Nikula <jani@nikula.org> wrote:
> On Jan 15, 2012 3:41 AM, "David Bremner" <david@tethera.net> wrote:
> >
> > From: David Bremner <bremner@debian.org>
> >
> > +    notmuch_opt_desc_t options[] = {
> > +       { NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
> > +       { NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
> > +       { NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
> > +       { NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
> > +       { NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
> 
> s/tag-len/seed/
> 

Fixed, thanks.

d

Re: [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup)

[Austin Clements]

Quoth David Bremner on Jan 14 at  9:40 pm:
> From: David Bremner <bremner@debian.org>
> 
> sup is the old format, and remains the default.
> 
> Each line of the notmuch format is "msg_id tag tag...tag" where each
> space seperated token is 'hex-encoded' to remove troubling characters.
> In particular this format won't have the same problem with e.g. spaces
> in message-ids or tags; they will be round-trip-able.

We definitely need a round-trip-able dump format.  Did you consider
using JSON to allow for future flexibility (e.g., expansion of what we
store in the database) and so we don't have to invent our own
encodings?  A JSON format wouldn't necessarily be a reason *not* to
also have this format, especially considering how
shell-script-friendly this is (versus how shell-script-unfriendly JSON
is), I'm just curious what trade-offs you're considering.

You might want to call this format something more self-descriptive
like "text" or "hextext" or something in case we do want to expand in
the future.  "sup" is probably fine for the legacy format since that's
set in stone at this point.

Re: [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup)

[David Bremner]

On Sun, 15 Jan 2012 15:35:11 -0500, Austin Clements <amdragon@MIT.EDU> wrote:

> We definitely need a round-trip-able dump format.  Did you consider
> using JSON to allow for future flexibility (e.g., expansion of what we
> store in the database) and so we don't have to invent our own
> encodings?  A JSON format wouldn't necessarily be a reason *not* to
> also have this format, especially considering how
> shell-script-friendly this is (versus how shell-script-unfriendly JSON
> is), I'm just curious what trade-offs you're considering.

I was looking for something fairly close to what we have, to allow
people to migrate their various scripts (e.g. nmbug) to the new format
without too much pain.  Maybe some small amount of header information at
the start of the file would support extensibility, while still being
shell script friendly.

I'm also not too sure how much overhead the JSON quoting would
induce. My tags file is currently about 10M, and on my old laptop takes
about 15s to dump. That's a long 15s when I'm trying to sync my mail.
For "normal" backup use, a little more overhead doesn't matter, although
the stories of non-linear slowdowns that people report suggest we
shouldn't get too cavalier about that.

> You might want to call this format something more self-descriptive
> like "text" or "hextext" or something in case we do want to expand in
> the future.  "sup" is probably fine for the legacy format since that's
> set in stone at this point.

yeah, I'm definitely open to better suggestions for a name

Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files

[Mark Walters]

On Sat, 14 Jan 2012 21:40:23 -0400, David Bremner <david@tethera.net> wrote:
> From: David Bremner <bremner@debian.org>
> 
> This binary creates a "torture test" dump file for the new dump
> format.
> ---
>  test/Makefile.local |    4 ++
>  test/basic          |    2 +-
>  test/random-dump.c  |  144 +++++++++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 149 insertions(+), 1 deletions(-)
>  create mode 100644 test/random-dump.c
> 
> diff --git a/test/Makefile.local b/test/Makefile.local
> index ba697f4..b59f837 100644
> --- a/test/Makefile.local
> +++ b/test/Makefile.local
> @@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
>  $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
>  	$(call quiet,CC) -I. $^ -o $@ -ltalloc
>  
> +$(dir)/random-dump:  $(dir)/random-dump.o command-line-arguments.o util/libutil.a
> +	$(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
> +
>  $(dir)/smtp-dummy: $(smtp_dummy_modules)
>  	$(call quiet,CC) $^ -o $@
>  
> @@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
>  .PHONY: test check
>  
>  test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
> +	$(dir)/random-dump \
>  	 $(dir)/smtp-dummy $(dir)/symbol-test
>  
>  test:	all test-binaries
> diff --git a/test/basic b/test/basic
> index af57026..e3a6cef 100755
> --- a/test/basic
> +++ b/test/basic
> @@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
>  eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
>  tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
>  available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
> -    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
> +    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d" | \
>      sort)
>  test_expect_equal "$tests_in_suite" "$available"
>  
> diff --git a/test/random-dump.c b/test/random-dump.c
> new file mode 100644
> index 0000000..1949425
> --- /dev/null
> +++ b/test/random-dump.c
> @@ -0,0 +1,144 @@
> +/*
> +   Generate a random dump file in 'notmuch' format.
> +   Generated message-id's and tags are intentionally nasty.
> +
> +   We restrict ourselves to 7 bit message-ids, because generating
> +   random valid UTF-8 seems like work. And invalid UTF-8 can't be
> +   round-tripped via Xapian.
> +
> + */
> +
> +#include <stdlib.h>
> +#include <assert.h>
> +#include <talloc.h>
> +#include <string.h>
> +#include "math.h"
> +#include "hex-escape.h"
> +#include "command-line-arguments.h"
> +
> +static void
> +hex_out (void *ctx, char *buf)
> +{
> +    static char *encoded_buf = NULL;
> +    static size_t encoded_buf_size = 0;
> +
> +    if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) != HEX_SUCCESS) {
> +	fprintf (stderr, "Hex encoding failed");
> +	exit (1);
> +    }
> +
> +    fputs (encoded_buf, stdout);
> +}
> +
> +static void
> +random_chars (char *buf, int from, int stop, int max_char,
> +	      const char *blacklist)
> +{
> +    int i;
> +
> +    for (i = from; i < stop; i++) {
> +	do {
> +	    buf[i] = ' ' + (random () % (max_char - ' '));
> +	} while (blacklist && strchr (blacklist, buf[i]));
> +    }
> +}
> +
> +static void
> +random_tag (void *ctx, size_t len)
> +{
> +    static char *buf = NULL;
> +    static size_t buf_len = 0;
> +
> +    int use = (random () % (len - 1)) + 1;
> +
> +    if (len > buf_len) {
> +	buf = talloc_realloc (ctx, buf, char, len);
> +	buf_len = len;
> +    }
> +
> +    random_chars (buf, 0, use, 255, NULL);
> +
> +    buf[use] = '\0';
> +
> +    hex_out (ctx, buf);
> +}
> +
> +static void
> +random_message_id (void *ctx, size_t len)
> +{
> +    static char *buf = NULL;
> +    static size_t buf_len = 0;
> +
> +    int lhs_len = (random () % (len / 2 - 1)) + 1;
> +
> +    int rhs_len = (random () % len / 2) + 1;
> +
> +    const char *blacklist = "\n\r@<>[]()";
> +
> +    if (len > buf_len) {
> +	buf = talloc_realloc (ctx, buf, char, len);
> +	buf_len = len;
> +    }
> +
> +    random_chars (buf, 0, lhs_len, 127, blacklist);
> +
> +    buf[lhs_len] = '@';
> +
> +    random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127, blacklist);
> +
> +    hex_out (ctx, buf);
> +}
> +
> +int
> +main (int argc, char **argv)
> +{
> +
> +    void *ctx = talloc_new (NULL);
> +    int num_lines = 500;
> +    int max_tags = 10;
> +    int message_id_len = 100;
> +    int tag_len = 50;
> +    int seed = 734569;
> +
> +    int pad_tag = 0, pad_mid = 0;
> +
> +    notmuch_opt_desc_t options[] = {
> +	{ NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
> +	{ NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
> +	{ NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
> +	{ NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
> +	{ NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
> +	{ 0, 0, 0, 0, 0 }
> +    };
> +
> +    int opt_index = parse_arguments (argc, argv, options, 1);
> +
> +    if (opt_index < 0)
> +	exit (1);
> +
> +    pad_mid = ((int) log10 (num_lines) + 1);
> +    pad_tag = ((int) log10 (max_tags)) + 1;
> +
> +    srandom (seed);
> +
> +    int line;
> +    for (line = 0; line < num_lines; line++) {
> +
> +	printf ("%0*d-", pad_mid, line);
> +
> +	random_message_id (ctx, message_id_len);
> +
> +	int num_tags = random () % (max_tags + 1);
> +
> +	int j;
> +	for (j = 0; j < num_tags; j++) {
> +	    printf (" %0*d-", pad_tag, j);
> +	    random_tag (ctx, tag_len);
> +	}
> +	putchar ('\n');
> +    }
> +
> +    talloc_free (ctx);
> +
> +    return 0;
> +}

Hi

Just a thought on this and the next test. Could you add messages with
the random ids and tags from the above code to the Xapian database
directly by calling whatever notmuch-new calls. Then test by doing dump,
restore and dump and check the two dumps are equal? It might avoid your
gmime concern from the next patch and you could have arbitrary
(non-null) strings including all sorts of malformed utf-8.

I guess Xapian might do bizarre things on the malformed utf-8 but, if it
does, it might mean the correct place to fix it is in notmuch-new.

Best wishes

Mark