* [PATCH v3 01/10] hex-escape: (en|de)code strings to/from restricted character set
2012-01-15 1:40 ` New dump/restore format David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 1:40 ` [PATCH v3 02/10] test/hex-xcode: new test binary David Bremner
` (8 subsequent siblings)
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
The character set is chosen to be suitable for pathnames, and the same
as that used by contrib/nmbug
---
util/Makefile.local | 2 +-
util/hex-escape.c | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++
util/hex-escape.h | 32 +++++++++++
3 files changed, 189 insertions(+), 1 deletions(-)
create mode 100644 util/hex-escape.c
create mode 100644 util/hex-escape.h
diff --git a/util/Makefile.local b/util/Makefile.local
index 26e4c3f..2e63932 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -3,7 +3,7 @@
dir := util
extra_cflags += -I$(srcdir)/$(dir)
-libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c
+libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c
libutil_modules := $(libutil_c_srcs:.c=.o)
diff --git a/util/hex-escape.c b/util/hex-escape.c
new file mode 100644
index 0000000..6c1260b
--- /dev/null
+++ b/util/hex-escape.c
@@ -0,0 +1,156 @@
+/* hex-escape.c - Manage encoding and decoding of byte strings into path names
+ *
+ * Copyright (c) 2011 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <talloc.h>
+#include "error_util.h"
+#include "hex-escape.h"
+
+static const size_t default_buf_size = 1024;
+
+static const char *output_charset =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.:,";
+
+static const int escape_char = '%';
+
+static int
+is_output (char c)
+{
+ return (strchr (output_charset, c) != NULL);
+}
+
+static int
+maybe_realloc (void *ctx, size_t needed, char **out, size_t *out_size)
+{
+ if (*out_size < needed) {
+
+ if (*out == NULL)
+ *out = talloc_size (ctx, needed);
+ else
+ *out = talloc_realloc (ctx, *out, char, needed);
+
+ if (*out == NULL)
+ return 0;
+
+ *out_size = needed;
+ }
+ return 1;
+}
+
+hex_status_t
+hex_encode (void *ctx, const char *in, char **out, size_t *out_size)
+{
+
+ const unsigned char *p;
+ char *q;
+
+ size_t escape_count = 0;
+ size_t len = 0;
+ size_t needed;
+
+ assert (ctx); assert (in); assert (out); assert (out_size);
+
+ for (p = (unsigned char *) in; *p; p++) {
+ escape_count += (!is_output (*p));
+ len++;
+ }
+
+ needed = len + escape_count * 2 + 1;
+
+ if (*out == NULL)
+ *out_size = 0;
+
+ if (!maybe_realloc (ctx, needed, out, out_size))
+ return HEX_OUT_OF_MEMORY;
+
+ q = *out;
+ p = (unsigned char *) in;
+
+ while (*p) {
+ if (is_output (*p)) {
+ *q++ = *p++;
+ } else {
+ sprintf (q, "%%%02x", *p++);
+ q += 3;
+ }
+ }
+
+ *q = '\0';
+ return HEX_SUCCESS;
+}
+
+hex_status_t
+hex_decode (void *ctx, const char *in, char **out, size_t * out_size)
+{
+
+ char buf[3];
+
+ const char *p;
+ unsigned char *q;
+
+ size_t escape_count = 0;
+ size_t needed = 0;
+
+ assert (ctx); assert (in); assert (out); assert (out_size);
+
+ size_t len = strlen (in);
+
+ for (p = in; *p; p++)
+ escape_count += (*p == escape_char);
+
+ needed = len - escape_count * 2 + 1;
+
+ if (!maybe_realloc (ctx, needed, out, out_size))
+ return HEX_OUT_OF_MEMORY;
+
+ p = in;
+ q = (unsigned char *) *out;
+ buf[2] = 0;
+
+ while (*p) {
+
+ if (*p == escape_char) {
+
+ char *endp;
+
+ if (len < 3)
+ return HEX_SYNTAX_ERROR;
+
+ buf[0] = p[1];
+ buf[1] = p[2];
+
+ *q = strtol (buf, &endp, 16);
+
+ if (endp != buf + 2)
+ return HEX_SYNTAX_ERROR;
+
+ len -= 3;
+ p += 3;
+ q++;
+ } else {
+ *q++ = *p++;
+ }
+ }
+
+ *q = '\0';
+
+ return HEX_SUCCESS;
+}
diff --git a/util/hex-escape.h b/util/hex-escape.h
new file mode 100644
index 0000000..e409626
--- /dev/null
+++ b/util/hex-escape.h
@@ -0,0 +1,32 @@
+#ifndef _HEX_ESCAPE_H
+#define _HEX_ESCAPE_H
+
+typedef enum hex_status {
+ HEX_SUCCESS = 0,
+ HEX_SYNTAX_ERROR,
+ HEX_OUT_OF_MEMORY
+} hex_status_t;
+
+/*
+ * The API is modelled on that for getline.
+ *
+ * If 'out' points to a NULL pointer a char array of the appropriate
+ * size is allocated using talloc, and out_size is updated.
+ *
+ * If 'out' points to a non-NULL pointer, it assumed to describe an
+ * existing char array, with the size given in *out_size. This array
+ * may be resized by talloc_realloc if needed; in this case *out_size
+ * will also be updated.
+ *
+ * Note that it is an error to pass a NULL pointer for any parameter
+ * of these routines.
+ */
+
+hex_status_t
+hex_encode (void *talloc_ctx, const char *in, char **out,
+ size_t *out_size);
+
+hex_status_t
+hex_decode (void *talloc_ctx, const char *in, char **out,
+ size_t *out_size);
+#endif
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* [PATCH v3 02/10] test/hex-xcode: new test binary
2012-01-15 1:40 ` New dump/restore format David Bremner
2012-01-15 1:40 ` [PATCH v3 01/10] hex-escape: (en|de)code strings to/from restricted character set David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 1:40 ` [PATCH v3 03/10] test/hex-escaping: new test for hex escaping routines David Bremner
` (7 subsequent siblings)
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
This program is used both as a test-bed/unit-tester for
../util/hex-escape.c, and also as a utility in future tests of dump
and restore.
---
test/.gitignore | 1 +
test/Makefile.local | 6 ++-
test/basic | 2 +-
test/hex-xcode.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 110 insertions(+), 2 deletions(-)
create mode 100644 test/hex-xcode.c
diff --git a/test/.gitignore b/test/.gitignore
index e63c689..be7ab5e 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -3,4 +3,5 @@ corpus.mail
smtp-dummy
symbol-test
arg-test
+hex-xcode
tmp.*
diff --git a/test/Makefile.local b/test/Makefile.local
index fa2df73..ba697f4 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -13,6 +13,9 @@ smtp_dummy_modules = $(smtp_dummy_srcs:.c=.o)
$(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
$(call quiet,CC) -I. $^ -o $@
+$(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
+ $(call quiet,CC) -I. $^ -o $@ -ltalloc
+
$(dir)/smtp-dummy: $(smtp_dummy_modules)
$(call quiet,CC) $^ -o $@
@@ -21,7 +24,8 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
.PHONY: test check
-test-binaries: $(dir)/arg-test $(dir)/smtp-dummy $(dir)/symbol-test
+test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
+ $(dir)/smtp-dummy $(dir)/symbol-test
test: all test-binaries
@${dir}/notmuch-test $(OPTIONS)
diff --git a/test/basic b/test/basic
index d6aed24..af57026 100755
--- a/test/basic
+++ b/test/basic
@@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
- sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test)$/d" | \
+ sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
sort)
test_expect_equal "$tests_in_suite" "$available"
diff --git a/test/hex-xcode.c b/test/hex-xcode.c
new file mode 100644
index 0000000..eec6541
--- /dev/null
+++ b/test/hex-xcode.c
@@ -0,0 +1,103 @@
+/* No, nothing to to with IDE from Apple Inc.
+ testbed for ../util/hex-escape.c.
+
+ usage:
+ hex-xcode [--direction=(encode|decode)] [--omit-newline] < file
+ hex-xcode [--direction=(encode|decode)] [--omit-newline] arg1 arg2 arg3 ...
+
+ */
+
+#include "notmuch-client.h"
+#include "hex-escape.h"
+#include <assert.h>
+
+
+enum direction {
+ ENCODE,
+ DECODE
+};
+
+static int
+xcode (void *ctx, enum direction dir, char *in, char **buf_p, size_t *size_p)
+{
+ hex_status_t status;
+
+ if (dir == ENCODE)
+ status = hex_encode (ctx, in, buf_p, size_p);
+ else
+ status = hex_decode (ctx, in, buf_p, size_p);
+
+ if (status == HEX_SUCCESS)
+ fputs (*buf_p, stdout);
+
+ return status;
+}
+
+
+int
+main (int argc, char **argv)
+{
+
+
+ enum direction dir = DECODE;
+ int omit_newline = FALSE;
+
+ notmuch_opt_desc_t options[] = {
+ { NOTMUCH_OPT_KEYWORD, &dir, "direction", 'd',
+ (notmuch_keyword_t []){ { "encode", ENCODE },
+ { "decode", DECODE },
+ { 0, 0 } } },
+ { NOTMUCH_OPT_BOOLEAN, &omit_newline, "omit-newline", 'n', 0 },
+ { 0, 0, 0, 0, 0 }
+ };
+
+ int opt_index = parse_arguments (argc, argv, options, 1);
+
+ if (opt_index < 0)
+ exit (1);
+
+ void *ctx = talloc_new (NULL);
+
+ char *line = NULL;
+ size_t line_size;
+ ssize_t line_len;
+
+ char *buffer = NULL;
+ size_t buf_size = 0;
+
+ notmuch_bool_t read_stdin = TRUE;
+
+ for (; opt_index < argc; opt_index++) {
+
+ if (xcode (ctx, dir, argv[opt_index],
+ &buffer, &buf_size) != HEX_SUCCESS)
+ return 1;
+
+ if (!omit_newline)
+ putchar ('\n');
+
+ read_stdin = FALSE;
+ }
+
+ if (!read_stdin)
+ return 0;
+
+ while ((line_len = getline (&line, &line_size, stdin)) != -1) {
+
+ chomp_newline (line);
+
+ if (xcode (ctx, dir, line, &buffer, &buf_size) != HEX_SUCCESS)
+ return 1;
+
+ if (!omit_newline)
+ putchar ('\n');
+
+ }
+
+ if (line)
+ free (line);
+
+ talloc_free (ctx);
+
+ return 0;
+}
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* [PATCH v3 03/10] test/hex-escaping: new test for hex escaping routines
2012-01-15 1:40 ` New dump/restore format David Bremner
2012-01-15 1:40 ` [PATCH v3 01/10] hex-escape: (en|de)code strings to/from restricted character set David Bremner
2012-01-15 1:40 ` [PATCH v3 02/10] test/hex-xcode: new test binary David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 1:40 ` [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup) David Bremner
` (6 subsequent siblings)
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
These are more like unit tests, to (try to) make sure the library
functionality is working before building more complicated things on
top of it.
---
test/hex-escaping | 26 ++++++++++++++++++++++++++
test/notmuch-test | 1 +
2 files changed, 27 insertions(+), 0 deletions(-)
create mode 100755 test/hex-escaping
diff --git a/test/hex-escaping b/test/hex-escaping
new file mode 100755
index 0000000..f34cc8c
--- /dev/null
+++ b/test/hex-escaping
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+test_description="hex encoding and decoding"
+. ./test-lib.sh
+
+test_begin_subtest "round trip"
+find $TEST_DIRECTORY/corpus -type f -print | sort | xargs cat > EXPECTED
+$TEST_DIRECTORY/hex-xcode --direction=encode < EXPECTED | $TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "punctuation"
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+tag_enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag1")
+test_expect_equal "$tag_enc1" "comic_swear=%24%26%5e%25%24%5e%25%5c%5c%2f%2f-+%24%5e%25%24"
+
+test_begin_subtest "round trip newlines"
+printf 'this\n tag\t has\n spaces\n' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=encode < EXPECTED.$test_count |\
+ $TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest "round trip 8bit chars"
+echo '%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=decode < EXPECTED.$test_count |\
+ $TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index 6a99ae3..cbe2ef9 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -52,6 +52,7 @@ TESTS="
python
hooks
argument-parsing
+ hex-escaping
"
TESTS=${NOTMUCH_TESTS:=$TESTS}
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup)
2012-01-15 1:40 ` New dump/restore format David Bremner
` (2 preceding siblings ...)
2012-01-15 1:40 ` [PATCH v3 03/10] test/hex-escaping: new test for hex escaping routines David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 20:35 ` Austin Clements
2012-01-15 1:40 ` [PATCH v3 05/10] test: add test for dump --format=notmuch David Bremner
` (5 subsequent siblings)
9 siblings, 1 reply; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
sup is the old format, and remains the default.
Each line of the notmuch format is "msg_id tag tag...tag" where each
space seperated token is 'hex-encoded' to remove troubling characters.
In particular this format won't have the same problem with e.g. spaces
in message-ids or tags; they will be round-trip-able.
---
dump-restore-private.h | 12 ++++++++++++
notmuch-dump.c | 47 +++++++++++++++++++++++++++++++++++++++--------
2 files changed, 51 insertions(+), 8 deletions(-)
create mode 100644 dump-restore-private.h
diff --git a/dump-restore-private.h b/dump-restore-private.h
new file mode 100644
index 0000000..34a5022
--- /dev/null
+++ b/dump-restore-private.h
@@ -0,0 +1,12 @@
+#ifndef DUMP_RESTORE_PRIVATE_H
+#define DUMP_RESTORE_PRIVATE_H
+
+#include "hex-escape.h"
+#include "command-line-arguments.h"
+
+typedef enum dump_formats {
+ DUMP_FORMAT_SUP,
+ DUMP_FORMAT_NOTMUCH
+} dump_format_t;
+
+#endif
diff --git a/notmuch-dump.c b/notmuch-dump.c
index a735875..0231db2 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -19,6 +19,7 @@
*/
#include "notmuch-client.h"
+#include "dump-restore-private.h"
int
notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
@@ -44,9 +45,15 @@ notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
char *output_file_name = NULL;
int opt_index;
+ int output_format = DUMP_FORMAT_SUP;
+
notmuch_opt_desc_t options[] = {
- { NOTMUCH_OPT_POSITION, &output_file_name, 0, 0, 0 },
- { 0, 0, 0, 0, 0 }
+ { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
+ (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
+ { "notmuch", DUMP_FORMAT_NOTMUCH },
+ {0, 0} } },
+ { NOTMUCH_OPT_POSITION, &output_file_name, 0, 0, 0 },
+ { 0, 0, 0, 0, 0 }
};
opt_index = parse_arguments (argc, argv, options, 1);
@@ -85,29 +92,53 @@ notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
*/
notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
+ char *buffer = NULL;
+ size_t buffer_size = 0;
+
for (messages = notmuch_query_search_messages (query);
notmuch_messages_valid (messages);
notmuch_messages_move_to_next (messages))
{
int first = 1;
- message = notmuch_messages_get (messages);
+ const char *message_id;
- fprintf (output,
- "%s (", notmuch_message_get_message_id (message));
+ message = notmuch_messages_get (messages);
+ message_id = notmuch_message_get_message_id (message);
+
+ if (output_format == DUMP_FORMAT_SUP) {
+ fprintf (output, "%s (", message_id);
+ } else {
+ if (hex_encode (notmuch, message_id,
+ &buffer, &buffer_size) != HEX_SUCCESS)
+ return 1;
+ fprintf (output, "%s ", buffer);
+ }
for (tags = notmuch_message_get_tags (message);
notmuch_tags_valid (tags);
notmuch_tags_move_to_next (tags))
{
+ const char *tag_str = notmuch_tags_get (tags);
+
if (! first)
- fprintf (output, " ");
+ fputs (" ", output);
- fprintf (output, "%s", notmuch_tags_get (tags));
+ if (output_format == DUMP_FORMAT_SUP) {
+ fputs (tag_str, output);
+ } else {
+ if (hex_encode (notmuch, tag_str,
+ &buffer, &buffer_size) != HEX_SUCCESS)
+ return 1;
+ fputs (buffer, output);
+ }
first = 0;
}
- fprintf (output, ")\n");
+ if (output_format == DUMP_FORMAT_SUP)
+ fputs (")\n", output);
+ else
+ fputs ("\n", output);
notmuch_message_destroy (message);
}
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* Re: [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup)
2012-01-15 1:40 ` [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup) David Bremner
@ 2012-01-15 20:35 ` Austin Clements
2012-01-15 23:40 ` David Bremner
0 siblings, 1 reply; 28+ messages in thread
From: Austin Clements @ 2012-01-15 20:35 UTC (permalink / raw)
To: David Bremner; +Cc: notmuch, David Bremner
Quoth David Bremner on Jan 14 at 9:40 pm:
> From: David Bremner <bremner@debian.org>
>
> sup is the old format, and remains the default.
>
> Each line of the notmuch format is "msg_id tag tag...tag" where each
> space seperated token is 'hex-encoded' to remove troubling characters.
> In particular this format won't have the same problem with e.g. spaces
> in message-ids or tags; they will be round-trip-able.
We definitely need a round-trip-able dump format. Did you consider
using JSON to allow for future flexibility (e.g., expansion of what we
store in the database) and so we don't have to invent our own
encodings? A JSON format wouldn't necessarily be a reason *not* to
also have this format, especially considering how
shell-script-friendly this is (versus how shell-script-unfriendly JSON
is), I'm just curious what trade-offs you're considering.
You might want to call this format something more self-descriptive
like "text" or "hextext" or something in case we do want to expand in
the future. "sup" is probably fine for the legacy format since that's
set in stone at this point.
^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup)
2012-01-15 20:35 ` Austin Clements
@ 2012-01-15 23:40 ` David Bremner
0 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 23:40 UTC (permalink / raw)
To: Austin Clements; +Cc: notmuch
On Sun, 15 Jan 2012 15:35:11 -0500, Austin Clements <amdragon@MIT.EDU> wrote:
> We definitely need a round-trip-able dump format. Did you consider
> using JSON to allow for future flexibility (e.g., expansion of what we
> store in the database) and so we don't have to invent our own
> encodings? A JSON format wouldn't necessarily be a reason *not* to
> also have this format, especially considering how
> shell-script-friendly this is (versus how shell-script-unfriendly JSON
> is), I'm just curious what trade-offs you're considering.
I was looking for something fairly close to what we have, to allow
people to migrate their various scripts (e.g. nmbug) to the new format
without too much pain. Maybe some small amount of header information at
the start of the file would support extensibility, while still being
shell script friendly.
I'm also not too sure how much overhead the JSON quoting would
induce. My tags file is currently about 10M, and on my old laptop takes
about 15s to dump. That's a long 15s when I'm trying to sync my mail.
For "normal" backup use, a little more overhead doesn't matter, although
the stories of non-linear slowdowns that people report suggest we
shouldn't get too cavalier about that.
> You might want to call this format something more self-descriptive
> like "text" or "hextext" or something in case we do want to expand in
> the future. "sup" is probably fine for the legacy format since that's
> set in stone at this point.
yeah, I'm definitely open to better suggestions for a name
^ permalink raw reply [flat|nested] 28+ messages in thread
* [PATCH v3 05/10] test: add test for dump --format=notmuch
2012-01-15 1:40 ` New dump/restore format David Bremner
` (3 preceding siblings ...)
2012-01-15 1:40 ` [PATCH v3 04/10] notmuch-dump: add --format=(notmuch|sup) David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 1:40 ` [PATCH v3 06/10] notmuch-restore: add 'notmuch format' support, auto detect David Bremner
` (4 subsequent siblings)
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
The first test is really to test our assumptions about the corpus,
namely that a certain set of message-id's is safe (i.e. doesn't change
under hex-escaping). We then check dump output as best we can without
functionality-to-come in notmuch-restore.
---
test/dump-restore | 12 ++++++++++++
1 files changed, 12 insertions(+), 0 deletions(-)
diff --git a/test/dump-restore b/test/dump-restore
index 439e998..0e210d3 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -82,4 +82,16 @@ test_begin_subtest "dump outfile -- from:cworth"
notmuch dump dump-outfile-dash-inbox.actual -- from:cworth
test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
+test_begin_subtest "Check for a safe set of message-ids"
+notmuch search --output=messages from:cworth > EXPECTED.$test_count
+notmuch search --output=messages from:cworth |\
+ $TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
+# we have observed that cworth has sane message-ids, and hopefully sane tags.
+test_begin_subtest "dump --format=notmuch -- from:cworth"
+notmuch dump --format=sup -- from:cworth | tr -d \(\) > EXPECTED.$test_count
+notmuch dump --format=notmuch -- from:cworth > OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
test_done
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* [PATCH v3 06/10] notmuch-restore: add 'notmuch format' support, auto detect
2012-01-15 1:40 ` New dump/restore format David Bremner
` (4 preceding siblings ...)
2012-01-15 1:40 ` [PATCH v3 05/10] test: add test for dump --format=notmuch David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 1:40 ` [PATCH v3 07/10] test: second set of dump/restore --format=notmuch tests David Bremner
` (3 subsequent siblings)
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
This is format is whitespace separated tokens, encoded by
util/hex-escape.c
The format detection heuristic relies on the fact that '(' is not part
of the character set used by hex-escape. Since hex-escape is designed
to be OK for pathnames (and shells), this seems like a reasonable
assumption.
In principle the --format argument to notmuch-restore is notmuch
needed at this point, but it adds literally 5 lines of argument
description, so I left it.
---
dump-restore-private.h | 5 +-
notmuch-restore.c | 111 ++++++++++++++++++++++++++++++++++++++---------
2 files changed, 92 insertions(+), 24 deletions(-)
diff --git a/dump-restore-private.h b/dump-restore-private.h
index 34a5022..67795e5 100644
--- a/dump-restore-private.h
+++ b/dump-restore-private.h
@@ -5,8 +5,9 @@
#include "command-line-arguments.h"
typedef enum dump_formats {
- DUMP_FORMAT_SUP,
- DUMP_FORMAT_NOTMUCH
+ DUMP_FORMAT_AUTO,
+ DUMP_FORMAT_NOTMUCH,
+ DUMP_FORMAT_SUP
} dump_format_t;
#endif
diff --git a/notmuch-restore.c b/notmuch-restore.c
index 87d9772..3fdfecc 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -19,6 +19,7 @@
*/
#include "notmuch-client.h"
+#include "dump-restore-private.h"
int
notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
@@ -35,6 +36,7 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
regex_t regex;
int rerr;
int opt_index;
+ int input_format = DUMP_FORMAT_AUTO;
config = notmuch_config_open (ctx, NULL, NULL);
if (config == NULL)
@@ -48,6 +50,11 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config);
notmuch_opt_desc_t options[] = {
+ { NOTMUCH_OPT_KEYWORD, &input_format, "format", 'f',
+ (notmuch_keyword_t []){ { "auto", DUMP_FORMAT_AUTO },
+ { "notmuch", DUMP_FORMAT_NOTMUCH },
+ { "sup", DUMP_FORMAT_SUP },
+ {0, 0} } },
{ NOTMUCH_OPT_POSITION, &input_file_name, 0, 0, 0 },
{ NOTMUCH_OPT_BOOLEAN, &accumulate, "accumulate", 'a', 0 },
{ 0, 0, 0, 0, 0 }
@@ -77,37 +84,85 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
return 1;
}
- /* Dump output is one line per message. We match a sequence of
- * non-space characters for the message-id, then one or more
- * spaces, then a list of space-separated tags as a sequence of
- * characters within literal '(' and ')'. */
- if ( xregcomp (®ex,
- "^([^ ]+) \\(([^)]*)\\)$",
- REG_EXTENDED) )
- INTERNAL_ERROR("compile time constant regex failed.");
+
+ /* These are out here to re-use the buffers with hex_decode */
+
+ char *message_id = NULL;
+ size_t message_id_size = 0;
+ char *tag = NULL;
+ size_t tag_size = 0;
+ notmuch_bool_t first_line = TRUE;
while ((line_len = getline (&line, &line_size, input)) != -1) {
regmatch_t match[3];
- char *message_id, *file_tags, *tag, *next;
+ char *file_tags, *next;
notmuch_message_t *message = NULL;
+
notmuch_status_t status;
notmuch_tags_t *db_tags;
char *db_tags_str;
chomp_newline (line);
+ if (first_line && input_format == DUMP_FORMAT_AUTO) {
+ char *p;
- rerr = xregexec (®ex, line, 3, match, 0);
- if (rerr == REG_NOMATCH)
- {
- fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
- line);
+ for (p = line; *p; p++) {
+ if (*p == '(')
+ input_format = DUMP_FORMAT_SUP;
+ }
+
+ if (input_format == DUMP_FORMAT_AUTO)
+ input_format = DUMP_FORMAT_NOTMUCH;
+
+ }
+
+ /* sup dump output is one line per message. We match a
+ * sequence of non-space characters for the message-id, then
+ * one or more spaces, then a list of space-separated tags as
+ * a sequence of characters within literal '(' and ')'. */
+ if (first_line && input_format == DUMP_FORMAT_SUP) {
+ if ( xregcomp (®ex,
+ "^([^ ]+) \\(([^)]*)\\)$",
+ REG_EXTENDED) )
+ INTERNAL_ERROR("compile time constant regex failed.");
+ }
+
+
+ /* Silently ignore blank lines */
+
+ if (line[0] == '\0') {
continue;
}
- message_id = xstrndup (line + match[1].rm_so,
- match[1].rm_eo - match[1].rm_so);
- file_tags = xstrndup (line + match[2].rm_so,
- match[2].rm_eo - match[2].rm_so);
+ if (input_format == DUMP_FORMAT_SUP) {
+ rerr = xregexec (®ex, line, 3, match, 0);
+ if (rerr == REG_NOMATCH)
+ {
+ fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
+ line);
+ continue;
+ }
+ message_id = talloc_strndup (notmuch, line + match[1].rm_so,
+ match[1].rm_eo - match[1].rm_so);
+ file_tags = talloc_strndup (notmuch, line + match[2].rm_so,
+ match[2].rm_eo - match[2].rm_so);
+ } else {
+ char *p = line;
+ char *raw_mid;
+
+ raw_mid = strsep (&p, " \t");
+
+ if (hex_decode (notmuch, raw_mid,
+ &message_id, &message_id_size) != HEX_SUCCESS)
+ return 1;
+
+ if (p)
+ file_tags = xstrdup (p);
+ else
+ file_tags = NULL;
+ }
+
+ first_line = FALSE;
status = notmuch_database_find_message (notmuch, message_id, &message);
if (status || message == NULL) {
@@ -153,7 +208,16 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
next = file_tags;
while (next) {
- tag = strsep (&next, " ");
+ char *raw_tag = strsep (&next, " ");
+
+ if (input_format == DUMP_FORMAT_NOTMUCH) {
+ if (hex_decode (notmuch, raw_tag,
+ &tag, &tag_size) != HEX_SUCCESS)
+ return 1;
+ } else {
+ tag = talloc_strdup (notmuch, raw_tag);
+ }
+
if (*tag == '\0')
continue;
status = notmuch_message_add_tag (message, tag);
@@ -175,11 +239,14 @@ notmuch_restore_command (unused (void *ctx), int argc, char *argv[])
if (message)
notmuch_message_destroy (message);
message = NULL;
- free (message_id);
- free (file_tags);
+ if (input_format == DUMP_FORMAT_SUP) {
+ talloc_free (message_id);
+ talloc_free (file_tags);
+ }
}
- regfree (®ex);
+ if (input_format == DUMP_FORMAT_SUP)
+ regfree (®ex);
if (line)
free (line);
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* [PATCH v3 07/10] test: second set of dump/restore --format=notmuch tests
2012-01-15 1:40 ` New dump/restore format David Bremner
` (5 preceding siblings ...)
2012-01-15 1:40 ` [PATCH v3 06/10] notmuch-restore: add 'notmuch format' support, auto detect David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 1:40 ` [PATCH v3 08/10] notmuch-{dump, restore}.1: document new format options David Bremner
` (2 subsequent siblings)
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
These one need the completed functionality in notmuch-restore. Fairly
exotic tags are tested, but no weird message id's.
We test each possible input to autodetection, both explicit (with
--format=auto) and implicit (without --format).
---
test/dump-restore | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 66 insertions(+), 0 deletions(-)
diff --git a/test/dump-restore b/test/dump-restore
index 0e210d3..60ae68e 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -94,4 +94,70 @@ notmuch dump --format=sup -- from:cworth | tr -d \(\) > EXPECTED.$test_count
notmuch dump --format=notmuch -- from:cworth > OUTPUT.$test_count
test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+test_begin_subtest "format=notmuch, # round-trip"
+notmuch dump --format=sup | sort > EXPECTED.$test_count
+notmuch dump --format=notmuch | notmuch restore --format=notmuch
+notmuch dump --format=sup | sort > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag1")
+
+tag2=$(printf 'this\n tag\t has\n spaces')
+enc2=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag2")
+
+enc3='%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a'
+tag3=$($TEST_DIRECTORY/hex-xcode --direction=decode $enc3)
+
+notmuch dump --format=notmuch > BACKUP
+
+notmuch tag +"$tag1" +"$tag2" +"$tag3" -inbox -unread "*"
+
+test_begin_subtest 'format=notmuch, round trip with strange tags'
+ notmuch dump --format=notmuch > EXPECTED.$test_count
+ notmuch dump --format=notmuch | notmuch restore --format=notmuch
+ notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, checking encoded output'
+ cp /dev/null EXPECTED.$test_count
+ notmuch dump --format=notmuch -- from:cworth |\
+ awk "{ print \$1 \" $enc1 $enc2 $enc3\" }" > EXPECTED.$test_count
+
+ notmuch dump --format=notmuch -- from:cworth > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'restoring sane tags'
+ notmuch restore --format=notmuch < BACKUP
+ notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file BACKUP OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, restore=auto'
+ notmuch dump --format=notmuch > EXPECTED.$test_count
+ notmuch tag -inbox -unread "*"
+ notmuch restore --format=auto < EXPECTED.$test_count
+ notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=auto'
+ notmuch dump --format=sup > EXPECTED.$test_count
+ notmuch tag -inbox -unread "*"
+ notmuch restore --format=auto < EXPECTED.$test_count
+ notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=notmuch, restore=default'
+ notmuch dump --format=notmuch > EXPECTED.$test_count
+ notmuch tag -inbox -unread "*"
+ notmuch restore < EXPECTED.$test_count
+ notmuch dump --format=notmuch > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=default'
+ notmuch dump --format=sup > EXPECTED.$test_count
+ notmuch tag -inbox -unread "*"
+ notmuch restore < EXPECTED.$test_count
+ notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
test_done
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* [PATCH v3 08/10] notmuch-{dump, restore}.1: document new format options
2012-01-15 1:40 ` New dump/restore format David Bremner
` (6 preceding siblings ...)
2012-01-15 1:40 ` [PATCH v3 07/10] test: second set of dump/restore --format=notmuch tests David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 1:40 ` [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files David Bremner
2012-01-15 1:40 ` [PATCH v3 10/10] test: new random message-id and tags dump/restore test David Bremner
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
More or less arbitrarily, notmuch-dump.1 gets the more detailed
description of the format.
---
man/man1/notmuch-dump.1 | 59 ++++++++++++++++++++++++++++++++++++++++++-
man/man1/notmuch-restore.1 | 60 ++++++++++++++++++++++++++++++++++++++-----
2 files changed, 111 insertions(+), 8 deletions(-)
diff --git a/man/man1/notmuch-dump.1 b/man/man1/notmuch-dump.1
index 9ccf35d..f4f7964 100644
--- a/man/man1/notmuch-dump.1
+++ b/man/man1/notmuch-dump.1
@@ -5,7 +5,8 @@ notmuch-dump \- Creates a plain-text dump of the tags of each message.
.SH SYNOPSIS
.B "notmuch dump"
-.RI "[ <" filename "> ] [--]"
+.RI "[ <" filename "> ]"
+.RB [ "\-\-format=(sup|notmuch)" "] [--]"
.RI "[ <" search-term ">...]"
.SH DESCRIPTION
@@ -20,6 +21,62 @@ recreated from the messages themselves. The output of notmuch dump is
therefore the only critical thing to backup (and much more friendly to
incremental backup than the native database files.)
+.TP 4
+.B \-\-format=(sup|notmuch)
+
+Notmuch restore supports two plain text dump formats, both with one message-id
+per line, followed by a list of tags.
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
+compatible with the format of files produced by sup-dump.
+So if you've previously been using sup for mail, then the
+.B "notmuch restore"
+command provides you a way to import all of your tags (or labels as
+sup calls them).
+Each line has the following form
+
+.RS 4
+.RI < message-id >
+.B (
+.RI < tag "> ..."
+.B )
+
+with zero or more tags are separated by spaces. Note that (malformed)
+message-ids may contained arbitrary non-null characters. Note also
+that tags with spaces will not be correctly restored with this format.
+
+.RE
+
+.RE
+.RS 4
+.TP 4
+.B notmuch
+
+The
+.B notmuch
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters.
+Each line has the form
+
+.RS 4
+.RI < encoded-message-id "> <" encoded-tag "> ..."
+
+where encoded means that every byte not matching the regex
+.B [A-Za-z0-9+-_@=.:,]
+is replace by
+.B %nn
+where nn is the two digit hex encoding.
+
+
+.RE
+
+
With no search terms, a dump of all messages in the database will be
generated. A "--" argument instructs notmuch that the
remaining arguments are search terms.
diff --git a/man/man1/notmuch-restore.1 b/man/man1/notmuch-restore.1
index 2191df0..3fb0e99 100644
--- a/man/man1/notmuch-restore.1
+++ b/man/man1/notmuch-restore.1
@@ -5,7 +5,7 @@ notmuch-restore \- Restores the tags from the given file (see notmuch dump).
.SH SYNOPSIS
.B "notmuch restore"
-.RB [ "--accumulate" ]
+.RI [ options "...]"
.RI "[ <" filename "> ]"
.SH DESCRIPTION
@@ -15,21 +15,67 @@ Restores the tags from the given file (see
The input is read from the given filename, if any, or from stdin.
-Note: The dump file format is specifically chosen to be
+
+Supported options for
+.B restore
+include
+.RS 4
+.TP 4
+.B \-\-accumulate
+
+The union of the existing and new tags is applied, instead of
+replacing each message's tags as they are read in from the dump file.
+
+.RE
+.RS 4
+.TP 4
+.B \-\-format=(sup|notmuch|auto)
+
+Notmuch restore supports two plain text dump formats, with one message-id
+per line, followed by a list of tags.
+For details of the actual formats, see \fBnotmuch-dump\fR(1).
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
compatible with the format of files produced by sup-dump.
So if you've previously been using sup for mail, then the
.B "notmuch restore"
command provides you a way to import all of your tags (or labels as
sup calls them).
-The --accumulate switch causes the union of the existing and new tags to be
-applied, instead of replacing each message's tags as they are read in from the
-dump file.
+.RE
+.RS 4
+.TP 4
+.B notmuch
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for <search-terms>.
+The
+.B notmuch
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters. This
+format hex-escapes all characters those outside of a small character
+set, intended to be suitable for e.g. pathnames in most UNIX-like
+systems.
.RE
+
+.RS 4
+.TP 4
+.B auto
+
+This option (the default) tries to guess the format from the
+input. For correctly formed input in either supported format, this
+heuristic, based the fact that notmuch format contains no parentheses,
+should be accurate.
+
+.RE
+
+.RE
+
.SH SEE ALSO
\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files
2012-01-15 1:40 ` New dump/restore format David Bremner
` (7 preceding siblings ...)
2012-01-15 1:40 ` [PATCH v3 08/10] notmuch-{dump, restore}.1: document new format options David Bremner
@ 2012-01-15 1:40 ` David Bremner
2012-01-15 8:05 ` Jani Nikula
2012-02-05 1:04 ` Mark Walters
2012-01-15 1:40 ` [PATCH v3 10/10] test: new random message-id and tags dump/restore test David Bremner
9 siblings, 2 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
This binary creates a "torture test" dump file for the new dump
format.
---
test/Makefile.local | 4 ++
test/basic | 2 +-
test/random-dump.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 149 insertions(+), 1 deletions(-)
create mode 100644 test/random-dump.c
diff --git a/test/Makefile.local b/test/Makefile.local
index ba697f4..b59f837 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
$(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
$(call quiet,CC) -I. $^ -o $@ -ltalloc
+$(dir)/random-dump: $(dir)/random-dump.o command-line-arguments.o util/libutil.a
+ $(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
+
$(dir)/smtp-dummy: $(smtp_dummy_modules)
$(call quiet,CC) $^ -o $@
@@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
.PHONY: test check
test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
+ $(dir)/random-dump \
$(dir)/smtp-dummy $(dir)/symbol-test
test: all test-binaries
diff --git a/test/basic b/test/basic
index af57026..e3a6cef 100755
--- a/test/basic
+++ b/test/basic
@@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
- sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
+ sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d" | \
sort)
test_expect_equal "$tests_in_suite" "$available"
diff --git a/test/random-dump.c b/test/random-dump.c
new file mode 100644
index 0000000..1949425
--- /dev/null
+++ b/test/random-dump.c
@@ -0,0 +1,144 @@
+/*
+ Generate a random dump file in 'notmuch' format.
+ Generated message-id's and tags are intentionally nasty.
+
+ We restrict ourselves to 7 bit message-ids, because generating
+ random valid UTF-8 seems like work. And invalid UTF-8 can't be
+ round-tripped via Xapian.
+
+ */
+
+#include <stdlib.h>
+#include <assert.h>
+#include <talloc.h>
+#include <string.h>
+#include "math.h"
+#include "hex-escape.h"
+#include "command-line-arguments.h"
+
+static void
+hex_out (void *ctx, char *buf)
+{
+ static char *encoded_buf = NULL;
+ static size_t encoded_buf_size = 0;
+
+ if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) != HEX_SUCCESS) {
+ fprintf (stderr, "Hex encoding failed");
+ exit (1);
+ }
+
+ fputs (encoded_buf, stdout);
+}
+
+static void
+random_chars (char *buf, int from, int stop, int max_char,
+ const char *blacklist)
+{
+ int i;
+
+ for (i = from; i < stop; i++) {
+ do {
+ buf[i] = ' ' + (random () % (max_char - ' '));
+ } while (blacklist && strchr (blacklist, buf[i]));
+ }
+}
+
+static void
+random_tag (void *ctx, size_t len)
+{
+ static char *buf = NULL;
+ static size_t buf_len = 0;
+
+ int use = (random () % (len - 1)) + 1;
+
+ if (len > buf_len) {
+ buf = talloc_realloc (ctx, buf, char, len);
+ buf_len = len;
+ }
+
+ random_chars (buf, 0, use, 255, NULL);
+
+ buf[use] = '\0';
+
+ hex_out (ctx, buf);
+}
+
+static void
+random_message_id (void *ctx, size_t len)
+{
+ static char *buf = NULL;
+ static size_t buf_len = 0;
+
+ int lhs_len = (random () % (len / 2 - 1)) + 1;
+
+ int rhs_len = (random () % len / 2) + 1;
+
+ const char *blacklist = "\n\r@<>[]()";
+
+ if (len > buf_len) {
+ buf = talloc_realloc (ctx, buf, char, len);
+ buf_len = len;
+ }
+
+ random_chars (buf, 0, lhs_len, 127, blacklist);
+
+ buf[lhs_len] = '@';
+
+ random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127, blacklist);
+
+ hex_out (ctx, buf);
+}
+
+int
+main (int argc, char **argv)
+{
+
+ void *ctx = talloc_new (NULL);
+ int num_lines = 500;
+ int max_tags = 10;
+ int message_id_len = 100;
+ int tag_len = 50;
+ int seed = 734569;
+
+ int pad_tag = 0, pad_mid = 0;
+
+ notmuch_opt_desc_t options[] = {
+ { NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
+ { NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
+ { NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
+ { NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
+ { NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
+ { 0, 0, 0, 0, 0 }
+ };
+
+ int opt_index = parse_arguments (argc, argv, options, 1);
+
+ if (opt_index < 0)
+ exit (1);
+
+ pad_mid = ((int) log10 (num_lines) + 1);
+ pad_tag = ((int) log10 (max_tags)) + 1;
+
+ srandom (seed);
+
+ int line;
+ for (line = 0; line < num_lines; line++) {
+
+ printf ("%0*d-", pad_mid, line);
+
+ random_message_id (ctx, message_id_len);
+
+ int num_tags = random () % (max_tags + 1);
+
+ int j;
+ for (j = 0; j < num_tags; j++) {
+ printf (" %0*d-", pad_tag, j);
+ random_tag (ctx, tag_len);
+ }
+ putchar ('\n');
+ }
+
+ talloc_free (ctx);
+
+ return 0;
+}
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread* Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files
2012-01-15 1:40 ` [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files David Bremner
@ 2012-01-15 8:05 ` Jani Nikula
2012-01-15 13:44 ` David Bremner
2012-02-05 1:04 ` Mark Walters
1 sibling, 1 reply; 28+ messages in thread
From: Jani Nikula @ 2012-01-15 8:05 UTC (permalink / raw)
To: David Bremner; +Cc: notmuch, David Bremner
[-- Attachment #1: Type: text/plain, Size: 5990 bytes --]
On Jan 15, 2012 3:41 AM, "David Bremner" <david@tethera.net> wrote:
>
> From: David Bremner <bremner@debian.org>
>
> This binary creates a "torture test" dump file for the new dump
> format.
> ---
> test/Makefile.local | 4 ++
> test/basic | 2 +-
> test/random-dump.c | 144
+++++++++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 149 insertions(+), 1 deletions(-)
> create mode 100644 test/random-dump.c
>
> diff --git a/test/Makefile.local b/test/Makefile.local
> index ba697f4..b59f837 100644
> --- a/test/Makefile.local
> +++ b/test/Makefile.local
> @@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o
command-line-arguments.o util/libutil.a
> $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o
util/libutil.a
> $(call quiet,CC) -I. $^ -o $@ -ltalloc
>
> +$(dir)/random-dump: $(dir)/random-dump.o command-line-arguments.o
util/libutil.a
> + $(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
> +
> $(dir)/smtp-dummy: $(smtp_dummy_modules)
> $(call quiet,CC) $^ -o $@
>
> @@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
> .PHONY: test check
>
> test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
> + $(dir)/random-dump \
> $(dir)/smtp-dummy $(dir)/symbol-test
>
> test: all test-binaries
> diff --git a/test/basic b/test/basic
> index af57026..e3a6cef 100755
> --- a/test/basic
> +++ b/test/basic
> @@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests
will be run by notmuch-test'
> eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
> tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
> available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable
-printf '%f\n' | \
> - sed -r -e
"/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d"
| \
> + sed -r -e
"/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d"
| \
> sort)
> test_expect_equal "$tests_in_suite" "$available"
>
> diff --git a/test/random-dump.c b/test/random-dump.c
> new file mode 100644
> index 0000000..1949425
> --- /dev/null
> +++ b/test/random-dump.c
> @@ -0,0 +1,144 @@
> +/*
> + Generate a random dump file in 'notmuch' format.
> + Generated message-id's and tags are intentionally nasty.
> +
> + We restrict ourselves to 7 bit message-ids, because generating
> + random valid UTF-8 seems like work. And invalid UTF-8 can't be
> + round-tripped via Xapian.
> +
> + */
> +
> +#include <stdlib.h>
> +#include <assert.h>
> +#include <talloc.h>
> +#include <string.h>
> +#include "math.h"
> +#include "hex-escape.h"
> +#include "command-line-arguments.h"
> +
> +static void
> +hex_out (void *ctx, char *buf)
> +{
> + static char *encoded_buf = NULL;
> + static size_t encoded_buf_size = 0;
> +
> + if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) !=
HEX_SUCCESS) {
> + fprintf (stderr, "Hex encoding failed");
> + exit (1);
> + }
> +
> + fputs (encoded_buf, stdout);
> +}
> +
> +static void
> +random_chars (char *buf, int from, int stop, int max_char,
> + const char *blacklist)
> +{
> + int i;
> +
> + for (i = from; i < stop; i++) {
> + do {
> + buf[i] = ' ' + (random () % (max_char - ' '));
> + } while (blacklist && strchr (blacklist, buf[i]));
> + }
> +}
> +
> +static void
> +random_tag (void *ctx, size_t len)
> +{
> + static char *buf = NULL;
> + static size_t buf_len = 0;
> +
> + int use = (random () % (len - 1)) + 1;
> +
> + if (len > buf_len) {
> + buf = talloc_realloc (ctx, buf, char, len);
> + buf_len = len;
> + }
> +
> + random_chars (buf, 0, use, 255, NULL);
> +
> + buf[use] = '\0';
> +
> + hex_out (ctx, buf);
> +}
> +
> +static void
> +random_message_id (void *ctx, size_t len)
> +{
> + static char *buf = NULL;
> + static size_t buf_len = 0;
> +
> + int lhs_len = (random () % (len / 2 - 1)) + 1;
> +
> + int rhs_len = (random () % len / 2) + 1;
> +
> + const char *blacklist = "\n\r@<>[]()";
> +
> + if (len > buf_len) {
> + buf = talloc_realloc (ctx, buf, char, len);
> + buf_len = len;
> + }
> +
> + random_chars (buf, 0, lhs_len, 127, blacklist);
> +
> + buf[lhs_len] = '@';
> +
> + random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127,
blacklist);
> +
> + hex_out (ctx, buf);
> +}
> +
> +int
> +main (int argc, char **argv)
> +{
> +
> + void *ctx = talloc_new (NULL);
> + int num_lines = 500;
> + int max_tags = 10;
> + int message_id_len = 100;
> + int tag_len = 50;
> + int seed = 734569;
> +
> + int pad_tag = 0, pad_mid = 0;
> +
> + notmuch_opt_desc_t options[] = {
> + { NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
> + { NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
> + { NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
> + { NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
> + { NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
s/tag-len/seed/
> + { 0, 0, 0, 0, 0 }
> + };
> +
> + int opt_index = parse_arguments (argc, argv, options, 1);
> +
> + if (opt_index < 0)
> + exit (1);
> +
> + pad_mid = ((int) log10 (num_lines) + 1);
> + pad_tag = ((int) log10 (max_tags)) + 1;
> +
> + srandom (seed);
> +
> + int line;
> + for (line = 0; line < num_lines; line++) {
> +
> + printf ("%0*d-", pad_mid, line);
> +
> + random_message_id (ctx, message_id_len);
> +
> + int num_tags = random () % (max_tags + 1);
> +
> + int j;
> + for (j = 0; j < num_tags; j++) {
> + printf (" %0*d-", pad_tag, j);
> + random_tag (ctx, tag_len);
> + }
> + putchar ('\n');
> + }
> +
> + talloc_free (ctx);
> +
> + return 0;
> +}
> --
> 1.7.7.3
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> http://notmuchmail.org/mailman/listinfo/notmuch
[-- Attachment #2: Type: text/html, Size: 8488 bytes --]
^ permalink raw reply [flat|nested] 28+ messages in thread* Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files
2012-01-15 8:05 ` Jani Nikula
@ 2012-01-15 13:44 ` David Bremner
0 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 13:44 UTC (permalink / raw)
To: Jani Nikula; +Cc: notmuch
On Sun, 15 Jan 2012 10:05:15 +0200, Jani Nikula <jani@nikula.org> wrote:
> On Jan 15, 2012 3:41 AM, "David Bremner" <david@tethera.net> wrote:
> >
> > From: David Bremner <bremner@debian.org>
> >
> > + notmuch_opt_desc_t options[] = {
> > + { NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
> > + { NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
> > + { NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
> > + { NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
> > + { NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
>
> s/tag-len/seed/
>
Fixed, thanks.
d
^ permalink raw reply [flat|nested] 28+ messages in thread
* Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files
2012-01-15 1:40 ` [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files David Bremner
2012-01-15 8:05 ` Jani Nikula
@ 2012-02-05 1:04 ` Mark Walters
1 sibling, 0 replies; 28+ messages in thread
From: Mark Walters @ 2012-02-05 1:04 UTC (permalink / raw)
To: David Bremner, notmuch; +Cc: David Bremner
On Sat, 14 Jan 2012 21:40:23 -0400, David Bremner <david@tethera.net> wrote:
> From: David Bremner <bremner@debian.org>
>
> This binary creates a "torture test" dump file for the new dump
> format.
> ---
> test/Makefile.local | 4 ++
> test/basic | 2 +-
> test/random-dump.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 149 insertions(+), 1 deletions(-)
> create mode 100644 test/random-dump.c
>
> diff --git a/test/Makefile.local b/test/Makefile.local
> index ba697f4..b59f837 100644
> --- a/test/Makefile.local
> +++ b/test/Makefile.local
> @@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
> $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
> $(call quiet,CC) -I. $^ -o $@ -ltalloc
>
> +$(dir)/random-dump: $(dir)/random-dump.o command-line-arguments.o util/libutil.a
> + $(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
> +
> $(dir)/smtp-dummy: $(smtp_dummy_modules)
> $(call quiet,CC) $^ -o $@
>
> @@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
> .PHONY: test check
>
> test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
> + $(dir)/random-dump \
> $(dir)/smtp-dummy $(dir)/symbol-test
>
> test: all test-binaries
> diff --git a/test/basic b/test/basic
> index af57026..e3a6cef 100755
> --- a/test/basic
> +++ b/test/basic
> @@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
> eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
> tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
> available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
> - sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
> + sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d" | \
> sort)
> test_expect_equal "$tests_in_suite" "$available"
>
> diff --git a/test/random-dump.c b/test/random-dump.c
> new file mode 100644
> index 0000000..1949425
> --- /dev/null
> +++ b/test/random-dump.c
> @@ -0,0 +1,144 @@
> +/*
> + Generate a random dump file in 'notmuch' format.
> + Generated message-id's and tags are intentionally nasty.
> +
> + We restrict ourselves to 7 bit message-ids, because generating
> + random valid UTF-8 seems like work. And invalid UTF-8 can't be
> + round-tripped via Xapian.
> +
> + */
> +
> +#include <stdlib.h>
> +#include <assert.h>
> +#include <talloc.h>
> +#include <string.h>
> +#include "math.h"
> +#include "hex-escape.h"
> +#include "command-line-arguments.h"
> +
> +static void
> +hex_out (void *ctx, char *buf)
> +{
> + static char *encoded_buf = NULL;
> + static size_t encoded_buf_size = 0;
> +
> + if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) != HEX_SUCCESS) {
> + fprintf (stderr, "Hex encoding failed");
> + exit (1);
> + }
> +
> + fputs (encoded_buf, stdout);
> +}
> +
> +static void
> +random_chars (char *buf, int from, int stop, int max_char,
> + const char *blacklist)
> +{
> + int i;
> +
> + for (i = from; i < stop; i++) {
> + do {
> + buf[i] = ' ' + (random () % (max_char - ' '));
> + } while (blacklist && strchr (blacklist, buf[i]));
> + }
> +}
> +
> +static void
> +random_tag (void *ctx, size_t len)
> +{
> + static char *buf = NULL;
> + static size_t buf_len = 0;
> +
> + int use = (random () % (len - 1)) + 1;
> +
> + if (len > buf_len) {
> + buf = talloc_realloc (ctx, buf, char, len);
> + buf_len = len;
> + }
> +
> + random_chars (buf, 0, use, 255, NULL);
> +
> + buf[use] = '\0';
> +
> + hex_out (ctx, buf);
> +}
> +
> +static void
> +random_message_id (void *ctx, size_t len)
> +{
> + static char *buf = NULL;
> + static size_t buf_len = 0;
> +
> + int lhs_len = (random () % (len / 2 - 1)) + 1;
> +
> + int rhs_len = (random () % len / 2) + 1;
> +
> + const char *blacklist = "\n\r@<>[]()";
> +
> + if (len > buf_len) {
> + buf = talloc_realloc (ctx, buf, char, len);
> + buf_len = len;
> + }
> +
> + random_chars (buf, 0, lhs_len, 127, blacklist);
> +
> + buf[lhs_len] = '@';
> +
> + random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127, blacklist);
> +
> + hex_out (ctx, buf);
> +}
> +
> +int
> +main (int argc, char **argv)
> +{
> +
> + void *ctx = talloc_new (NULL);
> + int num_lines = 500;
> + int max_tags = 10;
> + int message_id_len = 100;
> + int tag_len = 50;
> + int seed = 734569;
> +
> + int pad_tag = 0, pad_mid = 0;
> +
> + notmuch_opt_desc_t options[] = {
> + { NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
> + { NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
> + { NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
> + { NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
> + { NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
> + { 0, 0, 0, 0, 0 }
> + };
> +
> + int opt_index = parse_arguments (argc, argv, options, 1);
> +
> + if (opt_index < 0)
> + exit (1);
> +
> + pad_mid = ((int) log10 (num_lines) + 1);
> + pad_tag = ((int) log10 (max_tags)) + 1;
> +
> + srandom (seed);
> +
> + int line;
> + for (line = 0; line < num_lines; line++) {
> +
> + printf ("%0*d-", pad_mid, line);
> +
> + random_message_id (ctx, message_id_len);
> +
> + int num_tags = random () % (max_tags + 1);
> +
> + int j;
> + for (j = 0; j < num_tags; j++) {
> + printf (" %0*d-", pad_tag, j);
> + random_tag (ctx, tag_len);
> + }
> + putchar ('\n');
> + }
> +
> + talloc_free (ctx);
> +
> + return 0;
> +}
Hi
Just a thought on this and the next test. Could you add messages with
the random ids and tags from the above code to the Xapian database
directly by calling whatever notmuch-new calls. Then test by doing dump,
restore and dump and check the two dumps are equal? It might avoid your
gmime concern from the next patch and you could have arbitrary
(non-null) strings including all sorts of malformed utf-8.
I guess Xapian might do bizarre things on the malformed utf-8 but, if it
does, it might mean the correct place to fix it is in notmuch-new.
Best wishes
Mark
^ permalink raw reply [flat|nested] 28+ messages in thread
* [PATCH v3 10/10] test: new random message-id and tags dump/restore test.
2012-01-15 1:40 ` New dump/restore format David Bremner
` (8 preceding siblings ...)
2012-01-15 1:40 ` [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files David Bremner
@ 2012-01-15 1:40 ` David Bremner
9 siblings, 0 replies; 28+ messages in thread
From: David Bremner @ 2012-01-15 1:40 UTC (permalink / raw)
To: notmuch; +Cc: David Bremner
From: David Bremner <bremner@debian.org>
The idea is to generate a random dump file, and then generate minimal
mail messages from that.
Currently this relies on the behaviour of the gmime message-id parser
that if it cannot find a leading '<' in the Message-Id field, it just
copies the string without attempting to parse it. The alternative is
to settle for some weaker notion of round-tripping in this test.
---
test/dump-restore | 25 +++++++++++++++++++++++++
1 files changed, 25 insertions(+), 0 deletions(-)
diff --git a/test/dump-restore b/test/dump-restore
index 60ae68e..c354a44 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -160,4 +160,29 @@ test_begin_subtest 'format=sup, restore=default'
notmuch dump --format=sup > OUTPUT.$test_count
test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+test_begin_subtest 'random message-ids and tags'
+
+ count=0
+
+ ${TEST_DIRECTORY}/random-dump sort > EXPECTED.$test_count
+
+ while read id tags ; do
+ file=${MAIL_DIR}/cur/dump-$count:2,
+ count=$((count + 1))
+ printf 'From: dump-test@example.com\n' > ${file}
+ printf 'To: example@example.net\n' > ${file}
+ printf 'Subject: StrangeMessageIDsAndTags\n' > ${file}
+ # the missing angle-braces are intentional, since they trigger
+ # gmime to pass the message-id through unparsed.
+ printf 'Message-Id: ' >> $file
+ ${TEST_DIRECTORY}/hex-xcode --direction=decode $id >> $file
+ done < EXPECTED.$test_count
+
+ notmuch new
+ notmuch restore < EXPECTED.$test_count
+ notmuch dump --format=notmuch -- subject:StrangeMessageIDsAndTags | \
+ sed 's/ *$//' | sort > OUTPUT.$test_count
+
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
test_done
--
1.7.7.3
^ permalink raw reply related [flat|nested] 28+ messages in thread