[RFC PATCH 05/14] new: use new URL-based filenames for messages

[RFC PATCH 05/14] new: use new URL-based filenames for messages

[Ethan Glasser-Camp]

This commit breaks a bunch of tests; fixes follow.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 notmuch-new.c |   27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/notmuch-new.c b/notmuch-new.c
index 938ae29..1f11b2c 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -287,7 +287,7 @@ add_files (notmuch_database_t *notmuch,
 {
     DIR *dir = NULL;
     struct dirent *entry = NULL;
-    char *next = NULL;
+    char *next = NULL, *path_uri = NULL;
     time_t fs_mtime, db_mtime;
     notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
     notmuch_message_t *message = NULL;
@@ -315,7 +315,16 @@ add_files (notmuch_database_t *notmuch,
 
     fs_mtime = st.st_mtime;
 
-    status = notmuch_database_get_directory (notmuch, path, &directory);
+    /* maildir URIs should never have a hostname component, but
+     * uriparser doesn't parse paths correctly if they start with //,
+     * as in scheme://host//path.
+     */
+    if (path[0] == '/')
+	path_uri = talloc_asprintf (notmuch, "maildir://%s", path);
+    else
+	path_uri = talloc_asprintf (notmuch, "maildir:///%s", path);
+
+    status = notmuch_database_get_directory (notmuch, path_uri, &directory);
     if (status) {
 	ret = status;
 	goto DONE;
@@ -423,7 +432,7 @@ add_files (notmuch_database_t *notmuch,
 	       strcmp (notmuch_filenames_get (db_files), entry->d_name) < 0)
 	{
 	    char *absolute = talloc_asprintf (state->removed_files,
-					      "%s/%s", path,
+					      "%s/%s", path_uri,
 					      notmuch_filenames_get (db_files));
 
 	    _filename_list_add (state->removed_files, absolute);
@@ -439,7 +448,7 @@ add_files (notmuch_database_t *notmuch,
 	    if (strcmp (filename, entry->d_name) < 0)
 	    {
 		char *absolute = talloc_asprintf (state->removed_directories,
-						  "%s/%s", path, filename);
+						  "%s/%s", path_uri, filename);
 
 		_filename_list_add (state->removed_directories, absolute);
 	    }
@@ -467,7 +476,7 @@ add_files (notmuch_database_t *notmuch,
 
 	/* We're now looking at a regular file that doesn't yet exist
 	 * in the database, so add it. */
-	next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
+	next = talloc_asprintf (notmuch, "%s/%s", path_uri, entry->d_name);
 
 	state->processed_files++;
 
@@ -559,7 +568,7 @@ add_files (notmuch_database_t *notmuch,
     while (notmuch_filenames_valid (db_files))
     {
 	char *absolute = talloc_asprintf (state->removed_files,
-					  "%s/%s", path,
+					  "%s/%s", path_uri,
 					  notmuch_filenames_get (db_files));
 
 	_filename_list_add (state->removed_files, absolute);
@@ -570,7 +579,7 @@ add_files (notmuch_database_t *notmuch,
     while (notmuch_filenames_valid (db_subdirs))
     {
 	char *absolute = talloc_asprintf (state->removed_directories,
-					  "%s/%s", path,
+					  "%s/%s", path_uri,
 					  notmuch_filenames_get (db_subdirs));
 
 	_filename_list_add (state->removed_directories, absolute);
@@ -584,9 +593,11 @@ add_files (notmuch_database_t *notmuch,
      * same second.  This may lead to unnecessary re-scans, but it
      * avoids overlooking messages. */
     if (fs_mtime != stat_time)
-	_filename_list_add (state->directory_mtimes, path)->mtime = fs_mtime;
+	_filename_list_add (state->directory_mtimes, path_uri)->mtime = fs_mtime;
 
   DONE:
+    if (path_uri)
+	talloc_free (path_uri);
     if (next)
 	talloc_free (next);
     if (dir)
-- 
1.7.9.5

[RFC PATCH 06/14] maildir URIs can be used in tags_to_maildir_flags

[Ethan Glasser-Camp]

A better fix would probably be based on scheme.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 lib/message.cc |   51 ++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/lib/message.cc b/lib/message.cc
index c9857f5..8ecec71 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -23,6 +23,7 @@
 
 #include <stdint.h>
 
+#include <uriparser/Uri.h>
 #include <gmime/gmime.h>
 
 struct visible _notmuch_message {
@@ -1093,7 +1094,6 @@ notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
     {
 	filename = notmuch_filenames_get (filenames);
 	dir = _filename_is_in_maildir (filename);
-
 	if (! dir)
 	    continue;
 
@@ -1304,12 +1304,46 @@ _new_maildir_filename (void *ctx,
     return filename_new;
 }
 
+/* Parses a maildir URI and returns the filename corresponding to its
+ * path.
+ *
+ * Returns NULL if either the URI couldn't be parsed or if the
+ * scheme isn't maildir:.
+ */
+static char *
+_get_maildir_filename (const char *filename)
+{
+    UriParserStateA parser_state;
+    UriUriA parsed;
+    char *path;
+    parser_state.uri = &parsed;
+
+    if (uriParseUriA (&parser_state, filename) != URI_SUCCESS) {
+	uriFreeUriMembersA (&parsed);
+	return NULL;
+    }
+
+    if (parsed.scheme.first != NULL &&
+	0 != strncmp(parsed.scheme.first, "maildir",
+		     parsed.scheme.afterLast-parsed.scheme.first)) {
+	/* Full URI with non-maildir scheme. */
+	uriFreeUriMembersA (&parsed);
+	return NULL;
+    }
+
+    path = (char *)parsed.pathHead->text.first - 1;
+    uriFreeUriMembersA (&parsed);
+    return path;
+
+}
+
+
 notmuch_status_t
 notmuch_message_tags_to_maildir_flags (notmuch_message_t *message)
 {
     notmuch_filenames_t *filenames;
     const char *filename;
-    char *filename_new;
+    char *filename_new, *filename_old, *filename_new_uri;
     char *to_set, *to_clear;
     notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
 
@@ -1324,16 +1358,22 @@ notmuch_message_tags_to_maildir_flags (notmuch_message_t *message)
 	if (! _filename_is_in_maildir (filename))
 	    continue;
 
-	filename_new = _new_maildir_filename (message, filename,
+	filename_old = _get_maildir_filename (filename);
+	if (filename_old == NULL)
+	    continue;
+
+	filename_new = _new_maildir_filename (message, filename_old,
 					      to_set, to_clear);
 	if (filename_new == NULL)
 	    continue;
 
+	filename_new_uri = talloc_asprintf (message, "maildir://%s", filename_new);
+
 	if (strcmp (filename, filename_new)) {
 	    int err;
 	    notmuch_status_t new_status;
 
-	    err = rename (filename, filename_new);
+	    err = rename (filename_old, filename_new);
 	    if (err)
 		continue;
 
@@ -1347,7 +1387,7 @@ notmuch_message_tags_to_maildir_flags (notmuch_message_t *message)
 	    }
 
 	    new_status = _notmuch_message_add_filename (message,
-							filename_new);
+							filename_new_uri);
 	    /* Hold on to only the first error. */
 	    if (! status && new_status) {
 		status = new_status;
@@ -1358,6 +1398,7 @@ notmuch_message_tags_to_maildir_flags (notmuch_message_t *message)
 	}
 
 	talloc_free (filename_new);
+	talloc_free (filename_new_uri);
     }
 
     talloc_free (to_set);
-- 
1.7.9.5

[RFC PATCH 07/14] Update tests that need to see filenames to use URIs

[Ethan Glasser-Camp]

This fixes all tests except atomicity, which should be next.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 test/emacs                   |    2 +-
 test/json                    |    4 ++--
 test/maildir-sync            |    7 ++++---
 test/multipart               |    4 ++--
 test/new                     |    6 +++---
 test/search-folder-coherence |    2 +-
 test/search-output           |    4 ++--
 test/test-lib.sh             |    3 +++
 8 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/test/emacs b/test/emacs
index e9f954c..c08791e 100755
--- a/test/emacs
+++ b/test/emacs
@@ -621,7 +621,7 @@ Stash my stashables
 id:"bought"
 bought
 inbox,stashtest
-${gen_msg_filename}
+${gen_msg_uri}
 http://mid.gmane.org/bought
 http://marc.info/?i=bought
 http://mail-archive.com/search?l=mid&q=bought
diff --git a/test/json b/test/json
index 6439788..be29fac 100755
--- a/test/json
+++ b/test/json
@@ -5,7 +5,7 @@ test_description="--format=json output"
 test_begin_subtest "Show message: json"
 add_message "[subject]=\"json-show-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"json-show-message\""
 output=$(notmuch show --format=json "json-show-message")
-test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"excluded\": false, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"excluded\": false, \"filename\": \"${gen_msg_uri}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-subject\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"json-show-message\n\"}]}, []]]]"
 
 test_begin_subtest "Search message: json"
 add_message "[subject]=\"json-search-subject\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"json-search-message\""
@@ -22,7 +22,7 @@ test_expect_equal "$output" "[{\"thread\": \"XXX\",
 test_begin_subtest "Show message: json, utf-8"
 add_message "[subject]=\"json-show-utf8-body-sübjéct\"" "[date]=\"Sat, 01 Jan 2000 12:00:00 -0000\"" "[body]=\"jsön-show-méssage\""
 output=$(notmuch show --format=json "jsön-show-méssage")
-test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"excluded\": false, \"filename\": \"${gen_msg_filename}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-utf8-body-sübjéct\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"jsön-show-méssage\n\"}]}, []]]]"
+test_expect_equal "$output" "[[[{\"id\": \"${gen_msg_id}\", \"match\": true, \"excluded\": false, \"filename\": \"${gen_msg_uri}\", \"timestamp\": 946728000, \"date_relative\": \"2000-01-01\", \"tags\": [\"inbox\",\"unread\"], \"headers\": {\"Subject\": \"json-show-utf8-body-sübjéct\", \"From\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"To\": \"Notmuch Test Suite <test_suite@notmuchmail.org>\", \"Date\": \"Sat, 01 Jan 2000 12:00:00 +0000\"}, \"body\": [{\"id\": 1, \"content-type\": \"text/plain\", \"content\": \"jsön-show-méssage\n\"}]}, []]]]"
 
 test_begin_subtest "Show message: json, inline attachment filename"
 subject='json-show-inline-attachment-filename'
diff --git a/test/maildir-sync b/test/maildir-sync
index 01348d3..a2e110e 100755
--- a/test/maildir-sync
+++ b/test/maildir-sync
@@ -8,7 +8,7 @@ test_description="maildir synchronization"
 # --format=json" output includes some newlines. Also, need to avoid
 # including the local value of MAIL_DIR in the result.
 filter_show_json() {
-    sed -e 's/, /,\n/g'  | sed -e "s|${MAIL_DIR}/|MAIL_DIR/|"
+    sed -e 's/, /,\n/g'  | sed -e "s|${MAIL_URI}/|MAIL_DIR/|"
     echo
 }
 
@@ -102,8 +102,9 @@ No new mail. Detected 1 file rename.
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Removing S flag (inbox unread)"
 
 test_begin_subtest "Removing info from filename leaves tags unchanged"
-add_message [subject]='"Message to lose maildir info"' [filename]='message-to-lose-maildir-info' [dir]=cur
-notmuch tag -unread subject:"Message to lose maildir info"
+generate_message [subject]='"Message to lose maildir info"' [filename]='message-to-lose-maildir-info' [dir]=cur
+notmuch new > hrngh.new
+notmuch tag -unread subject:"Message to lose maildir info" > hrngh.tag
 mv "$MAIL_DIR/cur/message-to-lose-maildir-info:2,S" "$MAIL_DIR/cur/message-without-maildir-info"
 output=$(NOTMUCH_NEW)
 output+="
diff --git a/test/multipart b/test/multipart
index 72d3927..ccf410b 100755
--- a/test/multipart
+++ b/test/multipart
@@ -109,7 +109,7 @@ notmuch new > /dev/null
 test_begin_subtest "--format=text --part=0, full message"
 notmuch show --format=text --part=0 'id:87liy5ap00.fsf@yoom.home.cworth.org' >OUTPUT
 cat <<EOF >EXPECTED
-\fmessage{ id:87liy5ap00.fsf@yoom.home.cworth.org depth:0 match:1 excluded:0 filename:${MAIL_DIR}/multipart
+\fmessage{ id:87liy5ap00.fsf@yoom.home.cworth.org depth:0 match:1 excluded:0 filename:${MAIL_URI}/multipart
 \fheader{
 Carl Worth <cworth@cworth.org> (2001-01-05) (attachment inbox signed unread)
 Subject: Multipart message
@@ -323,7 +323,7 @@ notmuch show --format=json --part=0 'id:87liy5ap00.fsf@yoom.home.cworth.org' | s
 echo >>OUTPUT # expect *no* newline at end of output
 cat <<EOF >EXPECTED
 
-{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "excluded": false, "filename": "${MAIL_DIR}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
+{"id": "87liy5ap00.fsf@yoom.home.cworth.org", "match": true, "excluded": false, "filename": "${MAIL_URI}/multipart", "timestamp": 978709437, "date_relative": "2001-01-05", "tags": ["attachment","inbox","signed","unread"], "headers": {"Subject": "Multipart message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:43:57 +0000"}, "body": [
 {"id": 1, "content-type": "multipart/signed", "content": [
 {"id": 2, "content-type": "multipart/mixed", "content": [
 {"id": 3, "content-type": "message/rfc822", "content": [{"headers": {"Subject": "html message", "From": "Carl Worth <cworth@cworth.org>", "To": "cworth@cworth.org", "Date": "Fri, 05 Jan 2001 15:42:57 +0000"}, "body": [
diff --git a/test/new b/test/new
index cab7c01..e7852ec 100755
--- a/test/new
+++ b/test/new
@@ -171,9 +171,9 @@ touch "${MAIL_DIR}"/ignored_file
 touch "${MAIL_DIR}"/.ignored_hidden_file
 output=$(NOTMUCH_NEW 2>&1)
 test_expect_equal "$output" \
-"Note: Ignoring non-mail file: ${MAIL_DIR}/.git/config
-Note: Ignoring non-mail file: ${MAIL_DIR}/.ignored_hidden_file
-Note: Ignoring non-mail file: ${MAIL_DIR}/ignored_file
+"Note: Ignoring non-mail file: ${MAIL_URI}/.git/config
+Note: Ignoring non-mail file: ${MAIL_URI}/.ignored_hidden_file
+Note: Ignoring non-mail file: ${MAIL_URI}/ignored_file
 Added 1 new message to the database."
 
 test_begin_subtest "Ignore files and directories specified in new.ignore"
diff --git a/test/search-folder-coherence b/test/search-folder-coherence
index 3f6ec76..08558db 100755
--- a/test/search-folder-coherence
+++ b/test/search-folder-coherence
@@ -27,7 +27,7 @@ cat <<EOF >EXPECTED
 MAIL_DIR/msg-001
 MAIL_DIR/spam/msg-001
 EOF
-notmuch search --output=files id:$id_x | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --output=files id:$id_x | sed -e "s,$MAIL_URI,MAIL_DIR," >OUTPUT
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "Test matches folder:spam"
diff --git a/test/search-output b/test/search-output
index 8b57a43..d038dc7 100755
--- a/test/search-output
+++ b/test/search-output
@@ -181,7 +181,7 @@ EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files"
-notmuch search --output=files '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --output=files '*' | sed -e "s,$MAIL_URI,MAIL_DIR," >OUTPUT
 cat <<EOF >EXPECTED
 MAIL_DIR/cur/52:2,
 MAIL_DIR/cur/53:2,
@@ -240,7 +240,7 @@ EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files --format=json"
-notmuch search --format=json --output=files '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --format=json --output=files '*' | sed -e "s,$MAIL_URI,MAIL_DIR," >OUTPUT
 cat <<EOF >EXPECTED
 ["MAIL_DIR/cur/52:2,",
 "MAIL_DIR/cur/53:2,",
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 06aaea2..195158c 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -287,6 +287,7 @@ generate_message ()
 
     gen_msg_cnt=$((gen_msg_cnt + 1))
     if [ -z "${template[filename]}" ]; then
+
 	gen_msg_name="msg-$(printf "%03d" $gen_msg_cnt)"
     else
 	gen_msg_name=${template[filename]}
@@ -304,6 +305,7 @@ generate_message ()
 	gen_msg_filename="${MAIL_DIR}/${template[dir]}/$gen_msg_name"
 	mkdir -p "$(dirname "$gen_msg_filename")"
     fi
+    gen_msg_uri=maildir://$gen_msg_filename
 
     if [ -z "${template[body]}" ]; then
 	template[body]="This is just a test message (#${gen_msg_cnt})"
@@ -1149,6 +1151,7 @@ export HOME="${TMP_DIRECTORY}/home"
 mkdir -p "${HOME}"
 
 MAIL_DIR="${TMP_DIRECTORY}/mail"
+MAIL_URI=maildir://$MAIL_DIR
 export GNUPGHOME="${TMP_DIRECTORY}/gnupg"
 export NOTMUCH_CONFIG="${TMP_DIRECTORY}/notmuch-config"
 
-- 
1.7.9.5

[RFC PATCH 08/14] Don't cache corpus.mail

[Ethan Glasser-Camp]

corpus.mail has already been processed by notmuch-new, so it seems
like a good target to cache, but since filenames are no longer being
stored relative to the database, it isn't. Recopy on each test, or
else filenames from other tests will show up.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 test/test-lib.sh |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test-lib.sh b/test/test-lib.sh
index 195158c..def3760 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -441,7 +441,7 @@ add_email_corpus ()
     else
 	cp -a $TEST_DIRECTORY/corpus ${MAIL_DIR}
 	notmuch new >/dev/null
-	cp -a ${MAIL_DIR} $TEST_DIRECTORY/corpus.mail
+	#cp -a ${MAIL_DIR} $TEST_DIRECTORY/corpus.mail
     fi
 }
 
-- 
1.7.9.5

[RFC PATCH 09/14] Fix atomicity test to work without relocatable mailstores

[Ethan Glasser-Camp]

Instead of assuming that the mailstore doesn't store its absolute
filenames, we use a symlink that can change back and forth. As long as
filenames contain this symlink, they can work in either the real
database, or the current snapshot.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 test/atomicity     |   10 +++++-----
 test/atomicity.gdb |   11 ++++++++---
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/test/atomicity b/test/atomicity
index 6df0a00..7b62ec7 100755
--- a/test/atomicity
+++ b/test/atomicity
@@ -49,13 +49,13 @@ if test_require_external_prereq gdb; then
     rm $MAIL_DIR/.remove-dir/remove-directory-duplicate:2,
     rmdir $MAIL_DIR/.remove-dir
 
-    # Prepare a snapshot of the updated maildir.  The gdb script will
-    # update the database in this snapshot as it goes.
+    # Copy the mail database. We will run on this database concurrently.
     cp -ra $MAIL_DIR $MAIL_DIR.snap
-    cp ${NOTMUCH_CONFIG} ${NOTMUCH_CONFIG}.snap
-    NOTMUCH_CONFIG=${NOTMUCH_CONFIG}.snap notmuch config set database.path $MAIL_DIR.snap
-
 
+    # Use a symlink instead of the real path. This way, we can change the symlink,
+    # without filenames having to change.
+    mv $MAIL_DIR $MAIL_DIR.real
+    ln -s $MAIL_DIR.real $MAIL_DIR
 
     # Execute notmuch new and, at every call to rename, snapshot the
     # database, run notmuch new again on the snapshot, and capture the
diff --git a/test/atomicity.gdb b/test/atomicity.gdb
index fd67525..3d4e210 100644
--- a/test/atomicity.gdb
+++ b/test/atomicity.gdb
@@ -38,12 +38,17 @@ shell mv backtrace backtrace.`cat outcount`
 # Snapshot the database
 shell rm -r $MAIL_DIR.snap/.notmuch
 shell cp -r $MAIL_DIR/.notmuch $MAIL_DIR.snap/.notmuch
+shell rm $MAIL_DIR
+shell ln -s $MAIL_DIR.snap $MAIL_DIR
 # Restore the mtime of $MAIL_DIR.snap, which we just changed
-shell touch -r $MAIL_DIR $MAIL_DIR.snap
+shell touch -r $MAIL_DIR.real $MAIL_DIR.snap
 # Run notmuch new to completion on the snapshot
-shell NOTMUCH_CONFIG=${NOTMUCH_CONFIG}.snap XAPIAN_FLUSH_THRESHOLD=1000 notmuch new > /dev/null
-shell NOTMUCH_CONFIG=${NOTMUCH_CONFIG}.snap notmuch search '*' > search.`cat outcount` 2>&1
+shell NOTMUCH_CONFIG=${NOTMUCH_CONFIG} XAPIAN_FLUSH_THRESHOLD=1000 notmuch new > /dev/null
+shell NOTMUCH_CONFIG=${NOTMUCH_CONFIG} notmuch search '*' > search.`cat outcount` 2>&1
 shell echo $(expr $(cat outcount) + 1) > outcount
+# restore symlink to correct database before resuming
+shell rm $MAIL_DIR
+shell ln -s $MAIL_DIR.real $MAIL_DIR
 cont
 end
 
-- 
1.7.9.5

[RFC PATCH 10/14] new: add "scan" option

[Ethan Glasser-Camp]

This is just a quick hack to get started on adding an mbox backend.

The fact that the default maildir is scanned "automagically" is a
little weird, but it doesn't do any harm unless you decide to put mail
there that you really don't want indexed.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 notmuch-client.h |    9 +++++++++
 notmuch-config.c |   30 +++++++++++++++++++++++++++++-
 notmuch-new.c    |   18 ++++++++++++++++++
 test/config      |    1 +
 4 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/notmuch-client.h b/notmuch-client.h
index 9b63eae..9d922fe 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -256,6 +256,15 @@ notmuch_config_set_new_ignore (notmuch_config_t *config,
 			       const char *new_ignore[],
 			       size_t length);
 
+const char **
+notmuch_config_get_new_scan (notmuch_config_t *config,
+			       size_t *length);
+
+void
+notmuch_config_set_new_scan (notmuch_config_t *config,
+			       const char *new_scan[],
+			       size_t length);
+
 notmuch_bool_t
 notmuch_config_get_maildir_synchronize_flags (notmuch_config_t *config);
 
diff --git a/notmuch-config.c b/notmuch-config.c
index 3e37a2d..e9d99ea 100644
--- a/notmuch-config.c
+++ b/notmuch-config.c
@@ -50,7 +50,10 @@ static const char new_config_comment[] =
     "\t	that will not be searched for messages by \"notmuch new\".\n"
     "\n"
     "\t	NOTE: *Every* file/directory that goes by one of those names will\n"
-    "\t	be ignored, independent of its depth/location in the mail store.\n";
+    "\t	be ignored, independent of its depth/location in the mail store.\n"
+    "\n"
+    "\tscan	A list (separated by ';') of mail URLs to scan.\n"
+    "\t	The maildir located at database.path, above, will automatically be added.\n";
 
 static const char user_config_comment[] =
     " User configuration\n"
@@ -113,6 +116,8 @@ struct _notmuch_config {
     size_t new_tags_length;
     const char **new_ignore;
     size_t new_ignore_length;
+    const char **new_scan;
+    size_t new_scan_length;
     notmuch_bool_t maildir_synchronize_flags;
     const char **search_exclude_tags;
     size_t search_exclude_tags_length;
@@ -274,6 +279,8 @@ notmuch_config_open (void *ctx,
     config->new_tags_length = 0;
     config->new_ignore = NULL;
     config->new_ignore_length = 0;
+    config->new_scan = NULL;
+    config->new_scan_length = 0;
     config->maildir_synchronize_flags = TRUE;
     config->search_exclude_tags = NULL;
     config->search_exclude_tags_length = 0;
@@ -375,6 +382,10 @@ notmuch_config_open (void *ctx,
 	notmuch_config_set_new_ignore (config, NULL, 0);
     }
 
+    if (notmuch_config_get_new_scan (config, &tmp) == NULL) {
+	notmuch_config_set_new_scan (config, NULL, 0);
+    }
+
     if (notmuch_config_get_search_exclude_tags (config, &tmp) == NULL) {
 	if (is_new) {
 	    const char *tags[] = { "deleted", "spam" };
@@ -631,6 +642,14 @@ notmuch_config_get_new_ignore (notmuch_config_t *config, size_t *length)
 			     &(config->new_ignore_length), length);
 }
 
+const char **
+notmuch_config_get_new_scan (notmuch_config_t *config, size_t *length)
+{
+    return _config_get_list (config, "new", "scan",
+			     &(config->new_scan),
+			     &(config->new_scan_length), length);
+}
+
 void
 notmuch_config_set_user_other_email (notmuch_config_t *config,
 				     const char *list[],
@@ -658,6 +677,15 @@ notmuch_config_set_new_ignore (notmuch_config_t *config,
 		     &(config->new_ignore));
 }
 
+void
+notmuch_config_set_new_scan (notmuch_config_t *config,
+			     const char *list[],
+			     size_t length)
+{
+    _config_set_list (config, "new", "scan", list, length,
+		     &(config->new_scan));
+}
+
 const char **
 notmuch_config_get_search_exclude_tags (notmuch_config_t *config, size_t *length)
 {
diff --git a/notmuch-new.c b/notmuch-new.c
index 1f11b2c..57b27bf 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -239,6 +239,16 @@ _entry_in_ignore_list (const char *entry, add_files_state_t *state)
     return FALSE;
 }
 
+/* Call out to the appropriate add_files function, based on the URI. */
+static notmuch_status_t
+add_files_uri (unused(notmuch_database_t *notmuch),
+	       unused(const char *uri),
+	       unused(add_files_state_t *state))
+{
+    /* Stub for now */
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 /* Examine 'path' recursively as follows:
  *
  *   o Ask the filesystem for the mtime of 'path' (fs_mtime)
@@ -843,6 +853,8 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
     int ret = 0;
     struct stat st;
     const char *db_path;
+    const char **new_scan;
+    size_t new_scan_length, new_scan_i;
     char *dot_notmuch_path;
     struct sigaction action;
     _filename_node_t *f;
@@ -941,6 +953,12 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
 	timer_is_active = TRUE;
     }
 
+    new_scan = notmuch_config_get_new_scan (config, &new_scan_length);
+
+    for (new_scan_i = 0; new_scan_i < new_scan_length; new_scan_i++) {
+	add_files_uri (notmuch, new_scan[new_scan_i], &add_files_state);
+    }
+
     ret = add_files (notmuch, db_path, &add_files_state);
     if (ret)
 	goto DONE;
diff --git a/test/config b/test/config
index 93ecb13..b0ad0c1 100755
--- a/test/config
+++ b/test/config
@@ -52,6 +52,7 @@ user.primary_email=test_suite@notmuchmail.org
 user.other_email=test_suite_other@notmuchmail.org;test_suite@otherdomain.org
 new.tags=unread;inbox;
 new.ignore=
+new.scan=
 search.exclude_tags=
 maildir.synchronize_flags=true
 foo.string=this is another string value
-- 
1.7.9.5

[RFC PATCH 11/14] notmuch-new: pull out useful bits of add_files_recursive

[Ethan Glasser-Camp]

This is part of notmuch-new refactor phase 1: make add_files stuff
safe for other backends. add_files_recursive is essentially a
maildir-crawling function that periodically adds files to the database
or adds filenames to remove_files or remove_directory lists. I don't
see an easy way to adapt add_files_recursive for other backends who
might not have concepts of directories with other directories inside
of them, so instead just provide an add_files method for each backend.

This patch pulls some bits out of add_files_recursive which will be
useful for other backends: two reporting functions
_report_before_adding_file and _report_added_file, as well as
_add_message, which actually does the message adding.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 notmuch-new.c |  192 +++++++++++++++++++++++++++++++++++----------------------
 1 file changed, 119 insertions(+), 73 deletions(-)

diff --git a/notmuch-new.c b/notmuch-new.c
index 57b27bf..1bf4e25 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -249,6 +249,122 @@ add_files_uri (unused(notmuch_database_t *notmuch),
     return NOTMUCH_STATUS_SUCCESS;
 }
 
+/* Progress-reporting function.
+ *
+ * Can be used by any mailstore-crawling function that wants to alert
+ * users what message it's about to add. Subsequent errors will be due
+ * to this message ;)
+ */
+static void
+_report_before_adding_file (add_files_state_t *state, const char *filename)
+{
+    state->processed_files++;
+
+    if (state->verbose) {
+	if (state->output_is_a_tty)
+	    printf("\r\033[K");
+
+	printf ("%i/%i: %s",
+		state->processed_files,
+		state->total_files,
+		filename);
+
+	putchar((state->output_is_a_tty) ? '\r' : '\n');
+	fflush (stdout);
+    }
+}
+
+/* Progress-reporting function.
+ *
+ * Call this to respond to the signal handler for SIGALRM.
+ */
+static void
+_report_added_file (add_files_state_t *state)
+{
+    if (do_print_progress) {
+	do_print_progress = 0;
+	generic_print_progress ("Processed", "files", state->tv_start,
+				state->processed_files, state->total_files);
+    }
+}
+
+
+/* Atomically handles adding a message to the database.
+ *
+ * Should be used by any mailstore-crawling function that finds a new
+ * message to add.
+ */
+static notmuch_status_t
+_add_message (add_files_state_t *state, notmuch_database_t *notmuch,
+	      const char *filename)
+{
+    notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_message_t *message;
+    const char **tag;
+
+    status = notmuch_database_begin_atomic (notmuch);
+    if (status) {
+	ret = status;
+	goto DONE;
+    }
+
+    status = notmuch_database_add_message (notmuch, filename, &message);
+
+    switch (status) {
+    /* success */
+    case NOTMUCH_STATUS_SUCCESS:
+	state->added_messages++;
+	notmuch_message_freeze (message);
+	for (tag=state->new_tags; *tag != NULL; tag++)
+	    notmuch_message_add_tag (message, *tag);
+	if (state->synchronize_flags == TRUE)
+	    notmuch_message_maildir_flags_to_tags (message);
+	notmuch_message_thaw (message);
+	break;
+    /* Non-fatal issues (go on to next file) */
+    case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
+	if (state->synchronize_flags == TRUE)
+	    notmuch_message_maildir_flags_to_tags (message);
+	break;
+    case NOTMUCH_STATUS_FILE_NOT_EMAIL:
+	fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
+		 filename);
+	break;
+    /* Fatal issues. Don't process anymore. */
+    case NOTMUCH_STATUS_READ_ONLY_DATABASE:
+    case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
+    case NOTMUCH_STATUS_OUT_OF_MEMORY:
+	fprintf (stderr, "Error: %s. Halting processing.\n",
+		 notmuch_status_to_string (status));
+	ret = status;
+	goto DONE;
+    default:
+    case NOTMUCH_STATUS_FILE_ERROR:
+    case NOTMUCH_STATUS_NULL_POINTER:
+    case NOTMUCH_STATUS_TAG_TOO_LONG:
+    case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
+    case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
+    case NOTMUCH_STATUS_LAST_STATUS:
+	INTERNAL_ERROR ("add_message returned unexpected value: %d",  status);
+	ret = status;
+	goto DONE;
+    }
+
+    status = notmuch_database_end_atomic (notmuch);
+    if (status) {
+	ret = status;
+	goto DONE;
+    }
+
+  DONE:
+    if (message) {
+	notmuch_message_destroy (message);
+	message = NULL;
+    }
+
+    return ret;
+}
+
 /* Examine 'path' recursively as follows:
  *
  *   o Ask the filesystem for the mtime of 'path' (fs_mtime)
@@ -300,7 +416,6 @@ add_files (notmuch_database_t *notmuch,
     char *next = NULL, *path_uri = NULL;
     time_t fs_mtime, db_mtime;
     notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
-    notmuch_message_t *message = NULL;
     struct dirent **fs_entries = NULL;
     int i, num_fs_entries = 0, entry_type;
     notmuch_directory_t *directory;
@@ -309,7 +424,6 @@ add_files (notmuch_database_t *notmuch,
     time_t stat_time;
     struct stat st;
     notmuch_bool_t is_maildir;
-    const char **tag;
 
     if (stat (path, &st)) {
 	fprintf (stderr, "Error reading directory %s: %s\n",
@@ -488,83 +602,15 @@ add_files (notmuch_database_t *notmuch,
 	 * in the database, so add it. */
 	next = talloc_asprintf (notmuch, "%s/%s", path_uri, entry->d_name);
 
-	state->processed_files++;
-
-	if (state->verbose) {
-	    if (state->output_is_a_tty)
-		printf("\r\033[K");
-
-	    printf ("%i/%i: %s",
-		    state->processed_files,
-		    state->total_files,
-		    next);
-
-	    putchar((state->output_is_a_tty) ? '\r' : '\n');
-	    fflush (stdout);
-	}
+	_report_before_adding_file (state, next);
 
-	status = notmuch_database_begin_atomic (notmuch);
+	status = _add_message (state, notmuch, next);
 	if (status) {
 	    ret = status;
 	    goto DONE;
 	}
 
-	status = notmuch_database_add_message (notmuch, next, &message);
-	switch (status) {
-	/* success */
-	case NOTMUCH_STATUS_SUCCESS:
-	    state->added_messages++;
-	    notmuch_message_freeze (message);
-	    for (tag=state->new_tags; *tag != NULL; tag++)
-	        notmuch_message_add_tag (message, *tag);
-	    if (state->synchronize_flags == TRUE)
-		notmuch_message_maildir_flags_to_tags (message);
-	    notmuch_message_thaw (message);
-	    break;
-	/* Non-fatal issues (go on to next file) */
-	case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
-	    if (state->synchronize_flags == TRUE)
-		notmuch_message_maildir_flags_to_tags (message);
-	    break;
-	case NOTMUCH_STATUS_FILE_NOT_EMAIL:
-	    fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
-		     next);
-	    break;
-	/* Fatal issues. Don't process anymore. */
-	case NOTMUCH_STATUS_READ_ONLY_DATABASE:
-	case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
-	case NOTMUCH_STATUS_OUT_OF_MEMORY:
-	    fprintf (stderr, "Error: %s. Halting processing.\n",
-		     notmuch_status_to_string (status));
-	    ret = status;
-	    goto DONE;
-	default:
-	case NOTMUCH_STATUS_FILE_ERROR:
-	case NOTMUCH_STATUS_NULL_POINTER:
-	case NOTMUCH_STATUS_TAG_TOO_LONG:
-	case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
-	case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
-	case NOTMUCH_STATUS_LAST_STATUS:
-	    INTERNAL_ERROR ("add_message returned unexpected value: %d",  status);
-	    goto DONE;
-	}
-
-	status = notmuch_database_end_atomic (notmuch);
-	if (status) {
-	    ret = status;
-	    goto DONE;
-	}
-
-	if (message) {
-	    notmuch_message_destroy (message);
-	    message = NULL;
-	}
-
-	if (do_print_progress) {
-	    do_print_progress = 0;
-	    generic_print_progress ("Processed", "files", state->tv_start,
-				    state->processed_files, state->total_files);
-	}
+	_report_added_file (state);
 
 	talloc_free (next);
 	next = NULL;
-- 
1.7.9.5

[RFC PATCH 12/14] mailstore: support for mbox:// URIs

[Ethan Glasser-Camp]

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 lib/mailstore.c |   85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/lib/mailstore.c b/lib/mailstore.c
index ae02c12..e8d9bc1 100644
--- a/lib/mailstore.c
+++ b/lib/mailstore.c
@@ -19,6 +19,7 @@
  */
 #include <uriparser/Uri.h>
 #include <stdio.h>
+#include <glib.h>
 
 #include "notmuch-private.h"
 
@@ -28,6 +29,74 @@ notmuch_mailstore_basic_open (const char *filename)
     return fopen (filename, "r");
 }
 
+/* Since we have to return a FILE*, we use fmemopen to turn buffers
+ * into FILE* streams. But when we close these streams, we have to
+ * free() the buffers. Use a hash to associate the two.
+ */
+static GHashTable *_mbox_files_to_strings = NULL;
+
+static void
+_ensure_mbox_files_to_strings () {
+    if (_mbox_files_to_strings == NULL)
+        _mbox_files_to_strings = g_hash_table_new (NULL, NULL);
+}
+
+static FILE *
+notmuch_mailstore_mbox_open (UriUriA *uri)
+{
+    FILE *ret = NULL, *mbox = NULL;
+    char *filename, *message, *length_s;
+    const char *error;
+    long int offset, length, this_read;
+    _ensure_mbox_files_to_strings ();
+
+    offset = strtol (uri->fragment.first, &length_s, 10);
+    length = strtol (length_s+1, NULL, 10);
+
+    filename = talloc_strndup (NULL, uri->pathHead->text.first-1,
+                               uri->pathTail->text.afterLast-uri->pathHead->text.first+1);
+
+    if (filename == NULL)
+        goto DONE;
+
+    mbox = fopen (filename, "r");
+    if (mbox == NULL) {
+        fprintf (stderr, "Couldn't open message %s: %s.\n", uri->scheme.first,
+                 strerror (errno));
+        goto DONE;
+    }
+
+    message = talloc_array (NULL, char, length);
+    fseek (mbox, offset, SEEK_SET);
+
+    this_read = fread (message, sizeof(char), length, mbox);
+    if (this_read != length) {
+        if (feof (mbox))
+            error = "end of file reached";
+        if (ferror (mbox))
+            error = strerror (ferror (mbox));
+
+        fprintf (stderr, "Couldn't read message %s: %s.\n", uri->scheme.first, error);
+        goto DONE;
+    }
+
+    ret = fmemopen (message, length, "r");
+    if (ret == NULL) {
+        /* No fclose will ever be called, so let's free message now */
+        talloc_free (message);
+        goto DONE;
+    }
+
+    g_hash_table_insert (_mbox_files_to_strings, ret, message);
+DONE:
+    if (filename)
+        talloc_free (filename);
+    if (mbox)
+        fclose (mbox);
+
+    return ret;
+}
+
 FILE *
 notmuch_mailstore_open (const char *filename)
 {
@@ -57,6 +126,14 @@ notmuch_mailstore_open (const char *filename)
         goto DONE;
     }
 
+    if (0 == strncmp (parsed.scheme.first, "mbox",
+                      parsed.scheme.afterLast-parsed.scheme.first)) {
+        /* mbox URI of the form mbox:///path/to/file#offset+length.
+         * Just pass the parsed URI. */
+        ret = notmuch_mailstore_mbox_open (&parsed);
+        goto DONE;
+    }
+
 DONE:
     uriFreeUriMembersA (&parsed);
     return ret;
@@ -65,5 +142,13 @@ DONE:
 int
 notmuch_mailstore_close (FILE *file)
 {
+    char *file_buffer;
+    if (_mbox_files_to_strings != NULL) {
+        file_buffer = g_hash_table_lookup (_mbox_files_to_strings, file);
+        if (file_buffer != NULL) {
+            talloc_free (file_buffer);
+        }
+        g_hash_table_remove (_mbox_files_to_strings, file);
+    }
     return fclose (file);
 }
-- 
1.7.9.5

[RFC PATCH 13/14] Tests for mbox support

[Ethan Glasser-Camp]

These need to be improved, rather than hard-coding byte offsets.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 test/mbox         |   59 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 test/notmuch-test |    1 +
 2 files changed, 60 insertions(+)
 create mode 100755 test/mbox

diff --git a/test/mbox b/test/mbox
new file mode 100755
index 0000000..f03f887
--- /dev/null
+++ b/test/mbox
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+
+test_description='basic mbox support'
+. ./test-lib.sh
+
+mkdir -p $MAIL_DIR/some-mboxes/subdir $MAIL_DIR/database $MAIL_DIR/corpus
+
+# The Content-Length headers here include the final newline (added later).
+generate_message '[body]="Mbox message 1."' '[header]="Content-Length: 16"' "[dir]=corpus"
+generate_message '[body]="Mbox message 2. Longer."' '[header]="Content-Length: 24"' "[dir]=corpus"
+generate_message '[body]="Mbox message 3."' "[dir]=corpus"
+generate_message '[body]="Mbox message 4."' "[dir]=corpus"
+generate_message '[body]="Mbox message 5. Last message."' '[header]="Content-Length: 30"' "[dir]=corpus"
+
+MBOX1=$MAIL_DIR/some-mboxes/first.mbox
+for x in $MAIL_DIR/corpus/*; do
+    echo "From MAILER-DAEMON Sat Jan  3 01:05:34 1996" >> $MBOX1
+    cat $x >> $MBOX1
+    # Final newline
+    echo >> $MBOX1
+done
+
+notmuch config set database.path $MAIL_DIR/database
+notmuch config set new.scan mbox://$MAIL_DIR/some-mboxes
+
+test_begin_subtest "read a small mbox (5 messages)"
+output=$(NOTMUCH_NEW)
+test_expect_equal "$output" "Added 5 new messages to the database."
+
+test_begin_subtest "search"
+output=$(notmuch search '*' | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Test message #1 (inbox unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Test message #2 (inbox unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Test message #3 (inbox unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Test message #4 (inbox unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Test message #5 (inbox unread)"
+
+test_begin_subtest "show (mboxcl)"
+output=$(notmuch show "Test message #1" | grep -o "filename:[^ ]*")
+test_expect_equal "$output" "filename:mbox://$MAIL_DIR/some-mboxes/first.mbox#44+246"
+
+test_begin_subtest "show doesn't append an extra space at the end (mboxcl)"
+output=$(notmuch show --format=raw "Test message #1" )
+original=$(cat $MAIL_DIR/corpus/msg-001)
+test_expect_equal "$output" "$original"
+
+test_begin_subtest "show (non-cl)"
+output=$(notmuch show "Test message #3" | grep -o "filename:[^ ]*")
+test_expect_equal "$output" "filename:mbox://$MAIL_DIR/some-mboxes/first.mbox#634+227"
+
+test_begin_subtest "show doesn't append an extra space at the end (non-cl)"
+output=$(notmuch show --format=raw "Test message #3" )
+original=$(cat $MAIL_DIR/corpus/msg-003)
+test_expect_equal "$output" "$original"
+
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index bfad5d3..8cbb2cd 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -47,6 +47,7 @@ TESTS="
   emacs-large-search-buffer
   emacs-subject-to-filename
   maildir-sync
+  mbox
   crypto
   symbol-hiding
   search-folder-coherence
-- 
1.7.9.5

[RFC PATCH 14/14] new: Add scan support for mbox:// URIs

[Ethan Glasser-Camp]

A lot of code is duplicated from maildir, I don't think I handled all
those errors correctly, and I didn't report any progress.

Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
---
 notmuch-new.c |  299 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 289 insertions(+), 10 deletions(-)

diff --git a/notmuch-new.c b/notmuch-new.c
index 1bf4e25..36fee34 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -19,6 +19,7 @@
  */
 
 #include "notmuch-client.h"
+#include <uriparser/Uri.h>
 
 #include <unistd.h>
 
@@ -239,16 +240,6 @@ _entry_in_ignore_list (const char *entry, add_files_state_t *state)
     return FALSE;
 }
 
-/* Call out to the appropriate add_files function, based on the URI. */
-static notmuch_status_t
-add_files_uri (unused(notmuch_database_t *notmuch),
-	       unused(const char *uri),
-	       unused(add_files_state_t *state))
-{
-    /* Stub for now */
-    return NOTMUCH_STATUS_SUCCESS;
-}
-
 /* Progress-reporting function.
  *
  * Can be used by any mailstore-crawling function that wants to alert
@@ -674,6 +665,294 @@ add_files (notmuch_database_t *notmuch,
     return ret;
 }
 
+/* Scan an mbox file for messages.
+ *
+ * We assume that mboxes grow monotonically only.
+ *
+ * The mtime of the mbox file is stored in a "directory" document in
+ * Xapian.
+ */
+static notmuch_status_t
+add_messages_mbox_file (notmuch_database_t *notmuch,
+			const char *path,
+			add_files_state_t *state)
+{
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, status;
+    struct stat st;
+    time_t fs_mtime, db_mtime, stat_time;
+    FILE *mbox;
+    char *line, *path_uri = NULL, *message_uri = NULL;
+    int line_len;
+    size_t offset, end_offset, line_size = 0;
+    notmuch_directory_t *directory;
+    int content_length = -1, is_headers;
+
+    if (stat (path, &st)) {
+	fprintf (stderr, "Error reading mbox file %s: %s\n",
+		 path, strerror (errno));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    stat_time = time (NULL);
+    if (! S_ISREG (st.st_mode)) {
+	fprintf (stderr, "Error: %s is not a file.\n", path);
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    fs_mtime = st.st_mtime;
+
+    path_uri = talloc_asprintf (notmuch, "mbox://%s", path);
+    status = notmuch_database_get_directory (notmuch, path_uri, &directory);
+    if (status) {
+	ret = status;
+	goto DONE;
+    }
+    db_mtime = directory ? notmuch_directory_get_mtime (directory) : 0;
+
+    if (directory && db_mtime == fs_mtime) {
+	goto DONE;
+    }
+
+    mbox = fopen (path, "r");
+    if (mbox == NULL) {
+	fprintf (stderr, "Error: couldn't open %s for reading.\n",
+		 path);
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    line_len = getline (&line, &line_size, mbox);
+
+    if (line_len == -1) {
+	fprintf (stderr, "Error: reading from %s failed: %s\n",
+		 path, strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    if (strncmp (line, "From ", 5) != 0) {
+	fprintf (stderr, "Note: Ignoring non-mbox file: %s\n",
+		 path);
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+    free(line);
+    line = NULL;
+
+    /* Loop invariant: At the beginning of the loop, we have just read
+     * a From_ line, but haven't yet read any of the headers.
+     */
+    while (! feof (mbox)) {
+	is_headers = 1;
+	offset = ftell (mbox);
+	content_length = -1;
+
+	/* Read lines until we either get to the next From_ header, or
+	 * we find a Content-Length header (mboxcl) and we run out of headers.
+	 */
+	do {
+	    /* Get the offset before we read, in case we got another From_ header. */
+	    end_offset = ftell (mbox);
+
+	    line_len = getline (&line, &line_size, mbox);
+
+	    /* Check to see if this line is a content-length header,
+	     * or the end of the headers. */
+	    if (is_headers && strncasecmp (line, "Content-Length: ",
+					   strlen ("Content-Length: ")) == 0)
+		content_length = strtol (line + strlen ("Content-Length: "),
+					 NULL, 10);
+
+	    if (is_headers && strlen (line) == 1 && *line == '\n') {
+		is_headers = 0;
+		/* If we got a content_length, skip the message body. */
+		if (content_length != -1) {
+		    fseek (mbox, content_length, SEEK_CUR);
+		    line_len = getline (&line, &line_size, mbox);
+
+		    /* We should be at the end of the message.  Sanity
+		     * check: there should be a blank line, and then
+		     * another From_ header. */
+		    if (strlen (line) != 1 || *line != '\n') {
+			fprintf (stderr, "Warning: message with Content-Length not "
+				 "immediately followed by blank line (%d)\n", offset);
+		    }
+
+		    end_offset = ftell (mbox);
+		    line_len = getline (&line, &line_size, mbox);
+
+		    if (line_len != -1 && strncmp (line, "From ", 5) != 0) {
+			fprintf (stderr, "Warning: message with Content-Length not "
+				 "immediately followed by another message (%d)\n", offset);
+			fprintf (stderr, "Line was: %s", line);
+		    }
+		}
+	    }
+
+	} while (! feof (mbox) && strncmp (line, "From ", 5) != 0);
+
+	/* end_offset is now after the \n but before the From_. */
+	message_uri = talloc_asprintf (notmuch, "mbox://%s#%d+%d",
+				       path, offset, (end_offset - 1) - offset);
+	status = _add_message (state, notmuch, message_uri);
+	talloc_free (message_uri);
+	message_uri = NULL;
+    }
+
+    if (fs_mtime != stat_time)
+	_filename_list_add (state->directory_mtimes, path_uri)->mtime = fs_mtime;
+
+DONE:
+    if (line)
+	free (line);
+    if (path_uri)
+	talloc_free (path_uri);
+    if (message_uri)
+	talloc_free (message_uri);
+    if (directory)
+	notmuch_directory_destroy (directory);
+
+    return ret;
+}
+
+/*
+ * Examine path recursively as follows:
+ *
+ * - Recurse on each subdirectory, as in add_files.
+ *
+ * - Call add_messages_mbox_file on every non-directory.
+ */
+static notmuch_status_t
+add_files_mbox (notmuch_database_t *notmuch,
+		const char *path,
+		add_files_state_t *state)
+{
+    struct dirent **fs_entries = NULL;
+    struct dirent *entry = NULL;
+    char *next = NULL;
+    int num_fs_entries = 0, i, entry_type;
+    struct stat st;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, status;
+
+    if (stat (path, &st)) {
+	fprintf (stderr, "Error reading directory %s: %s\n",
+		 path, strerror (errno));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    num_fs_entries = scandir (path, &fs_entries, 0,
+			      dirent_sort_inode);
+
+    if (num_fs_entries == -1) {
+	fprintf (stderr, "Error opening directory %s: %s\n",
+		 path, strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    for (i = 0; i < num_fs_entries; i++) {
+	if (interrupted)
+	    break;
+
+	entry = fs_entries[i];
+
+	entry_type = dirent_type (path, entry);
+	if (entry_type == -1) {
+	    /* Be pessimistic, e.g. so we don't lose lots of mail just
+	     * because a user broke a symlink. */
+	    fprintf (stderr, "Error reading file %s/%s: %s\n",
+		     path, entry->d_name, strerror (errno));
+	    return NOTMUCH_STATUS_FILE_ERROR;
+	} else if (entry_type != S_IFDIR) {
+	    continue;
+	}
+
+	/* Ignore special directories to avoid infinite recursion.
+	 * Also ignore the .notmuch directory, any "tmp" directory
+	 * that appears within a maildir and files/directories
+	 * the user has configured to be ignored.
+	 */
+	if (strcmp (entry->d_name, ".") == 0 ||
+	    strcmp (entry->d_name, "..") == 0 ||
+	    strcmp (entry->d_name, ".notmuch") == 0 ||
+	    _entry_in_ignore_list (entry->d_name, state))
+	{
+	    continue;
+	}
+
+	next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
+	status = add_files_mbox (notmuch, next, state);
+	if (status) {
+	    ret = status;
+	    goto DONE;
+	}
+	talloc_free (next);
+	next = NULL;
+    }
+
+    /* Pass 2: Scan for new files, removed files, and removed directories. */
+    for (i = 0; i < num_fs_entries; i++)
+    {
+	if (interrupted)
+	    break;
+
+        entry = fs_entries[i];
+
+	/* Ignore files & directories user has configured to be ignored */
+	if (_entry_in_ignore_list (entry->d_name, state))
+	    continue;
+
+	/* Only add regular files (and symlinks to regular files). */
+	entry_type = dirent_type (path, entry);
+	if (entry_type == -1) {
+	    fprintf (stderr, "Error reading file %s/%s: %s\n",
+		     path, entry->d_name, strerror (errno));
+	    return NOTMUCH_STATUS_FILE_ERROR;
+	} else if (entry_type != S_IFREG) {
+	    continue;
+	}
+
+	next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);
+	status = add_messages_mbox_file (notmuch, next, state);
+	talloc_free (next);
+	next = NULL;
+
+	if (status) {
+	    ret = status;
+	    goto DONE;
+	}
+    }
+
+DONE:
+    if (next)
+	talloc_free (next);
+    return ret;
+}
+
+/* Call out to the appropriate add_files function, based on the URI. */
+static notmuch_status_t
+add_files_uri (notmuch_database_t *notmuch,
+	       const char *uri,
+	       add_files_state_t *state)
+{
+    UriUriA parsed;
+    UriParserStateA parser;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    parser.uri = &parsed;
+    if (uriParseUriA (&parser, uri) != URI_SUCCESS)
+	goto DONE;
+
+    if (strncmp (parsed.scheme.first, "mbox",
+		 parsed.scheme.afterLast - parsed.scheme.first) == 0) {
+	ret = add_files_mbox (notmuch, parsed.pathHead->text.first - 1, state);
+	goto DONE;
+    }
+
+DONE:
+    uriFreeUriMembersA (&parsed);
+    return ret;
+}
+
 static void
 setup_progress_printing_timer (void)
 {
-- 
1.7.9.5

Re: [RFC PATCH 10/14] new: add "scan" option

[Mark Walters]

On Mon, 25 Jun 2012, Ethan Glasser-Camp <ethan.glasser.camp@gmail.com> wrote:
> This is just a quick hack to get started on adding an mbox backend.
>
> The fact that the default maildir is scanned "automagically" is a
> little weird, but it doesn't do any harm unless you decide to put mail
> there that you really don't want indexed.

I was a bit caught out by the way this worked. I expected that I would
add 
scan=mbox:///full_path_of_mbox 
but instead it seems that I need to add
scan=mbox:///full_path_of_directory_containing_the_mbox

I am happy with either (and yours is probably easier) but it should be
documented somewhere and I found the line
> +    "\tscan	A list (separated by ';') of mail URLs to scan.\n"

confusing.

Best wishes

Mark

> Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>
> ---
>  notmuch-client.h |    9 +++++++++
>  notmuch-config.c |   30 +++++++++++++++++++++++++++++-
>  notmuch-new.c    |   18 ++++++++++++++++++
>  test/config      |    1 +
>  4 files changed, 57 insertions(+), 1 deletion(-)
>
> diff --git a/notmuch-client.h b/notmuch-client.h
> index 9b63eae..9d922fe 100644
> --- a/notmuch-client.h
> +++ b/notmuch-client.h
> @@ -256,6 +256,15 @@ notmuch_config_set_new_ignore (notmuch_config_t *config,
>  			       const char *new_ignore[],
>  			       size_t length);
>  
> +const char **
> +notmuch_config_get_new_scan (notmuch_config_t *config,
> +			       size_t *length);
> +
> +void
> +notmuch_config_set_new_scan (notmuch_config_t *config,
> +			       const char *new_scan[],
> +			       size_t length);
> +
>  notmuch_bool_t
>  notmuch_config_get_maildir_synchronize_flags (notmuch_config_t *config);
>  
> diff --git a/notmuch-config.c b/notmuch-config.c
> index 3e37a2d..e9d99ea 100644
> --- a/notmuch-config.c
> +++ b/notmuch-config.c
> @@ -50,7 +50,10 @@ static const char new_config_comment[] =
>      "\t	that will not be searched for messages by \"notmuch new\".\n"
>      "\n"
>      "\t	NOTE: *Every* file/directory that goes by one of those names will\n"
> -    "\t	be ignored, independent of its depth/location in the mail store.\n";
> +    "\t	be ignored, independent of its depth/location in the mail store.\n"
> +    "\n"
> +    "\tscan	A list (separated by ';') of mail URLs to scan.\n"
> +    "\t	The maildir located at database.path, above, will automatically be added.\n";
>  
>  static const char user_config_comment[] =
>      " User configuration\n"
> @@ -113,6 +116,8 @@ struct _notmuch_config {
>      size_t new_tags_length;
>      const char **new_ignore;
>      size_t new_ignore_length;
> +    const char **new_scan;
> +    size_t new_scan_length;
>      notmuch_bool_t maildir_synchronize_flags;
>      const char **search_exclude_tags;
>      size_t search_exclude_tags_length;
> @@ -274,6 +279,8 @@ notmuch_config_open (void *ctx,
>      config->new_tags_length = 0;
>      config->new_ignore = NULL;
>      config->new_ignore_length = 0;
> +    config->new_scan = NULL;
> +    config->new_scan_length = 0;
>      config->maildir_synchronize_flags = TRUE;
>      config->search_exclude_tags = NULL;
>      config->search_exclude_tags_length = 0;
> @@ -375,6 +382,10 @@ notmuch_config_open (void *ctx,
>  	notmuch_config_set_new_ignore (config, NULL, 0);
>      }
>  
> +    if (notmuch_config_get_new_scan (config, &tmp) == NULL) {
> +	notmuch_config_set_new_scan (config, NULL, 0);
> +    }
> +
>      if (notmuch_config_get_search_exclude_tags (config, &tmp) == NULL) {
>  	if (is_new) {
>  	    const char *tags[] = { "deleted", "spam" };
> @@ -631,6 +642,14 @@ notmuch_config_get_new_ignore (notmuch_config_t *config, size_t *length)
>  			     &(config->new_ignore_length), length);
>  }
>  
> +const char **
> +notmuch_config_get_new_scan (notmuch_config_t *config, size_t *length)
> +{
> +    return _config_get_list (config, "new", "scan",
> +			     &(config->new_scan),
> +			     &(config->new_scan_length), length);
> +}
> +
>  void
>  notmuch_config_set_user_other_email (notmuch_config_t *config,
>  				     const char *list[],
> @@ -658,6 +677,15 @@ notmuch_config_set_new_ignore (notmuch_config_t *config,
>  		     &(config->new_ignore));
>  }
>  
> +void
> +notmuch_config_set_new_scan (notmuch_config_t *config,
> +			     const char *list[],
> +			     size_t length)
> +{
> +    _config_set_list (config, "new", "scan", list, length,
> +		     &(config->new_scan));
> +}
> +
>  const char **
>  notmuch_config_get_search_exclude_tags (notmuch_config_t *config, size_t *length)
>  {
> diff --git a/notmuch-new.c b/notmuch-new.c
> index 1f11b2c..57b27bf 100644
> --- a/notmuch-new.c
> +++ b/notmuch-new.c
> @@ -239,6 +239,16 @@ _entry_in_ignore_list (const char *entry, add_files_state_t *state)
>      return FALSE;
>  }
>  
> +/* Call out to the appropriate add_files function, based on the URI. */
> +static notmuch_status_t
> +add_files_uri (unused(notmuch_database_t *notmuch),
> +	       unused(const char *uri),
> +	       unused(add_files_state_t *state))
> +{
> +    /* Stub for now */
> +    return NOTMUCH_STATUS_SUCCESS;
> +}
> +
>  /* Examine 'path' recursively as follows:
>   *
>   *   o Ask the filesystem for the mtime of 'path' (fs_mtime)
> @@ -843,6 +853,8 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
>      int ret = 0;
>      struct stat st;
>      const char *db_path;
> +    const char **new_scan;
> +    size_t new_scan_length, new_scan_i;
>      char *dot_notmuch_path;
>      struct sigaction action;
>      _filename_node_t *f;
> @@ -941,6 +953,12 @@ notmuch_new_command (void *ctx, int argc, char *argv[])
>  	timer_is_active = TRUE;
>      }
>  
> +    new_scan = notmuch_config_get_new_scan (config, &new_scan_length);
> +
> +    for (new_scan_i = 0; new_scan_i < new_scan_length; new_scan_i++) {
> +	add_files_uri (notmuch, new_scan[new_scan_i], &add_files_state);
> +    }
> +
>      ret = add_files (notmuch, db_path, &add_files_state);
>      if (ret)
>  	goto DONE;
> diff --git a/test/config b/test/config
> index 93ecb13..b0ad0c1 100755
> --- a/test/config
> +++ b/test/config
> @@ -52,6 +52,7 @@ user.primary_email=test_suite@notmuchmail.org
>  user.other_email=test_suite_other@notmuchmail.org;test_suite@otherdomain.org
>  new.tags=unread;inbox;
>  new.ignore=
> +new.scan=
>  search.exclude_tags=
>  maildir.synchronize_flags=true
>  foo.string=this is another string value
> -- 
> 1.7.9.5
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> http://notmuchmail.org/mailman/listinfo/notmuch