[PATCH] WIP: support XDG database directory

[PATCH] WIP: support XDG database directory

[David Bremner]

---

Just a rebase against current master, based on discussion in IRC
today.  AFAIK, the general approach could be extended to support a
"NOTMUCH_DATABASE_DIRECTORY" environment variable, which if think is
what Tomi was suggesting previously.

 lib/database.cc        | 65 ++++++++++++++++++++++++++++++++++++++++----------
 test/T560-lib-error.sh |  2 +-
 test/T590-libconfig.sh | 35 +++++++++++++++++++++++++++
 3 files changed, 89 insertions(+), 13 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index 02444e09..521949cd 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -800,6 +800,23 @@ notmuch_database_open (const char *path,
     return status;
 }
 
+static char *
+_xdg_database_path (void *ctx) {
+
+    const char *data_dir = NULL;
+
+    data_dir = getenv ("XDG_DATA_HOME");
+
+    if (! data_dir) {
+	const char *home = getenv ("HOME");
+	if (! home)
+	    return NULL;
+
+	data_dir = talloc_asprintf (ctx, "%s/.local/share", home);
+    }
+    return talloc_asprintf (ctx, "%s/notmuch", data_dir);
+}
+
 notmuch_status_t
 notmuch_database_open_verbose (const char *path,
 			       notmuch_database_mode_t mode,
@@ -810,6 +827,7 @@ notmuch_database_open_verbose (const char *path,
     void *local = talloc_new (NULL);
     notmuch_database_t *notmuch = NULL;
     char *notmuch_path, *xapian_path, *incompat_features;
+    char *xdg_path = NULL;
     char *message = NULL;
     struct stat st;
     int err;
@@ -817,21 +835,29 @@ notmuch_database_open_verbose (const char *path,
     static int initialized = 0;
 
     if (path == NULL) {
-	message = strdup ("Error: Cannot open a database for a NULL path.\n");
-	status = NOTMUCH_STATUS_NULL_POINTER;
-	goto DONE;
+	xdg_path = _xdg_database_path (local);
+	if (! xdg_path) {
+	    message = strdup ("Error: NULL path, and cannot compute XDG_DATA_HOME.\n");
+	    status = NOTMUCH_STATUS_NULL_POINTER;
+	    goto DONE;
+	}
     }
 
-    if (path[0] != '/') {
+    if (path && path[0] != '/') {
 	message = strdup ("Error: Database path must be absolute.\n");
 	status = NOTMUCH_STATUS_PATH_ERROR;
 	goto DONE;
     }
 
-    if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) {
-	message = strdup ("Out of memory\n");
-	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
-	goto DONE;
+    if (xdg_path) {
+	notmuch_path = xdg_path;
+    } else {
+	notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch");
+	if (! (notmuch_path)) {
+	    message = strdup ("Out of memory\n");
+	    status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	    goto DONE;
+	}
     }
 
     err = stat (notmuch_path, &st);
@@ -862,9 +888,12 @@ notmuch_database_open_verbose (const char *path,
     notmuch = talloc_zero (NULL, notmuch_database_t);
     notmuch->exception_reported = false;
     notmuch->status_string = NULL;
-    notmuch->path = talloc_strdup (notmuch, path);
-
-    strip_trailing(notmuch->path, '/');
+    if (path) {
+	notmuch->path = talloc_strdup (notmuch, path);
+        strip_trailing(notmuch->path, '/');
+    } else {
+	notmuch->path = NULL;
+    }
 
     notmuch->mode = mode;
     notmuch->atomic_nesting = 0;
@@ -1254,7 +1283,19 @@ notmuch_database_destroy (notmuch_database_t *notmuch)
 const char *
 notmuch_database_get_path (notmuch_database_t *notmuch)
 {
-    return notmuch->path;
+    char *path = NULL;
+    notmuch_status_t status;
+
+    if (notmuch->path)
+	return notmuch->path;
+
+    status = notmuch_database_get_config (notmuch, "maildir_root", &path);
+    if (status) {
+	_notmuch_database_log (notmuch, "unable to find maildir_root\n");
+	return NULL;
+    }
+
+    return path;
 }
 
 unsigned int
diff --git a/test/T560-lib-error.sh b/test/T560-lib-error.sh
index 06a6b860..115a0a35 100755
--- a/test/T560-lib-error.sh
+++ b/test/T560-lib-error.sh
@@ -22,7 +22,7 @@ EOF
 cat <<'EOF' >EXPECTED
 == stdout ==
 == stderr ==
-Error: Cannot open a database for a NULL path.
+Error opening database at CWD/home/.local/share/notmuch: No such file or directory
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
index 46f3a76d..eaaec43f 100755
--- a/test/T590-libconfig.sh
+++ b/test/T590-libconfig.sh
@@ -132,4 +132,39 @@ notmuch restore --include=config <EXPECTED
 notmuch dump --include=config >OUTPUT
 test_expect_equal_file EXPECTED OUTPUT
 
+XDG_DIR=$HOME/.local/share/notmuch
+test_begin_subtest "Split database and maildir"
+xapian-metadata set ${MAIL_DIR}/.notmuch/xapian Cmaildir_root ${MAIL_DIR}
+mkdir -p $XDG_DIR
+mv ${MAIL_DIR}/.notmuch/xapian $XDG_DIR
+test_C <<EOF >OUTPUT
+#include <stdio.h>
+#include <notmuch.h>
+
+int main (int argc, char** argv)
+{
+   notmuch_database_t *db;
+   char *val;
+   notmuch_status_t stat;
+   notmuch_message_t *message;
+
+   stat=notmuch_database_open (NULL, NOTMUCH_DATABASE_MODE_READ_WRITE, &db);
+   printf("database_open status = %d\n", stat);
+   stat = notmuch_database_find_message (db, "87ocn0qh6d.fsf@yoom.home.cworth.org", &message);
+   printf("find_message status = %d\n", stat);
+   printf("found message = %d\n", message != NULL);
+   printf("filename = %s\n",notmuch_message_get_filename (message));
+}
+EOF
+
+cat <<EOF >EXPECTED
+== stdout ==
+database_open status = 0
+find_message status = 0
+found message = 1
+filename = MAIL_DIR/cur/41:2,
+== stderr ==
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done
-- 
2.15.1

Re: [PATCH] WIP: support XDG database directory

[Daniel Kahn Gillmor]

[-- Attachment #1: Type: text/plain, Size: 1147 bytes --]

On Sat 2017-12-30 16:07:40 -0400, David Bremner wrote:

> Just a rebase against current master, based on discussion in IRC
> today.  AFAIK, the general approach could be extended to support a
> "NOTMUCH_DATABASE_DIRECTORY" environment variable, which if think is
> what Tomi was suggesting previously.

I'm not sure i understand the rationale here -- it looks like this might
move the notmuch database directory away from its historic location
alongside the set of maildirs (then "mailstore").  is that right?

where would each such notmuch database live if there were two different
mailstores?  how is any given mailstore bound to the associated notmuch
database itself?  does the user point explicitly to both the mailstore
and the database? or does pointing to the mailstore reveal the location
of the database? or vice versa?

is it possible (or should it be) to have one mailstore that is indexed
by multiple databases?

i ask because i've been thinking about ways to protect the index itself,
but i want to make sure i understand all the different ways that the
mailstore and the database are (or are not) coupled to each other.

     --dkg

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 832 bytes --]

Re: [PATCH] WIP: support XDG database directory

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

> On Sat 2017-12-30 16:07:40 -0400, David Bremner wrote:
>
>> Just a rebase against current master, based on discussion in IRC
>> today.  AFAIK, the general approach could be extended to support a 
>> "NOTMUCH_DATABASE_DIRECTORY" environment variable, which if think is
>> what Tomi was suggesting previously.
>
> I'm not sure i understand the rationale here -- it looks like this might
> move the notmuch database directory away from its historic location
> alongside the set of maildirs (then "mailstore").  is that right?
>

Correct. This is an often reguested feature.

> where would each such notmuch database live if there were two different
> mailstores?  how is any given mailstore bound to the associated notmuch
> database itself?

Each database would point to the associated maildir_root in this version.

> does the user point explicitly to both the mailstore
> and the database?

at the library level, I'm thinking the right API is probably to take
both parameters (database and maildir), and if

- database_path is NULL => look in traditional .notmuch location
  current notmuch_database_open would call something like
          notmuch_database_open2 (maildir, NULL).
  Presumable with a more informative name.
  
- maildir_path is NULL => look in the database for a configuration

- both are NULL => look in $ENV{NOTMUCH_DATABASE_DIRECTORY} and then
  under $ENV{XDG_DATA_HOME}/notmuch, using the database to find the
  maildir (essentially this patch)

  This last case is aimed at allowing users of the library to open a
  "default" database without the current parsing of .notmuch-config
  
> is it possible (or should it be) to have one mailstore that is indexed
> by multiple databases?

Yes, this would possible, at least in terms of switching databases
associated with a maildir. It doesn't cost extra implimentation
effort. Whether it's a good thing or not, I don't know. As I write this
I'm reminded that Xapian can search across multiple databases. I'm not
sure if that's worth thinking about here.

> i ask because i've been thinking about ways to protect the index itself,
> but i want to make sure i understand all the different ways that the
> mailstore and the database are (or are not) coupled to each other.

I guess the initial proposal would be not at all? That probably allows
a few new kinds of user error; I'm imagining runing notmuch new with
mismatched database and maildir, and happily deleteing all of the
database documents.

d

Re: [PATCH] WIP: support XDG database directory

[Daniel Kahn Gillmor]

On Tue 2018-01-02 20:00:33 -0400, David Bremner wrote:
> Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:
>> where would each such notmuch database live if there were two different
>> mailstores?  how is any given mailstore bound to the associated notmuch
>> database itself?
>
> Each database would point to the associated maildir_root in this version.
>
>> does the user point explicitly to both the mailstore
>> and the database?
>
> at the library level, I'm thinking the right API is probably to take
> both parameters (database and maildir), and if
>
> - database_path is NULL => look in traditional .notmuch location
>   current notmuch_database_open would call something like
>           notmuch_database_open2 (maildir, NULL).
>   Presumable with a more informative name.
>   
> - maildir_path is NULL => look in the database for a configuration
>
> - both are NULL => look in $ENV{NOTMUCH_DATABASE_DIRECTORY} and then
>   under $ENV{XDG_DATA_HOME}/notmuch, using the database to find the
>   maildir (essentially this patch)
>
>   This last case is aimed at allowing users of the library to open a
>   "default" database without the current parsing of .notmuch-config

i like the sound of this, but...

>> i ask because i've been thinking about ways to protect the index itself,
>> but i want to make sure i understand all the different ways that the
>> mailstore and the database are (or are not) coupled to each other.
>
> I guess the initial proposal would be not at all?

this doesn't seem like a correct characterization of it, unless we're
misunderstanding each other.

it looks to me like each notmuch database *would* be coupled with
exactly one mailstore, given this proposal.  is that right?

if the library tried to open the database on its own, it would look at
the database's stored maildir_root path.

or am i missing something?

> That probably allows a few new kinds of user error; I'm imagining
> runing notmuch new with mismatched database and maildir, and happily
> deleteing all of the database documents.

yikes!  still, i agree that in general i'd rather point notmuch at the
database and have it discover everything it needs to from that; rather
than the current approach of the split config file and database, which
seems cumbersome and error-prone in other ways.

that said, i *definitely* prefer the database identifying the mailstore
(as you've done here) rather than the other way around.

one final question: for portable databases, which live on a removable
volume, or which are on networked-accessible storage and might be
mounted in different places on different computers, do we have a way to
protect them from the "whoops, mailstore is missing because you are on
the wrong host" or ("…because you mounted the USB stick at /Media/foo
instead of /Volumes/foo") scenario?

        --dkg

Re: [PATCH] WIP: support XDG database directory

[David Bremner]

Daniel Kahn Gillmor <dkg@fifthhorseman.net> writes:

>
> this doesn't seem like a correct characterization of it, unless we're
> misunderstanding each other.
>
> it looks to me like each notmuch database *would* be coupled with
> exactly one mailstore, given this proposal.  is that right?
>
> if the library tried to open the database on its own, it would look at
> the database's stored maildir_root path.

Right, with the possibility of the user overriding the maildir, at least
at the library level. 

> or am i missing something?
>
>> That probably allows a few new kinds of user error; I'm imagining
>> runing notmuch new with mismatched database and maildir, and happily
>> deleteing all of the database documents.
>
> yikes!  still, i agree that in general i'd rather point notmuch at the
> database and have it discover everything it needs to from that; rather
> than the current approach of the split config file and database, which
> seems cumbersome and error-prone in other ways.
>
> that said, i *definitely* prefer the database identifying the mailstore
> (as you've done here) rather than the other way around.
>
> one final question: for portable databases, which live on a removable
> volume, or which are on networked-accessible storage and might be
> mounted in different places on different computers, do we have a way to
> protect them from the "whoops, mailstore is missing because you are on
> the wrong host" or ("…because you mounted the USB stick at /Media/foo
> instead of /Volumes/foo") scenario?

That's something that needs to be thought through. Essentially the same
issue I mentioned.

d