WIP: git-remote-notmuch

WIP: git-remote-notmuch

[David Bremner]

I wanted to try out Felipe's idea of using a remote helper to replace
/ augment the existing notmuch-git script, but I didn't want to
require language bindings. Porting Felipe's ruby script to C led to
a predictable amount of increase in the number of lines of code, but
it seems to work. There are a couple of points to note

0) Currently this is not used by notmuch-git at all.

1) Felipe's original script didn't have an explicit license grant, but
I added the standard notmuch GPL3+ license to my rewrite. Hopefully
that is OK for Felipe.

2) I renamed to git-remote-notmuch, so the corresponding URLs start with
   notmuch:: instead of nm::
   
3) The original script took a path to mail_root, but this doesn't
   really cover the possible split configurations; in particular it
   doesn't handle my (XDG) layout. The current code just looks for the
   "default" notmuch setup, and ignores the URL.  I am open to ideas
   for how a notmuch URL should look like. I'd imagine they
   should have the profile in them, but I'm not sure how much flexibility we need / want to suppot.
   
4) the code is a bit rough and ready, with error handling mainly
consisting of asserts.

5) There is currently no documentation, see test/T860 and
perf-test/T08 for hints.

6) The remote helper is potentially complimentary to git-notmuch/nmbug
as it provides a more standard (to git users) interface.
git-notmuch could probably transition to a wrapper.

[PATCH 1/5] util: refactor sync_dir and mkdir_recursive

[David Bremner]

Moving these functions to libnotmuch_util will allow re-user from
either multiple CLI compilation units or from the library. To avoid
future surprises, replace printing to stderr with the usual status
string mechanism.
---
 notmuch-insert.c | 84 +++++------------------------------------
 util/path-util.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++-
 util/path-util.h |  8 ++++
 3 files changed, 114 insertions(+), 75 deletions(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index e44607ad..66c4f434 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -21,13 +21,13 @@
  * Author: Peter Wang <novalazy@gmail.com>
  */
 
-#include "notmuch-client.h"
-#include "tag-util.h"
 
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <fcntl.h>
+
+#include "notmuch-client.h"
+#include "tag-util.h"
 #include "string-util.h"
+#include "path-util.h"
 
 static volatile sig_atomic_t interrupted;
 
@@ -64,26 +64,6 @@ safe_gethostname (char *hostname, size_t len)
     }
 }
 
-/* Call fsync() on a directory path. */
-static bool
-sync_dir (const char *dir)
-{
-    int fd, r;
-
-    fd = open (dir, O_RDONLY);
-    if (fd == -1) {
-	fprintf (stderr, "Error: open %s: %s\n", dir, strerror (errno));
-	return false;
-    }
-
-    r = fsync (fd);
-    if (r)
-	fprintf (stderr, "Error: fsync %s: %s\n", dir, strerror (errno));
-
-    close (fd);
-
-    return r == 0;
-}
 
 /*
  * Check the specified folder name does not contain a directory
@@ -105,54 +85,6 @@ is_valid_folder_name (const char *folder)
     }
 }
 
-/*
- * Make the given directory and its parents as necessary, using the
- * given mode. Return true on success, false otherwise. Partial
- * results are not cleaned up on errors.
- */
-static bool
-mkdir_recursive (const void *ctx, const char *path, int mode)
-{
-    struct stat st;
-    int r;
-    char *parent = NULL, *slash;
-
-    /* First check the common case: directory already exists. */
-    r = stat (path, &st);
-    if (r == 0) {
-	if (! S_ISDIR (st.st_mode)) {
-	    fprintf (stderr, "Error: '%s' is not a directory: %s\n",
-		     path, strerror (EEXIST));
-	    return false;
-	}
-
-	return true;
-    } else if (errno != ENOENT) {
-	fprintf (stderr, "Error: stat '%s': %s\n", path, strerror (errno));
-	return false;
-    }
-
-    /* mkdir parents, if any */
-    slash = strrchr (path, '/');
-    if (slash && slash != path) {
-	parent = talloc_strndup (ctx, path, slash - path);
-	if (! parent) {
-	    fprintf (stderr, "Error: %s\n", strerror (ENOMEM));
-	    return false;
-	}
-
-	if (! mkdir_recursive (ctx, parent, mode))
-	    return false;
-    }
-
-    if (mkdir (path, mode)) {
-	fprintf (stderr, "Error: mkdir '%s': %s\n", path, strerror (errno));
-	return false;
-    }
-
-    return parent ? sync_dir (parent) : true;
-}
-
 /*
  * Create the given maildir folder, i.e. maildir and its
  * subdirectories cur/new/tmp. Return true on success, false
@@ -165,6 +97,7 @@ maildir_create_folder (const void *ctx, const char *maildir, bool world_readable
     const int mode = (world_readable ? 0755 : 0700);
     char *subdir;
     unsigned int i;
+    char **status_string = NULL;
 
     for (i = 0; i < ARRAY_SIZE (subdirs); i++) {
 	subdir = talloc_asprintf (ctx, "%s/%s", maildir, subdirs[i]);
@@ -173,7 +106,7 @@ maildir_create_folder (const void *ctx, const char *maildir, bool world_readable
 	    return false;
 	}
 
-	if (! mkdir_recursive (ctx, subdir, mode))
+	if (mkdir_recursive (ctx, subdir, mode, status_string))
 	    return false;
     }
 
@@ -347,6 +280,7 @@ static char *
 maildir_write_new (const void *ctx, int fdin, const char *maildir, bool world_readable)
 {
     char *cleanpath, *tmppath, *newpath, *newdir;
+    char *status_string = NULL;
 
     tmppath = maildir_write_tmp (ctx, fdin, maildir, world_readable);
     if (! tmppath)
@@ -375,13 +309,15 @@ maildir_write_new (const void *ctx, int fdin, const char *maildir, bool world_re
 	goto FAIL;
     }
 
-    if (! sync_dir (newdir))
+    if (sync_dir (newdir, &status_string))
 	goto FAIL;
 
     return newpath;
 
   FAIL:
     unlink (cleanpath);
+    if (status_string)
+	fputs (status_string, stderr);
 
     return NULL;
 }
diff --git a/util/path-util.c b/util/path-util.c
index 3267a967..b7114b18 100644
--- a/util/path-util.c
+++ b/util/path-util.c
@@ -5,10 +5,17 @@
 #define _GNU_SOURCE
 
 #include "path-util.h"
-
+#include "compat.h"
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
 #include <limits.h>
 #include <stdlib.h>
 
+#include <talloc.h>
 
 char *
 notmuch_canonicalize_file_name (const char *path)
@@ -25,3 +32,91 @@ notmuch_canonicalize_file_name (const char *path)
 #error undefined PATH_MAX _and_ missing canonicalize_file_name not supported
 #endif
 }
+
+/* Call fsync() on a directory path. */
+notmuch_status_t 
+sync_dir (const char *dir, char **status_string)
+{
+    int fd, r;
+
+    fd = open (dir, O_RDONLY);
+    if (fd == -1) {
+	if (status_string)
+	    IGNORE_RESULT (asprintf (status_string,
+				     "Error: open %s: %s\n", dir, strerror (errno)));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    r = fsync (fd);
+    if (r && status_string)
+	IGNORE_RESULT (asprintf (status_string,
+				 "Error: fsync %s: %s\n", dir, strerror (errno)));
+
+    close (fd);
+
+    return r == 0 ? NOTMUCH_STATUS_SUCCESS : NOTMUCH_STATUS_FILE_ERROR;
+}
+
+/*
+ * Make the given directory and its parents as necessary, using the
+ * given mode. Partial results are not cleaned up on errors.
+ */
+notmuch_status_t
+mkdir_recursive (const void *ctx, const char *path, int mode,
+		 char **status_string)
+{
+    notmuch_status_t status;
+    struct stat st;
+    int r;
+    char *parent = NULL, *slash;
+
+    /* First check the common case: directory already exists. */
+    r = stat (path, &st);
+    if (r == 0) {
+	if (! S_ISDIR (st.st_mode)) {
+	    if (status_string)
+		IGNORE_RESULT(asprintf (status_string, "Error: '%s' is not a directory: %s\n",
+					path, strerror (EEXIST)));
+	    return NOTMUCH_STATUS_FILE_ERROR;
+	}
+
+	return NOTMUCH_STATUS_SUCCESS;
+    } else if (errno != ENOENT) {
+	    if (status_string)
+		IGNORE_RESULT(asprintf (status_string,
+					"Error: stat '%s': %s\n", path, strerror (errno)));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    /* mkdir parents, if any */
+    slash = strrchr (path, '/');
+    if (slash && slash != path) {
+	parent = talloc_strndup (ctx, path, slash - path);
+	if (! parent) {
+	    if (status_string)
+		IGNORE_RESULT(asprintf (status_string,
+					"Error: %s\n",
+					strerror (ENOMEM)));
+	    return NOTMUCH_STATUS_FILE_ERROR;
+	}
+
+	status = mkdir_recursive (ctx, parent, mode, status_string);
+	if (status)
+	    return status;
+    }
+
+    if (mkdir (path, mode)) {
+	    if (status_string)
+		IGNORE_RESULT(asprintf (status_string,
+					"Error: mkdir '%s': %s\n",
+					path, strerror (errno)));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    if (parent) {
+	status = sync_dir (parent, status_string);
+	if (status)
+	    return status;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/util/path-util.h b/util/path-util.h
index ac85f696..fcdaf626 100644
--- a/util/path-util.h
+++ b/util/path-util.h
@@ -5,6 +5,8 @@
 #ifndef NOTMUCH_UTIL_PATH_UTIL_H_
 #define NOTMUCH_UTIL_PATH_UTIL_H_
 
+#include "notmuch.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -12,6 +14,12 @@ extern "C" {
 char *
 notmuch_canonicalize_file_name (const char *path);
 
+notmuch_status_t
+mkdir_recursive (const void *ctx, const char *path, int mode, char **status_string);
+
+notmuch_status_t
+sync_dir (const char *path, char **status_string);
+    
 #ifdef __cplusplus
 }
 #endif
-- 
2.43.0

[PATCH 2/5] cli: start remote helper for git.

[David Bremner]

This is closely based on git-remote-nm (in ruby) by Felipe Contreras.
Initially just implement the commands 'capabilites' and 'list'.  This
isn't enough to do anything useful so start some unit tests.
---
 Makefile.local          |   7 +-
 git-remote-notmuch.c    | 181 ++++++++++++++++++++++++++++++++++++++++
 test/T860-git-remote.sh |  45 ++++++++++
 3 files changed, 232 insertions(+), 1 deletion(-)
 create mode 100644 git-remote-notmuch.c
 create mode 100755 test/T860-git-remote.sh

diff --git a/Makefile.local b/Makefile.local
index 7699c208..2ac494b8 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -1,7 +1,8 @@
 # -*- makefile-gmake -*-
 
 .PHONY: all
-all: notmuch notmuch-shared build-man build-info ruby-bindings python-cffi-bindings notmuch-git nmbug
+all: notmuch notmuch-shared git-remote-notmuch \
+	build-man build-info ruby-bindings python-cffi-bindings notmuch-git nmbug
 ifeq ($(MAKECMDGOALS),)
 ifeq ($(shell cat .first-build-message 2>/dev/null),)
 	@NOTMUCH_FIRST_BUILD=1 $(MAKE) --no-print-directory all
@@ -274,6 +275,9 @@ notmuch: $(notmuch_client_modules) lib/libnotmuch.a util/libnotmuch_util.a parse
 notmuch-shared: $(notmuch_client_modules) lib/$(LINKER_NAME)
 	$(call quiet,$(FINAL_NOTMUCH_LINKER) $(CFLAGS)) $(notmuch_client_modules) $(FINAL_NOTMUCH_LDFLAGS) -o $@
 
+git-remote-notmuch: git-remote-notmuch.o status.o
+	$(call quiet,$(FINAL_NOTMUCH_LINKER) $(CFLAGS)) $^ $(FINAL_NOTMUCH_LDFLAGS) -o $@
+
 .PHONY: install
 install: all install-man install-info
 	mkdir -p "$(DESTDIR)$(prefix)/bin/"
@@ -302,6 +306,7 @@ endif
 
 SRCS  := $(SRCS) $(notmuch_client_srcs)
 CLEAN := $(CLEAN) notmuch notmuch-shared $(notmuch_client_modules)
+CLEAN := $(CLEAN) git-remote-notmuch git-remote-notmuch.o
 CLEAN := $(CLEAN) version.stamp notmuch-*.tar.gz.tmp
 CLEAN := $(CLEAN) .deps
 
diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
new file mode 100644
index 00000000..cfc43a68
--- /dev/null
+++ b/git-remote-notmuch.c
@@ -0,0 +1,181 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2023 Felipe Contreras
+ * Copyright © 2024 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Authors: Felipe Contreras
+ *	    David Bremner <david@tethera.net>
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <notmuch.h>
+#include "notmuch-client.h"
+#include "path-util.h"
+#include "hex-escape.h"
+#include "string-util.h"
+
+#define ASSERT(x) assert((x))
+
+/* File scope globals */
+const char *alias = NULL;
+const char *nm_dir = NULL;
+const char *url = NULL;
+const char* debug_flags = NULL;
+unsigned long lastmod;
+notmuch_database_t *db;
+FILE *log_file = NULL;
+
+static void
+flog (const char *format, ...) {
+    va_list va_args;
+
+    if (log_file) {
+	va_start (va_args, format);
+	vfprintf (log_file, format, va_args);
+	fflush (log_file);
+	va_end (va_args);
+    }
+}
+
+static unsigned long read_lastmod (const void *ctx, const char *dir) {
+    char *filename = NULL;
+    unsigned long num = 0;
+
+    FILE *in;
+
+    ASSERT(filename = talloc_asprintf (ctx, "%s/lastmod", dir));
+
+    in = fopen (filename, "r");
+    if (in) {
+	ASSERT(fscanf (in, "%zu", &num) == 1);
+    } else {
+	if (errno != ENOENT) {
+	    fprintf (stderr, "error opening lastmod file");
+	    exit(EXIT_FAILURE);
+	}
+    }
+
+    flog ("loaded lastmod = %zu\n", num);
+
+    return num;
+}
+
+static void
+cmd_capabilities () {
+    fputs("import\nexport\nrefspec refs/heads/*:refs/notmuch/*\n\n", stdout);
+    fflush (stdout);
+}
+
+static void
+cmd_list () {
+    unsigned long current_lastmod;
+    current_lastmod = notmuch_database_get_revision (db, NULL);
+    printf("? refs/heads/master%s\n\n",
+	   lastmod == current_lastmod ? " unchanged" : "");
+    fflush (stdout);
+}
+
+static void
+usage() {
+    fprintf (stderr, "usage: git-remote-nm ALIAS URL\n");
+    exit(EXIT_FAILURE);
+}
+
+int
+main (int argc, char *argv[])
+{
+  notmuch_status_t status;
+  char *status_string = NULL;
+  const char* git_dir;
+  ssize_t nread;
+  size_t len = 0;
+  const char *log_file_name;
+
+  char *line = NULL;
+
+  debug_flags = getenv ("GIT_REMOTE_NM_DEBUG");
+  log_file_name = getenv ("GIT_REMOTE_NM_LOG");
+
+  if (log_file_name)
+      log_file = fopen (log_file_name, "w");
+
+  if (argc != 3)
+    usage();
+  /* setup globals */
+  alias = argv[1];
+  url = argv[2];
+
+  status = notmuch_database_open_with_config (NULL,
+					      NOTMUCH_DATABASE_MODE_READ_WRITE,
+					      NULL,
+					      NULL,
+					      &db,
+					      &status_string);
+  if (status) {
+      if (status_string) {
+	  fputs (status_string, stderr);
+	  free (status_string);
+	  status_string = NULL;
+      }
+      return EXIT_FAILURE;
+  }
+
+  git_dir = getenv ("GIT_DIR");
+  if (! git_dir) {
+      fprintf (stderr, "GIT_DIR not set\n");
+      exit(EXIT_FAILURE);
+  }
+  flog ("GIT_DIR=%s\n", git_dir);
+
+  ASSERT(nm_dir = talloc_asprintf(db, "%s/%s", git_dir, "notmuch"));
+
+  status = mkdir_recursive (db, nm_dir, 0700, &status_string);
+  if (status) {
+      if (status_string)
+	  fputs(status_string, stderr);
+
+      exit (EXIT_FAILURE);
+  }
+
+  lastmod = read_lastmod (db, nm_dir);
+
+  while ((nread = getline(&line, &len, stdin)) != -1) {
+      size_t count=0;
+      char *s = line;
+      flog ("command = %s\n", line);
+
+      /* skip leading space */
+      while (*s && isspace(*s))  s++;
+      while (s[count]  && ! isspace(s[count])) count++;
+
+      if (count == 0)
+	  break;
+
+      if (STRNCMP_LITERAL (s, "capabilities")== 0)
+	  cmd_capabilities ();
+      else if (STRNCMP_LITERAL (s, "list") == 0)
+	  cmd_list ();
+
+      fflush(stdout);
+      flog ("finished command = %s\n", s);
+  }
+  flog ("finished loop\n");
+
+  notmuch_database_destroy (db);
+}
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
new file mode 100755
index 00000000..7b2b6b49
--- /dev/null
+++ b/test/T860-git-remote.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+test_description='git-remote-notmuch'
+. $(dirname "$0")/test-lib.sh || exit 1
+
+notmuch_sanitize_git() {
+    sed 's/^committer \(.*\) \(<[^>]*>\) [1-9][0-9]* [-+][0-9]*/committer \1 \2 TIMESTAMP TIMEZONE/'
+}
+
+add_email_corpus
+
+mkdir repo
+
+git_tmp=$(mktemp -d gitXXXXXXXX)
+
+run_helper () {
+    GIT_DIR=${git_tmp} git-remote-notmuch dummy-alias dummy-url
+}
+
+export GIT_COMMITTER_NAME="Notmuch Test Suite"
+export GIT_COMMITTER_EMAIL="notmuch@example.com"
+export GIT_REMOTE_NM_DEBUG="s"
+export GIT_REMOTE_NM_LOG=grn-log.txt
+EXPECTED=$NOTMUCH_SRCDIR/test/git-remote-nm.expected-output
+
+TAG_FILE="87/b1/4EFC743A.3060609@april.org/tags"
+
+test_begin_subtest 'capabilities'
+echo capabilities | run_helper > OUTPUT
+cat <<EOF > EXPECTED
+import
+export
+refspec refs/heads/*:refs/notmuch/*
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest 'list'
+echo list | run_helper > OUTPUT
+cat <<EOF > EXPECTED
+? refs/heads/master
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_done
-- 
2.43.0
\r

[PATCH 3/5] cli/git-remote: add support for import command

[David Bremner]

The output in default.import is based on a modified version
of Felipe's git-remote-nm with Blake2 hashing replaced by SHA1
(for portability). This enable git-pull, so test that as well.
---
 git-remote-notmuch.c    | 129 ++++++++++++++++++++++++++++++++++++++++
 test/T860-git-remote.sh |  38 +++++++++++-
 2 files changed, 166 insertions(+), 1 deletion(-)

diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index cfc43a68..a4ed98e2 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -76,6 +76,54 @@ static unsigned long read_lastmod (const void *ctx, const char *dir) {
     return num;
 }
 
+static void
+store_lastmod (notmuch_database_t *notmuch, const char *dir) {
+    char *filename = NULL;
+    FILE *out;
+
+    ASSERT(filename = talloc_asprintf (notmuch, "%s/lastmod", dir));
+
+    ASSERT(out = fopen (filename, "w"));
+
+    ASSERT(fprintf (out, "%zu\n", notmuch_database_get_revision (notmuch, NULL)) > 0);
+}
+
+static void
+read_one_line (FILE *stream, char **line_p, size_t *len_p) {
+    ssize_t nread;
+
+    nread = getline(line_p, len_p, stream);
+    if (nread < 0) {
+	perror ("getline");
+	exit (EXIT_FAILURE);
+    }
+    ASSERT(line_p);
+
+    chomp_newline(*line_p);
+}
+
+static const char*
+shell2string (const char* command) {
+    FILE *out;
+    char *line = NULL;
+    size_t len;
+
+    out = popen (command,"r");
+    if (out == NULL) {
+	perror("popen");
+	exit(EXIT_FAILURE);
+    }
+
+    read_one_line (out, &line, &len);
+    return line;
+}
+
+static void
+wr_data(const char *data) {
+    printf ("data %zu\n", strlen(data));
+    fputs (data, stdout);
+}
+
 static void
 cmd_capabilities () {
     fputs("import\nexport\nrefspec refs/heads/*:refs/notmuch/*\n\n", stdout);
@@ -97,6 +145,85 @@ usage() {
     exit(EXIT_FAILURE);
 }
 
+static void
+cmd_import (notmuch_database_t *notmuch) {
+    const char* ident = NULL;
+    const char* lastmod_str = NULL;
+    notmuch_messages_t *messages;
+    notmuch_status_t status;
+    notmuch_query_t *query;
+    char *mid_buf = NULL;
+    size_t mid_buf_len = 0;
+
+    ident = shell2string("git var GIT_COMMITTER_IDENT");
+
+    printf ("feature done\ncommit refs/notmuch/master\nmark :1\ncommitter %s\n", ident);
+
+    ASSERT(lastmod_str = talloc_asprintf (notmuch, "lastmod: %zu\n", lastmod));
+    wr_data (lastmod_str);
+    if (lastmod > 0)
+	puts("from refs/notmuch/master^0");
+    puts("deleteall");
+
+    status = notmuch_query_create_with_syntax (notmuch,
+					       "",
+					       NOTMUCH_QUERY_SYNTAX_XAPIAN,
+					       &query);
+
+    if (print_status_database ("git-remote-nm", notmuch, status))
+	exit(EXIT_FAILURE);
+
+    if (debug_flags && strchr (debug_flags, 's'))
+	notmuch_query_set_sort (query, NOTMUCH_SORT_NEWEST_FIRST);
+    else
+	notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
+
+    status = notmuch_query_search_messages (query, &messages);
+    if (print_status_query ("git-remote-nm", query, status))
+	exit(EXIT_FAILURE);
+
+    for (;
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+
+	const char* tag_buf = "";
+	const char* mid;
+
+	const char* hash;
+
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	mid = notmuch_message_get_message_id (message);
+
+	if (hex_encode (notmuch, mid, &mid_buf, &mid_buf_len) != HEX_SUCCESS) {
+	    fprintf (stderr, "Error: failed to hex-encode message-id %s\n", mid);
+	    exit(EXIT_FAILURE);
+	}
+
+	/* we can't use _notmuch_sha1_from_string because we don't want
+	 * to include the null terminator */
+	GChecksum *sha1;
+	sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+	g_checksum_update (sha1, (const guchar *) mid, strlen (mid));
+	hash = g_checksum_get_string (sha1);
+	printf ("M 644 inline %2.2s/%2.2s/%s/tags\n", hash, hash+2, mid_buf);
+
+	g_checksum_free (sha1);
+
+	for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+	     notmuch_tags_valid (tags);
+	     notmuch_tags_move_to_next (tags)) {
+	    const char *tag_str = notmuch_tags_get (tags);
+	    ASSERT(tag_buf=talloc_asprintf (message, "%s%s\n", tag_buf, tag_str));
+	}
+	wr_data (tag_buf);
+	notmuch_message_destroy (message);
+    }
+    puts("");
+    puts("done");
+    fflush (stdout);
+    store_lastmod(notmuch, nm_dir);
+}
+
 int
 main (int argc, char *argv[])
 {
@@ -169,6 +296,8 @@ main (int argc, char *argv[])
 
       if (STRNCMP_LITERAL (s, "capabilities")== 0)
 	  cmd_capabilities ();
+      else if (STRNCMP_LITERAL (s, "import") == 0)
+	  cmd_import (db);
       else if (STRNCMP_LITERAL (s, "list") == 0)
 	  cmd_list ();
 
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index 7b2b6b49..d5809e6b 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -20,7 +20,7 @@ export GIT_COMMITTER_NAME="Notmuch Test Suite"
 export GIT_COMMITTER_EMAIL="notmuch@example.com"
 export GIT_REMOTE_NM_DEBUG="s"
 export GIT_REMOTE_NM_LOG=grn-log.txt
-EXPECTED=$NOTMUCH_SRCDIR/test/git-remote-nm.expected-output
+EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
 
 TAG_FILE="87/b1/4EFC743A.3060609@april.org/tags"
 
@@ -42,4 +42,40 @@ cat <<EOF > EXPECTED
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest 'import writes lastmod file'
+echo import | run_helper dummy-alias dummy-url > /dev/null
+lastmod=$(notmuch count --lastmod '*' | cut -f3)
+test_expect_equal "${lastmod}" "$(cat ${git_tmp}/notmuch/lastmod)"
+
+# note that this test must not be the first time import is run,
+# because it depends on the lastmod file
+test_begin_subtest 'import produces expected output'
+echo import | run_helper | notmuch_sanitize_git > OUTPUT
+test_expect_equal_file $EXPECTED/default.import OUTPUT
+
+test_begin_subtest "clone has every message"
+git clone notmuch::currently-ignored repo
+find repo -name tags -type f | sed -e s,repo/../../,id:, -e s,/tags$,, | sort > OUTPUT
+notmuch search --output=messages '*' | sort > EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "pull get new tag"
+notmuch tag +zznew -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+unread
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
+test_begin_subtest "pull sees deletion"
+notmuch tag -unread -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
 test_done
-- 
2.43.0

[PATCH 4/5] cli/git-remote: add export command

[David Bremner]

This enables the push command, and the helper is now feature complete.
---
 git-remote-notmuch.c    | 157 ++++++++++++++++++++++++++++++++++++++++
 test/T860-git-remote.sh |  69 ++++++++++++++++++
 test/make-export.py     |  44 +++++++++++
 3 files changed, 270 insertions(+)
 create mode 100644 test/make-export.py

diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index a4ed98e2..35579004 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -224,6 +224,161 @@ cmd_import (notmuch_database_t *notmuch) {
     store_lastmod(notmuch, nm_dir);
 }
 
+static GString *
+get_tok (const char *line, size_t index) {
+
+    const char *tok = line;
+    size_t tok_len =0;
+
+    for (size_t count = 0; count<=index; count++) {
+	ASSERT(tok = strsplit_len (tok+tok_len, ' ', &tok_len));
+    }
+
+    return g_string_new_len (tok, tok_len);
+}
+
+static GString *
+read_data (char **line_p, size_t *len_p) {
+    ssize_t nread;
+    size_t bytes;
+    size_t data_size;
+    const char *tok;
+    size_t tok_len =0;
+
+    ASSERT (line_p);
+    ASSERT (len_p);
+    ASSERT((nread = getline(line_p, len_p, stdin) != -1));
+
+    tok=*line_p;
+    for (size_t count = 0; count<=1; count++) {
+	ASSERT(tok = strsplit_len (tok+tok_len, ' ', &tok_len));
+    }
+
+    /* TODO: don't ignore tok_len */
+    ASSERT(sscanf (tok, "%zu", &data_size) == 1);
+
+    *line_p = realloc (*line_p, data_size+1);
+    bytes = fread (*line_p, 1, data_size, stdin);
+    ASSERT (bytes == data_size);
+
+    *len_p = data_size;
+
+    return g_string_new_len (*line_p, *len_p);
+}
+
+static void free_string (GString *str) {
+    g_string_free (str, true);
+}
+
+static void
+cmd_export (notmuch_database_t *notmuch)
+{
+  ssize_t nread;
+  size_t len = 0;
+  char *line = NULL;
+  GHashTable* blobs;
+
+  ASSERT(blobs = g_hash_table_new_full ((GHashFunc)g_string_hash,
+					(GEqualFunc)g_string_equal,
+					(GDestroyNotify)free_string, (GDestroyNotify)free_string));
+
+  while ((nread = getline(&line, &len, stdin)) != -1) {
+      flog ("\texport %s\n", line);
+      if (STRNCMP_LITERAL (line, "done") == 0) {
+	  break;
+      }
+      else if (STRNCMP_LITERAL (line, "blob") == 0) {
+	  GString *mark;
+	  GString *data;
+
+	  flog ("export blob\n");
+	  read_one_line (stdin, &line, &len);
+	  mark = get_tok (line, 1);
+
+	  data = read_data (&line, &len);
+
+	  g_hash_table_insert (blobs, mark, data);
+	  read_one_line (stdin, &line, &len);
+	  /* TODO free things */
+      }
+
+      else if (STRNCMP_LITERAL (line, "commit") == 0) {
+	  char *mid = NULL;
+	  size_t mid_len = 0;
+
+	  flog ("export commit\n");
+
+	  /* mark for commit (ignored) */
+	  read_one_line (stdin, &line, &len);
+	  /* author (ignored) */
+	  read_one_line (stdin, &line, &len);
+	  /* committer (ignored) */
+	  read_one_line (stdin, &line, &len);
+
+	  /* commit message (ignored) */
+	  (void)read_data (&line, &len);
+
+	  while (strlen (line) > 0) {
+	      GString *mark, *path;
+	      GBytes *blob;
+	      char *basename;
+	      notmuch_message_t *message;
+	      char *tag;
+	      const char *tok;
+	      size_t tok_len;
+	      size_t max_tok_len;
+
+	      read_one_line (stdin, &line, &len);
+	      if (strlen (line) ==0)
+		  break;
+
+	      ASSERT(mark = get_tok (line, 2));
+	      ASSERT(blob = g_hash_table_lookup (blobs, mark));
+	      free_string (mark);
+
+	      ASSERT(path = get_tok (line, 3));
+
+	      basename = g_path_get_dirname (path->str+6);
+	      ASSERT(HEX_SUCCESS ==
+		     hex_decode (notmuch, basename, &mid, &mid_len));
+	      g_free (basename);
+	      free_string (path);
+
+	      ASSERT(NOTMUCH_STATUS_SUCCESS ==
+		     notmuch_database_find_message (notmuch, mid, &message));
+	      ASSERT(message);
+
+
+	      ASSERT(NOTMUCH_STATUS_SUCCESS ==
+		     notmuch_message_freeze (message));
+
+	      ASSERT(NOTMUCH_STATUS_SUCCESS ==
+		     notmuch_message_remove_all_tags (message));
+
+	      tok = g_bytes_get_data (blob, &max_tok_len);
+	      tok_len = 0;
+	      while ((tok_len < max_tok_len) &&
+		     (tok = strsplit_len (tok + tok_len, '\n', &tok_len)) != NULL) {
+		  tag = strndup (tok, tok_len);
+		  ASSERT(NOTMUCH_STATUS_SUCCESS ==
+			 notmuch_message_add_tag (message, tag));
+		  free (tag);
+	      }
+
+	      ASSERT(NOTMUCH_STATUS_SUCCESS ==
+		     notmuch_message_thaw (message));
+
+	      notmuch_message_destroy (message);
+	  }
+	  puts("ok refs/heads/master");
+      }
+
+  }
+  store_lastmod(notmuch, nm_dir);
+  puts("");
+  g_hash_table_destroy (blobs);
+}
+
 int
 main (int argc, char *argv[])
 {
@@ -296,6 +451,8 @@ main (int argc, char *argv[])
 
       if (STRNCMP_LITERAL (s, "capabilities")== 0)
 	  cmd_capabilities ();
+      else if (STRNCMP_LITERAL (s, "export") == 0)
+	  cmd_export (db);
       else if (STRNCMP_LITERAL (s, "import") == 0)
 	  cmd_import (db);
       else if (STRNCMP_LITERAL (s, "list") == 0)
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index d5809e6b..6dc55391 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -21,6 +21,7 @@ export GIT_COMMITTER_EMAIL="notmuch@example.com"
 export GIT_REMOTE_NM_DEBUG="s"
 export GIT_REMOTE_NM_LOG=grn-log.txt
 EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
+MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
 
 TAG_FILE="87/b1/4EFC743A.3060609@april.org/tags"
 
@@ -78,4 +79,72 @@ zznew
 EOF
 test_expect_equal_file EXPECTED repo/$TAG_FILE
 
+test_begin_subtest 'export runs'
+run_helper <<EOF | notmuch_sanitize_git > OUTPUT
+export
+blob
+mark :1
+data 10
+tag1
+tag2
+
+commit refs/heads/master
+mark :2
+author Notmuch Test Suite <notmuch@example.com> 1234 +0000
+committer Notmuch Test Suite <notmuch@example.com> 1234 +0000
+data 8
+ignored
+M 100644 :1 $TAG_FILE
+
+done
+
+EOF
+cat <<EOF > EXPECTED
+ok refs/heads/master
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+# this test depends on the previous one
+test_begin_subtest 'export modifies database'
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest 'restore via export'
+notmuch dump > BEFORE
+python3 $MAKE_EXPORT_PY > export.in
+notmuch tag +transient -- id:4EFC743A.3060609@april.org
+run_helper < export.in > OUTPUT
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "push updates database"
+git -C repo push origin master
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "adding tag via repo"
+cat<<EOF >repo/$TAG_FILE
+tag1
+tag2
+tag3
+EOF
+git -C repo add $TAG_FILE
+git -C repo commit -m 'testing push'
+git -C repo push origin master
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 +tag3 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done
diff --git a/test/make-export.py b/test/make-export.py
new file mode 100644
index 00000000..3837dc3a
--- /dev/null
+++ b/test/make-export.py
@@ -0,0 +1,44 @@
+# generate a test input for the 'export' subcommand of the
+# git-remote-notmuch helper
+
+from notmuch2 import Database
+from time import time
+from hashlib import sha1
+
+def hexencode(str):
+    output_charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.,"
+    out = ""
+    for char in str:
+        if not char in output_charset:
+            out+= f"%{ord(char):x}"
+        else:
+            out+= char
+    return out
+
+db = Database(config=Database.CONFIG.SEARCH)
+
+count=1
+print("export")
+mark={}
+
+for msg in db.messages(""):
+    mark[msg.messageid]=count
+    blob=""
+    for tag in msg.tags:
+        blob += f"{tag}\n"
+    print (f"blob\nmark :{count}");
+    print (f"data {len(blob)}\n{blob}")
+    count=count+1
+
+print (f"\ncommit refs/heads/master\nmark :{count+1}")
+ctime = int(time())
+print (f"author Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
+print (f"committer Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
+print (f"data 8\nignored")
+
+for msg in db.messages(""):
+    digest = sha1(msg.messageid.encode('utf8')).hexdigest()
+    filename = hexencode(msg.messageid)
+    print (f"M 100644 :{mark[msg.messageid]} {digest[0:2]}/{digest[2:4]}/{filename}/tags")
+
+print("\ndone\n")
-- 
2.43.0

[PATCH 5/5] cli/git-remote: add time performance test

[David Bremner]

Memory tests already existed (for correctness) but this helps sanity
check the performance to make sure we are in the same range as the
existing notmuch-git script.
---
 performance-test/T08-git-remote.sh | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100755 performance-test/T08-git-remote.sh

diff --git a/performance-test/T08-git-remote.sh b/performance-test/T08-git-remote.sh
new file mode 100755
index 00000000..799e221b
--- /dev/null
+++ b/performance-test/T08-git-remote.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+test_description='git-remote-notmuch'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+export GIT_COMMITTER_NAME="Notmuch Test Suite"
+export GIT_COMMITTER_EMAIL="notmuch@example.com"
+
+time_start
+
+time_run 'clone --bare' "git clone --quiet --bare notmuch::default default.git"
+time_run 'clone' "git clone --quiet notmuch::default"
+
+cd default
+hash=$(git hash-object --stdin -w < /dev/null)
+# replace all files with empty files
+git ls-tree -r HEAD | sed "s/blob [^\t]*/blob $hash/" | git update-index --index-info
+git commit -m'zero tags' 2>>log.txt 1>&2
+
+time_run "push" "git push --quiet origin master"
+
+time_done
-- 
2.43.0

Re: WIP: git-remote-notmuch

[David Bremner]

David Bremner <david@tethera.net> writes:

> 3) The original script took a path to mail_root, but this doesn't
> really cover the possible split configurations; in particular it
> doesn't handle my (XDG) layout. The current code just looks for the
> "default" notmuch setup, and ignores the URL.  I am open to ideas for
> how a notmuch URL should look like. I'd imagine they should have the
> profile in them, but I'm not sure how much flexibility we need / want
> to suppot.

I guess some kind of key value scheme like

  notmuch://?profile=foo
  notmuch::?profile=foo
  
or

  notmuch://?config=.bar?database=fub/zub

would work, basically a thin wrapper around
notmuch_database_open_with_config.

There is still room for bikeshedding about the syntax.  We could throw
in "localhost" for future-proofing,

notmuch://localhost

would pass NULL to all 3 relevant parameters of n_d_o_w_c