* [PATCH 1/5] util: refactor sync_dir and mkdir_recursive
2024-08-28 15:45 WIP: git-remote-notmuch David Bremner
@ 2024-08-28 15:45 ` David Bremner
2024-08-28 15:45 ` [PATCH 2/5] cli: start remote helper for git David Bremner
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: David Bremner @ 2024-08-28 15:45 UTC (permalink / raw)
To: notmuch
Moving these functions to libnotmuch_util will allow re-user from
either multiple CLI compilation units or from the library. To avoid
future surprises, replace printing to stderr with the usual status
string mechanism.
---
notmuch-insert.c | 84 +++++------------------------------------
util/path-util.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++-
util/path-util.h | 8 ++++
3 files changed, 114 insertions(+), 75 deletions(-)
diff --git a/notmuch-insert.c b/notmuch-insert.c
index e44607ad..66c4f434 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -21,13 +21,13 @@
* Author: Peter Wang <novalazy@gmail.com>
*/
-#include "notmuch-client.h"
-#include "tag-util.h"
-#include <sys/types.h>
-#include <sys/stat.h>
#include <fcntl.h>
+
+#include "notmuch-client.h"
+#include "tag-util.h"
#include "string-util.h"
+#include "path-util.h"
static volatile sig_atomic_t interrupted;
@@ -64,26 +64,6 @@ safe_gethostname (char *hostname, size_t len)
}
}
-/* Call fsync() on a directory path. */
-static bool
-sync_dir (const char *dir)
-{
- int fd, r;
-
- fd = open (dir, O_RDONLY);
- if (fd == -1) {
- fprintf (stderr, "Error: open %s: %s\n", dir, strerror (errno));
- return false;
- }
-
- r = fsync (fd);
- if (r)
- fprintf (stderr, "Error: fsync %s: %s\n", dir, strerror (errno));
-
- close (fd);
-
- return r == 0;
-}
/*
* Check the specified folder name does not contain a directory
@@ -105,54 +85,6 @@ is_valid_folder_name (const char *folder)
}
}
-/*
- * Make the given directory and its parents as necessary, using the
- * given mode. Return true on success, false otherwise. Partial
- * results are not cleaned up on errors.
- */
-static bool
-mkdir_recursive (const void *ctx, const char *path, int mode)
-{
- struct stat st;
- int r;
- char *parent = NULL, *slash;
-
- /* First check the common case: directory already exists. */
- r = stat (path, &st);
- if (r == 0) {
- if (! S_ISDIR (st.st_mode)) {
- fprintf (stderr, "Error: '%s' is not a directory: %s\n",
- path, strerror (EEXIST));
- return false;
- }
-
- return true;
- } else if (errno != ENOENT) {
- fprintf (stderr, "Error: stat '%s': %s\n", path, strerror (errno));
- return false;
- }
-
- /* mkdir parents, if any */
- slash = strrchr (path, '/');
- if (slash && slash != path) {
- parent = talloc_strndup (ctx, path, slash - path);
- if (! parent) {
- fprintf (stderr, "Error: %s\n", strerror (ENOMEM));
- return false;
- }
-
- if (! mkdir_recursive (ctx, parent, mode))
- return false;
- }
-
- if (mkdir (path, mode)) {
- fprintf (stderr, "Error: mkdir '%s': %s\n", path, strerror (errno));
- return false;
- }
-
- return parent ? sync_dir (parent) : true;
-}
-
/*
* Create the given maildir folder, i.e. maildir and its
* subdirectories cur/new/tmp. Return true on success, false
@@ -165,6 +97,7 @@ maildir_create_folder (const void *ctx, const char *maildir, bool world_readable
const int mode = (world_readable ? 0755 : 0700);
char *subdir;
unsigned int i;
+ char **status_string = NULL;
for (i = 0; i < ARRAY_SIZE (subdirs); i++) {
subdir = talloc_asprintf (ctx, "%s/%s", maildir, subdirs[i]);
@@ -173,7 +106,7 @@ maildir_create_folder (const void *ctx, const char *maildir, bool world_readable
return false;
}
- if (! mkdir_recursive (ctx, subdir, mode))
+ if (mkdir_recursive (ctx, subdir, mode, status_string))
return false;
}
@@ -347,6 +280,7 @@ static char *
maildir_write_new (const void *ctx, int fdin, const char *maildir, bool world_readable)
{
char *cleanpath, *tmppath, *newpath, *newdir;
+ char *status_string = NULL;
tmppath = maildir_write_tmp (ctx, fdin, maildir, world_readable);
if (! tmppath)
@@ -375,13 +309,15 @@ maildir_write_new (const void *ctx, int fdin, const char *maildir, bool world_re
goto FAIL;
}
- if (! sync_dir (newdir))
+ if (sync_dir (newdir, &status_string))
goto FAIL;
return newpath;
FAIL:
unlink (cleanpath);
+ if (status_string)
+ fputs (status_string, stderr);
return NULL;
}
diff --git a/util/path-util.c b/util/path-util.c
index 3267a967..b7114b18 100644
--- a/util/path-util.c
+++ b/util/path-util.c
@@ -5,10 +5,17 @@
#define _GNU_SOURCE
#include "path-util.h"
-
+#include "compat.h"
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
#include <limits.h>
#include <stdlib.h>
+#include <talloc.h>
char *
notmuch_canonicalize_file_name (const char *path)
@@ -25,3 +32,91 @@ notmuch_canonicalize_file_name (const char *path)
#error undefined PATH_MAX _and_ missing canonicalize_file_name not supported
#endif
}
+
+/* Call fsync() on a directory path. */
+notmuch_status_t
+sync_dir (const char *dir, char **status_string)
+{
+ int fd, r;
+
+ fd = open (dir, O_RDONLY);
+ if (fd == -1) {
+ if (status_string)
+ IGNORE_RESULT (asprintf (status_string,
+ "Error: open %s: %s\n", dir, strerror (errno)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ r = fsync (fd);
+ if (r && status_string)
+ IGNORE_RESULT (asprintf (status_string,
+ "Error: fsync %s: %s\n", dir, strerror (errno)));
+
+ close (fd);
+
+ return r == 0 ? NOTMUCH_STATUS_SUCCESS : NOTMUCH_STATUS_FILE_ERROR;
+}
+
+/*
+ * Make the given directory and its parents as necessary, using the
+ * given mode. Partial results are not cleaned up on errors.
+ */
+notmuch_status_t
+mkdir_recursive (const void *ctx, const char *path, int mode,
+ char **status_string)
+{
+ notmuch_status_t status;
+ struct stat st;
+ int r;
+ char *parent = NULL, *slash;
+
+ /* First check the common case: directory already exists. */
+ r = stat (path, &st);
+ if (r == 0) {
+ if (! S_ISDIR (st.st_mode)) {
+ if (status_string)
+ IGNORE_RESULT(asprintf (status_string, "Error: '%s' is not a directory: %s\n",
+ path, strerror (EEXIST)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ return NOTMUCH_STATUS_SUCCESS;
+ } else if (errno != ENOENT) {
+ if (status_string)
+ IGNORE_RESULT(asprintf (status_string,
+ "Error: stat '%s': %s\n", path, strerror (errno)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ /* mkdir parents, if any */
+ slash = strrchr (path, '/');
+ if (slash && slash != path) {
+ parent = talloc_strndup (ctx, path, slash - path);
+ if (! parent) {
+ if (status_string)
+ IGNORE_RESULT(asprintf (status_string,
+ "Error: %s\n",
+ strerror (ENOMEM)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ status = mkdir_recursive (ctx, parent, mode, status_string);
+ if (status)
+ return status;
+ }
+
+ if (mkdir (path, mode)) {
+ if (status_string)
+ IGNORE_RESULT(asprintf (status_string,
+ "Error: mkdir '%s': %s\n",
+ path, strerror (errno)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ if (parent) {
+ status = sync_dir (parent, status_string);
+ if (status)
+ return status;
+ }
+ return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/util/path-util.h b/util/path-util.h
index ac85f696..fcdaf626 100644
--- a/util/path-util.h
+++ b/util/path-util.h
@@ -5,6 +5,8 @@
#ifndef NOTMUCH_UTIL_PATH_UTIL_H_
#define NOTMUCH_UTIL_PATH_UTIL_H_
+#include "notmuch.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -12,6 +14,12 @@ extern "C" {
char *
notmuch_canonicalize_file_name (const char *path);
+notmuch_status_t
+mkdir_recursive (const void *ctx, const char *path, int mode, char **status_string);
+
+notmuch_status_t
+sync_dir (const char *path, char **status_string);
+
#ifdef __cplusplus
}
#endif
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 2/5] cli: start remote helper for git.
2024-08-28 15:45 WIP: git-remote-notmuch David Bremner
2024-08-28 15:45 ` [PATCH 1/5] util: refactor sync_dir and mkdir_recursive David Bremner
@ 2024-08-28 15:45 ` David Bremner
2024-08-28 15:45 ` [PATCH 3/5] cli/git-remote: add support for import command David Bremner
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: David Bremner @ 2024-08-28 15:45 UTC (permalink / raw)
To: notmuch
This is closely based on git-remote-nm (in ruby) by Felipe Contreras.
Initially just implement the commands 'capabilites' and 'list'. This
isn't enough to do anything useful so start some unit tests.
---
Makefile.local | 7 +-
git-remote-notmuch.c | 181 ++++++++++++++++++++++++++++++++++++++++
test/T860-git-remote.sh | 45 ++++++++++
3 files changed, 232 insertions(+), 1 deletion(-)
create mode 100644 git-remote-notmuch.c
create mode 100755 test/T860-git-remote.sh
diff --git a/Makefile.local b/Makefile.local
index 7699c208..2ac494b8 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -1,7 +1,8 @@
# -*- makefile-gmake -*-
.PHONY: all
-all: notmuch notmuch-shared build-man build-info ruby-bindings python-cffi-bindings notmuch-git nmbug
+all: notmuch notmuch-shared git-remote-notmuch \
+ build-man build-info ruby-bindings python-cffi-bindings notmuch-git nmbug
ifeq ($(MAKECMDGOALS),)
ifeq ($(shell cat .first-build-message 2>/dev/null),)
@NOTMUCH_FIRST_BUILD=1 $(MAKE) --no-print-directory all
@@ -274,6 +275,9 @@ notmuch: $(notmuch_client_modules) lib/libnotmuch.a util/libnotmuch_util.a parse
notmuch-shared: $(notmuch_client_modules) lib/$(LINKER_NAME)
$(call quiet,$(FINAL_NOTMUCH_LINKER) $(CFLAGS)) $(notmuch_client_modules) $(FINAL_NOTMUCH_LDFLAGS) -o $@
+git-remote-notmuch: git-remote-notmuch.o status.o
+ $(call quiet,$(FINAL_NOTMUCH_LINKER) $(CFLAGS)) $^ $(FINAL_NOTMUCH_LDFLAGS) -o $@
+
.PHONY: install
install: all install-man install-info
mkdir -p "$(DESTDIR)$(prefix)/bin/"
@@ -302,6 +306,7 @@ endif
SRCS := $(SRCS) $(notmuch_client_srcs)
CLEAN := $(CLEAN) notmuch notmuch-shared $(notmuch_client_modules)
+CLEAN := $(CLEAN) git-remote-notmuch git-remote-notmuch.o
CLEAN := $(CLEAN) version.stamp notmuch-*.tar.gz.tmp
CLEAN := $(CLEAN) .deps
diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
new file mode 100644
index 00000000..cfc43a68
--- /dev/null
+++ b/git-remote-notmuch.c
@@ -0,0 +1,181 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2023 Felipe Contreras
+ * Copyright © 2024 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see https://www.gnu.org/licenses/ .
+ *
+ * Authors: Felipe Contreras
+ * David Bremner <david@tethera.net>
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <notmuch.h>
+#include "notmuch-client.h"
+#include "path-util.h"
+#include "hex-escape.h"
+#include "string-util.h"
+
+#define ASSERT(x) assert((x))
+
+/* File scope globals */
+const char *alias = NULL;
+const char *nm_dir = NULL;
+const char *url = NULL;
+const char* debug_flags = NULL;
+unsigned long lastmod;
+notmuch_database_t *db;
+FILE *log_file = NULL;
+
+static void
+flog (const char *format, ...) {
+ va_list va_args;
+
+ if (log_file) {
+ va_start (va_args, format);
+ vfprintf (log_file, format, va_args);
+ fflush (log_file);
+ va_end (va_args);
+ }
+}
+
+static unsigned long read_lastmod (const void *ctx, const char *dir) {
+ char *filename = NULL;
+ unsigned long num = 0;
+
+ FILE *in;
+
+ ASSERT(filename = talloc_asprintf (ctx, "%s/lastmod", dir));
+
+ in = fopen (filename, "r");
+ if (in) {
+ ASSERT(fscanf (in, "%zu", &num) == 1);
+ } else {
+ if (errno != ENOENT) {
+ fprintf (stderr, "error opening lastmod file");
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ flog ("loaded lastmod = %zu\n", num);
+
+ return num;
+}
+
+static void
+cmd_capabilities () {
+ fputs("import\nexport\nrefspec refs/heads/*:refs/notmuch/*\n\n", stdout);
+ fflush (stdout);
+}
+
+static void
+cmd_list () {
+ unsigned long current_lastmod;
+ current_lastmod = notmuch_database_get_revision (db, NULL);
+ printf("? refs/heads/master%s\n\n",
+ lastmod == current_lastmod ? " unchanged" : "");
+ fflush (stdout);
+}
+
+static void
+usage() {
+ fprintf (stderr, "usage: git-remote-nm ALIAS URL\n");
+ exit(EXIT_FAILURE);
+}
+
+int
+main (int argc, char *argv[])
+{
+ notmuch_status_t status;
+ char *status_string = NULL;
+ const char* git_dir;
+ ssize_t nread;
+ size_t len = 0;
+ const char *log_file_name;
+
+ char *line = NULL;
+
+ debug_flags = getenv ("GIT_REMOTE_NM_DEBUG");
+ log_file_name = getenv ("GIT_REMOTE_NM_LOG");
+
+ if (log_file_name)
+ log_file = fopen (log_file_name, "w");
+
+ if (argc != 3)
+ usage();
+ /* setup globals */
+ alias = argv[1];
+ url = argv[2];
+
+ status = notmuch_database_open_with_config (NULL,
+ NOTMUCH_DATABASE_MODE_READ_WRITE,
+ NULL,
+ NULL,
+ &db,
+ &status_string);
+ if (status) {
+ if (status_string) {
+ fputs (status_string, stderr);
+ free (status_string);
+ status_string = NULL;
+ }
+ return EXIT_FAILURE;
+ }
+
+ git_dir = getenv ("GIT_DIR");
+ if (! git_dir) {
+ fprintf (stderr, "GIT_DIR not set\n");
+ exit(EXIT_FAILURE);
+ }
+ flog ("GIT_DIR=%s\n", git_dir);
+
+ ASSERT(nm_dir = talloc_asprintf(db, "%s/%s", git_dir, "notmuch"));
+
+ status = mkdir_recursive (db, nm_dir, 0700, &status_string);
+ if (status) {
+ if (status_string)
+ fputs(status_string, stderr);
+
+ exit (EXIT_FAILURE);
+ }
+
+ lastmod = read_lastmod (db, nm_dir);
+
+ while ((nread = getline(&line, &len, stdin)) != -1) {
+ size_t count=0;
+ char *s = line;
+ flog ("command = %s\n", line);
+
+ /* skip leading space */
+ while (*s && isspace(*s)) s++;
+ while (s[count] && ! isspace(s[count])) count++;
+
+ if (count == 0)
+ break;
+
+ if (STRNCMP_LITERAL (s, "capabilities")== 0)
+ cmd_capabilities ();
+ else if (STRNCMP_LITERAL (s, "list") == 0)
+ cmd_list ();
+
+ fflush(stdout);
+ flog ("finished command = %s\n", s);
+ }
+ flog ("finished loop\n");
+
+ notmuch_database_destroy (db);
+}
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
new file mode 100755
index 00000000..7b2b6b49
--- /dev/null
+++ b/test/T860-git-remote.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+test_description='git-remote-notmuch'
+. $(dirname "$0")/test-lib.sh || exit 1
+
+notmuch_sanitize_git() {
+ sed 's/^committer \(.*\) \(<[^>]*>\) [1-9][0-9]* [-+][0-9]*/committer \1 \2 TIMESTAMP TIMEZONE/'
+}
+
+add_email_corpus
+
+mkdir repo
+
+git_tmp=$(mktemp -d gitXXXXXXXX)
+
+run_helper () {
+ GIT_DIR=${git_tmp} git-remote-notmuch dummy-alias dummy-url
+}
+
+export GIT_COMMITTER_NAME="Notmuch Test Suite"
+export GIT_COMMITTER_EMAIL="notmuch@example.com"
+export GIT_REMOTE_NM_DEBUG="s"
+export GIT_REMOTE_NM_LOG=grn-log.txt
+EXPECTED=$NOTMUCH_SRCDIR/test/git-remote-nm.expected-output
+
+TAG_FILE="87/b1/4EFC743A.3060609@april.org/tags"
+
+test_begin_subtest 'capabilities'
+echo capabilities | run_helper > OUTPUT
+cat <<EOF > EXPECTED
+import
+export
+refspec refs/heads/*:refs/notmuch/*
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest 'list'
+echo list | run_helper > OUTPUT
+cat <<EOF > EXPECTED
+? refs/heads/master
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_done
--
2.43.0
\r
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 3/5] cli/git-remote: add support for import command
2024-08-28 15:45 WIP: git-remote-notmuch David Bremner
2024-08-28 15:45 ` [PATCH 1/5] util: refactor sync_dir and mkdir_recursive David Bremner
2024-08-28 15:45 ` [PATCH 2/5] cli: start remote helper for git David Bremner
@ 2024-08-28 15:45 ` David Bremner
2024-08-28 15:45 ` [PATCH 4/5] cli/git-remote: add export command David Bremner
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: David Bremner @ 2024-08-28 15:45 UTC (permalink / raw)
To: notmuch
The output in default.import is based on a modified version
of Felipe's git-remote-nm with Blake2 hashing replaced by SHA1
(for portability). This enable git-pull, so test that as well.
---
git-remote-notmuch.c | 129 ++++++++++++++++++++++++++++++++++++++++
test/T860-git-remote.sh | 38 +++++++++++-
2 files changed, 166 insertions(+), 1 deletion(-)
diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index cfc43a68..a4ed98e2 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -76,6 +76,54 @@ static unsigned long read_lastmod (const void *ctx, const char *dir) {
return num;
}
+static void
+store_lastmod (notmuch_database_t *notmuch, const char *dir) {
+ char *filename = NULL;
+ FILE *out;
+
+ ASSERT(filename = talloc_asprintf (notmuch, "%s/lastmod", dir));
+
+ ASSERT(out = fopen (filename, "w"));
+
+ ASSERT(fprintf (out, "%zu\n", notmuch_database_get_revision (notmuch, NULL)) > 0);
+}
+
+static void
+read_one_line (FILE *stream, char **line_p, size_t *len_p) {
+ ssize_t nread;
+
+ nread = getline(line_p, len_p, stream);
+ if (nread < 0) {
+ perror ("getline");
+ exit (EXIT_FAILURE);
+ }
+ ASSERT(line_p);
+
+ chomp_newline(*line_p);
+}
+
+static const char*
+shell2string (const char* command) {
+ FILE *out;
+ char *line = NULL;
+ size_t len;
+
+ out = popen (command,"r");
+ if (out == NULL) {
+ perror("popen");
+ exit(EXIT_FAILURE);
+ }
+
+ read_one_line (out, &line, &len);
+ return line;
+}
+
+static void
+wr_data(const char *data) {
+ printf ("data %zu\n", strlen(data));
+ fputs (data, stdout);
+}
+
static void
cmd_capabilities () {
fputs("import\nexport\nrefspec refs/heads/*:refs/notmuch/*\n\n", stdout);
@@ -97,6 +145,85 @@ usage() {
exit(EXIT_FAILURE);
}
+static void
+cmd_import (notmuch_database_t *notmuch) {
+ const char* ident = NULL;
+ const char* lastmod_str = NULL;
+ notmuch_messages_t *messages;
+ notmuch_status_t status;
+ notmuch_query_t *query;
+ char *mid_buf = NULL;
+ size_t mid_buf_len = 0;
+
+ ident = shell2string("git var GIT_COMMITTER_IDENT");
+
+ printf ("feature done\ncommit refs/notmuch/master\nmark :1\ncommitter %s\n", ident);
+
+ ASSERT(lastmod_str = talloc_asprintf (notmuch, "lastmod: %zu\n", lastmod));
+ wr_data (lastmod_str);
+ if (lastmod > 0)
+ puts("from refs/notmuch/master^0");
+ puts("deleteall");
+
+ status = notmuch_query_create_with_syntax (notmuch,
+ "",
+ NOTMUCH_QUERY_SYNTAX_XAPIAN,
+ &query);
+
+ if (print_status_database ("git-remote-nm", notmuch, status))
+ exit(EXIT_FAILURE);
+
+ if (debug_flags && strchr (debug_flags, 's'))
+ notmuch_query_set_sort (query, NOTMUCH_SORT_NEWEST_FIRST);
+ else
+ notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
+
+ status = notmuch_query_search_messages (query, &messages);
+ if (print_status_query ("git-remote-nm", query, status))
+ exit(EXIT_FAILURE);
+
+ for (;
+ notmuch_messages_valid (messages);
+ notmuch_messages_move_to_next (messages)) {
+
+ const char* tag_buf = "";
+ const char* mid;
+
+ const char* hash;
+
+ notmuch_message_t *message = notmuch_messages_get (messages);
+ mid = notmuch_message_get_message_id (message);
+
+ if (hex_encode (notmuch, mid, &mid_buf, &mid_buf_len) != HEX_SUCCESS) {
+ fprintf (stderr, "Error: failed to hex-encode message-id %s\n", mid);
+ exit(EXIT_FAILURE);
+ }
+
+ /* we can't use _notmuch_sha1_from_string because we don't want
+ * to include the null terminator */
+ GChecksum *sha1;
+ sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+ g_checksum_update (sha1, (const guchar *) mid, strlen (mid));
+ hash = g_checksum_get_string (sha1);
+ printf ("M 644 inline %2.2s/%2.2s/%s/tags\n", hash, hash+2, mid_buf);
+
+ g_checksum_free (sha1);
+
+ for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+ notmuch_tags_valid (tags);
+ notmuch_tags_move_to_next (tags)) {
+ const char *tag_str = notmuch_tags_get (tags);
+ ASSERT(tag_buf=talloc_asprintf (message, "%s%s\n", tag_buf, tag_str));
+ }
+ wr_data (tag_buf);
+ notmuch_message_destroy (message);
+ }
+ puts("");
+ puts("done");
+ fflush (stdout);
+ store_lastmod(notmuch, nm_dir);
+}
+
int
main (int argc, char *argv[])
{
@@ -169,6 +296,8 @@ main (int argc, char *argv[])
if (STRNCMP_LITERAL (s, "capabilities")== 0)
cmd_capabilities ();
+ else if (STRNCMP_LITERAL (s, "import") == 0)
+ cmd_import (db);
else if (STRNCMP_LITERAL (s, "list") == 0)
cmd_list ();
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index 7b2b6b49..d5809e6b 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -20,7 +20,7 @@ export GIT_COMMITTER_NAME="Notmuch Test Suite"
export GIT_COMMITTER_EMAIL="notmuch@example.com"
export GIT_REMOTE_NM_DEBUG="s"
export GIT_REMOTE_NM_LOG=grn-log.txt
-EXPECTED=$NOTMUCH_SRCDIR/test/git-remote-nm.expected-output
+EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
TAG_FILE="87/b1/4EFC743A.3060609@april.org/tags"
@@ -42,4 +42,40 @@ cat <<EOF > EXPECTED
EOF
test_expect_equal_file EXPECTED OUTPUT
+test_begin_subtest 'import writes lastmod file'
+echo import | run_helper dummy-alias dummy-url > /dev/null
+lastmod=$(notmuch count --lastmod '*' | cut -f3)
+test_expect_equal "${lastmod}" "$(cat ${git_tmp}/notmuch/lastmod)"
+
+# note that this test must not be the first time import is run,
+# because it depends on the lastmod file
+test_begin_subtest 'import produces expected output'
+echo import | run_helper | notmuch_sanitize_git > OUTPUT
+test_expect_equal_file $EXPECTED/default.import OUTPUT
+
+test_begin_subtest "clone has every message"
+git clone notmuch::currently-ignored repo
+find repo -name tags -type f | sed -e s,repo/../../,id:, -e s,/tags$,, | sort > OUTPUT
+notmuch search --output=messages '*' | sort > EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "pull get new tag"
+notmuch tag +zznew -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+unread
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
+test_begin_subtest "pull sees deletion"
+notmuch tag -unread -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
test_done
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 4/5] cli/git-remote: add export command
2024-08-28 15:45 WIP: git-remote-notmuch David Bremner
` (2 preceding siblings ...)
2024-08-28 15:45 ` [PATCH 3/5] cli/git-remote: add support for import command David Bremner
@ 2024-08-28 15:45 ` David Bremner
2024-08-28 15:45 ` [PATCH 5/5] cli/git-remote: add time performance test David Bremner
2024-08-28 21:11 ` WIP: git-remote-notmuch David Bremner
5 siblings, 0 replies; 7+ messages in thread
From: David Bremner @ 2024-08-28 15:45 UTC (permalink / raw)
To: notmuch
This enables the push command, and the helper is now feature complete.
---
git-remote-notmuch.c | 157 ++++++++++++++++++++++++++++++++++++++++
test/T860-git-remote.sh | 69 ++++++++++++++++++
test/make-export.py | 44 +++++++++++
3 files changed, 270 insertions(+)
create mode 100644 test/make-export.py
diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index a4ed98e2..35579004 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -224,6 +224,161 @@ cmd_import (notmuch_database_t *notmuch) {
store_lastmod(notmuch, nm_dir);
}
+static GString *
+get_tok (const char *line, size_t index) {
+
+ const char *tok = line;
+ size_t tok_len =0;
+
+ for (size_t count = 0; count<=index; count++) {
+ ASSERT(tok = strsplit_len (tok+tok_len, ' ', &tok_len));
+ }
+
+ return g_string_new_len (tok, tok_len);
+}
+
+static GString *
+read_data (char **line_p, size_t *len_p) {
+ ssize_t nread;
+ size_t bytes;
+ size_t data_size;
+ const char *tok;
+ size_t tok_len =0;
+
+ ASSERT (line_p);
+ ASSERT (len_p);
+ ASSERT((nread = getline(line_p, len_p, stdin) != -1));
+
+ tok=*line_p;
+ for (size_t count = 0; count<=1; count++) {
+ ASSERT(tok = strsplit_len (tok+tok_len, ' ', &tok_len));
+ }
+
+ /* TODO: don't ignore tok_len */
+ ASSERT(sscanf (tok, "%zu", &data_size) == 1);
+
+ *line_p = realloc (*line_p, data_size+1);
+ bytes = fread (*line_p, 1, data_size, stdin);
+ ASSERT (bytes == data_size);
+
+ *len_p = data_size;
+
+ return g_string_new_len (*line_p, *len_p);
+}
+
+static void free_string (GString *str) {
+ g_string_free (str, true);
+}
+
+static void
+cmd_export (notmuch_database_t *notmuch)
+{
+ ssize_t nread;
+ size_t len = 0;
+ char *line = NULL;
+ GHashTable* blobs;
+
+ ASSERT(blobs = g_hash_table_new_full ((GHashFunc)g_string_hash,
+ (GEqualFunc)g_string_equal,
+ (GDestroyNotify)free_string, (GDestroyNotify)free_string));
+
+ while ((nread = getline(&line, &len, stdin)) != -1) {
+ flog ("\texport %s\n", line);
+ if (STRNCMP_LITERAL (line, "done") == 0) {
+ break;
+ }
+ else if (STRNCMP_LITERAL (line, "blob") == 0) {
+ GString *mark;
+ GString *data;
+
+ flog ("export blob\n");
+ read_one_line (stdin, &line, &len);
+ mark = get_tok (line, 1);
+
+ data = read_data (&line, &len);
+
+ g_hash_table_insert (blobs, mark, data);
+ read_one_line (stdin, &line, &len);
+ /* TODO free things */
+ }
+
+ else if (STRNCMP_LITERAL (line, "commit") == 0) {
+ char *mid = NULL;
+ size_t mid_len = 0;
+
+ flog ("export commit\n");
+
+ /* mark for commit (ignored) */
+ read_one_line (stdin, &line, &len);
+ /* author (ignored) */
+ read_one_line (stdin, &line, &len);
+ /* committer (ignored) */
+ read_one_line (stdin, &line, &len);
+
+ /* commit message (ignored) */
+ (void)read_data (&line, &len);
+
+ while (strlen (line) > 0) {
+ GString *mark, *path;
+ GBytes *blob;
+ char *basename;
+ notmuch_message_t *message;
+ char *tag;
+ const char *tok;
+ size_t tok_len;
+ size_t max_tok_len;
+
+ read_one_line (stdin, &line, &len);
+ if (strlen (line) ==0)
+ break;
+
+ ASSERT(mark = get_tok (line, 2));
+ ASSERT(blob = g_hash_table_lookup (blobs, mark));
+ free_string (mark);
+
+ ASSERT(path = get_tok (line, 3));
+
+ basename = g_path_get_dirname (path->str+6);
+ ASSERT(HEX_SUCCESS ==
+ hex_decode (notmuch, basename, &mid, &mid_len));
+ g_free (basename);
+ free_string (path);
+
+ ASSERT(NOTMUCH_STATUS_SUCCESS ==
+ notmuch_database_find_message (notmuch, mid, &message));
+ ASSERT(message);
+
+
+ ASSERT(NOTMUCH_STATUS_SUCCESS ==
+ notmuch_message_freeze (message));
+
+ ASSERT(NOTMUCH_STATUS_SUCCESS ==
+ notmuch_message_remove_all_tags (message));
+
+ tok = g_bytes_get_data (blob, &max_tok_len);
+ tok_len = 0;
+ while ((tok_len < max_tok_len) &&
+ (tok = strsplit_len (tok + tok_len, '\n', &tok_len)) != NULL) {
+ tag = strndup (tok, tok_len);
+ ASSERT(NOTMUCH_STATUS_SUCCESS ==
+ notmuch_message_add_tag (message, tag));
+ free (tag);
+ }
+
+ ASSERT(NOTMUCH_STATUS_SUCCESS ==
+ notmuch_message_thaw (message));
+
+ notmuch_message_destroy (message);
+ }
+ puts("ok refs/heads/master");
+ }
+
+ }
+ store_lastmod(notmuch, nm_dir);
+ puts("");
+ g_hash_table_destroy (blobs);
+}
+
int
main (int argc, char *argv[])
{
@@ -296,6 +451,8 @@ main (int argc, char *argv[])
if (STRNCMP_LITERAL (s, "capabilities")== 0)
cmd_capabilities ();
+ else if (STRNCMP_LITERAL (s, "export") == 0)
+ cmd_export (db);
else if (STRNCMP_LITERAL (s, "import") == 0)
cmd_import (db);
else if (STRNCMP_LITERAL (s, "list") == 0)
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index d5809e6b..6dc55391 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -21,6 +21,7 @@ export GIT_COMMITTER_EMAIL="notmuch@example.com"
export GIT_REMOTE_NM_DEBUG="s"
export GIT_REMOTE_NM_LOG=grn-log.txt
EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
+MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
TAG_FILE="87/b1/4EFC743A.3060609@april.org/tags"
@@ -78,4 +79,72 @@ zznew
EOF
test_expect_equal_file EXPECTED repo/$TAG_FILE
+test_begin_subtest 'export runs'
+run_helper <<EOF | notmuch_sanitize_git > OUTPUT
+export
+blob
+mark :1
+data 10
+tag1
+tag2
+
+commit refs/heads/master
+mark :2
+author Notmuch Test Suite <notmuch@example.com> 1234 +0000
+committer Notmuch Test Suite <notmuch@example.com> 1234 +0000
+data 8
+ignored
+M 100644 :1 $TAG_FILE
+
+done
+
+EOF
+cat <<EOF > EXPECTED
+ok refs/heads/master
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+# this test depends on the previous one
+test_begin_subtest 'export modifies database'
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest 'restore via export'
+notmuch dump > BEFORE
+python3 $MAKE_EXPORT_PY > export.in
+notmuch tag +transient -- id:4EFC743A.3060609@april.org
+run_helper < export.in > OUTPUT
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "push updates database"
+git -C repo push origin master
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "adding tag via repo"
+cat<<EOF >repo/$TAG_FILE
+tag1
+tag2
+tag3
+EOF
+git -C repo add $TAG_FILE
+git -C repo commit -m 'testing push'
+git -C repo push origin master
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 +tag3 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
test_done
diff --git a/test/make-export.py b/test/make-export.py
new file mode 100644
index 00000000..3837dc3a
--- /dev/null
+++ b/test/make-export.py
@@ -0,0 +1,44 @@
+# generate a test input for the 'export' subcommand of the
+# git-remote-notmuch helper
+
+from notmuch2 import Database
+from time import time
+from hashlib import sha1
+
+def hexencode(str):
+ output_charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.,"
+ out = ""
+ for char in str:
+ if not char in output_charset:
+ out+= f"%{ord(char):x}"
+ else:
+ out+= char
+ return out
+
+db = Database(config=Database.CONFIG.SEARCH)
+
+count=1
+print("export")
+mark={}
+
+for msg in db.messages(""):
+ mark[msg.messageid]=count
+ blob=""
+ for tag in msg.tags:
+ blob += f"{tag}\n"
+ print (f"blob\nmark :{count}");
+ print (f"data {len(blob)}\n{blob}")
+ count=count+1
+
+print (f"\ncommit refs/heads/master\nmark :{count+1}")
+ctime = int(time())
+print (f"author Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
+print (f"committer Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
+print (f"data 8\nignored")
+
+for msg in db.messages(""):
+ digest = sha1(msg.messageid.encode('utf8')).hexdigest()
+ filename = hexencode(msg.messageid)
+ print (f"M 100644 :{mark[msg.messageid]} {digest[0:2]}/{digest[2:4]}/{filename}/tags")
+
+print("\ndone\n")
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 5/5] cli/git-remote: add time performance test
2024-08-28 15:45 WIP: git-remote-notmuch David Bremner
` (3 preceding siblings ...)
2024-08-28 15:45 ` [PATCH 4/5] cli/git-remote: add export command David Bremner
@ 2024-08-28 15:45 ` David Bremner
2024-08-28 21:11 ` WIP: git-remote-notmuch David Bremner
5 siblings, 0 replies; 7+ messages in thread
From: David Bremner @ 2024-08-28 15:45 UTC (permalink / raw)
To: notmuch
Memory tests already existed (for correctness) but this helps sanity
check the performance to make sure we are in the same range as the
existing notmuch-git script.
---
performance-test/T08-git-remote.sh | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100755 performance-test/T08-git-remote.sh
diff --git a/performance-test/T08-git-remote.sh b/performance-test/T08-git-remote.sh
new file mode 100755
index 00000000..799e221b
--- /dev/null
+++ b/performance-test/T08-git-remote.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+test_description='git-remote-notmuch'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+export GIT_COMMITTER_NAME="Notmuch Test Suite"
+export GIT_COMMITTER_EMAIL="notmuch@example.com"
+
+time_start
+
+time_run 'clone --bare' "git clone --quiet --bare notmuch::default default.git"
+time_run 'clone' "git clone --quiet notmuch::default"
+
+cd default
+hash=$(git hash-object --stdin -w < /dev/null)
+# replace all files with empty files
+git ls-tree -r HEAD | sed "s/blob [^\t]*/blob $hash/" | git update-index --index-info
+git commit -m'zero tags' 2>>log.txt 1>&2
+
+time_run "push" "git push --quiet origin master"
+
+time_done
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: WIP: git-remote-notmuch
2024-08-28 15:45 WIP: git-remote-notmuch David Bremner
` (4 preceding siblings ...)
2024-08-28 15:45 ` [PATCH 5/5] cli/git-remote: add time performance test David Bremner
@ 2024-08-28 21:11 ` David Bremner
5 siblings, 0 replies; 7+ messages in thread
From: David Bremner @ 2024-08-28 21:11 UTC (permalink / raw)
To: notmuch
David Bremner <david@tethera.net> writes:
> 3) The original script took a path to mail_root, but this doesn't
> really cover the possible split configurations; in particular it
> doesn't handle my (XDG) layout. The current code just looks for the
> "default" notmuch setup, and ignores the URL. I am open to ideas for
> how a notmuch URL should look like. I'd imagine they should have the
> profile in them, but I'm not sure how much flexibility we need / want
> to suppot.
I guess some kind of key value scheme like
notmuch://?profile=foo
notmuch::?profile=foo
or
notmuch://?config=.bar?database=fub/zub
would work, basically a thin wrapper around
notmuch_database_open_with_config.
There is still room for bikeshedding about the syntax. We could throw
in "localhost" for future-proofing,
notmuch://localhost
would pass NULL to all 3 relevant parameters of n_d_o_w_c
^ permalink raw reply [flat|nested] 7+ messages in thread