* [PATCH v2 1/4] util: refactor sync_dir and mkdir_recursive
2024-09-08 11:27 David Bremner
@ 2024-09-08 11:27 ` David Bremner
2024-09-08 11:27 ` [PATCH v2 2/4] cli: start remote helper for git David Bremner
` (3 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: David Bremner @ 2024-09-08 11:27 UTC (permalink / raw)
To: notmuch
Moving these functions to libnotmuch_util will allow re-user from
either multiple CLI compilation units or from the library. To avoid
future surprises, replace printing to stderr with the usual status
string mechanism.
---
notmuch-insert.c | 84 +++++------------------------------------
util/path-util.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++-
util/path-util.h | 8 ++++
3 files changed, 114 insertions(+), 75 deletions(-)
diff --git a/notmuch-insert.c b/notmuch-insert.c
index e44607ad..66c4f434 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -21,13 +21,13 @@
* Author: Peter Wang <novalazy@gmail.com>
*/
-#include "notmuch-client.h"
-#include "tag-util.h"
-#include <sys/types.h>
-#include <sys/stat.h>
#include <fcntl.h>
+
+#include "notmuch-client.h"
+#include "tag-util.h"
#include "string-util.h"
+#include "path-util.h"
static volatile sig_atomic_t interrupted;
@@ -64,26 +64,6 @@ safe_gethostname (char *hostname, size_t len)
}
}
-/* Call fsync() on a directory path. */
-static bool
-sync_dir (const char *dir)
-{
- int fd, r;
-
- fd = open (dir, O_RDONLY);
- if (fd == -1) {
- fprintf (stderr, "Error: open %s: %s\n", dir, strerror (errno));
- return false;
- }
-
- r = fsync (fd);
- if (r)
- fprintf (stderr, "Error: fsync %s: %s\n", dir, strerror (errno));
-
- close (fd);
-
- return r == 0;
-}
/*
* Check the specified folder name does not contain a directory
@@ -105,54 +85,6 @@ is_valid_folder_name (const char *folder)
}
}
-/*
- * Make the given directory and its parents as necessary, using the
- * given mode. Return true on success, false otherwise. Partial
- * results are not cleaned up on errors.
- */
-static bool
-mkdir_recursive (const void *ctx, const char *path, int mode)
-{
- struct stat st;
- int r;
- char *parent = NULL, *slash;
-
- /* First check the common case: directory already exists. */
- r = stat (path, &st);
- if (r == 0) {
- if (! S_ISDIR (st.st_mode)) {
- fprintf (stderr, "Error: '%s' is not a directory: %s\n",
- path, strerror (EEXIST));
- return false;
- }
-
- return true;
- } else if (errno != ENOENT) {
- fprintf (stderr, "Error: stat '%s': %s\n", path, strerror (errno));
- return false;
- }
-
- /* mkdir parents, if any */
- slash = strrchr (path, '/');
- if (slash && slash != path) {
- parent = talloc_strndup (ctx, path, slash - path);
- if (! parent) {
- fprintf (stderr, "Error: %s\n", strerror (ENOMEM));
- return false;
- }
-
- if (! mkdir_recursive (ctx, parent, mode))
- return false;
- }
-
- if (mkdir (path, mode)) {
- fprintf (stderr, "Error: mkdir '%s': %s\n", path, strerror (errno));
- return false;
- }
-
- return parent ? sync_dir (parent) : true;
-}
-
/*
* Create the given maildir folder, i.e. maildir and its
* subdirectories cur/new/tmp. Return true on success, false
@@ -165,6 +97,7 @@ maildir_create_folder (const void *ctx, const char *maildir, bool world_readable
const int mode = (world_readable ? 0755 : 0700);
char *subdir;
unsigned int i;
+ char **status_string = NULL;
for (i = 0; i < ARRAY_SIZE (subdirs); i++) {
subdir = talloc_asprintf (ctx, "%s/%s", maildir, subdirs[i]);
@@ -173,7 +106,7 @@ maildir_create_folder (const void *ctx, const char *maildir, bool world_readable
return false;
}
- if (! mkdir_recursive (ctx, subdir, mode))
+ if (mkdir_recursive (ctx, subdir, mode, status_string))
return false;
}
@@ -347,6 +280,7 @@ static char *
maildir_write_new (const void *ctx, int fdin, const char *maildir, bool world_readable)
{
char *cleanpath, *tmppath, *newpath, *newdir;
+ char *status_string = NULL;
tmppath = maildir_write_tmp (ctx, fdin, maildir, world_readable);
if (! tmppath)
@@ -375,13 +309,15 @@ maildir_write_new (const void *ctx, int fdin, const char *maildir, bool world_re
goto FAIL;
}
- if (! sync_dir (newdir))
+ if (sync_dir (newdir, &status_string))
goto FAIL;
return newpath;
FAIL:
unlink (cleanpath);
+ if (status_string)
+ fputs (status_string, stderr);
return NULL;
}
diff --git a/util/path-util.c b/util/path-util.c
index 3267a967..5511c09b 100644
--- a/util/path-util.c
+++ b/util/path-util.c
@@ -5,10 +5,17 @@
#define _GNU_SOURCE
#include "path-util.h"
-
+#include "compat.h"
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
#include <limits.h>
#include <stdlib.h>
+#include <talloc.h>
char *
notmuch_canonicalize_file_name (const char *path)
@@ -25,3 +32,91 @@ notmuch_canonicalize_file_name (const char *path)
#error undefined PATH_MAX _and_ missing canonicalize_file_name not supported
#endif
}
+
+/* Call fsync() on a directory path. */
+notmuch_status_t
+sync_dir (const char *dir, char **status_string)
+{
+ int fd, r;
+
+ fd = open (dir, O_RDONLY);
+ if (fd == -1) {
+ if (status_string)
+ IGNORE_RESULT (asprintf (status_string,
+ "Error: open %s: %s\n", dir, strerror (errno)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ r = fsync (fd);
+ if (r && status_string)
+ IGNORE_RESULT (asprintf (status_string,
+ "Error: fsync %s: %s\n", dir, strerror (errno)));
+
+ close (fd);
+
+ return r == 0 ? NOTMUCH_STATUS_SUCCESS : NOTMUCH_STATUS_FILE_ERROR;
+}
+
+/*
+ * Make the given directory and its parents as necessary, using the
+ * given mode. Partial results are not cleaned up on errors.
+ */
+notmuch_status_t
+mkdir_recursive (const void *ctx, const char *path, int mode,
+ char **status_string)
+{
+ notmuch_status_t status;
+ struct stat st;
+ int r;
+ char *parent = NULL, *slash;
+
+ /* First check the common case: directory already exists. */
+ r = stat (path, &st);
+ if (r == 0) {
+ if (! S_ISDIR (st.st_mode)) {
+ if (status_string)
+ IGNORE_RESULT (asprintf (status_string, "Error: '%s' is not a directory: %s\n",
+ path, strerror (EEXIST)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ return NOTMUCH_STATUS_SUCCESS;
+ } else if (errno != ENOENT) {
+ if (status_string)
+ IGNORE_RESULT (asprintf (status_string,
+ "Error: stat '%s': %s\n", path, strerror (errno)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ /* mkdir parents, if any */
+ slash = strrchr (path, '/');
+ if (slash && slash != path) {
+ parent = talloc_strndup (ctx, path, slash - path);
+ if (! parent) {
+ if (status_string)
+ IGNORE_RESULT (asprintf (status_string,
+ "Error: %s\n",
+ strerror (ENOMEM)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ status = mkdir_recursive (ctx, parent, mode, status_string);
+ if (status)
+ return status;
+ }
+
+ if (mkdir (path, mode)) {
+ if (status_string)
+ IGNORE_RESULT (asprintf (status_string,
+ "Error: mkdir '%s': %s\n",
+ path, strerror (errno)));
+ return NOTMUCH_STATUS_FILE_ERROR;
+ }
+
+ if (parent) {
+ status = sync_dir (parent, status_string);
+ if (status)
+ return status;
+ }
+ return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/util/path-util.h b/util/path-util.h
index ac85f696..673c3b6f 100644
--- a/util/path-util.h
+++ b/util/path-util.h
@@ -5,6 +5,8 @@
#ifndef NOTMUCH_UTIL_PATH_UTIL_H_
#define NOTMUCH_UTIL_PATH_UTIL_H_
+#include "notmuch.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -12,6 +14,12 @@ extern "C" {
char *
notmuch_canonicalize_file_name (const char *path);
+notmuch_status_t
+mkdir_recursive (const void *ctx, const char *path, int mode, char **status_string);
+
+notmuch_status_t
+sync_dir (const char *path, char **status_string);
+
#ifdef __cplusplus
}
#endif
--
2.43.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH v2 2/4] cli: start remote helper for git.
2024-09-08 11:27 David Bremner
2024-09-08 11:27 ` [PATCH v2 1/4] util: refactor sync_dir and mkdir_recursive David Bremner
@ 2024-09-08 11:27 ` David Bremner
2024-10-03 11:22 ` David Bremner
2024-09-08 11:27 ` [PATCH v2 3/4] cli/git-remote: add import command David Bremner
` (2 subsequent siblings)
4 siblings, 1 reply; 8+ messages in thread
From: David Bremner @ 2024-09-08 11:27 UTC (permalink / raw)
To: notmuch
This is closely based on git-remote-nm (in ruby) by Felipe Contreras.
Initially just implement the commands 'capabilites' and 'list'. This
isn't enough to do anything useful so start some unit tests. Testing
of URL passing will be done after clone (import command) support is
added.
---
Makefile.local | 7 +-
git-remote-notmuch.c | 314 ++++++++++++++++++++++++++++++++++++++++
test/T860-git-remote.sh | 46 ++++++
3 files changed, 366 insertions(+), 1 deletion(-)
create mode 100644 git-remote-notmuch.c
create mode 100755 test/T860-git-remote.sh
diff --git a/Makefile.local b/Makefile.local
index 7699c208..ffeb0d00 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -1,7 +1,8 @@
# -*- makefile-gmake -*-
.PHONY: all
-all: notmuch notmuch-shared build-man build-info ruby-bindings python-cffi-bindings notmuch-git nmbug
+all: notmuch notmuch-shared git-remote-notmuch \
+ build-man build-info ruby-bindings python-cffi-bindings notmuch-git nmbug
ifeq ($(MAKECMDGOALS),)
ifeq ($(shell cat .first-build-message 2>/dev/null),)
@NOTMUCH_FIRST_BUILD=1 $(MAKE) --no-print-directory all
@@ -274,6 +275,9 @@ notmuch: $(notmuch_client_modules) lib/libnotmuch.a util/libnotmuch_util.a parse
notmuch-shared: $(notmuch_client_modules) lib/$(LINKER_NAME)
$(call quiet,$(FINAL_NOTMUCH_LINKER) $(CFLAGS)) $(notmuch_client_modules) $(FINAL_NOTMUCH_LDFLAGS) -o $@
+git-remote-notmuch: git-remote-notmuch.o status.o tag-util.o query-string.o
+ $(call quiet,$(FINAL_NOTMUCH_LINKER) $(CFLAGS)) $^ $(FINAL_NOTMUCH_LDFLAGS) -o $@
+
.PHONY: install
install: all install-man install-info
mkdir -p "$(DESTDIR)$(prefix)/bin/"
@@ -302,6 +306,7 @@ endif
SRCS := $(SRCS) $(notmuch_client_srcs)
CLEAN := $(CLEAN) notmuch notmuch-shared $(notmuch_client_modules)
+CLEAN := $(CLEAN) git-remote-notmuch git-remote-notmuch.o
CLEAN := $(CLEAN) version.stamp notmuch-*.tar.gz.tmp
CLEAN := $(CLEAN) .deps
diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
new file mode 100644
index 00000000..4afd198c
--- /dev/null
+++ b/git-remote-notmuch.c
@@ -0,0 +1,314 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2023 Felipe Contreras
+ * Copyright © 2024 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see https://www.gnu.org/licenses/ .
+ *
+ * Authors: Felipe Contreras
+ * David Bremner <david@tethera.net>
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <notmuch.h>
+#include "notmuch-client.h"
+#include "path-util.h"
+#include "hex-escape.h"
+#include "string-util.h"
+#include "tag-util.h"
+
+#define ASSERT(x) assert ((x))
+
+/* File scope globals */
+const char *debug_flags = NULL;
+FILE *log_file = NULL;
+
+/* For use with getline. */
+char *buffer = NULL;
+size_t buffer_len = 0;
+
+static inline bool
+equal_lastmod (const char *uuid1, unsigned long counter1,
+ const char *uuid2, unsigned long counter2)
+{
+ return (strcmp_null (uuid1, uuid2) == 0) && (counter1 == counter2);
+}
+
+/* Error handling */
+static void
+ensure (bool condition, const char *format, ...)
+{
+ va_list va_args;
+
+ if (! condition) {
+ va_start (va_args, format);
+ vfprintf (stderr, format, va_args);
+ va_end (va_args);
+ fprintf (stderr, "\n");
+ exit (EXIT_FAILURE);
+ }
+}
+
+/* It is a (protocol) error to call this at/after EOF */
+static void
+buffer_line (FILE *stream)
+{
+ ssize_t nread;
+
+ nread = getline (&buffer, &buffer_len, stream);
+ ensure (nread >= 0, "getline %s", strerror (errno));
+ chomp_newline (buffer);
+}
+
+static GStrv
+tokenize_buffer ()
+{
+ char *tok = buffer;
+ size_t tok_len = 0;
+
+ g_autoptr (GStrvBuilder) builder = g_strv_builder_new ();
+
+ while ((tok = strtok_len (tok + tok_len, " \t\n", &tok_len))) {
+ g_strv_builder_take (builder, g_strndup (tok, tok_len));
+ }
+
+ return g_strv_builder_end (builder);
+}
+
+static void
+flog (const char *format, ...)
+{
+ va_list va_args;
+
+ if (log_file) {
+ va_start (va_args, format);
+ vfprintf (log_file, format, va_args);
+ fflush (log_file);
+ va_end (va_args);
+ }
+}
+
+static const char *
+gmessage (GError *err)
+{
+ if (err)
+ return err->message;
+ else
+ return NULL;
+}
+
+static void
+str2ul (const char *str, unsigned long int *num_p)
+{
+ gboolean ret;
+
+ g_autoptr (GError) gerror = NULL;
+
+ ret = g_ascii_string_to_unsigned (str, 10, 0, G_MAXUINT64, num_p, &gerror);
+ ensure (ret, "converting %s to unsigned long: %s", str, gmessage (gerror));
+}
+
+static void
+read_lastmod (const char *dir, char **uuid_out, unsigned long *counter_out)
+{
+ g_autoptr (GString) filename = g_string_new (dir);
+ unsigned long num = 0;
+ FILE *in;
+
+ assert (uuid_out);
+ assert (counter_out);
+
+ g_string_append (filename, "/lastmod");
+
+ in = fopen (filename->str, "r");
+ if (! in) {
+ ensure (errno == ENOENT, "error opening lastmod file");
+ *uuid_out = NULL;
+ *counter_out = 0;
+ } else {
+ g_auto (GStrv) tokens = NULL;
+ buffer_line (in);
+
+ tokens = tokenize_buffer ();
+
+ *uuid_out = tokens[0];
+ str2ul (tokens[1], &num);
+
+ flog ("loaded uuid = %s\tlastmod = %zu\n", tokens[0], num);
+ }
+
+ *counter_out = num;
+
+}
+
+static void
+cmd_capabilities ()
+{
+ fputs ("import\nexport\nrefspec refs/heads/*:refs/notmuch/*\n\n", stdout);
+ fflush (stdout);
+}
+
+static void
+cmd_list (notmuch_database_t *db, const char *uuid, unsigned long lastmod)
+{
+ unsigned long db_lastmod;
+ const char *db_uuid;
+
+ db_lastmod = notmuch_database_get_revision (db, &db_uuid);
+
+ printf ("? refs/heads/master%s\n\n",
+ equal_lastmod (uuid, lastmod, db_uuid, db_lastmod) ? " unchanged" : "");
+}
+
+/* stubs since we cannot link with notmuch.o */
+const notmuch_opt_desc_t notmuch_shared_options[] = {
+ { }
+};
+
+const char *notmuch_requested_db_uuid = NULL;
+
+void
+notmuch_process_shared_options (unused (notmuch_database_t *notmuch),
+ unused (const char *dummy))
+{
+}
+
+int
+notmuch_minimal_options (unused (const char *subcommand),
+ unused (int argc),
+ unused (char **argv))
+{
+ return 0;
+}
+
+static notmuch_database_t *
+open_database (const char *arg)
+{
+ notmuch_status_t status;
+ notmuch_database_t *notmuch;
+ const char *path = NULL;
+ const char *config = NULL;
+ const char *profile = NULL;
+ const char *scheme = NULL;
+ const char *uriquery = NULL;
+ g_autofree char *status_string = NULL;
+
+ g_autoptr (GUri) uri = NULL;
+ g_autoptr (GHashTable) params = NULL;
+ g_autoptr (GError) gerror = NULL;
+ g_autoptr (GString) address = NULL;
+
+ address = g_string_new (arg);
+
+ scheme = g_uri_peek_scheme (address->str);
+ if (! scheme || (strcmp (scheme, "notmuch") != 0)) {
+ ASSERT (g_string_prepend (address, "notmuch://"));
+ }
+
+ uri = g_uri_parse (address->str, G_URI_FLAGS_ENCODED_QUERY, &gerror);
+ ensure (uri, "unable to parse URL/address %s: %s\n", address->str, gmessage (gerror));
+
+ uriquery = g_uri_get_query (uri);
+ if (uriquery) {
+ flog ("uriquery = %s\n", uriquery);
+ params = g_uri_parse_params (uriquery, -1, "&", G_URI_PARAMS_NONE, &gerror);
+ ensure (params, "unable to parse parameters %s: %s\n", uriquery, gmessage (gerror));
+ }
+
+ if (strlen (g_uri_get_path (uri)) > 0) {
+ path = g_uri_get_path (uri);
+ config = "";
+ }
+
+ if (params) {
+ if (! path)
+ path = g_hash_table_lookup (params, "path");
+ config = g_hash_table_lookup (params, "config");
+ profile = g_hash_table_lookup (params, "profile");
+ }
+
+ flog ("url = %s\npath = %s\nconfig = %s\nprofile = %s\n",
+ address->str, path, config, profile);
+
+ status = notmuch_database_open_with_config (path,
+ NOTMUCH_DATABASE_MODE_READ_WRITE,
+ config,
+ profile,
+ ¬much,
+ &status_string);
+
+ ensure (status == 0, "open database: %s", status_string);
+
+ return notmuch;
+}
+
+int
+main (int argc, char *argv[])
+{
+ notmuch_status_t status;
+ notmuch_database_t *db;
+ unsigned long lastmod = 0;
+ char *uuid = NULL;
+ const char *nm_dir = NULL;
+ g_autofree char *status_string = NULL;
+ const char *git_dir;
+ ssize_t nread;
+ const char *log_file_name;
+
+ debug_flags = getenv ("GIT_REMOTE_NM_DEBUG");
+ log_file_name = getenv ("GIT_REMOTE_NM_LOG");
+
+ if (log_file_name)
+ log_file = fopen (log_file_name, "w");
+
+ ensure (argc >= 3, "usage: %s ALIAS URL\n", argv[0]);
+
+ db = open_database (argv[2]);
+
+ git_dir = getenv ("GIT_DIR");
+ ensure (git_dir, "GIT_DIR not set");
+ flog ("GIT_DIR=%s\n", git_dir);
+
+ ASSERT (nm_dir = talloc_asprintf (db, "%s/%s", git_dir, "notmuch"));
+
+ status = mkdir_recursive (db, nm_dir, 0700, &status_string);
+ ensure (status == 0, "mkdir: %s", status_string);
+
+ read_lastmod (nm_dir, &uuid, &lastmod);
+
+ while ((nread = getline (&buffer, &buffer_len, stdin)) != -1) {
+ char *s = buffer;
+ flog ("command = %s\n", buffer);
+
+ /* skip leading space */
+ while (*s && isspace (*s)) s++;
+
+ if (! *s)
+ break;
+
+ if (STRNCMP_LITERAL (s, "capabilities") == 0)
+ cmd_capabilities ();
+ else if (STRNCMP_LITERAL (s, "list") == 0)
+ cmd_list (db, uuid, lastmod);
+
+ fflush (stdout);
+ flog ("finished command = %s\n", s);
+ }
+ flog ("finished loop\n");
+
+ notmuch_database_destroy (db);
+}
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
new file mode 100755
index 00000000..1a087611
--- /dev/null
+++ b/test/T860-git-remote.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+test_description='git-remote-notmuch'
+. $(dirname "$0")/test-lib.sh || exit 1
+
+notmuch_sanitize_git() {
+ sed 's/^committer \(.*\) \(<[^>]*>\) [1-9][0-9]* [-+][0-9]*/committer \1 \2 TIMESTAMP TIMEZONE/'
+}
+
+add_email_corpus
+
+mkdir repo
+
+git_tmp=$(mktemp -d gitXXXXXXXX)
+
+run_helper () {
+ env -u NOTMUCH_CONFIG GIT_DIR=${git_tmp} git-remote-notmuch dummy-alias "?config=${NOTMUCH_CONFIG}"
+}
+
+export GIT_COMMITTER_NAME="Notmuch Test Suite"
+export GIT_COMMITTER_EMAIL="notmuch@example.com"
+export GIT_REMOTE_NM_DEBUG="s"
+export GIT_REMOTE_NM_LOG=grn-log.txt
+EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
+MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
+
+TAG_FILE="87/b1/4EFC743A.3060609@april.org/tags"
+
+test_begin_subtest 'capabilities'
+echo capabilities | run_helper > OUTPUT
+cat <<EOF > EXPECTED
+import
+export
+refspec refs/heads/*:refs/notmuch/*
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest 'list'
+echo list | run_helper > OUTPUT
+cat <<EOF > EXPECTED
+? refs/heads/master
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_done
--
2.43.0
\r
^ permalink raw reply related [flat|nested] 8+ messages in thread* Re: [PATCH v2 2/4] cli: start remote helper for git.
2024-09-08 11:27 ` [PATCH v2 2/4] cli: start remote helper for git David Bremner
@ 2024-10-03 11:22 ` David Bremner
2024-10-03 11:26 ` David Bremner
0 siblings, 1 reply; 8+ messages in thread
From: David Bremner @ 2024-10-03 11:22 UTC (permalink / raw)
To: notmuch
[-- Attachment #1: Type: text/plain, Size: 422 bytes --]
David Bremner <david@tethera.net> writes:
> +EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
> +MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
I see I lost the script make-export.py going from v1 to v2. I will
re-roll the series to include it, but I want try a refactor that puts
the URL parsing into the library in a new notmuch_database_open_url
function that can be re-used by e.g. neomutt (hopefully).
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: make-export.py --]
[-- Type: text/x-python, Size: 1226 bytes --]
# generate a test input for the 'export' subcommand of the
# git-remote-notmuch helper
from notmuch2 import Database
from time import time
from hashlib import sha1
def hexencode(str):
output_charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.,"
out = ""
for char in str:
if not char in output_charset:
out+= f"%{ord(char):x}"
else:
out+= char
return out
db = Database(config=Database.CONFIG.SEARCH)
count=1
print("export")
mark={}
for msg in db.messages(""):
mark[msg.messageid]=count
blob=""
for tag in msg.tags:
blob += f"{tag}\n"
print (f"blob\nmark :{count}");
print (f"data {len(blob)}\n{blob}")
count=count+1
print (f"\ncommit refs/heads/master\nmark :{count+1}")
ctime = int(time())
print (f"author Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
print (f"committer Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
print (f"data 8\nignored")
for msg in db.messages(""):
digest = sha1(msg.messageid.encode('utf8')).hexdigest()
filename = hexencode(msg.messageid)
print (f"M 100644 :{mark[msg.messageid]}
digest[0:2]}/{digest[2:4]}/{filename}/tags")
print("\ndone\n")
[-- Attachment #3: Type: text/plain, Size: 0 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread* Re: [PATCH v2 2/4] cli: start remote helper for git.
2024-10-03 11:22 ` David Bremner
@ 2024-10-03 11:26 ` David Bremner
0 siblings, 0 replies; 8+ messages in thread
From: David Bremner @ 2024-10-03 11:26 UTC (permalink / raw)
To: notmuch
[-- Attachment #1: Type: text/plain, Size: 586 bytes --]
David Bremner <david@tethera.net> writes:
> David Bremner <david@tethera.net> writes:
>
>> +EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
>> +MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
>
> I see I lost the script make-export.py going from v1 to v2. I will
> re-roll the series to include it, but I want try a refactor that puts
> the URL parsing into the library in a new notmuch_database_open_url
> function that can be re-used by e.g. neomutt (hopefully).
Ugh. And that version of the script was broken by screen scraping from
notmuch. This one is hopefully ok.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: make-export.py --]
[-- Type: text/x-python, Size: 1227 bytes --]
# generate a test input for the 'export' subcommand of the
# git-remote-notmuch helper
from notmuch2 import Database
from time import time
from hashlib import sha1
def hexencode(str):
output_charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.,"
out = ""
for char in str:
if not char in output_charset:
out+= f"%{ord(char):x}"
else:
out+= char
return out
db = Database(config=Database.CONFIG.SEARCH)
count=1
print("export")
mark={}
for msg in db.messages(""):
mark[msg.messageid]=count
blob=""
for tag in msg.tags:
blob += f"{tag}\n"
print (f"blob\nmark :{count}");
print (f"data {len(blob)}\n{blob}")
count=count+1
print (f"\ncommit refs/heads/master\nmark :{count+1}")
ctime = int(time())
print (f"author Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
print (f"committer Notmuch Test Suite <notmuch@example.com> {ctime} +0000")
print (f"data 8\nignored")
for msg in db.messages(""):
digest = sha1(msg.messageid.encode('utf8')).hexdigest()
filename = hexencode(msg.messageid)
print (f"M 100644 :{mark[msg.messageid]} {digest[0:2]}/{digest[2:4]}/{filename}/tags")
print("\ndone\n")
[-- Attachment #3: Type: text/plain, Size: 0 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* [PATCH v2 3/4] cli/git-remote: add import command
2024-09-08 11:27 David Bremner
2024-09-08 11:27 ` [PATCH v2 1/4] util: refactor sync_dir and mkdir_recursive David Bremner
2024-09-08 11:27 ` [PATCH v2 2/4] cli: start remote helper for git David Bremner
@ 2024-09-08 11:27 ` David Bremner
2024-09-08 11:27 ` [PATCH v2 4/4] cli/git-remote: add export command David Bremner
2024-09-08 12:07 ` git-remote-notmuch v2 David Bremner
4 siblings, 0 replies; 8+ messages in thread
From: David Bremner @ 2024-09-08 11:27 UTC (permalink / raw)
To: notmuch
The output in default.import is based on a modified version
of Felipe's git-remote-nm with Blake2 hashing replaced by SHA1
(for portability). This enable fetch/pull/clone, so test that as well.
---
git-remote-notmuch.c | 108 +++++++++
performance-test/M07-git-remote.sh | 16 ++
performance-test/T08-git-remote.sh | 12 +
test/T860-git-remote.sh | 69 ++++++
.../git-remote.expected-output/default.import | 229 ++++++++++++++++++
5 files changed, 434 insertions(+)
create mode 100755 performance-test/M07-git-remote.sh
create mode 100755 performance-test/T08-git-remote.sh
create mode 100644 test/git-remote.expected-output/default.import
diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index 4afd198c..39ba6bf3 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -155,6 +155,30 @@ read_lastmod (const char *dir, char **uuid_out, unsigned long *counter_out)
}
+static void
+store_lastmod (notmuch_database_t *notmuch, const char *dir)
+{
+ char *filename = NULL;
+ FILE *out;
+ unsigned long lastmod;
+ const char *uuid;
+
+ ASSERT (filename = talloc_asprintf (notmuch, "%s/lastmod", dir));
+
+ out = fopen (filename, "w");
+ ensure (out, "error opening %s for writing: %s", filename, strerror (errno));
+
+ lastmod = notmuch_database_get_revision (notmuch, &uuid);
+ ASSERT (fprintf (out, "%s\t%zu\n", uuid, lastmod) > 0);
+}
+
+static void
+write_data (const char *data)
+{
+ printf ("data %zu\n", strlen (data));
+ fputs (data, stdout);
+}
+
static void
cmd_capabilities ()
{
@@ -174,6 +198,88 @@ cmd_list (notmuch_database_t *db, const char *uuid, unsigned long lastmod)
equal_lastmod (uuid, lastmod, db_uuid, db_lastmod) ? " unchanged" : "");
}
+static void
+cmd_import (notmuch_database_t *notmuch,
+ const char *nm_dir,
+ const char *uuid,
+ unsigned long lastmod)
+{
+ const char *ident = NULL;
+ const char *lastmod_str = NULL;
+ notmuch_messages_t *messages;
+ notmuch_status_t status;
+ notmuch_query_t *query;
+ char *mid_buf = NULL;
+ size_t mid_buf_len = 0;
+
+ ident = talloc_asprintf (notmuch, "%s <%s> %zu +0000",
+ notmuch_config_get (notmuch, NOTMUCH_CONFIG_USER_NAME),
+ notmuch_config_get (notmuch, NOTMUCH_CONFIG_PRIMARY_EMAIL),
+ time (NULL));
+
+
+ printf ("feature done\ncommit refs/notmuch/master\nmark :1\ncommitter %s\n", ident);
+
+ ASSERT (lastmod_str = talloc_asprintf (notmuch, "lastmod: %zu\n", lastmod));
+ write_data (lastmod_str);
+ if (uuid)
+ puts ("from refs/notmuch/master^0");
+ puts ("deleteall");
+
+ status = notmuch_query_create_with_syntax (notmuch,
+ "",
+ NOTMUCH_QUERY_SYNTAX_XAPIAN,
+ &query);
+
+ if (print_status_database ("git-remote-nm", notmuch, status))
+ exit (EXIT_FAILURE);
+
+ if (debug_flags && strchr (debug_flags, 's'))
+ notmuch_query_set_sort (query, NOTMUCH_SORT_NEWEST_FIRST);
+ else
+ notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
+
+ status = notmuch_query_search_messages (query, &messages);
+ if (print_status_query ("git-remote-nm", query, status))
+ exit (EXIT_FAILURE);
+
+ for (;
+ notmuch_messages_valid (messages);
+ notmuch_messages_move_to_next (messages)) {
+ const char *tag_buf = "";
+ const char *mid;
+ const char *hash;
+ int ret;
+
+ notmuch_message_t *message = notmuch_messages_get (messages);
+ mid = notmuch_message_get_message_id (message);
+
+ ret = hex_encode (notmuch, mid, &mid_buf, &mid_buf_len);
+ ensure (ret == HEX_SUCCESS, "failed to hex-encode message-id %s\n", mid);
+
+ /* we can't use _notmuch_sha1_from_string because we don't want
+ * to include the null terminator */
+ g_autoptr (GChecksum) sha1 = NULL;
+ sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+ g_checksum_update (sha1, (const guchar *) mid, strlen (mid));
+ hash = g_checksum_get_string (sha1);
+ printf ("M 644 inline %2.2s/%2.2s/%s/tags\n", hash, hash + 2, mid_buf);
+
+ for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+ notmuch_tags_valid (tags);
+ notmuch_tags_move_to_next (tags)) {
+ const char *tag_str = notmuch_tags_get (tags);
+ ASSERT (tag_buf = talloc_asprintf (message, "%s%s\n", tag_buf, tag_str));
+ }
+ write_data (tag_buf);
+ notmuch_message_destroy (message);
+ }
+ puts ("");
+ puts ("done");
+ fflush (stdout);
+ store_lastmod (notmuch, nm_dir);
+}
+
/* stubs since we cannot link with notmuch.o */
const notmuch_opt_desc_t notmuch_shared_options[] = {
{ }
@@ -302,6 +408,8 @@ main (int argc, char *argv[])
if (STRNCMP_LITERAL (s, "capabilities") == 0)
cmd_capabilities ();
+ else if (STRNCMP_LITERAL (s, "import") == 0)
+ cmd_import (db, nm_dir, uuid, lastmod);
else if (STRNCMP_LITERAL (s, "list") == 0)
cmd_list (db, uuid, lastmod);
diff --git a/performance-test/M07-git-remote.sh b/performance-test/M07-git-remote.sh
new file mode 100755
index 00000000..efce18a6
--- /dev/null
+++ b/performance-test/M07-git-remote.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+test_description='search'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+mkdir repo
+export GIT_DIR=`pwd`/repo
+
+memory_start
+
+echo "import refs/heads/master" > import.in
+
+memory_run "import" "git-remote-notmuch origin notmuch:// >import.out <import.in"
+
+memory_done
diff --git a/performance-test/T08-git-remote.sh b/performance-test/T08-git-remote.sh
new file mode 100755
index 00000000..df03d978
--- /dev/null
+++ b/performance-test/T08-git-remote.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+test_description='git-remote-notmuch'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+time_start
+
+time_run 'clone --bare' "git clone --quiet --bare -b master notmuch::default default.git"
+time_run 'clone' "git clone --quiet -b master notmuch:// repo"
+
+time_done
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index 1a087611..97b8e4f9 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -43,4 +43,73 @@ cat <<EOF > EXPECTED
EOF
test_expect_equal_file EXPECTED OUTPUT
+test_begin_subtest 'import writes lastmod file'
+echo import | run_helper dummy-alias dummy-url > /dev/null
+lastmod=$(notmuch count --lastmod '*' | cut -f2-)
+test_expect_equal "${lastmod}" "$(cat < ${git_tmp}/notmuch/lastmod)"
+
+# note that this test must not be the first time import is run,
+# because it depends on the lastmod file
+test_begin_subtest 'import produces expected output'
+echo import | run_helper | notmuch_sanitize_git > OUTPUT
+test_expect_equal_file $EXPECTED/default.import OUTPUT
+
+test_begin_subtest "clone notmuch://"
+test_expect_success "git clone notmuch:// $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch://?config=notmuch-config"
+test_expect_success "git clone notmuch://?config=notmuch-config $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch://?profile=default"
+test_expect_success "git clone notmuch://?profile=default $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch://?config=notmuch-config&profile=default"
+test_expect_success "git clone notmuch://?config=notmuch-config\&profile=default $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch://`pwd`/mail'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch://`pwd`/mail/?config=`pwd`/notmuch-config'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail?config=`pwd`/notmuch-config $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch://.../mail/?config=.../notmuch-config&profile=default'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail/?config=`pwd`/notmuch-config\&profile=default $(mktemp -d clone XXX)"
+
+test_begin_subtest 'clone notmuch://?path=.../mail/&config=.../notmuch-config&profile=default'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://?path=`pwd`/mail\&config=notmuch-config\&profile=default $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch::"
+test_expect_success "git clone notmuch:: $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch::`pwd`/mail'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch::`pwd`/mail $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch::`pwd`/mail?config=`pwd`/notmuch-config'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch::`pwd`/mail?config=`pwd`/notmuch-config $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone has every message"
+git clone notmuch:: repo
+find repo -name tags -type f | sed -e s,repo/../../,id:, -e s,/tags$,, | sort > OUTPUT
+notmuch search --output=messages '*' | sort > EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "pull get new tag"
+notmuch tag +zznew -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+unread
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
+test_begin_subtest "pull sees deletion"
+notmuch tag -unread -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
test_done
diff --git a/test/git-remote.expected-output/default.import b/test/git-remote.expected-output/default.import
new file mode 100644
index 00000000..2d59861e
--- /dev/null
+++ b/test/git-remote.expected-output/default.import
@@ -0,0 +1,229 @@
+feature done
+commit refs/notmuch/master
+mark :1
+committer Notmuch Test Suite <test_suite@notmuchmail.org> TIMESTAMP TIMEZONE
+data 12
+lastmod: 53
+from refs/notmuch/master^0
+deleteall
+M 644 inline 87/b1/4EFC743A.3060609@april.org/tags
+data 13
+inbox
+unread
+M 644 inline 9a/a0/877h1wv7mg.fsf@inf-8657.int-evry.fr/tags
+data 13
+inbox
+unread
+M 644 inline a8/61/1258544095-16616-1-git-send-email-chris@chris-wilson.co.uk/tags
+data 13
+inbox
+unread
+M 644 inline 50/8c/877htoqdbo.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 50/69/878we4qdqf.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 94/67/87aaykqe24.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline bd/10/87bpj0qeng.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 27/84/87fx8cqf8v.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 3c/98/87hbssqfix.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline dd/a4/87iqd8qgiz.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 02/28/87k4xoqgnl.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 61/f6/87ocn0qh6d.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 9a/6b/87pr7gqidx.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline cc/ab/867hto2p0t.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline ba/76/1258532999-9316-1-git-send-email-keithp@keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline ec/65/86aayk2rbj.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline 15/a0/86d43g2w3y.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline 54/e7/ddd65cda0911172214t60d22b63hcfeb5a19ab54a39b@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline 60/cc/86einw2xof.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline 68/39/736613.51770.qm@web113505.mail.gq1.yahoo.com/tags
+data 13
+inbox
+unread
+M 644 inline 1c/7b/1258520223-15328-1-git-send-email-jan@ryngle.com/tags
+data 13
+inbox
+unread
+M 644 inline 46/60/ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline 78/87/1258510940-7018-1-git-send-email-stewart@flamingspork.com/tags
+data 13
+inbox
+unread
+M 644 inline 8c/92/yunzl6kd1w0.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline cf/e1/yun1vjwegii.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline d3/55/yun3a4cegoa.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline fa/e4/1258509400-32511-1-git-send-email-stewart@flamingspork.com/tags
+data 13
+inbox
+unread
+M 644 inline c2/7f/1258506353-20352-1-git-send-email-stewart@flamingspork.com/tags
+data 13
+inbox
+unread
+M 644 inline 13/dd/20091118010116.GC25380@dottiness.seas.harvard.edu/tags
+data 31
+attachment
+inbox
+signed
+unread
+M 644 inline 97/cf/20091118005829.GB25380@dottiness.seas.harvard.edu/tags
+data 31
+attachment
+inbox
+signed
+unread
+M 644 inline cf/81/20091118005040.GA25380@dottiness.seas.harvard.edu/tags
+data 20
+inbox
+signed
+unread
+M 644 inline ca/dc/cf0c4d610911171623q3e27a0adx802e47039b57604b@mail.gmail.com/tags
+data 24
+attachment
+inbox
+unread
+M 644 inline 42/c5/1258500222-32066-1-git-send-email-ingmar@exherbo.org/tags
+data 13
+inbox
+unread
+M 644 inline 47/c1/20091117232137.GA7669@griffis1.net/tags
+data 13
+inbox
+unread
+M 644 inline d7/7c/20091118002059.067214ed@hikari/tags
+data 20
+inbox
+signed
+unread
+M 644 inline ae/4b/1258498485-sup-142@elly/tags
+data 13
+inbox
+unread
+M 644 inline 41/fa/f35dbb950911171438k5df6eb56k77b6c0944e2e79ae@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline e4/81/f35dbb950911171435ieecd458o853c873e35f4be95@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline fe/4b/1258496327-12086-1-git-send-email-jan@ryngle.com/tags
+data 13
+inbox
+unread
+M 644 inline 2c/3b/1258493565-13508-1-git-send-email-keithp@keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline 0c/a5/yunaayketfm.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline 21/a3/yunbpj0etua.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline b8/fa/1258491078-29658-1-git-send-email-dottedmag@dottedmag.net/tags
+data 13
+inbox
+unread
+M 644 inline 08/50/87fx8can9z.fsf@vertex.dottedmag/tags
+data 13
+inbox
+unread
+M 644 inline d3/76/20091117203301.GV3165@dottiness.seas.harvard.edu/tags
+data 20
+inbox
+signed
+unread
+M 644 inline 52/bb/87lji4lx9v.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline df/26/cf0c4d610911171136h1713aa59w9cf9aa31f052ad0a@mail.gmail.com/tags
+data 24
+attachment
+inbox
+unread
+M 644 inline ab/f7/87iqd9rn3l.fsf@vertex.dottedmag/tags
+data 20
+inbox
+signed
+unread
+M 644 inline c5/19/20091117190054.GU3165@dottiness.seas.harvard.edu/tags
+data 20
+inbox
+signed
+unread
+M 644 inline 15/c2/87lji5cbwo.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline de/77/1258471718-6781-2-git-send-email-dottedmag@dottedmag.net/tags
+data 13
+inbox
+unread
+M 644 inline 77/76/1258471718-6781-1-git-send-email-dottedmag@dottedmag.net/tags
+data 13
+inbox
+unread
+
+done
--
2.43.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH v2 4/4] cli/git-remote: add export command
2024-09-08 11:27 David Bremner
` (2 preceding siblings ...)
2024-09-08 11:27 ` [PATCH v2 3/4] cli/git-remote: add import command David Bremner
@ 2024-09-08 11:27 ` David Bremner
2024-09-08 12:07 ` git-remote-notmuch v2 David Bremner
4 siblings, 0 replies; 8+ messages in thread
From: David Bremner @ 2024-09-08 11:27 UTC (permalink / raw)
To: notmuch
This enables the push command, and the helper is now feature complete.
---
git-remote-notmuch.c | 136 +++++++++++++++++++++++++++++
performance-test/M07-git-remote.sh | 4 +
performance-test/T08-git-remote.sh | 41 +++++++++
test/T860-git-remote.sh | 68 +++++++++++++++
4 files changed, 249 insertions(+)
diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index 39ba6bf3..de689474 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -280,6 +280,140 @@ cmd_import (notmuch_database_t *notmuch,
store_lastmod (notmuch, nm_dir);
}
+static GString *
+read_data ()
+{
+ ssize_t nread;
+ size_t bytes;
+ size_t data_size;
+
+ g_auto (GStrv) tokens = NULL;
+
+ ASSERT ((nread = getline (&buffer, &buffer_len, stdin) != -1));
+
+ tokens = tokenize_buffer ();
+
+ str2ul (tokens[1], &data_size);
+
+ buffer = realloc (buffer, data_size + 1);
+ bytes = fread (buffer, 1, data_size, stdin);
+ ASSERT (bytes == data_size);
+
+ buffer_len = data_size;
+
+ return g_string_new_len (buffer, buffer_len);
+}
+
+static void
+free_string (GString *str)
+{
+ g_string_free (str, true);
+}
+
+static void
+cmd_export (notmuch_database_t *notmuch, const char *nm_dir)
+{
+ ssize_t nread;
+
+ g_autoptr (GHashTable) blobs = NULL;
+
+ ASSERT (blobs = g_hash_table_new_full ((GHashFunc) g_str_hash,
+ (GEqualFunc) g_str_equal,
+ g_free, (GDestroyNotify) free_string)
+ );
+
+ while ((nread = getline (&buffer, &buffer_len, stdin)) != -1) {
+ flog ("\texport %s\n", buffer);
+ if (STRNCMP_LITERAL (buffer, "done") == 0) {
+ break;
+ } else if (STRNCMP_LITERAL (buffer, "blob") == 0) {
+ GString *data;
+ g_auto (GStrv) tokens = NULL;
+
+
+ flog ("export blob\n");
+ buffer_line (stdin);
+
+ tokens = tokenize_buffer ();
+
+ data = read_data ();
+
+ flog ("\tmark%s\n", tokens[1]);
+ g_hash_table_insert (blobs, g_strdup (tokens[1]), data);
+ buffer_line (stdin);
+ } else if (STRNCMP_LITERAL (buffer, "commit") == 0) {
+ char *mid = NULL;
+ size_t mid_len = 0;
+
+ flog ("export commit\n");
+
+ /* mark for commit (ignored) */
+ buffer_line (stdin);
+ /* author (ignored) */
+ buffer_line (stdin);
+ /* committer (ignored) */
+ buffer_line (stdin);
+
+ /* commit message (ignored) */
+ (void) read_data ();
+
+ while (strlen (buffer) > 0) {
+ g_autoptr (GString) mark = NULL;
+ g_autoptr (GString) path = NULL;
+ const GString *blob;
+ g_autofree char *basename = NULL;
+ notmuch_message_t *message;
+ const char *tok;
+ size_t tok_len;
+ size_t max_tok_len;
+ tag_op_list_t *tag_ops;
+ g_auto (GStrv) tokens = NULL;
+
+ buffer_line (stdin);
+ if (strlen (buffer) == 0)
+ break;
+
+ tokens = tokenize_buffer ();
+ flog ("looking for blob |%s|\n", tokens[2]);
+ ASSERT (blob = g_hash_table_lookup (blobs, tokens[2]));
+
+ basename = g_path_get_dirname (tokens[3] + 6);
+ ASSERT (HEX_SUCCESS ==
+ hex_decode (notmuch, basename, &mid, &mid_len));
+ ASSERT (NOTMUCH_STATUS_SUCCESS ==
+ notmuch_database_find_message (notmuch, mid, &message));
+ ASSERT (message);
+
+ ASSERT (NOTMUCH_STATUS_SUCCESS ==
+ notmuch_message_freeze (message));
+
+ tag_ops = tag_op_list_create (message);
+ tok = blob->str;
+ max_tok_len = blob->len;
+ tok_len = 0;
+ while ((tok_len < max_tok_len) &&
+ (tok = strsplit_len (tok + tok_len, '\n', &tok_len)) != NULL) {
+ const char *tag = talloc_strndup (message, tok, tok_len);
+ ASSERT (0 == tag_op_list_append (tag_ops, tag, false));
+ }
+
+ ASSERT (NOTMUCH_STATUS_SUCCESS ==
+ tag_op_list_apply (message, tag_ops, TAG_FLAG_REMOVE_ALL));
+
+ ASSERT (NOTMUCH_STATUS_SUCCESS ==
+ notmuch_message_thaw (message));
+
+ notmuch_message_destroy (message);
+ }
+ puts ("ok refs/heads/master");
+ }
+
+ }
+ store_lastmod (notmuch, nm_dir);
+ puts ("");
+}
+
+
/* stubs since we cannot link with notmuch.o */
const notmuch_opt_desc_t notmuch_shared_options[] = {
{ }
@@ -408,6 +542,8 @@ main (int argc, char *argv[])
if (STRNCMP_LITERAL (s, "capabilities") == 0)
cmd_capabilities ();
+ else if (STRNCMP_LITERAL (s, "export") == 0)
+ cmd_export (db, nm_dir);
else if (STRNCMP_LITERAL (s, "import") == 0)
cmd_import (db, nm_dir, uuid, lastmod);
else if (STRNCMP_LITERAL (s, "list") == 0)
diff --git a/performance-test/M07-git-remote.sh b/performance-test/M07-git-remote.sh
index efce18a6..526cd856 100755
--- a/performance-test/M07-git-remote.sh
+++ b/performance-test/M07-git-remote.sh
@@ -6,6 +6,7 @@ test_description='search'
mkdir repo
export GIT_DIR=`pwd`/repo
+MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
memory_start
@@ -13,4 +14,7 @@ echo "import refs/heads/master" > import.in
memory_run "import" "git-remote-notmuch origin notmuch:// >import.out <import.in"
+python3 $MAKE_EXPORT_PY > export.in
+memory_run "export" "git-remote-notmuch origin notmuch:// >export.out <export.in"
+
memory_done
diff --git a/performance-test/T08-git-remote.sh b/performance-test/T08-git-remote.sh
index df03d978..00ee1702 100755
--- a/performance-test/T08-git-remote.sh
+++ b/performance-test/T08-git-remote.sh
@@ -4,9 +4,50 @@ test_description='git-remote-notmuch'
. $(dirname "$0")/perf-test-lib.sh || exit 1
+add_tags() {
+ local dir=$1
+ local denom=$2
+ local olddir=$(pwd)
+
+ cd $dir
+ find . -name tags -type f |
+ while read -r path; do
+ if [ $(($RANDOM % $denom)) -eq 0 ]; then
+ echo $RANDOM >> $path
+ fi
+ done
+
+ cd $olddir
+}
+
time_start
time_run 'clone --bare' "git clone --quiet --bare -b master notmuch::default default.git"
time_run 'clone' "git clone --quiet -b master notmuch:// repo"
+time_run "push (no changes)" "git -C repo push --quiet origin master"
+
+add_tags repo 10
+git -C repo add -u
+git -C repo commit --quiet -m'add tags to 10% of messages'
+time_run "push (10% changed)" "git -C repo push --quiet origin master"
+
+add_tags repo 4
+git -C repo add -u
+git -C repo commit --quiet -m'add tags to 25% of messages'
+time_run "push (25% changed)" "git -C repo push --quiet origin master"
+
+add_tags repo 2
+git -C repo add -u
+git -C repo commit --quiet -m'add tags to 50% of messages'
+time_run "push (50% changed)" "git -C repo push --quiet origin master"
+
+hash=$(git -C repo hash-object --stdin -w < /dev/null)
+# replace all files with empty files
+git -C repo ls-tree -r HEAD | sed "s/blob [^\t]*/blob $hash/" \
+ | git -C repo update-index --index-info
+git -C repo commit --quiet -m'zero tags' 2>>log.txt 1>&2
+
+time_run "push (rem. all tags)" "git -C repo push --quiet origin master"
+
time_done
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index 97b8e4f9..014ffc2b 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -112,4 +112,72 @@ zznew
EOF
test_expect_equal_file EXPECTED repo/$TAG_FILE
+test_begin_subtest 'export runs'
+run_helper <<EOF | notmuch_sanitize_git > OUTPUT
+export
+blob
+mark :1
+data 10
+tag1
+tag2
+
+commit refs/heads/master
+mark :2
+author Notmuch Test Suite <notmuch@example.com> 1234 +0000
+committer Notmuch Test Suite <notmuch@example.com> 1234 +0000
+data 8
+ignored
+M 100644 :1 $TAG_FILE
+
+done
+
+EOF
+cat <<EOF > EXPECTED
+ok refs/heads/master
+
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+# this test depends on the previous one
+test_begin_subtest 'export modifies database'
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest 'restore via export'
+notmuch dump > BEFORE
+python3 $MAKE_EXPORT_PY > export.in
+notmuch tag +transient -- id:4EFC743A.3060609@april.org
+run_helper < export.in > OUTPUT
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "push updates database"
+git -C repo push origin master
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "adding tag via repo"
+cat<<EOF >repo/$TAG_FILE
+tag1
+tag2
+tag3
+EOF
+git -C repo add $TAG_FILE
+git -C repo commit -m 'testing push'
+git -C repo push origin master
+notmuch dump id:4EFC743A.3060609@april.org | tail -n 1 > OUTPUT
+cat <<EOF > EXPECTED
++tag1 +tag2 +tag3 -- id:4EFC743A.3060609@april.org
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
test_done
--
2.43.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* git-remote-notmuch v2
2024-09-08 11:27 David Bremner
` (3 preceding siblings ...)
2024-09-08 11:27 ` [PATCH v2 4/4] cli/git-remote: add export command David Bremner
@ 2024-09-08 12:07 ` David Bremner
4 siblings, 0 replies; 8+ messages in thread
From: David Bremner @ 2024-09-08 12:07 UTC (permalink / raw)
To: notmuch
Apologies for the lack of subject. Still caffeine deficient this
morning...
David Bremner <david@tethera.net> writes:
> 7) This code makes heavier use of GLib than most notmuch code. In
> particular there is extensive use of g_auto macros for memory
> management. These could be replaced with more use of talloc, but that
> would require a few more low level utility routines to be written.
The other thing it makes me think is that GLib is arguably badly
re-inventing C++, so that's something to ponder. I don't know if argues
for just using C++, or for going the extra mile and using talloc.
> 8) Performance has been optimized for the case of few changes to the
> database since last git-push (analogous notmuch restore)
9) I replaced the shelling out to "git var" with interrogating the
notmuch configuration database. The behaviour is slightly different, in
particular GIT_COMMITTER_* environment variables are currently
ignored. Obeying those could be added fairly simply, I think. There are
probably other subtlties to how git constructs GIT_COMMITTER_IDENT; I'm
not sure if those are important.
10) I observed there are still a few relics of the "git-remote-nm" name
in the code.
11) I think this should eventually save and restore configuration
information and properties like dump / restore do. I don't see any
conceptual barrier to doing so, just more code.
^ permalink raw reply [flat|nested] 8+ messages in thread