notmuch-git

[David Bremner <david@tethera.net>]

This series obsolates the WIP series at [1]. The rather long interdiff
is given below [2].

At this point I think the tool is feature complete, at least enough
for initial release. The main changes since the last series are the
addition of configuration variables and a "safe_fraction" check that
asks the user to override any operation that would change "lots" of
the database or git repo.  There is also a bunch more documentation
and tests. Finally I dropped support for old python versions (older
than 3.2, including python2). I'll try to get those into master
relatively quickly, to make reviewing this series easier.

This series needs to be applied on top of a couple of patches not yet in master [3]

[1]: id:20220423133848.3852688-1-david@tethera.net

[2]: interdiff

[3]: definitely id:20220513122929.1981190-1-david@tethera.net and
probably id:20220514122406.2220621-1-david@tethera.net

diff --git a/Makefile.local b/Makefile.local
index ca2310f4..0fadfb26 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -1,7 +1,7 @@
 # -*- makefile-gmake -*-
 
 .PHONY: all
-all: notmuch notmuch-shared build-man build-info ruby-bindings python-cffi-bindings notmuch-git
+all: notmuch notmuch-shared build-man build-info ruby-bindings python-cffi-bindings notmuch-git nmbug
 ifeq ($(MAKECMDGOALS),)
 ifeq ($(shell cat .first-build-message 2>/dev/null),)
 	@NOTMUCH_FIRST_BUILD=1 $(MAKE) --no-print-directory all
@@ -307,9 +307,10 @@ cppcheck:
 	@echo "No cppcheck found during configure; skipping static checking"
 endif
 
-notmuch-git: notmuch-git.in
+nmbug notmuch-git: notmuch-git.in
 	sed s/@NOTMUCH_VERSION@/${VERSION}/ < notmuch-git.in > notmuch-git
 	chmod ugo+rx notmuch-git
+	ln -sf notmuch-git nmbug
 
 DEPS := $(SRCS:%.c=.deps/%.d)
 DEPS := $(DEPS:%.cc=.deps/%.d)
diff --git a/debian/notmuch-git.install b/debian/notmuch-git.install
index ca632418..2be08276 100644
--- a/debian/notmuch-git.install
+++ b/debian/notmuch-git.install
@@ -1 +1,2 @@
 notmuch-git /usr/bin
+nmbug /usr/bin
diff --git a/debian/notmuch-git.manpages b/debian/notmuch-git.manpages
new file mode 100644
index 00000000..e0895c86
--- /dev/null
+++ b/debian/notmuch-git.manpages
@@ -0,0 +1,2 @@
+usr/share/man/man1/notmuch-git.1.gz
+usr/share/man/man1/nmbug.1.gz
diff --git a/doc/Makefile.local b/doc/Makefile.local
index d43ef269..2f67f4de 100644
--- a/doc/Makefile.local
+++ b/doc/Makefile.local
@@ -131,6 +131,7 @@ install-man: ${MAN_GZIP_FILES}
 	install -m0644 $(filter %.5.gz,$(MAN_GZIP_FILES)) $(DESTDIR)/$(mandir)/man5
 	install -m0644 $(filter %.7.gz,$(MAN_GZIP_FILES)) $(DESTDIR)/$(mandir)/man7
 	cd $(DESTDIR)/$(mandir)/man1 && ln -sf notmuch.1.gz notmuch-setup.1.gz
+	cd $(DESTDIR)/$(mandir)/man1 && ln -sf notmuch-git.1.gz nmbug.1.gz
 endif
 
 ifneq ($(HAVE_SPHINX)$(HAVE_MAKEINFO),11)
diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 36d48725..388315f6 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -107,6 +107,21 @@ paths are presumed relative to `$HOME` for items in section
 
     Default: see :ref:`database`
 
+.. nmconfig:: git.path
+
+    Default location for git repository for :any:`notmuch-git`.
+
+.. nmconfig:: git.safe_fraction
+
+   Some :any:`notmuch-git` operations check that the fraction of
+   messages changed (in the database or in git, as appropriate) is not
+   too large. This item controls what fraction of total messages is
+   considered "not too large".
+
+.. nmconfig:: git.tag_prefix
+
+    Default tag prefix (filter) for :any:`notmuch-git`.
+
 .. nmconfig:: index.decrypt
 
     Policy for decrypting encrypted messages during indexing.  Must be
diff --git a/doc/man1/notmuch-git.rst b/doc/man1/notmuch-git.rst
index 4877f22d..fa7a748e 100644
--- a/doc/man1/notmuch-git.rst
+++ b/doc/man1/notmuch-git.rst
@@ -1,23 +1,25 @@
 .. _notmuch-git(1):
 
-============
+===========
 notmuch-git
-============
+===========
 
 SYNOPSIS
 ========
 
-**notmuch** **git** [-h] [-C REPO] [-p PREFIX] [-v] [-l *log level*] *subcommand*
+**notmuch** **git** [-h] [-N] [-C *repo*] [-p *prefix*] [-v] [-l *log level*] *subcommand*
+
+**nmbug** [-h] [-C *repo*] [-p *prefix*] [-v] [-l *log level*] *subcommand*
 
 DESCRIPTION
 ===========
 
 Manage notmuch tags with Git.
 
-Options
+OPTIONS
 -------
 
-Supported options for **notmuch-git** include
+Supported options for `notmuch git` include
 
 .. program:: notmuch-git
 
@@ -25,77 +27,271 @@ Supported options for **notmuch-git** include
 
    show help message and exit
 
-.. option:: -C repo, --git-dir repo
+.. option:: -N, --nmbug
+
+   Set defaults for :option:`--tag-prefix` and :option:`--git-dir` suitable for the
+   :any:`notmuch` bug tracker
 
-   Operate on git repository *repo*
+.. option:: -C <repo>, --git-dir <repo>
 
-.. option::  -p prefix, --tag-prefix prefix
+   Operate on git repository *repo*. See :ref:`repo_location` for
+   defaults.
 
-   Operate only on tags with prefix *prefix*
+.. option:: -p <prefix>, --tag-prefix <prefix>
+
+   Operate only on tags with prefix *prefix*. See :ref:`prefix_val` for
+   defaults.
 
 .. option::   -v, --version
 
    show notmuch-git's version number and exit
 
-.. option::   -l *level*, --log-level *level* {critical,error,warning,info,debug}
+.. option::   -l <level>, --log-level <level>
 
-   Log verbosity. Defaults to 'warning'.
+   Log verbosity, one of: `critical`, `error`, `warning`, `info`,
+   `debug`. Defaults to `warning`.
 
-Subcommands
+SUBCOMMANDS
 -----------
 
 For help on a particular subcommand, run: 'notmuch-git ... <command> --help'.
 
-.. option:: archive [TREE-ISH] [ARG ...]
+.. program:: notmuch-git
 
-Dump a tar archive of the current nmbug tag set using 'git archive'.
+.. option:: archive [tree-ish] [arg ...]
 
-For each tag *tag* for message with Message-Id *id* an empty file
+Dump a tar archive of a committed tag set using 'git archive'. See
+:any:`format` for details of the archive contents.
 
-  tags/encode(*id*)/encode(*tag*)
+   .. describe:: tree-ish
 
-is written to the output.
+   The tree or commit to produce an archive for. Defaults to 'HEAD'.
 
-The encoding preserves alphanumerics, and the characters
-"+-_@=.:," (not the quotes).  All other octets are replaced with
-'%%' followed by a two digit hex number.
+   .. describe:: arg
 
-positional arguments:
-  TREE-ISH    The tree or commit to produce an archive for. Defaults to
-              'HEAD'.
-  ARG         Argument passed through to 'git archive'. Set anything before
-              <tree-ish>, see any:`git-archive(1)` for details.
+   If present, any optional arguments are passed through to
+   :manpage:`git-archive(1)`. Arguments to `git-archive` are reordered
+   so that *tree-ish* comes last.
 
-.. option:: checkout
+.. option:: checkout [-f|--force]
 
 Update the notmuch database from Git.
 
 This is mainly useful to discard your changes in notmuch relative
 to Git.
-Create a local nmbug repository from a remote source.
 
-.. option:: clone repository
+   .. describe:: [-f|--force]
+
+   Override checks that prevent modifying tags for large fractions of
+   messages in the database. See also :nmconfig:`git.safe_fraction`.
+
+.. option:: clone <repository>
+
+Create a local `notmuch git` repository from a remote source.
 
 This wraps 'git clone', adding some options to avoid creating a
 working tree while preserving remote-tracking branches and
 upstreams.
 
-positional arguments:
-  repository  The (possibly remote) repository to clone from. See the URLS section of git-clone(1) for more information on specifying repositories.
+    .. describe:: repository
+
+    The (possibly remote) repository to clone from. See the URLS
+    section of :manpage:`git-clone(1)` for more information on
+    specifying repositories.
+
+.. option:: commit [-f|--force] [message]
+
+Commit prefix-matching tags from the notmuch database to Git.
+
+   .. describe:: message
+
+   Optional text for the commit message.
+
+   .. describe:: -f|--force
+
+   Override checks that prevent modifying tags for large fractions of
+   messages in the database. See also :nmconfig:`git.safe_fraction`.
+
+.. option:: fetch [remote]
+
+Fetch changes from the remote repository.
+
+    .. describe:: remote
+
+    Override the default configured in `branch.<name>.remote` to fetch
+    from a particular remote repository (e.g. `origin`).
+
+.. option:: help
+
+Show brief help for an `notmuch git` command.
+
+.. option:: init
+
+Create an empty `notmuch git` repository.
+
+This wraps 'git init' with a few extra steps to support subsequent
+status and commit commands.
+
+.. option:: log [arg ...]
+
+A wrapper for 'git log'.
+
+   .. describe:: arg
+
+   Additional arguments are passed through to 'git log'.
+
+After running `notmuch git fetch`, you can inspect the changes with
+
+::
+
+   $ notmuch git log HEAD..@{upstream}
+
+.. option:: merge [reference]
+
+Merge changes from 'reference' into HEAD and load the result into notmuch.
+
+   .. describe:: reference
+
+   Reference, usually other branch heads, to merge into our
+   branch. Defaults to `@{upstream}`.
+
+.. option:: pull [repository] [refspec ...]
+
+Pull (merge) remote repository changes to notmuch.
+
+**pull** is equivalent to **fetch** followed by **merge**.  We use the
+Git-configured repository for your current branch
+(`branch.<name>.repository`, likely `origin`, and `branch.<name>.merge`,
+likely `master` or `main`).
+
+   .. describe:: repository
+
+   The "remote" repository that is the source of the pull. This parameter
+   can be either a URL (see the section GIT URLS in :manpage:`git-pull(1)`) or the
+   name of a remote (see the section REMOTES in :manpage:`git-pull(1)`).
+
+   .. describe:: refspec
+
+   Refspec (usually a branch name) to fetch and merge. See the
+   *refspec* entry in the OPTIONS section of :manpage:`git-pull(1`) for
+   other possibilities.
+
+.. option:: push [repository] [refspec]
+
+Push the local `notmuch git` Git state to a remote repository.
+
+    .. describe::  repository
+
+    The "remote" repository that is the destination of the push. This
+    parameter can be either a URL (see the section GIT URLS in
+    :manpage:`git-push(1)`) or the name of a remote (see the section
+    REMOTES in :manpage:`git-push(1)`).
+
+    .. describe:: refspec
+
+    Refspec (usually a branch name) to push. See the *refspec* entry in the OPTIONS section of
+    :manpage:`git-push(1)` for other possibilities.
+
+.. option:: status
+
+Show pending updates in notmuch or git repo.
+
+Prints lines of the form
+
+|  ng Message-Id tag
+
+where n is a single character representing notmuch database status
+
+   .. describe:: A
+
+   Tag is present in notmuch database, but not committed to nmbug
+   (equivalently, tag has been deleted in nmbug repo, e.g. by a
+   pull, but not restored to notmuch database).
+
+   .. describe:: D
+
+   Tag is present in nmbug repo, but not restored to notmuch
+   database (equivalently, tag has been deleted in notmuch).
+
+   .. describe:: U
+
+   Message is unknown (missing from local notmuch database).
+
+The second character *g* (if present) represents a difference between
+local and upstream branches. Typically `notmuch git fetch` needs to be
+run to update this.
+
+   .. describe:: a
+
+   Tag is present in upstream, but not in the local Git branch.
+
+   .. describe:: d
+
+   Tag is present in local Git branch, but not upstream.
+
+.. _format:
+
+REPOSITORY CONTENTS
+===================
+
+The tags are stored in the git repo (and exported) as a set of empty
+files. For a message with Message-Id *id*, for each tag *tag*, there
+is an empty file with path
+
+       tags/ `encode` (*id*) / `encode` (*tag*)
+
+The encoding preserves alphanumerics, and the characters `+-_@=.,:`.
+All other octets are replaced with `%` followed by a two digit hex
+number.
+
+.. _repo_location:
+
+REPOSITORY LOCATION
+===================
+
+:any:`notmuch-git` uses the first of the following with a non-empty
+value to locate the git repository.
+
+- Option :option:`--git-dir`.
+
+- Environment variable :envvar:`NOTMUCH_GIT_DIR`.
+
+- Configuration item :nmconfig:`git.path`
+
+- If invoked as `nmbug` or with the :option:`--nmbug` option,
+  :code:`$HOME/.nmbug`; otherwise
+  :code:`$XDG_DATA_HOME/notmuch/$NOTMUCH_PROFILE/git`.
+
+.. _prefix_val:
+
+PREFIX VALUE
+============
+
+:any:`notmuch-git` uses the first of the following with a non-null
+value to define the tag prefix.
+
+- Option :option:`--tag-prefix`.
+
+- Environment variable :envvar:`NOTMUCH_GIT_PREFIX`.
+
+- Configuration item :nmconfig:`git.tag_prefix`.
+
+- If invoked as `nmbug` or with the :option:`--nmbug` option,
+  :code:`notmuch::`, otherwise the empty string.
+
+ENVIRONMENT
+===========
+
+Variable :envvar:`NOTMUCH_PROFILE` influences :ref:`repo_location`.
+If it is unset, 'default' is assumed.
 
-    clone               Create a local nmbug repository from a remote source.
-    commit              Commit prefix-matching tags from the notmuch database to Git.
-    fetch               Fetch changes from the remote repository.
-    help                Show help for an nmbug command.
-    init                Create an empty nmbug repository.
-    log                 A simple wrapper for 'git log'.
-    merge               Merge changes from 'reference' into HEAD and load the result into notmuch.
-    pull                Pull (merge) remote repository changes to notmuch.
-    push                Push the local nmbug Git state to a remote repository.
-    status              Show pending updates in notmuch or git repo.
+.. envvar:: NOTMUCH_GIT_DIR
 
+   Default location of git repository. Overriden by :option:`--git-dir`.
 
+.. envvar:: NOTMUCH_GIT_PREFIX
 
+   Default tag prefix (filter). Overriden by :option:`--tag-prefix`.
 
 SEE ALSO
 ========
diff --git a/notmuch-git.in b/notmuch-git.in
index 261b3f85..6505c2e5 100755
--- a/notmuch-git.in
+++ b/notmuch-git.in
@@ -18,13 +18,6 @@
 
 """
 Manage notmuch tags with Git
-
-Environment variables:
-
-* NMBGIT specifies the location of the git repository used by nmbug.
-  If not specified $HOME/.nmbug is used.
-* NMBPREFIX specifies the prefix in the notmuch database for tags of
-  interest to nmbug. If not specified 'notmuch::' is used.
 """
 
 from __future__ import print_function
@@ -43,25 +36,17 @@ import subprocess as _subprocess
 import sys as _sys
 import tempfile as _tempfile
 import textwrap as _textwrap
-try:  # Python 3
-    from urllib.parse import quote as _quote
-    from urllib.parse import unquote as _unquote
-except ImportError:  # Python 2
-    from urllib import quote as _quote
-    from urllib import unquote as _unquote
-
+from urllib.parse import quote as _quote
+from urllib.parse import unquote as _unquote
 import json as _json
 
-# hopefully big enough, handle 32 bit hosts
-MAX_LASTMOD=2**32
-
 __version__ = '@NOTMUCH_VERSION@'
 
 _LOG = _logging.getLogger('nmbug')
 _LOG.setLevel(_logging.WARNING)
 _LOG.addHandler(_logging.StreamHandler())
 
-NMBGIT = None
+NOTMUCH_GIT_DIR = None
 TAG_PREFIX = None
 
 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
@@ -71,31 +56,6 @@ _TAG_FILE_REGEX = _re.compile(_TAG_DIRECTORY + '(?P<id>[^/]*)/(?P<tag>[^/]*)')
 # magic hash for Git (git hash-object -t blob /dev/null)
 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
 
-
-try:
-    getattr(_tempfile, 'TemporaryDirectory')
-except AttributeError:  # Python < 3.2
-    class _TemporaryDirectory(object):
-        """
-        Fallback context manager for Python < 3.2
-
-        See PEP 343 for details on context managers [1].
-
-        [1]: https://www.python.org/dev/peps/pep-0343/
-        """
-        def __init__(self, **kwargs):
-            self.name = _tempfile.mkdtemp(**kwargs)
-
-        def __enter__(self):
-            return self.name
-
-        def __exit__(self, type, value, traceback):
-            _shutil.rmtree(self.name)
-
-
-    _tempfile.TemporaryDirectory = _TemporaryDirectory
-
-
 def _hex_quote(string, safe='+@=:,'):
     """
     quote('abc def') -> 'abc%20def'.
@@ -248,7 +208,7 @@ def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
 
 
 def _git(args, **kwargs):
-    args = ['git', '--git-dir', NMBGIT] + list(args)
+    args = ['git', '--git-dir', NOTMUCH_GIT_DIR] + list(args)
     return _spawn(args=args, **kwargs)
 
 
@@ -281,6 +241,16 @@ def _tag_query(prefix=None):
         prefix = TAG_PREFIX
     return '(tag (starts-with "{:s}"))'.format(prefix.replace('"','\\\"'))
 
+def count_messages(prefix=None):
+    "count messages with a given prefix."
+    (status, stdout, stderr) = _spawn(
+        args=['notmuch', 'count', '--query=sexp', _tag_query(prefix)],
+        stdout=_subprocess.PIPE, wait=True)
+    if status != 0:
+        _LOG.error("failed to run notmuch config")
+        sys.exit(1)
+    return int(stdout.rstrip())
+
 def get_tags(prefix=None):
     "Get a list of tags with a given prefix."
     (status, stdout, stderr) = _spawn(
@@ -317,7 +287,7 @@ def clone(repository):
     with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
         _spawn(
             args=[
-                'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
+                'git', 'clone', '--no-checkout', '--separate-git-dir', NOTMUCH_GIT_DIR,
                 repository, workdir],
             wait=True)
     _git(args=['config', '--unset', 'core.worktree'], wait=True, expect=(0, 5))
@@ -399,7 +369,22 @@ class CachedIndex:
         _git(args=['read-tree', self.current_treeish], wait=True)
 
 
-def commit(treeish='HEAD', message=None):
+def check_safe_fraction(status):
+    safe = 0.1
+    conf = _notmuch_config_get ('git.safe_fraction')
+    if conf and conf != '':
+        safe=float(conf)
+
+    total = count_messages (TAG_PREFIX)
+    change = len(status['added'])+len(status['deleted'])+len(status['missing'])
+    fraction = change/total
+    _LOG.debug('total messages {:d}, change: {:d}, fraction: {:f}'.format(total,change,fraction))
+    if fraction > safe:
+        _LOG.error('safe fraction {:f} exceeded, stopping.'.format(safe))
+        _LOG.error('Use --force to override or reconfigure git.safe_fraction.')
+        exit(1)
+
+def commit(treeish='HEAD', message=None, force=False):
     """
     Commit prefix-matching tags from the notmuch database to Git.
     """
@@ -410,7 +395,10 @@ def commit(treeish='HEAD', message=None):
         _LOG.warning('Nothing to commit')
         return
 
-    with CachedIndex(NMBGIT, treeish) as index:
+    if not force:
+        check_safe_fraction (status)
+
+    with CachedIndex(NOTMUCH_GIT_DIR, treeish) as index:
         try:
             _update_index(status=status)
             (_, tree, _) = _git(
@@ -467,7 +455,14 @@ def init(remote=None):
     This wraps 'git init' with a few extra steps to support subsequent
     status and commit commands.
     """
-    _spawn(args=['git', '--git-dir', NMBGIT, 'init',
+    from pathlib import Path
+    parent = Path(NOTMUCH_GIT_DIR).parent
+    try:
+        _os.makedirs(parent)
+    except FileExistsError:
+        pass
+
+    _spawn(args=['git', '--git-dir', NOTMUCH_GIT_DIR, 'init',
                  '--initial-branch=master', '--quiet', '--bare'], wait=True)
     _git(args=['config', 'core.logallrefupdates', 'true'], wait=True)
     # create an empty blob (e69de29bb2d1d6434b8b29ae775ad8c2e48c5391)
@@ -476,11 +471,11 @@ def init(remote=None):
         args=[
             'commit', '--allow-empty', '-m', 'Start a new nmbug repository'
         ],
-        additional_env={'GIT_WORK_TREE': NMBGIT},
+        additional_env={'GIT_WORK_TREE': NOTMUCH_GIT_DIR},
         wait=True)
 
 
-def checkout():
+def checkout(force=None):
     """
     Update the notmuch database from Git.
 
@@ -488,6 +483,10 @@ def checkout():
     to Git.
     """
     status = get_status()
+
+    if not force:
+        check_safe_fraction(status)
+
     with _spawn(
             args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
         for id, tags in status['added'].items():
@@ -682,7 +681,7 @@ def get_status():
         'deleted': {},
         'missing': {},
         }
-    with PrivateIndex(repo=NMBGIT, prefix=TAG_PREFIX) as index:
+    with PrivateIndex(repo=NOTMUCH_GIT_DIR, prefix=TAG_PREFIX) as index:
         maybe_deleted = index.diff(filter='D')
         for id, tags in maybe_deleted.items():
             (_, stdout, stderr) = _spawn(
@@ -897,6 +896,24 @@ def _help(parser, command=None):
     else:
         parser.parse_args(['--help'])
 
+def _notmuch_config_get(key):
+    (status, stdout, stderr) = _spawn(
+        args=['notmuch', 'config', 'get', key],
+        stdout=_subprocess.PIPE, wait=True)
+    if status != 0:
+        _LOG.error("failed to run notmuch config")
+        _sys.exit(1)
+    return stdout.rstrip()
+
+# based on BaseDirectory.save_data_path from pyxdg (LGPL2+)
+def xdg_data_path(profile):
+    resource = _os.path.join('notmuch',profile,'git')
+    assert not resource.startswith('/')
+    _home = _os.path.expanduser('~')
+    xdg_data_home = _os.environ.get('XDG_DATA_HOME') or \
+        _os.path.join(_home, '.local', 'share')
+    path = _os.path.join(xdg_data_home, resource)
+    return path
 
 if __name__ == '__main__':
     import argparse
@@ -909,8 +926,11 @@ if __name__ == '__main__':
         help='Git repository to operate on.')
     parser.add_argument(
         '-p', '--tag-prefix', metavar='PREFIX',
-        default = _os.getenv('NMBPREFIX', 'notmuch::'),
+        default = None,
         help='Prefix of tags to operate on.')
+    parser.add_argument(
+        '-N', '--nmbug', action='store_true',
+        help='Set defaults for --tag-prefix and --git-dir for the notmuch bug tracker')
     parser.add_argument(
         '-v', '--version', action='version',
         version='%(prog)s {}'.format(__version__))
@@ -960,6 +980,10 @@ if __name__ == '__main__':
                 help=(
                     "Argument passed through to 'git archive'.  Set anything "
                     'before <tree-ish>, see git-archive(1) for details.'))
+        elif command == 'checkout':
+            subparser.add_argument(
+                '-f', '--force', action='store_true',
+                help='checkout a large fraction of tags.')
         elif command == 'clone':
             subparser.add_argument(
                 'repository',
@@ -968,6 +992,9 @@ if __name__ == '__main__':
                     'URLS section of git-clone(1) for more information on '
                     'specifying repositories.'))
         elif command == 'commit':
+            subparser.add_argument(
+                '-f', '--force', action='store_true',
+                help='commit a large fraction of tags.')
             subparser.add_argument(
                 'message', metavar='MESSAGE', default='', nargs='?',
                 help='Text for the commit message.')
@@ -1024,35 +1051,60 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
+    nmbug_mode = False
+    notmuch_profile = _os.getenv('NOTMUCH_PROFILE','default')
+
+    if args.nmbug or _os.path.basename(__file__) == 'nmbug':
+        nmbug_mode = True
+
     if args.git_dir:
-        NMBGIT = args.git_dir
+        NOTMUCH_GIT_DIR = args.git_dir
     else:
-        NMBGIT = _os.path.expanduser(
-        _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
-        _NMBGIT = _os.path.join(NMBGIT, '.git')
-        if _os.path.isdir(_NMBGIT):
-            NMBGIT = _NMBGIT
+        if nmbug_mode:
+            default = _os.path.join('~', '.nmbug')
+        else:
+            default = _notmuch_config_get ('git.path')
+            if default == '':
+                default = xdg_data_path(notmuch_profile)
+
+        NOTMUCH_GIT_DIR = _os.path.expanduser(_os.getenv('NOTMUCH_GIT_DIR', default))
+
+    _NOTMUCH_GIT_DIR = _os.path.join(NOTMUCH_GIT_DIR, '.git')
+    if _os.path.isdir(_NOTMUCH_GIT_DIR):
+        NOTMUCH_GIT_DIR = _NOTMUCH_GIT_DIR
+
+    if args.tag_prefix:
+        TAG_PREFIX = args.tag_prefix
+    else:
+        if nmbug_mode:
+            prefix = 'notmuch::'
+        else:
+            prefix = _notmuch_config_get ('git.tag_prefix')
+
+        TAG_PREFIX =  _os.getenv('NOTMUCH_GIT_PREFIX', prefix)
 
-    TAG_PREFIX = args.tag_prefix
     _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,')  # quote ':'
 
     if args.log_level:
         level = getattr(_logging, args.log_level.upper())
         _LOG.setLevel(level)
 
-    (status, stdout, stderr) = _spawn(
-        args=['notmuch', 'config', 'get', 'built_with.sexp_queries'],
-        stdout=_subprocess.PIPE, wait=True)
-    if status != 0:
-        _LOG.error("failed to run notmuch")
-        sys.exit(1)
-    if stdout != "true\n":
+    # for test suite
+    for var in ['NOTMUCH_GIT_DIR', 'NOTMUCH_GIT_PREFIX', 'NOTMUCH_PROFILE', 'NOTMUCH_CONFIG' ]:
+        _LOG.debug('env {:s} = {:s}'.format(var, _os.getenv(var,'%None%')))
+
+    if _notmuch_config_get('built_with.sexp_queries') != 'true':
         _LOG.error("notmuch git needs sexp query support")
+        _sys.exit(1)
 
     if not getattr(args, 'func', None):
         parser.print_usage()
         _sys.exit(1)
 
+    # The following two lines are used by the test suite.
+    _LOG.debug('prefix = {:s}'.format(TAG_PREFIX))
+    _LOG.debug('repository = {:s}'.format(NOTMUCH_GIT_DIR))
+
     if args.func == help:
         arg_names = ['command']
     else:
diff --git a/notmuch.c b/notmuch.c
index ac25ae18..8c53fb80 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -201,6 +201,8 @@ static const command_t commands[] = {
     { "emacs-mua", NULL, 0,
       "send mail with notmuch and emacs." },
 #endif
+    { "git", NULL, 0,
+      "manage notmuch tags with git" },
     { "help", notmuch_help_command, NOTMUCH_COMMAND_CONFIG_CREATE, /* create but don't save config */
       "This message, or more detailed help for the named command." }
 };
diff --git a/test/T850-git.sh b/test/T850-git.sh
index 2358690f..508615e1 100755
--- a/test/T850-git.sh
+++ b/test/T850-git.sh
@@ -2,45 +2,65 @@
 test_description='"notmuch git" to save and restore tags'
 . $(dirname "$0")/test-lib.sh || exit 1
 
-add_git_repos () {
-    notmuch git -C remote.git -p '' init
-    notmuch git -C tags.git -p '' clone remote.git
-}
-
 if [ $NOTMUCH_HAVE_SFSEXP -ne 1 ]; then
     printf "Skipping due to missing sfsexp library\n"
     test_done
 fi
 
 add_email_corpus
-add_git_repos
+
+git config --global user.email notmuch@example.org
+git config --global user.name  "Notmuch Test Suite"
+
+test_begin_subtest "init"
+test_expect_success "notmuch git -p '' -C remote.git init"
+
+test_begin_subtest "init (git.path)"
+notmuch config set git.path configured.git
+notmuch git init
+notmuch config set git.path
+output=$(git -C configured.git rev-parse --is-bare-repository)
+test_expect_equal "$output" "true"
 
 test_begin_subtest "clone"
-test_expect_success "notmuch git -C clone.git clone tags.git"
+test_expect_success "notmuch git -p '' -C tags.git clone remote.git"
+
+test_begin_subtest "initial commit needs force"
+test_expect_code 1 "notmuch git -C tags.git commit"
 
 test_begin_subtest "commit"
-notmuch git -C tags.git -p '' commit
+notmuch git -C tags.git commit --force
 git -C tags.git ls-tree -r --name-only HEAD | xargs dirname | sort -u | sed s,tags/,id:, > OUTPUT
 notmuch search --output=messages '*' | sort > EXPECTED
 test_expect_equal_file_nonempty EXPECTED OUTPUT
 
+test_begin_subtest "commit --force succeeds"
+notmuch git -C force.git init
+test_expect_success "notmuch git -C force.git commit --force"
+
+test_begin_subtest "changing git.safe_fraction succeeds"
+notmuch config set git.safe_fraction 1
+notmuch git -C force2.git init
+test_expect_success "notmuch git -C force2.git commit"
+notmuch config set git.safe_fraction
+
 test_begin_subtest "commit, with quoted tag"
-notmuch git -C clone2.git -p '' clone tags.git
+notmuch git -C clone2.git clone tags.git
 git -C clone2.git ls-tree -r --name-only HEAD | grep /inbox > BEFORE
 notmuch tag '+"quoted tag"' '*'
-notmuch git -C clone2.git -p '' commit
+notmuch git -C clone2.git commit
 notmuch tag '-"quoted tag"' '*'
 git -C clone2.git ls-tree -r --name-only HEAD | grep /inbox > AFTER
 test_expect_equal_file_nonempty BEFORE AFTER
 
 test_begin_subtest "commit (incremental)"
 notmuch tag +test id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p '' commit
+notmuch git -C tags.git commit
 git -C tags.git ls-tree -r --name-only HEAD |
     grep 20091117190054 | sort > OUTPUT
 echo "--------------------------------------------------" >> OUTPUT
 notmuch tag -test id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p '' commit
+notmuch git -C tags.git commit
 git -C tags.git ls-tree -r --name-only HEAD |
     grep 20091117190054 | sort >> OUTPUT
 cat <<EOF > EXPECTED
@@ -57,12 +77,12 @@ test_expect_equal_file_nonempty EXPECTED OUTPUT
 
 test_begin_subtest "commit (change prefix)"
 notmuch tag +test::one id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p 'test::' commit
+notmuch git -C tags.git -p 'test::' commit --force
 git -C tags.git ls-tree -r --name-only HEAD |
     grep 20091117190054 | sort > OUTPUT
 echo "--------------------------------------------------" >> OUTPUT
 notmuch tag -test::one id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p '' commit
+notmuch git -C tags.git commit --force
 git -C tags.git ls-tree -r --name-only HEAD |
     grep 20091117190054 | sort >> OUTPUT
 cat <<EOF > EXPECTED
@@ -74,15 +94,31 @@ tags/20091117190054.GU3165@dottiness.seas.harvard.edu/unread
 EOF
 test_expect_equal_file_nonempty EXPECTED OUTPUT
 
+backup_database
+test_begin_subtest "large checkout needs --force"
+notmuch tag -inbox '*'
+test_expect_code 1 "notmuch git -C tags.git checkout"
+restore_database
+
+test_begin_subtest "checkout (git.safe_fraction)"
+notmuch git -C force3.git clone tags.git
+notmuch dump > BEFORE
+notmuch tag -inbox '*'
+notmuch config set git.safe_fraction 1
+notmuch git -C force3.git checkout
+notmuch config set git.safe_fraction
+notmuch dump > AFTER
+test_expect_equal_file_nonempty BEFORE AFTER
+
 test_begin_subtest "checkout"
 notmuch dump > BEFORE
 notmuch tag -inbox '*'
-notmuch git -C tags.git -p '' checkout
+notmuch git -C tags.git checkout --force
 notmuch dump > AFTER
 test_expect_equal_file_nonempty BEFORE AFTER
 
 test_begin_subtest "archive"
-notmuch git -C tags.git -p '' archive | tar tf - | \
+notmuch git -C tags.git archive | tar tf - | \
     grep 20091117190054.GU3165@dottiness.seas.harvard.edu | sort > OUTPUT
 cat <<EOF > EXPECTED
 tags/20091117190054.GU3165@dottiness.seas.harvard.edu/
@@ -90,32 +126,32 @@ tags/20091117190054.GU3165@dottiness.seas.harvard.edu/inbox
 tags/20091117190054.GU3165@dottiness.seas.harvard.edu/signed
 tags/20091117190054.GU3165@dottiness.seas.harvard.edu/unread
 EOF
-notmuch git -C tags.git -p '' checkout
+notmuch git -C tags.git checkout
 test_expect_equal_file EXPECTED OUTPUT
 
 test_begin_subtest "status"
 notmuch tag +test id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p '' status > OUTPUT
+notmuch git -C tags.git status > OUTPUT
 cat <<EOF > EXPECTED
 A	20091117190054.GU3165@dottiness.seas.harvard.edu	test
 EOF
-notmuch git -C tags.git -p '' checkout
+notmuch git -C tags.git checkout
 test_expect_equal_file EXPECTED OUTPUT
 
 test_begin_subtest "fetch"
 notmuch tag +test2 id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C remote.git -p '' commit
+notmuch git -C remote.git commit --force
 notmuch tag -test2 id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p '' fetch
-notmuch git -C tags.git -p '' status > OUTPUT
+notmuch git -C tags.git fetch
+notmuch git -C tags.git status > OUTPUT
 cat <<EOF > EXPECTED
  a	20091117190054.GU3165@dottiness.seas.harvard.edu	test2
 EOF
-notmuch git -C tags.git -p '' checkout
+notmuch git -C tags.git checkout
 test_expect_equal_file EXPECTED OUTPUT
 
 test_begin_subtest "merge"
-notmuch git -C tags.git -p '' merge
+notmuch git -C tags.git merge
 notmuch dump id:20091117190054.GU3165@dottiness.seas.harvard.edu | grep -v '^#' > OUTPUT
 cat <<EOF > EXPECTED
 +inbox +signed +test2 +unread -- id:20091117190054.GU3165@dottiness.seas.harvard.edu
@@ -124,14 +160,148 @@ test_expect_equal_file EXPECTED OUTPUT
 
 test_begin_subtest "push"
 notmuch tag +test3 id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p '' commit
+notmuch git -C tags.git commit
 notmuch tag -test3 id:20091117190054.GU3165@dottiness.seas.harvard.edu
-notmuch git -C tags.git -p '' push
-notmuch git -C remote.git -p '' checkout
+notmuch git -C tags.git push
+notmuch git -C remote.git checkout
 notmuch dump id:20091117190054.GU3165@dottiness.seas.harvard.edu | grep -v '^#' > OUTPUT
 cat <<EOF > EXPECTED
 +inbox +signed +test2 +test3 +unread -- id:20091117190054.GU3165@dottiness.seas.harvard.edu
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "environment passed through when run as 'notmuch git'"
+env NOTMUCH_GIT_DIR=foo NOTMUCH_GIT_PREFIX=bar NOTMUCH_PROFILE=default notmuch git -C tags.git -p '' -ldebug status |& \
+    grep '^env ' | notmuch_dir_sanitize > OUTPUT
+cat <<EOF > EXPECTED
+env NOTMUCH_GIT_DIR = foo
+env NOTMUCH_GIT_PREFIX = bar
+env NOTMUCH_PROFILE = default
+env NOTMUCH_CONFIG = CWD/notmuch-config
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "--nmbug argument sets defaults"
+notmuch git -ldebug --nmbug status |& grep '^\(prefix\|repository\)' | notmuch_dir_sanitize > OUTPUT
+cat <<EOF > EXPECTED
+prefix = notmuch::
+repository = CWD/home/.nmbug
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "invoke as nmbug sets defaults"
+"$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^\(prefix\|repository\)' | notmuch_dir_sanitize > OUTPUT
+cat <<EOF > EXPECTED
+prefix = notmuch::
+repository = CWD/home/.nmbug
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "env variable NOTMUCH_GIT_DIR works when invoked as nmbug"
+NOTMUCH_GIT_DIR=`pwd`/foo "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
+cat <<EOF > EXPECTED
+repository = CWD/foo
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "env variable NOTMUCH_GIT_DIR works when invoked as 'notmuch git'"
+NOTMUCH_GIT_DIR=`pwd`/remote.git notmuch git -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
+cat <<EOF > EXPECTED
+repository = CWD/remote.git
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+
+test_begin_subtest "env variable NOTMUCH_GIT_DIR overrides config when invoked as 'nmbug'"
+notmuch config set git.path `pwd`/bar
+NOTMUCH_GIT_DIR=`pwd`/remote.git  "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
+notmuch config set git.path
+cat <<EOF > EXPECTED
+repository = CWD/remote.git
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "env variable NOTMUCH_GIT_DIR overrides config when invoked as 'notmuch git'"
+notmuch config set git.path `pwd`/bar
+NOTMUCH_GIT_DIR=`pwd`/remote.git notmuch git -ldebug status |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
+notmuch config set git.path
+cat <<EOF > EXPECTED
+repository = CWD/remote.git
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "env variable NOTMUCH_GIT_PREFIX works when invoked as 'nmbug'"
+NOTMUCH_GIT_PREFIX=env:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT
+cat <<EOF > EXPECTED
+prefix = env::
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "env variable NOTMUCH_GIT_PREFIX works when invoked as nmbug"
+NOTMUCH_GIT_PREFIX=foo:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT
+cat <<EOF > EXPECTED
+prefix = foo::
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "env variable NOTMUCH_GIT_PREFIX overrides config when invoked as 'nmbug'"
+notmuch config set git.tag_prefix config::
+NOTMUCH_GIT_PREFIX=env:: "$NOTMUCH_BUILDDIR"/nmbug -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT
+notmuch config set git.path
+cat <<EOF > EXPECTED
+prefix = env::
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "env variable NOTMUCH_GIT_PREFIX overrides config when invoked as 'notmuch git'"
+notmuch config set git.tag_prefix config::
+NOTMUCH_GIT_PREFIX=env:: notmuch git -ldebug status |& grep '^prefix' | notmuch_dir_sanitize > OUTPUT
+notmuch config set git.path
+cat <<EOF > EXPECTED
+prefix = env::
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+
+test_begin_subtest "init, xdg default location"
+repo=home/.local/share/notmuch/default/git
+notmuch git -ldebug init |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
+git -C $repo rev-parse --absolute-git-dir | notmuch_dir_sanitize >> OUTPUT
+cat <<EOF > EXPECTED
+repository = CWD/$repo
+CWD/$repo
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "init, xdg default location, with profile"
+repo=home/.local/share/notmuch/work/git
+NOTMUCH_PROFILE=work notmuch git -ldebug init |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
+git -C $repo rev-parse --absolute-git-dir | notmuch_dir_sanitize >> OUTPUT
+cat <<EOF > EXPECTED
+repository = CWD/$repo
+CWD/$repo
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "init, configured location"
+repo=configured-tags
+notmuch config set git.path `pwd`/$repo
+notmuch git -ldebug init |& grep '^repository' | notmuch_dir_sanitize > OUTPUT
+notmuch config set git.path
+git -C $repo rev-parse --absolute-git-dir | notmuch_dir_sanitize >> OUTPUT
+cat <<EOF > EXPECTED
+repository = CWD/$repo
+CWD/$repo
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "configured tag prefix"
+notmuch config set git.tag_prefix test::
+notmuch git -ldebug status |& grep '^prefix' > OUTPUT
+notmuch config set git.tag_prefix
+cat <<EOF > EXPECTED
+prefix = test::
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
 test_done