Drop support for pre-1.4 Xapian, prepare for 1.5.x (v2)

Drop support for pre-1.4 Xapian, prepare for 1.5.x (v2)

[David Bremner]

This obsoletes id:20190414124445.3394-1-david@tethera.net

Compared to the previous version it

    - is rebased against master
    - has some unfortunate whitespace changes to INSTALL reverted.
    - now includes a removal of T530-upgrade.sh and supporting machinery.

I'm not sure about the last point. It is potentially possible to
regenerate a v1 database with a non-chert xapian backend, but I'm not
sure it's worth the trouble. I did notice that devel/gen-testdb.sh is
currently broken due to changes in the test framework.

[PATCH 1/4] build: require Xapian >= 1.4.0

[David Bremner]

This is necessary before we can support Xapian 1.5, which drops the
old compact API, and replaces ValueRangeProcessor classes with
RangeProcessor classes. It also simplifies maintainability as 3
previously optional features are now part of the baseline Xapian
version.
---
 INSTALL   |  7 +----
 NEWS      |  6 +++++
 configure | 78 +++++++++++++------------------------------------------
 3 files changed, 25 insertions(+), 66 deletions(-)

diff --git a/INSTALL b/INSTALL
index f1236e71..4cc7d67c 100644
--- a/INSTALL
+++ b/INSTALL
@@ -32,12 +32,7 @@ Talloc, and zlib which are each described below:
 
 	Xapian is available from https://xapian.org
 
-	Note: Notmuch will work best with Xapian 1.0.18 (or later) or
-	Xapian 1.1.4 (or later). Previous versions of Xapian (whether
-	1.0 or 1.1) had a performance bug that made notmuch very slow
-	when modifying tags. This would cause distracting pauses when
-	reading mail while notmuch would wait for Xapian when removing
-	the "inbox" and "unread" tags from messages in a thread.
+	Notmuch needs a Xapian version at least 1.4.0.
 
 	GMime
 	-----
diff --git a/NEWS b/NEWS
index 26b8160c..330901a0 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,12 @@ Emacs
 Support for GNU Emacs older than 25.1 is deprecated with this release,
 and may be removed in a future release.
 
+Dependencies
+------------
+
+As of this release, support for versions of Xapian before 1.4.0 is
+removed. It was deprecated in release 0.27.
+
 Notmuch 0.28.4 (2019-05-05)
 ===========================
 
diff --git a/configure b/configure
index 8b80f0e0..c4056d7a 100755
--- a/configure
+++ b/configure
@@ -401,75 +401,33 @@ else
     have_pkg_config=0
 fi
 
-printf "Checking for Xapian development files... "
+XAPIAN_MINVER=1.4
+
+printf "Checking for Xapian development files (>= $XAPIAN_MINVER)..."
 have_xapian=0
-for xapian_config in ${XAPIAN_CONFIG} xapian-config xapian-config-1.3; do
-    if ${xapian_config} --version > /dev/null 2>&1; then
-	xapian_version=$(${xapian_config} --version | sed -e 's/.* //')
-	printf "Yes (%s).\n" ${xapian_version}
-	have_xapian=1
-	xapian_cxxflags=$(${xapian_config} --cxxflags)
-	xapian_ldflags=$(${xapian_config} --libs)
-	break
-    fi
-done
+if pkg-config --atleast-version=${XAPIAN_MINVER} xapian-core; then
+   for xapian_config in ${XAPIAN_CONFIG} xapian-config; do
+       if ${xapian_config} --version > /dev/null 2>&1; then
+	   xapian_version=$(${xapian_config} --version | sed -e 's/.* //')
+	   printf "Yes (%s).\n" ${xapian_version}
+	   have_xapian=1
+           have_xapian_compact=1
+           have_xapian_field_processor=1
+           have_xapian_db_retry_lock=1
+	   xapian_cxxflags=$(${xapian_config} --cxxflags)
+	   xapian_ldflags=$(${xapian_config} --libs)
+	   break
+       fi
+   done
+fi
 if [ ${have_xapian} = "0" ]; then
     printf "No.\n"
     errors=$((errors + 1))
 fi
 
-have_xapian_compact=0
-have_xapian_field_processor=0
 if [ ${have_xapian} = "1" ]; then
-    printf "Checking for Xapian compaction support... "
-    cat>_compact.cc<<EOF
-#include <xapian.h>
-class TestCompactor : public Xapian::Compactor { };
-EOF
-    if ${CXX} ${CXXFLAGS_for_sh} ${xapian_cxxflags} -c _compact.cc -o _compact.o > /dev/null 2>&1
-    then
-	have_xapian_compact=1
-	printf "Yes.\n"
-    else
-	printf "No.\n"
-	errors=$((errors + 1))
-    fi
-
-    rm -f _compact.o _compact.cc
-
-    printf "Checking for Xapian FieldProcessor API... "
-    cat>_field_processor.cc<<EOF
-#include <xapian.h>
-class TitleFieldProcessor : public Xapian::FieldProcessor { };
-EOF
-    if ${CXX} ${CXXFLAGS_for_sh} ${xapian_cxxflags} -c _field_processor.cc -o _field_processor.o > /dev/null 2>&1
-    then
-	have_xapian_field_processor=1
-	printf "Yes.\n"
-    else
-	printf "No. (optional)\n"
-    fi
-
-    rm -f _field_processor.o _field_processor.cc
 
     default_xapian_backend=""
-    # DB_RETRY_LOCK is only supported on Xapian > 1.3.2
-    have_xapian_db_retry_lock=0
-    if [ $WITH_RETRY_LOCK = "1" ]; then
-	printf "Checking for Xapian lock retry support... "
-	cat>_retry.cc<<EOF
-#include <xapian.h>
-int flag = Xapian::DB_RETRY_LOCK;
-EOF
-	if ${CXX} ${CXXFLAGS_for_sh} ${xapian_cxxflags} -c _retry.cc -o _retry.o > /dev/null 2>&1
-	then
-	    have_xapian_db_retry_lock=1
-	    printf "Yes.\n"
-	else
-	    printf "No. (optional)\n"
-	fi
-	rm -f _retry.o _retry.cc
-    fi
 
     printf "Testing default Xapian backend... "
     cat >_default_backend.cc <<EOF
-- 
2.20.1

[PATCH 2/4] lib: migrate to post Xapian 1.3.4 compact support

[David Bremner]

The old API was deprecated in Xapian 1.3.4 and (will be) removed in 1.5.0
---
 lib/database.cc | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index 1753117f..e4f38d6a 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -1211,11 +1211,7 @@ notmuch_database_compact (const char *path,
 
     try {
 	NotmuchCompactor compactor (status_cb, closure);
-
-	compactor.set_renumber (false);
-	compactor.add_source (xapian_path);
-	compactor.set_destdir (compact_xapian_path);
-	compactor.compact ();
+	notmuch->xapian_db->compact (compact_xapian_path, Xapian::DBCOMPACT_NO_RENUMBER, 0, compactor);
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch, "Error while compacting: %s\n", error.get_msg().c_str());
 	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
-- 
2.20.1

[PATCH 3/4] lib: migrate from Xapian ValueRangeProcessor to RangeProcessor

[David Bremner]

This will be mandatory as of Xapian 1.5.  The API is also more
consistent with the FieldProcessor API, which helps code re-use a bit.

Note that this switches to using the built-in Xapian support for
prefixes on ranges (i.e. deleted code at beginning of
ParseTimeRangeProcessor::operator(), added prefix to constructor).

Another side effect of the migration is that we are generating smaller
queries, using one OP_VALUE_RANGE instead of an AND of two OP_VALUE_*
queries.
---
 lib/database-private.h |  6 ++--
 lib/database.cc        | 23 +++++----------
 lib/parse-time-vrp.cc  | 64 ++++++++++++++++++++----------------------
 lib/parse-time-vrp.h   | 17 +++++------
 4 files changed, 50 insertions(+), 60 deletions(-)

diff --git a/lib/database-private.h b/lib/database-private.h
index 293f2db4..037499b8 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -218,9 +218,9 @@ struct _notmuch_database {
     unsigned long view;
     Xapian::QueryParser *query_parser;
     Xapian::TermGenerator *term_gen;
-    Xapian::ValueRangeProcessor *value_range_processor;
-    Xapian::ValueRangeProcessor *date_range_processor;
-    Xapian::ValueRangeProcessor *last_mod_range_processor;
+    Xapian::RangeProcessor *value_range_processor;
+    Xapian::RangeProcessor *date_range_processor;
+    Xapian::RangeProcessor *last_mod_range_processor;
 };
 
 /* Prior to database version 3, features were implied by the database
diff --git a/lib/database.cc b/lib/database.cc
index e4f38d6a..b2cf4ce1 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -322,7 +322,6 @@ _setup_query_field_default (const prefix_t *prefix, notmuch_database_t *notmuch)
 	notmuch->query_parser->add_boolean_prefix (prefix->name, prefix->prefix);
 }
 
-#if HAVE_XAPIAN_FIELD_PROCESSOR
 static void
 _setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
 {
@@ -330,7 +329,7 @@ _setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
 	Xapian::FieldProcessor *fp;
 
 	if (STRNCMP_LITERAL (prefix->name, "date") == 0)
-	    fp = (new DateFieldProcessor())->release ();
+	    fp = (new DateFieldProcessor(NOTMUCH_VALUE_TIMESTAMP))->release ();
 	else if (STRNCMP_LITERAL(prefix->name, "query") == 0)
 	    fp = (new QueryFieldProcessor (*notmuch->query_parser, notmuch))->release ();
 	else if (STRNCMP_LITERAL(prefix->name, "thread") == 0)
@@ -347,13 +346,6 @@ _setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
 	_setup_query_field_default (prefix, notmuch);
     }
 }
-#else
-static inline void
-_setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
-{
-    _setup_query_field_default (prefix, notmuch);
-}
-#endif
 
 const char *
 _find_prefix (const char *name)
@@ -968,17 +960,16 @@ notmuch_database_open_verbose (const char *path,
 	notmuch->query_parser = new Xapian::QueryParser;
 	notmuch->term_gen = new Xapian::TermGenerator;
 	notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
-	notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
-	notmuch->date_range_processor = new ParseTimeValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
-	notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
-
+	notmuch->value_range_processor = new Xapian::NumberRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
+	notmuch->date_range_processor = new ParseTimeRangeProcessor (NOTMUCH_VALUE_TIMESTAMP, "date:");
+	notmuch->last_mod_range_processor = new Xapian::NumberRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
 	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
 	notmuch->query_parser->set_database (*notmuch->xapian_db);
 	notmuch->query_parser->set_stemmer (Xapian::Stem ("english"));
 	notmuch->query_parser->set_stemming_strategy (Xapian::QueryParser::STEM_SOME);
-	notmuch->query_parser->add_valuerangeprocessor (notmuch->value_range_processor);
-	notmuch->query_parser->add_valuerangeprocessor (notmuch->date_range_processor);
-	notmuch->query_parser->add_valuerangeprocessor (notmuch->last_mod_range_processor);
+	notmuch->query_parser->add_rangeprocessor (notmuch->value_range_processor);
+	notmuch->query_parser->add_rangeprocessor (notmuch->date_range_processor);
+	notmuch->query_parser->add_rangeprocessor (notmuch->last_mod_range_processor);
 
 	for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
 	    const prefix_t *prefix = &prefix_table[i];
diff --git a/lib/parse-time-vrp.cc b/lib/parse-time-vrp.cc
index dd691494..a64e9461 100644
--- a/lib/parse-time-vrp.cc
+++ b/lib/parse-time-vrp.cc
@@ -24,48 +24,49 @@
 #include "parse-time-vrp.h"
 #include "parse-time-string.h"
 
-#define PREFIX "date:"
+static Xapian::Query _make_query (Xapian::valueno slot, time_t from, time_t to) {
+    if (to < 0) {
+	return Xapian::Query (Xapian::Query::OP_VALUE_GE, slot,
+			      Xapian::sortable_serialise ((double) from));
+    } else if (from < 0) {
+	return Xapian::Query (Xapian::Query::OP_VALUE_LE, slot,
+			      Xapian::sortable_serialise ((double) to));
+    } else {
+	return Xapian::Query (Xapian::Query::OP_VALUE_RANGE, slot,
+			      Xapian::sortable_serialise ((double) from),
+			      Xapian::sortable_serialise ((double) to));
+    }
+}
 
-/* See *ValueRangeProcessor in xapian-core/api/valuerangeproc.cc */
-Xapian::valueno
-ParseTimeValueRangeProcessor::operator() (std::string &begin, std::string &end)
+Xapian::Query
+ParseTimeRangeProcessor::operator() (const std::string &begin, const std::string &end)
 {
-    time_t t, now;
-    std::string b;
-
-    /* Require date: prefix in start of the range... */
-    if (STRNCMP_LITERAL (begin.c_str (), PREFIX))
-	return Xapian::BAD_VALUENO;
-
-    /* ...and remove it. */
-    begin.erase (0, sizeof (PREFIX) - 1);
-    b = begin;
+    time_t from = -1, to = -1, now = -1;
+    std::string str;
 
     /* Use the same 'now' for begin and end. */
     if (time (&now) == (time_t) -1)
-	return Xapian::BAD_VALUENO;
+	throw Xapian::QueryParserError ("unable to get current time");
 
-    if (!begin.empty ()) {
-	if (parse_time_string (begin.c_str (), &t, &now, PARSE_TIME_ROUND_DOWN))
-	    return Xapian::BAD_VALUENO;
+    if (!begin.empty ())
+	if (parse_time_string (begin.c_str (), &from, &now, PARSE_TIME_ROUND_DOWN))
+	    throw Xapian::QueryParserError ("Didn't understand date specification '" + begin + "'");
 
-	begin.assign (Xapian::sortable_serialise ((double) t));
-    }
 
     if (!end.empty ()) {
-	if (end == "!" && ! b.empty ())
-	    end = b;
-
-	if (parse_time_string (end.c_str (), &t, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
-	    return Xapian::BAD_VALUENO;
-
-	end.assign (Xapian::sortable_serialise ((double) t));
+	if (end == "!" && ! begin.empty ())
+	    str = begin;
+	else
+	    str = end;
+
+	if (parse_time_string (str.c_str (), &to, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
+	    if (parse_time_string (str.c_str (), &to, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
+		throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
     }
 
-    return valno;
+    return _make_query (slot, from, to);
 }
 
-#if HAVE_XAPIAN_FIELD_PROCESSOR
 /* XXX TODO: is throwing an exception the right thing to do here? */
 Xapian::Query DateFieldProcessor::operator()(const std::string & str) {
     time_t from, to, now;
@@ -80,8 +81,5 @@ Xapian::Query DateFieldProcessor::operator()(const std::string & str) {
     if (parse_time_string (str.c_str (), &to, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
 	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
 
-    return Xapian::Query(Xapian::Query::OP_AND,
-			 Xapian::Query(Xapian::Query::OP_VALUE_GE, 0, Xapian::sortable_serialise ((double) from)),
-			 Xapian::Query(Xapian::Query::OP_VALUE_LE, 0, Xapian::sortable_serialise ((double) to)));
+    return _make_query (slot, from, to);
 }
-#endif
diff --git a/lib/parse-time-vrp.h b/lib/parse-time-vrp.h
index c024dba2..f495e716 100644
--- a/lib/parse-time-vrp.h
+++ b/lib/parse-time-vrp.h
@@ -26,20 +26,21 @@
 #include <xapian.h>
 
 /* see *ValueRangeProcessor in xapian-core/include/xapian/queryparser.h */
-class ParseTimeValueRangeProcessor : public Xapian::ValueRangeProcessor {
-protected:
-    Xapian::valueno valno;
+class ParseTimeRangeProcessor : public Xapian::RangeProcessor {
 
 public:
-    ParseTimeValueRangeProcessor (Xapian::valueno slot_)
-	: valno(slot_) { }
+    ParseTimeRangeProcessor (Xapian::valueno slot_, const std::string prefix_)
+	:  Xapian::RangeProcessor(slot_, prefix_, 0) { }
 
-    Xapian::valueno operator() (std::string &begin, std::string &end);
+    Xapian::Query operator() (const std::string &begin, const std::string &end);
 };
 
-#if HAVE_XAPIAN_FIELD_PROCESSOR
 class DateFieldProcessor : public Xapian::FieldProcessor {
+private:
+    Xapian::valueno slot;
+public:
+    DateFieldProcessor(Xapian::valueno slot_) : slot(slot_) { };
     Xapian::Query operator()(const std::string & str);
 };
-#endif
+
 #endif /* NOTMUCH_PARSE_TIME_VRP_H */
-- 
2.20.1

[PATCH 4/4] test: drop upgrade tests.

[David Bremner]

These rely on a pre-generated v1 database which happens to be chert
format. This backend is not supported by Xapian 1.5.
---
 .travis.yml                                   |   1 -
 configure                                     |   2 +-
 test/T530-upgrade.sh                          | 136 ------------------
 test/test-databases/.gitignore                |   1 -
 test/test-databases/Makefile                  |   7 -
 test/test-databases/Makefile.local            |  20 ---
 test/test-databases/database-v1.tar.xz.sha256 |   1 -
 7 files changed, 1 insertion(+), 167 deletions(-)
 delete mode 100755 test/T530-upgrade.sh
 delete mode 100644 test/test-databases/.gitignore
 delete mode 100644 test/test-databases/Makefile
 delete mode 100644 test/test-databases/Makefile.local
 delete mode 100644 test/test-databases/database-v1.tar.xz.sha256

diff --git a/.travis.yml b/.travis.yml
index f9516bde..f9d99203 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,6 @@ addons:
 
 script:
   - ./configure
-  - make download-test-databases
   - make test
 
 notifications:
diff --git a/configure b/configure
index c4056d7a..ae912fa6 100755
--- a/configure
+++ b/configure
@@ -27,7 +27,7 @@ srcdir=$(dirname "$0")
 NOTMUCH_SRCDIR=$(cd "$srcdir" && pwd)
 
 subdirs="util compat lib parse-time-string completion doc emacs"
-subdirs="${subdirs} performance-test test test/test-databases"
+subdirs="${subdirs} performance-test test"
 subdirs="${subdirs} bindings"
 
 # For a non-srcdir configure invocation (such as ../configure), create
diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
deleted file mode 100755
index 2124dde2..00000000
--- a/test/T530-upgrade.sh
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env bash
-test_description="database upgrade"
-
-. $(dirname "$0")/test-lib.sh || exit 1
-
-dbtarball=database-v1.tar.xz
-
-# XXX: Accomplish the same with test lib helpers
-if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
-    test_subtest_missing_external_prereq_["${dbtarball} - fetch with 'make download-test-databases'"]=t
-fi
-
-test_begin_subtest "database checksum"
-test_expect_success \
-    '( cd $TEST_DIRECTORY/test-databases &&
-       sha256sum --quiet --check --status ${dbtarball}.sha256 )'
-
-tar xf $TEST_DIRECTORY/test-databases/${dbtarball} -C ${MAIL_DIR} --strip-components=1
-
-test_begin_subtest "folder: search does not work with old database version"
-output=$(notmuch search folder:foo)
-test_expect_equal "$output" ""
-
-test_begin_subtest "path: search does not work with old database version"
-output=$(notmuch search path:foo)
-test_expect_equal "$output" ""
-
-test_begin_subtest "pre upgrade dump"
-test_expect_success 'notmuch dump | sort > pre-upgrade-dump'
-
-test_begin_subtest "database upgrade from format version 1"
-output=$(notmuch new | sed -e 's/^Backing up tags to .*$/Backing up tags to FILENAME/')
-test_expect_equal "$output" "\
-Welcome to a new version of notmuch! Your database will now be upgraded.
-This process is safe to interrupt.
-Backing up tags to FILENAME
-Your notmuch database has now been upgraded.
-No new mail."
-
-test_begin_subtest "tag backup matches pre-upgrade dump"
-gunzip -c ${MAIL_DIR}/.notmuch/dump-*.gz | sort > backup-dump
-test_expect_equal_file pre-upgrade-dump backup-dump
-
-test_begin_subtest "folder: no longer matches in the middle of path"
-output=$(notmuch search folder:baz)
-test_expect_equal "$output" ""
-
-test_begin_subtest "folder: search"
-output=$(notmuch search --output=files folder:foo | notmuch_search_files_sanitize | sort)
-test_expect_equal "$output" "MAIL_DIR/foo/06:2,
-MAIL_DIR/foo/cur/07:2,
-MAIL_DIR/foo/cur/08:2,
-MAIL_DIR/foo/new/03:2,
-MAIL_DIR/foo/new/09:2,
-MAIL_DIR/foo/new/10:2,"
-
-test_begin_subtest "top level folder: search"
-output=$(notmuch search --output=files folder:'""' | notmuch_search_files_sanitize | sort)
-# bar/18:2, is a duplicate of cur/51:2,
-test_expect_equal "$output" "MAIL_DIR/01:2,
-MAIL_DIR/02:2,
-MAIL_DIR/bar/18:2,
-MAIL_DIR/cur/29:2,
-MAIL_DIR/cur/30:2,
-MAIL_DIR/cur/31:2,
-MAIL_DIR/cur/32:2,
-MAIL_DIR/cur/33:2,
-MAIL_DIR/cur/34:2,
-MAIL_DIR/cur/35:2,
-MAIL_DIR/cur/36:2,
-MAIL_DIR/cur/37:2,
-MAIL_DIR/cur/38:2,
-MAIL_DIR/cur/39:2,
-MAIL_DIR/cur/40:2,
-MAIL_DIR/cur/41:2,
-MAIL_DIR/cur/42:2,
-MAIL_DIR/cur/43:2,
-MAIL_DIR/cur/44:2,
-MAIL_DIR/cur/45:2,
-MAIL_DIR/cur/46:2,
-MAIL_DIR/cur/47:2,
-MAIL_DIR/cur/48:2,
-MAIL_DIR/cur/49:2,
-MAIL_DIR/cur/50:2,
-MAIL_DIR/cur/51:2,
-MAIL_DIR/cur/52:2,
-MAIL_DIR/cur/53:2,
-MAIL_DIR/new/04:2,"
-
-test_begin_subtest "path: search"
-output=$(notmuch search --output=files path:"bar" | notmuch_search_files_sanitize | sort)
-# cur/51:2, is a duplicate of bar/18:2,
-test_expect_equal "$output" "MAIL_DIR/bar/17:2,
-MAIL_DIR/bar/18:2,
-MAIL_DIR/cur/51:2,"
-
-test_begin_subtest "top level path: search"
-output=$(notmuch search --output=files path:'""' | notmuch_search_files_sanitize | sort)
-test_expect_equal "$output" "MAIL_DIR/01:2,
-MAIL_DIR/02:2,"
-
-test_begin_subtest "recursive path: search"
-output=$(notmuch search --output=files path:"bar/**" | notmuch_search_files_sanitize | sort)
-# cur/51:2, is a duplicate of bar/18:2,
-test_expect_equal "$output" "MAIL_DIR/bar/17:2,
-MAIL_DIR/bar/18:2,
-MAIL_DIR/bar/baz/05:2,
-MAIL_DIR/bar/baz/23:2,
-MAIL_DIR/bar/baz/24:2,
-MAIL_DIR/bar/baz/cur/25:2,
-MAIL_DIR/bar/baz/cur/26:2,
-MAIL_DIR/bar/baz/new/27:2,
-MAIL_DIR/bar/baz/new/28:2,
-MAIL_DIR/bar/cur/19:2,
-MAIL_DIR/bar/cur/20:2,
-MAIL_DIR/bar/new/21:2,
-MAIL_DIR/bar/new/22:2,
-MAIL_DIR/cur/51:2,"
-
-test_begin_subtest "body: same as unprefixed before reindex"
-notmuch search --output=messages body:close > OUTPUT
-notmuch search --output=messages close  > EXPECTED
-test_expect_equal_file EXPECTED OUTPUT
-
-test_begin_subtest "body: subset of unprefixed after reindex"
-notmuch reindex '*'
-notmuch search --output=messages body:close | sort > BODY
-notmuch search --output=messages close | sort > UNPREFIXED
-diff -e UNPREFIXED BODY | cut -c2- > OUTPUT
-cat <<EOF > EXPECTED
-d
-d
-EOF
-test_expect_equal_file EXPECTED OUTPUT
-
-test_done
diff --git a/test/test-databases/.gitignore b/test/test-databases/.gitignore
deleted file mode 100644
index 9452199f..00000000
--- a/test/test-databases/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/*.tar.xz
diff --git a/test/test-databases/Makefile b/test/test-databases/Makefile
deleted file mode 100644
index b250a8be..00000000
--- a/test/test-databases/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-# See Makefile.local for the list of files to be compiled in this
-# directory.
-all:
-	$(MAKE) -C ../.. all
-
-.DEFAULT:
-	$(MAKE) -C ../.. $@
diff --git a/test/test-databases/Makefile.local b/test/test-databases/Makefile.local
deleted file mode 100644
index 7aedff70..00000000
--- a/test/test-databases/Makefile.local
+++ /dev/null
@@ -1,20 +0,0 @@
-# -*- makefile -*-
-
-TEST_DATABASE_MIRROR=https://notmuchmail.org/releases/test-databases
-
-dir := test/test-databases
-
-test_databases := $(dir)/database-v1.tar.xz
-
-%.tar.xz:
-	@exec 1>&2 ;\
-	if command -v wget >/dev/null ;\
-	then set -x; wget -nv -O $@ ${TEST_DATABASE_MIRROR}/$(notdir $@) ;\
-	elif command -v curl >/dev/null ;\
-	then set -x; curl -L -s -o $@ ${TEST_DATABASE_MIRROR}/$(notdir $@) ;\
-	else echo Cannot fetch databases, no wget nor curl available; exit 1 ;\
-	fi
-
-download-test-databases: ${test_databases}
-
-DATACLEAN := $(DATACLEAN) ${test_databases}
diff --git a/test/test-databases/database-v1.tar.xz.sha256 b/test/test-databases/database-v1.tar.xz.sha256
deleted file mode 100644
index 2cc4f965..00000000
--- a/test/test-databases/database-v1.tar.xz.sha256
+++ /dev/null
@@ -1 +0,0 @@
-4299e051b10e1fa7b33ea2862790a09ebfe96859681804e5251e130f800e69d2  database-v1.tar.xz
-- 
2.20.1