[PATCH v4 00/13] boolean folder: and path: searches

[PATCH v4 00/13] boolean folder: and path: searches

[Jani Nikula]

This is v4 of id:1394313585-28422-1-git-send-email-david@tethera.net.

Changes since v3, mostly addressing Austin's review and comments on IRC:

* Removed the extra corpus for folder tests. Instead, arranged the
  existing corpus into more folders. (Patch 01.)

* Changed the folder/path tests to reflect the above, and incorporated
  them into existing folder tests instead of adding a new one. (Patch
  07.)

* Added a new patch to do notmuch_search_files_sanitize. (Patch 02.)

* Folded test updates to folder: update to make tests pass at every
  commit. (Patch 06.)

* Fixed indent fails and added comment about XFOLDER: prefix,
  spotted/suggested by Austin. (Patch 06.)

* Used Austin's man page suggestion verbatim, but this may have to be
  redone if David's documentation changes get merged first. (Patch 09.)

* Fixed a stray make dependency. (Patch 13.)

* Notably *no* functional code changes.

Finally, take heed of David's warning id:874n37a017.fsf@zancas.localnet.

BR,
Jani.


David Bremner (2):
  test: commit folders-v1.tar.xz checksum, ignore actual databases
  test: add machinery to download and verify databases

Jani Nikula (11):
  test: rearrange the test corpus into subfolders
  test: add notmuch_search_files_sanitize and use it
  lib: refactor folder term update after filename removal
  lib: add support for path: prefix searches
  test: make insert test use the path: prefix
  lib: make folder: prefix literal
  test: add tests for the new boolean folder: and path: prefixes
  test: add database upgrade test from format version 1 to 2
  man: update man pages for folder: and path: search terms
  man: try to clarify the folder: and path: vs. --output=files confusion
  devel: add script to generate test databases

 Makefile                                     |   3 +-
 devel/gen-testdb.sh                          | 131 ++++++++++++++
 lib/database.cc                              |  51 +++++-
 lib/message.cc                               | 249 ++++++++++++++++++---------
 lib/notmuch-private.h                        |   3 +
 man/man1/notmuch-search.1                    |  10 +-
 man/man7/notmuch-search-terms.7              |  41 ++++-
 test/README                                  |   8 +
 test/T070-insert.sh                          |  10 +-
 test/T090-search-output.sh                   | 176 +++++++++----------
 test/T100-search-by-folder.sh                | 117 ++++++++++++-
 test/T370-search-folder-coherence.sh         |   2 +-
 test/T530-upgrade.sh                         | 115 +++++++++++++
 test/corpus/{cur => }/01:2,                  |   0
 test/corpus/{cur => }/02:2,                  |   0
 test/corpus/{cur => bar}/17:2,               |   0
 test/corpus/{cur => bar}/18:2,               |   0
 test/corpus/{cur => bar/baz}/05:2,           |   0
 test/corpus/{cur => bar/baz}/23:2,           |   0
 test/corpus/{cur => bar/baz}/24:2,           |   0
 test/corpus/{ => bar/baz}/cur/25:2,          |   0
 test/corpus/{ => bar/baz}/cur/26:2,          |   0
 test/corpus/{cur => bar/baz/new}/27:2,       |   0
 test/corpus/{cur => bar/baz/new}/28:2,       |   0
 test/corpus/{ => bar}/cur/19:2,              |   0
 test/corpus/{ => bar}/cur/20:2,              |   0
 test/corpus/{cur => bar/new}/21:2,           |   0
 test/corpus/{cur => bar/new}/22:2,           |   0
 test/corpus/{cur => foo}/06:2,               |   0
 test/corpus/{cur => foo/baz}/11:2,           |   0
 test/corpus/{cur => foo/baz}/12:2,           |   0
 test/corpus/{ => foo/baz}/cur/13:2,          |   0
 test/corpus/{ => foo/baz}/cur/14:2,          |   0
 test/corpus/{cur => foo/baz/new}/15:2,       |   0
 test/corpus/{cur => foo/baz/new}/16:2,       |   0
 test/corpus/{ => foo}/cur/07:2,              |   0
 test/corpus/{ => foo}/cur/08:2,              |   0
 test/corpus/{cur => foo/new}/03:2,           |   0
 test/corpus/{cur => foo/new}/09:2,           |   0
 test/corpus/{cur => foo/new}/10:2,           |   0
 test/corpus/{cur => new}/04:2,               |   0
 test/test-databases/.gitignore               |   1 +
 test/test-databases/Makefile                 |   7 +
 test/test-databases/Makefile.local           |  14 ++
 test/test-databases/folders-v1.tar.xz.sha256 |   1 +
 test/test-lib.sh                             |   5 +
 46 files changed, 753 insertions(+), 191 deletions(-)
 create mode 100755 devel/gen-testdb.sh
 create mode 100755 test/T530-upgrade.sh
 rename test/corpus/{cur => }/01:2, (100%)
 rename test/corpus/{cur => }/02:2, (100%)
 rename test/corpus/{cur => bar}/17:2, (100%)
 rename test/corpus/{cur => bar}/18:2, (100%)
 rename test/corpus/{cur => bar/baz}/05:2, (100%)
 rename test/corpus/{cur => bar/baz}/23:2, (100%)
 rename test/corpus/{cur => bar/baz}/24:2, (100%)
 rename test/corpus/{ => bar/baz}/cur/25:2, (100%)
 rename test/corpus/{ => bar/baz}/cur/26:2, (100%)
 rename test/corpus/{cur => bar/baz/new}/27:2, (100%)
 rename test/corpus/{cur => bar/baz/new}/28:2, (100%)
 rename test/corpus/{ => bar}/cur/19:2, (100%)
 rename test/corpus/{ => bar}/cur/20:2, (100%)
 rename test/corpus/{cur => bar/new}/21:2, (100%)
 rename test/corpus/{cur => bar/new}/22:2, (100%)
 rename test/corpus/{cur => foo}/06:2, (100%)
 rename test/corpus/{cur => foo/baz}/11:2, (100%)
 rename test/corpus/{cur => foo/baz}/12:2, (100%)
 rename test/corpus/{ => foo/baz}/cur/13:2, (100%)
 rename test/corpus/{ => foo/baz}/cur/14:2, (100%)
 rename test/corpus/{cur => foo/baz/new}/15:2, (100%)
 rename test/corpus/{cur => foo/baz/new}/16:2, (100%)
 rename test/corpus/{ => foo}/cur/07:2, (100%)
 rename test/corpus/{ => foo}/cur/08:2, (100%)
 rename test/corpus/{cur => foo/new}/03:2, (100%)
 rename test/corpus/{cur => foo/new}/09:2, (100%)
 rename test/corpus/{cur => foo/new}/10:2, (100%)
 rename test/corpus/{cur => new}/04:2, (100%)
 create mode 100644 test/test-databases/.gitignore
 create mode 100644 test/test-databases/Makefile
 create mode 100644 test/test-databases/Makefile.local
 create mode 100644 test/test-databases/folders-v1.tar.xz.sha256

-- 
1.9.0

[PATCH v4 01/13] test: rearrange the test corpus into subfolders

[Jani Nikula]

We will need this for improved folder search tests, but having some
folders should exercise our code paths better anyway.

Modify the relevant test accordingly to make it pass.
---
 test/T090-search-output.sh             | 168 ++++++++++++++++-----------------
 test/corpus/{cur => }/01:2,            |   0
 test/corpus/{cur => }/02:2,            |   0
 test/corpus/{cur => bar}/17:2,         |   0
 test/corpus/{cur => bar}/18:2,         |   0
 test/corpus/{cur => bar/baz}/05:2,     |   0
 test/corpus/{cur => bar/baz}/23:2,     |   0
 test/corpus/{cur => bar/baz}/24:2,     |   0
 test/corpus/{ => bar/baz}/cur/25:2,    |   0
 test/corpus/{ => bar/baz}/cur/26:2,    |   0
 test/corpus/{cur => bar/baz/new}/27:2, |   0
 test/corpus/{cur => bar/baz/new}/28:2, |   0
 test/corpus/{ => bar}/cur/19:2,        |   0
 test/corpus/{ => bar}/cur/20:2,        |   0
 test/corpus/{cur => bar/new}/21:2,     |   0
 test/corpus/{cur => bar/new}/22:2,     |   0
 test/corpus/{cur => foo}/06:2,         |   0
 test/corpus/{cur => foo/baz}/11:2,     |   0
 test/corpus/{cur => foo/baz}/12:2,     |   0
 test/corpus/{ => foo/baz}/cur/13:2,    |   0
 test/corpus/{ => foo/baz}/cur/14:2,    |   0
 test/corpus/{cur => foo/baz/new}/15:2, |   0
 test/corpus/{cur => foo/baz/new}/16:2, |   0
 test/corpus/{ => foo}/cur/07:2,        |   0
 test/corpus/{ => foo}/cur/08:2,        |   0
 test/corpus/{cur => foo/new}/03:2,     |   0
 test/corpus/{cur => foo/new}/09:2,     |   0
 test/corpus/{cur => foo/new}/10:2,     |   0
 test/corpus/{cur => new}/04:2,         |   0
 29 files changed, 84 insertions(+), 84 deletions(-)
 rename test/corpus/{cur => }/01:2, (100%)
 rename test/corpus/{cur => }/02:2, (100%)
 rename test/corpus/{cur => bar}/17:2, (100%)
 rename test/corpus/{cur => bar}/18:2, (100%)
 rename test/corpus/{cur => bar/baz}/05:2, (100%)
 rename test/corpus/{cur => bar/baz}/23:2, (100%)
 rename test/corpus/{cur => bar/baz}/24:2, (100%)
 rename test/corpus/{ => bar/baz}/cur/25:2, (100%)
 rename test/corpus/{ => bar/baz}/cur/26:2, (100%)
 rename test/corpus/{cur => bar/baz/new}/27:2, (100%)
 rename test/corpus/{cur => bar/baz/new}/28:2, (100%)
 rename test/corpus/{ => bar}/cur/19:2, (100%)
 rename test/corpus/{ => bar}/cur/20:2, (100%)
 rename test/corpus/{cur => bar/new}/21:2, (100%)
 rename test/corpus/{cur => bar/new}/22:2, (100%)
 rename test/corpus/{cur => foo}/06:2, (100%)
 rename test/corpus/{cur => foo/baz}/11:2, (100%)
 rename test/corpus/{cur => foo/baz}/12:2, (100%)
 rename test/corpus/{ => foo/baz}/cur/13:2, (100%)
 rename test/corpus/{ => foo/baz}/cur/14:2, (100%)
 rename test/corpus/{cur => foo/baz/new}/15:2, (100%)
 rename test/corpus/{cur => foo/baz/new}/16:2, (100%)
 rename test/corpus/{ => foo}/cur/07:2, (100%)
 rename test/corpus/{ => foo}/cur/08:2, (100%)
 rename test/corpus/{cur => foo/new}/03:2, (100%)
 rename test/corpus/{cur => foo/new}/09:2, (100%)
 rename test/corpus/{cur => foo/new}/10:2, (100%)
 rename test/corpus/{cur => new}/04:2, (100%)

diff --git a/test/T090-search-output.sh b/test/T090-search-output.sh
index 86544ac956a1..ef28c3d157aa 100755
--- a/test/T090-search-output.sh
+++ b/test/T090-search-output.sh
@@ -207,35 +207,35 @@ MAIL_DIR/cur/32:2,
 MAIL_DIR/cur/31:2,
 MAIL_DIR/cur/30:2,
 MAIL_DIR/cur/29:2,
-MAIL_DIR/cur/28:2,
-MAIL_DIR/cur/27:2,
-MAIL_DIR/cur/26:2,
-MAIL_DIR/cur/25:2,
-MAIL_DIR/cur/24:2,
-MAIL_DIR/cur/23:2,
-MAIL_DIR/cur/22:2,
-MAIL_DIR/cur/21:2,
-MAIL_DIR/cur/19:2,
-MAIL_DIR/cur/18:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/18:2,
 MAIL_DIR/cur/51:2,
-MAIL_DIR/cur/20:2,
-MAIL_DIR/cur/17:2,
-MAIL_DIR/cur/16:2,
-MAIL_DIR/cur/15:2,
-MAIL_DIR/cur/14:2,
-MAIL_DIR/cur/13:2,
-MAIL_DIR/cur/12:2,
-MAIL_DIR/cur/11:2,
-MAIL_DIR/cur/10:2,
-MAIL_DIR/cur/09:2,
-MAIL_DIR/cur/08:2,
-MAIL_DIR/cur/06:2,
-MAIL_DIR/cur/05:2,
-MAIL_DIR/cur/04:2,
-MAIL_DIR/cur/03:2,
-MAIL_DIR/cur/07:2,
-MAIL_DIR/cur/02:2,
-MAIL_DIR/cur/01:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/17:2,
+MAIL_DIR/foo/baz/new/16:2,
+MAIL_DIR/foo/baz/new/15:2,
+MAIL_DIR/foo/baz/cur/14:2,
+MAIL_DIR/foo/baz/cur/13:2,
+MAIL_DIR/foo/baz/12:2,
+MAIL_DIR/foo/baz/11:2,
+MAIL_DIR/foo/new/10:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/06:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/new/04:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/02:2,
+MAIL_DIR/01:2,
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
@@ -266,34 +266,34 @@ MAIL_DIR/cur/32:2,
 MAIL_DIR/cur/31:2,
 MAIL_DIR/cur/30:2,
 MAIL_DIR/cur/29:2,
-MAIL_DIR/cur/28:2,
-MAIL_DIR/cur/27:2,
-MAIL_DIR/cur/26:2,
-MAIL_DIR/cur/25:2,
-MAIL_DIR/cur/24:2,
-MAIL_DIR/cur/23:2,
-MAIL_DIR/cur/22:2,
-MAIL_DIR/cur/21:2,
-MAIL_DIR/cur/19:2,
-MAIL_DIR/cur/18:2,
-MAIL_DIR/cur/20:2,
-MAIL_DIR/cur/17:2,
-MAIL_DIR/cur/16:2,
-MAIL_DIR/cur/15:2,
-MAIL_DIR/cur/14:2,
-MAIL_DIR/cur/13:2,
-MAIL_DIR/cur/12:2,
-MAIL_DIR/cur/11:2,
-MAIL_DIR/cur/10:2,
-MAIL_DIR/cur/09:2,
-MAIL_DIR/cur/08:2,
-MAIL_DIR/cur/06:2,
-MAIL_DIR/cur/05:2,
-MAIL_DIR/cur/04:2,
-MAIL_DIR/cur/03:2,
-MAIL_DIR/cur/07:2,
-MAIL_DIR/cur/02:2,
-MAIL_DIR/cur/01:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/17:2,
+MAIL_DIR/foo/baz/new/16:2,
+MAIL_DIR/foo/baz/new/15:2,
+MAIL_DIR/foo/baz/cur/14:2,
+MAIL_DIR/foo/baz/cur/13:2,
+MAIL_DIR/foo/baz/12:2,
+MAIL_DIR/foo/baz/11:2,
+MAIL_DIR/foo/new/10:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/06:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/new/04:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/02:2,
+MAIL_DIR/01:2,
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
@@ -324,35 +324,35 @@ cat <<EOF >EXPECTED
 "MAIL_DIR/cur/31:2,",
 "MAIL_DIR/cur/30:2,",
 "MAIL_DIR/cur/29:2,",
-"MAIL_DIR/cur/28:2,",
-"MAIL_DIR/cur/27:2,",
-"MAIL_DIR/cur/26:2,",
-"MAIL_DIR/cur/25:2,",
-"MAIL_DIR/cur/24:2,",
-"MAIL_DIR/cur/23:2,",
-"MAIL_DIR/cur/22:2,",
-"MAIL_DIR/cur/21:2,",
-"MAIL_DIR/cur/19:2,",
-"MAIL_DIR/cur/18:2,",
+"MAIL_DIR/bar/baz/new/28:2,",
+"MAIL_DIR/bar/baz/new/27:2,",
+"MAIL_DIR/bar/baz/cur/26:2,",
+"MAIL_DIR/bar/baz/cur/25:2,",
+"MAIL_DIR/bar/baz/24:2,",
+"MAIL_DIR/bar/baz/23:2,",
+"MAIL_DIR/bar/new/22:2,",
+"MAIL_DIR/bar/new/21:2,",
+"MAIL_DIR/bar/cur/19:2,",
+"MAIL_DIR/bar/18:2,",
 "MAIL_DIR/cur/51:2,",
-"MAIL_DIR/cur/20:2,",
-"MAIL_DIR/cur/17:2,",
-"MAIL_DIR/cur/16:2,",
-"MAIL_DIR/cur/15:2,",
-"MAIL_DIR/cur/14:2,",
-"MAIL_DIR/cur/13:2,",
-"MAIL_DIR/cur/12:2,",
-"MAIL_DIR/cur/11:2,",
-"MAIL_DIR/cur/10:2,",
-"MAIL_DIR/cur/09:2,",
-"MAIL_DIR/cur/08:2,",
-"MAIL_DIR/cur/06:2,",
-"MAIL_DIR/cur/05:2,",
-"MAIL_DIR/cur/04:2,",
-"MAIL_DIR/cur/03:2,",
-"MAIL_DIR/cur/07:2,",
-"MAIL_DIR/cur/02:2,",
-"MAIL_DIR/cur/01:2,"]
+"MAIL_DIR/bar/cur/20:2,",
+"MAIL_DIR/bar/17:2,",
+"MAIL_DIR/foo/baz/new/16:2,",
+"MAIL_DIR/foo/baz/new/15:2,",
+"MAIL_DIR/foo/baz/cur/14:2,",
+"MAIL_DIR/foo/baz/cur/13:2,",
+"MAIL_DIR/foo/baz/12:2,",
+"MAIL_DIR/foo/baz/11:2,",
+"MAIL_DIR/foo/new/10:2,",
+"MAIL_DIR/foo/new/09:2,",
+"MAIL_DIR/foo/cur/08:2,",
+"MAIL_DIR/foo/06:2,",
+"MAIL_DIR/bar/baz/05:2,",
+"MAIL_DIR/new/04:2,",
+"MAIL_DIR/foo/new/03:2,",
+"MAIL_DIR/foo/cur/07:2,",
+"MAIL_DIR/02:2,",
+"MAIL_DIR/01:2,"]
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
diff --git a/test/corpus/cur/01:2, b/test/corpus/01:2,
similarity index 100%
rename from test/corpus/cur/01:2,
rename to test/corpus/01:2,
diff --git a/test/corpus/cur/02:2, b/test/corpus/02:2,
similarity index 100%
rename from test/corpus/cur/02:2,
rename to test/corpus/02:2,
diff --git a/test/corpus/cur/17:2, b/test/corpus/bar/17:2,
similarity index 100%
rename from test/corpus/cur/17:2,
rename to test/corpus/bar/17:2,
diff --git a/test/corpus/cur/18:2, b/test/corpus/bar/18:2,
similarity index 100%
rename from test/corpus/cur/18:2,
rename to test/corpus/bar/18:2,
diff --git a/test/corpus/cur/05:2, b/test/corpus/bar/baz/05:2,
similarity index 100%
rename from test/corpus/cur/05:2,
rename to test/corpus/bar/baz/05:2,
diff --git a/test/corpus/cur/23:2, b/test/corpus/bar/baz/23:2,
similarity index 100%
rename from test/corpus/cur/23:2,
rename to test/corpus/bar/baz/23:2,
diff --git a/test/corpus/cur/24:2, b/test/corpus/bar/baz/24:2,
similarity index 100%
rename from test/corpus/cur/24:2,
rename to test/corpus/bar/baz/24:2,
diff --git a/test/corpus/cur/25:2, b/test/corpus/bar/baz/cur/25:2,
similarity index 100%
rename from test/corpus/cur/25:2,
rename to test/corpus/bar/baz/cur/25:2,
diff --git a/test/corpus/cur/26:2, b/test/corpus/bar/baz/cur/26:2,
similarity index 100%
rename from test/corpus/cur/26:2,
rename to test/corpus/bar/baz/cur/26:2,
diff --git a/test/corpus/cur/27:2, b/test/corpus/bar/baz/new/27:2,
similarity index 100%
rename from test/corpus/cur/27:2,
rename to test/corpus/bar/baz/new/27:2,
diff --git a/test/corpus/cur/28:2, b/test/corpus/bar/baz/new/28:2,
similarity index 100%
rename from test/corpus/cur/28:2,
rename to test/corpus/bar/baz/new/28:2,
diff --git a/test/corpus/cur/19:2, b/test/corpus/bar/cur/19:2,
similarity index 100%
rename from test/corpus/cur/19:2,
rename to test/corpus/bar/cur/19:2,
diff --git a/test/corpus/cur/20:2, b/test/corpus/bar/cur/20:2,
similarity index 100%
rename from test/corpus/cur/20:2,
rename to test/corpus/bar/cur/20:2,
diff --git a/test/corpus/cur/21:2, b/test/corpus/bar/new/21:2,
similarity index 100%
rename from test/corpus/cur/21:2,
rename to test/corpus/bar/new/21:2,
diff --git a/test/corpus/cur/22:2, b/test/corpus/bar/new/22:2,
similarity index 100%
rename from test/corpus/cur/22:2,
rename to test/corpus/bar/new/22:2,
diff --git a/test/corpus/cur/06:2, b/test/corpus/foo/06:2,
similarity index 100%
rename from test/corpus/cur/06:2,
rename to test/corpus/foo/06:2,
diff --git a/test/corpus/cur/11:2, b/test/corpus/foo/baz/11:2,
similarity index 100%
rename from test/corpus/cur/11:2,
rename to test/corpus/foo/baz/11:2,
diff --git a/test/corpus/cur/12:2, b/test/corpus/foo/baz/12:2,
similarity index 100%
rename from test/corpus/cur/12:2,
rename to test/corpus/foo/baz/12:2,
diff --git a/test/corpus/cur/13:2, b/test/corpus/foo/baz/cur/13:2,
similarity index 100%
rename from test/corpus/cur/13:2,
rename to test/corpus/foo/baz/cur/13:2,
diff --git a/test/corpus/cur/14:2, b/test/corpus/foo/baz/cur/14:2,
similarity index 100%
rename from test/corpus/cur/14:2,
rename to test/corpus/foo/baz/cur/14:2,
diff --git a/test/corpus/cur/15:2, b/test/corpus/foo/baz/new/15:2,
similarity index 100%
rename from test/corpus/cur/15:2,
rename to test/corpus/foo/baz/new/15:2,
diff --git a/test/corpus/cur/16:2, b/test/corpus/foo/baz/new/16:2,
similarity index 100%
rename from test/corpus/cur/16:2,
rename to test/corpus/foo/baz/new/16:2,
diff --git a/test/corpus/cur/07:2, b/test/corpus/foo/cur/07:2,
similarity index 100%
rename from test/corpus/cur/07:2,
rename to test/corpus/foo/cur/07:2,
diff --git a/test/corpus/cur/08:2, b/test/corpus/foo/cur/08:2,
similarity index 100%
rename from test/corpus/cur/08:2,
rename to test/corpus/foo/cur/08:2,
diff --git a/test/corpus/cur/03:2, b/test/corpus/foo/new/03:2,
similarity index 100%
rename from test/corpus/cur/03:2,
rename to test/corpus/foo/new/03:2,
diff --git a/test/corpus/cur/09:2, b/test/corpus/foo/new/09:2,
similarity index 100%
rename from test/corpus/cur/09:2,
rename to test/corpus/foo/new/09:2,
diff --git a/test/corpus/cur/10:2, b/test/corpus/foo/new/10:2,
similarity index 100%
rename from test/corpus/cur/10:2,
rename to test/corpus/foo/new/10:2,
diff --git a/test/corpus/cur/04:2, b/test/corpus/new/04:2,
similarity index 100%
rename from test/corpus/cur/04:2,
rename to test/corpus/new/04:2,
-- 
1.9.0

[PATCH v4 02/13] test: add notmuch_search_files_sanitize and use it

[Jani Nikula]

We do this in a lot of places, so make it a helper in the test-lib.
---
 test/T090-search-output.sh           | 8 ++++----
 test/T370-search-folder-coherence.sh | 2 +-
 test/test-lib.sh                     | 5 +++++
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/test/T090-search-output.sh b/test/T090-search-output.sh
index ef28c3d157aa..e0bf8c16aecb 100755
--- a/test/T090-search-output.sh
+++ b/test/T090-search-output.sh
@@ -181,7 +181,7 @@ EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files"
-notmuch search --output=files '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --output=files '*' | notmuch_search_files_sanitize >OUTPUT
 cat <<EOF >EXPECTED
 MAIL_DIR/cur/52:2,
 MAIL_DIR/cur/53:2,
@@ -240,7 +240,7 @@ EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files --duplicate=1"
-notmuch search --output=files --duplicate=1 '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --output=files --duplicate=1 '*' | notmuch_search_files_sanitize >OUTPUT
 cat <<EOF >EXPECTED
 MAIL_DIR/cur/52:2,
 MAIL_DIR/cur/53:2,
@@ -298,7 +298,7 @@ EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files --format=json"
-notmuch search --format=json --output=files '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --format=json --output=files '*' | notmuch_search_files_sanitize >OUTPUT
 cat <<EOF >EXPECTED
 ["MAIL_DIR/cur/52:2,",
 "MAIL_DIR/cur/53:2,",
@@ -357,7 +357,7 @@ EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files --format=json --duplicate=2"
-notmuch search --format=json --output=files --duplicate=2 '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --format=json --output=files --duplicate=2 '*' | notmuch_search_files_sanitize >OUTPUT
 cat <<EOF >EXPECTED
 ["MAIL_DIR/cur/51:2,"]
 EOF
diff --git a/test/T370-search-folder-coherence.sh b/test/T370-search-folder-coherence.sh
index 3f6ec76348c0..5e72a6ccfd3a 100755
--- a/test/T370-search-folder-coherence.sh
+++ b/test/T370-search-folder-coherence.sh
@@ -27,7 +27,7 @@ cat <<EOF >EXPECTED
 MAIL_DIR/msg-001
 MAIL_DIR/spam/msg-001
 EOF
-notmuch search --output=files id:$id_x | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --output=files id:$id_x | notmuch_search_files_sanitize >OUTPUT
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "Test matches folder:spam"
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 66edb7c99f24..837c669ee757 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -661,6 +661,11 @@ notmuch_search_sanitize ()
     perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'
 }
 
+notmuch_search_files_sanitize()
+{
+    sed -e "s,$MAIL_DIR,MAIL_DIR,"
+}
+
 NOTMUCH_SHOW_FILENAME_SQUELCH='s,filename:.*/mail,filename:/XXX/mail,'
 notmuch_show_sanitize ()
 {
-- 
1.9.0

[PATCH v4 03/13] lib: refactor folder term update after filename removal

[Jani Nikula]

Abstract some blocks of code for reuse. No functional changes.
---
 lib/message.cc | 135 ++++++++++++++++++++++++++++-----------------------------
 1 file changed, 66 insertions(+), 69 deletions(-)

diff --git a/lib/message.cc b/lib/message.cc
index c91f3a59836f..7aff4ae5111a 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -481,6 +481,68 @@ notmuch_message_get_replies (notmuch_message_t *message)
     return _notmuch_messages_create (message->replies);
 }
 
+static void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
+{
+    Xapian::TermIterator i;
+    size_t prefix_len = strlen (prefix);
+
+    while (1) {
+	i = message->doc.termlist_begin ();
+	i.skip_to (prefix);
+
+	/* Terminate loop when no terms remain with desired prefix. */
+	if (i == message->doc.termlist_end () ||
+	    strncmp ((*i).c_str (), prefix, prefix_len))
+	    break;
+
+	try {
+	    message->doc.remove_term ((*i));
+	} catch (const Xapian::InvalidArgumentError) {
+	    /* Ignore failure to remove non-existent term. */
+	}
+    }
+}
+
+/* Add directory based terms for all filenames of the message. */
+static notmuch_status_t
+_notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message)
+{
+    const char *direntry_prefix = _find_prefix ("file-direntry");
+    int direntry_prefix_len = strlen (direntry_prefix);
+    Xapian::TermIterator i = message->doc.termlist_begin ();
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    for (i.skip_to (direntry_prefix); i != message->doc.termlist_end (); i++) {
+	unsigned int directory_id;
+	const char *direntry, *directory;
+	char *colon;
+
+	/* Terminate loop at first term without desired prefix. */
+	if (strncmp ((*i).c_str (), direntry_prefix, direntry_prefix_len))
+	    break;
+
+	/* Indicate that there are filenames remaining. */
+	status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+	direntry = (*i).c_str ();
+	direntry += direntry_prefix_len;
+
+	directory_id = strtol (direntry, &colon, 10);
+
+	if (colon == NULL || *colon != ':')
+	    INTERNAL_ERROR ("malformed direntry");
+
+	directory = _notmuch_database_get_directory_path (ctx,
+							  message->notmuch,
+							  directory_id);
+	if (strlen (directory))
+	    _notmuch_message_gen_terms (message, "folder", directory);
+    }
+
+    return status;
+}
+
 /* Add an additional 'filename' for 'message'.
  *
  * This change will not be reflected in the database until the next
@@ -536,17 +598,12 @@ notmuch_status_t
 _notmuch_message_remove_filename (notmuch_message_t *message,
 				  const char *filename)
 {
-    const char *direntry_prefix = _find_prefix ("file-direntry");
-    int direntry_prefix_len = strlen (direntry_prefix);
-    const char *folder_prefix = _find_prefix ("folder");
-    int folder_prefix_len = strlen (folder_prefix);
     void *local = talloc_new (message);
+    const char *folder_prefix = _find_prefix ("folder");
     char *zfolder_prefix = talloc_asprintf(local, "Z%s", folder_prefix);
-    int zfolder_prefix_len = strlen (zfolder_prefix);
     char *direntry;
     notmuch_private_status_t private_status;
     notmuch_status_t status;
-    Xapian::TermIterator i, last;
 
     status = _notmuch_database_filename_to_direntry (
 	local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
@@ -567,73 +624,13 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
      *  3. adding back terms for all remaining filenames of the message. */
 
     /* 1. removing all "folder:" terms */
-    while (1) {
-	i = message->doc.termlist_begin ();
-	i.skip_to (folder_prefix);
-
-	/* Terminate loop when no terms remain with desired prefix. */
-	if (i == message->doc.termlist_end () ||
-	    strncmp ((*i).c_str (), folder_prefix, folder_prefix_len))
-	{
-	    break;
-	}
-
-	try {
-	    message->doc.remove_term ((*i));
-	} catch (const Xapian::InvalidArgumentError) {
-	    /* Ignore failure to remove non-existent term. */
-	}
-    }
+    _notmuch_message_remove_terms (message, folder_prefix);
 
     /* 2. removing all "folder:" stemmed terms */
-    while (1) {
-	i = message->doc.termlist_begin ();
-	i.skip_to (zfolder_prefix);
-
-	/* Terminate loop when no terms remain with desired prefix. */
-	if (i == message->doc.termlist_end () ||
-	    strncmp ((*i).c_str (), zfolder_prefix, zfolder_prefix_len))
-	{
-	    break;
-	}
-
-	try {
-	    message->doc.remove_term ((*i));
-	} catch (const Xapian::InvalidArgumentError) {
-	    /* Ignore failure to remove non-existent term. */
-	}
-    }
+    _notmuch_message_remove_terms (message, zfolder_prefix);
 
     /* 3. adding back terms for all remaining filenames of the message. */
-    i = message->doc.termlist_begin ();
-    i.skip_to (direntry_prefix);
-
-    for (; i != message->doc.termlist_end (); i++) {
-	unsigned int directory_id;
-	const char *direntry, *directory;
-	char *colon;
-
-	/* Terminate loop at first term without desired prefix. */
-	if (strncmp ((*i).c_str (), direntry_prefix, direntry_prefix_len))
-	    break;
-
-	/* Indicate that there are filenames remaining. */
-	status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-
-	direntry = (*i).c_str ();
-	direntry += direntry_prefix_len;
-
-	directory_id = strtol (direntry, &colon, 10);
-
-	if (colon == NULL || *colon != ':')
-	    INTERNAL_ERROR ("malformed direntry");
-
-	directory = _notmuch_database_get_directory_path (local,
-							  message->notmuch,
-							  directory_id);
-	if (strlen (directory))
-	    _notmuch_message_gen_terms (message, "folder", directory);
-    }
+    status = _notmuch_message_add_directory_terms (local, message);
 
     talloc_free (local);
 
-- 
1.9.0

[PATCH v4 04/13] lib: add support for path: prefix searches

[Jani Nikula]

The path: prefix is a literal boolean prefix matching the paths,
relative from the maildir root, of the message files.

path:foo matches all message files in foo (but not in foo/new or
foo/cur).

path:foo/new matches all message files in foo/new.

path:"" matches all message files in the top level maildir.

path:foo/** matches all message files in foo and recursively in all
subdirectories of foo.

path:** matches all message files recursively, i.e. all messages.
---
 lib/database.cc |  7 ++++---
 lib/message.cc  | 52 +++++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index f395061e3a73..93cc7f57e9db 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -100,8 +100,8 @@ typedef struct {
  * In addition, terms from the content of the message are added with
  * "from", "to", "attachment", and "subject" prefixes for use by the
  * user in searching. Similarly, terms from the path of the mail
- * message are added with a "folder" prefix. But the database doesn't
- * really care itself about any of these.
+ * message are added with "folder" and "path" prefixes. But the
+ * database doesn't really care itself about any of these.
  *
  * The data portion of a mail document is empty.
  *
@@ -208,7 +208,8 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "thread",			"G" },
     { "tag",			"K" },
     { "is",			"K" },
-    { "id",			"Q" }
+    { "id",			"Q" },
+    { "path",			"P" },
 };
 
 static prefix_t PROBABILISTIC_PREFIX[]= {
diff --git a/lib/message.cc b/lib/message.cc
index 7aff4ae5111a..21abe8e12b9d 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -504,6 +504,40 @@ _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
     }
 }
 
+#define RECURSIVE_SUFFIX "/**"
+
+/* Add "path:" terms for directory. */
+static notmuch_status_t
+_notmuch_message_add_path_terms (notmuch_message_t *message,
+				 const char *directory)
+{
+    /* Add exact "path:" term. */
+    _notmuch_message_add_term (message, "path", directory);
+
+    if (strlen (directory)) {
+	char *path, *p;
+
+	path = talloc_asprintf (NULL, "%s%s", directory, RECURSIVE_SUFFIX);
+	if (! path)
+	    return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+	/* Add recursive "path:" terms for directory and all parents. */
+	for (p = path + strlen (path) - 1; p > path; p--) {
+	    if (*p == '/') {
+		strcpy (p, RECURSIVE_SUFFIX);
+		_notmuch_message_add_term (message, "path", path);
+	    }
+	}
+
+	talloc_free (path);
+    }
+
+    /* Recursive all-matching path:** for consistency. */
+    _notmuch_message_add_term (message, "path", "**");
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 /* Add directory based terms for all filenames of the message. */
 static notmuch_status_t
 _notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message)
@@ -538,6 +572,8 @@ _notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message)
 							  directory_id);
 	if (strlen (directory))
 	    _notmuch_message_gen_terms (message, "folder", directory);
+
+	_notmuch_message_add_path_terms (message, directory);
     }
 
     return status;
@@ -577,6 +613,8 @@ _notmuch_message_add_filename (notmuch_message_t *message,
     /* New terms allow user to search with folder: specification. */
     _notmuch_message_gen_terms (message, "folder", directory);
 
+    _notmuch_message_add_path_terms (message, directory);
+
     talloc_free (local);
 
     return NOTMUCH_STATUS_SUCCESS;
@@ -618,18 +656,18 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
     if (status)
 	return status;
 
-    /* Re-synchronize "folder:" terms for this message. This requires:
-     *  1. removing all "folder:" terms
-     *  2. removing all "folder:" stemmed terms
-     *  3. adding back terms for all remaining filenames of the message. */
+    /* Re-synchronize "folder:" and "path:" terms for this message. */
 
-    /* 1. removing all "folder:" terms */
+    /* Remove all "folder:" terms. */
     _notmuch_message_remove_terms (message, folder_prefix);
 
-    /* 2. removing all "folder:" stemmed terms */
+    /* Remove all "folder:" stemmed terms. */
     _notmuch_message_remove_terms (message, zfolder_prefix);
 
-    /* 3. adding back terms for all remaining filenames of the message. */
+    /* Remove all "path:" terms. */
+    _notmuch_message_remove_terms (message, _find_prefix ("path"));
+
+    /* Add back terms for all remaining filenames of the message. */
     status = _notmuch_message_add_directory_terms (local, message);
 
     talloc_free (local);
-- 
1.9.0

[PATCH v4 05/13] test: make insert test use the path: prefix

[Jani Nikula]

This is a more strict test for the insert test.
---
 test/T070-insert.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/T070-insert.sh b/test/T070-insert.sh
index b77c5e13c87f..ea9db07e2fa2 100755
--- a/test/T070-insert.sh
+++ b/test/T070-insert.sh
@@ -126,14 +126,14 @@ test_expect_equal "$dirname" "$MAIL_DIR/new"
 test_begin_subtest "Insert message into folder"
 gen_insert_msg
 notmuch insert --folder=Drafts < "$gen_msg_filename"
-output=$(notmuch search --output=files folder:Drafts)
+output=$(notmuch search --output=files path:Drafts/new)
 dirname=$(dirname "$output")
 test_expect_equal "$dirname" "$MAIL_DIR/Drafts/new"
 
 test_begin_subtest "Insert message into folder, add/remove tags"
 gen_insert_msg
 notmuch insert --folder=Drafts +draft -unread < "$gen_msg_filename"
-output=$(notmuch search --output=messages folder:Drafts tag:draft NOT tag:unread)
+output=$(notmuch search --output=messages path:Drafts/cur tag:draft NOT tag:unread)
 test_expect_equal "$output" "id:$gen_msg_id"
 
 gen_insert_msg
@@ -143,21 +143,21 @@ test_expect_code 1 "Insert message into non-existent folder" \
 test_begin_subtest "Insert message, create folder"
 gen_insert_msg
 notmuch insert --folder=F --create-folder +folder < "$gen_msg_filename"
-output=$(notmuch search --output=files folder:F tag:folder)
+output=$(notmuch search --output=files path:F/new tag:folder)
 basename=$(basename "$output")
 test_expect_equal_file "$gen_msg_filename" "$MAIL_DIR/F/new/${basename}"
 
 test_begin_subtest "Insert message, create subfolder"
 gen_insert_msg
 notmuch insert --folder=F/G/H/I/J --create-folder +folder < "$gen_msg_filename"
-output=$(notmuch search --output=files folder:F/G/H/I/J tag:folder)
+output=$(notmuch search --output=files path:F/G/H/I/J/new tag:folder)
 basename=$(basename "$output")
 test_expect_equal_file "$gen_msg_filename" "${MAIL_DIR}/F/G/H/I/J/new/${basename}"
 
 test_begin_subtest "Insert message, create existing subfolder"
 gen_insert_msg
 notmuch insert --folder=F/G/H/I/J --create-folder +folder < "$gen_msg_filename"
-output=$(notmuch count folder:F/G/H/I/J tag:folder)
+output=$(notmuch count path:F/G/H/I/J/new tag:folder)
 test_expect_equal "$output" "2"
 
 gen_insert_msg
-- 
1.9.0

[PATCH v4 06/13] lib: make folder: prefix literal

[Jani Nikula]

In xapian terms, convert folder: prefix from probabilistic to boolean
prefix, matching the paths, relative from the maildir root, of the
message files, ignoring the maildir new and cur leaf directories.

folder:foo matches all message files in foo, foo/new, and foo/cur.

folder:foo/new does *not* match message files in foo/new.

folder:"" matches all message files in the top level maildir and its
new and cur subdirectories.

This change constitutes a database change: bump the database version
and add database upgrade support for folder: terms. The upgrade also
adds path: terms.

Finally, fix the folder search test for literal folder: search, as
some of the folder: matching capabilities are lost in the
probabilistic to boolean prefix change.
---
 lib/database.cc               | 44 ++++++++++++++++++++++--
 lib/message.cc                | 80 +++++++++++++++++++++++++++++++++++++------
 lib/notmuch-private.h         |  3 ++
 test/T100-search-by-folder.sh | 24 +++++++++++--
 4 files changed, 135 insertions(+), 16 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index 93cc7f57e9db..aef748f75b5d 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -42,7 +42,7 @@ typedef struct {
     const char *prefix;
 } prefix_t;
 
-#define NOTMUCH_DATABASE_VERSION 1
+#define NOTMUCH_DATABASE_VERSION 2
 
 #define STRINGIFY(s) _SUB_STRINGIFY(s)
 #define _SUB_STRINGIFY(s) #s
@@ -210,6 +210,13 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "is",			"K" },
     { "id",			"Q" },
     { "path",			"P" },
+    /*
+     * Without the ":", since this is a multi-letter prefix, Xapian
+     * will add a colon itself if the first letter of the path is
+     * upper-case ASCII. Including the ":" forces there to always be a
+     * colon, which keeps our own logic simpler.
+     */
+    { "folder",			"XFOLDER:" },
 };
 
 static prefix_t PROBABILISTIC_PREFIX[]= {
@@ -217,7 +224,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= {
     { "to",			"XTO" },
     { "attachment",		"XATTACHMENT" },
     { "subject",		"XSUBJECT"},
-    { "folder",			"XFOLDER"}
 };
 
 const char *
@@ -1168,6 +1174,40 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
 	}
     }
 
+    /*
+     * Prior to version 2, the "folder:" prefix was probabilistic and
+     * stemmed. Change it to the current boolean prefix. Add "path:"
+     * prefixes while at it.
+     */
+    if (version < 2) {
+	notmuch_query_t *query = notmuch_query_create (notmuch, "");
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+
+	count = 0;
+	total = notmuch_query_count_messages (query);
+
+	for (messages = notmuch_query_search_messages (query);
+	     notmuch_messages_valid (messages);
+	     notmuch_messages_move_to_next (messages)) {
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    _notmuch_message_upgrade_folder (message);
+	    _notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+
+	    count++;
+	}
+
+	notmuch_query_destroy (query);
+    }
+
     db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
     db->flush ();
 
diff --git a/lib/message.cc b/lib/message.cc
index 21abe8e12b9d..9243b769d2b0 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -504,6 +504,56 @@ _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
     }
 }
 
+/* Return true if p points at "new" or "cur". */
+static bool is_maildir (const char *p)
+{
+    return strcmp (p, "cur") == 0 || strcmp (p, "new") == 0;
+}
+
+/* Add "folder:" term for directory. */
+static notmuch_status_t
+_notmuch_message_add_folder_terms (notmuch_message_t *message,
+				   const char *directory)
+{
+    char *folder, *last;
+
+    folder = talloc_strdup (NULL, directory);
+    if (! folder)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    /*
+     * If the message file is in a leaf directory named "new" or
+     * "cur", presume maildir and index the parent directory. Thus a
+     * "folder:" prefix search matches messages in the specified
+     * maildir folder, i.e. in the specified directory and its "new"
+     * and "cur" subdirectories.
+     *
+     * Note that this means the "folder:" prefix can't be used for
+     * distinguishing between message files in "new" or "cur". The
+     * "path:" prefix needs to be used for that.
+     *
+     * Note the deliberate difference to _filename_is_in_maildir(). We
+     * don't want to index different things depending on the existence
+     * or non-existence of all maildir sibling directories "new",
+     * "cur", and "tmp". Doing so would be surprising, and difficult
+     * for the user to fix in case all subdirectories were not in
+     * place during indexing.
+     */
+    last = strrchr (folder, '/');
+    if (last) {
+	if (is_maildir (last + 1))
+	    *last = '\0';
+    } else if (is_maildir (folder)) {
+	*folder = '\0';
+    }
+
+    _notmuch_message_add_term (message, "folder", folder);
+
+    talloc_free (folder);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 #define RECURSIVE_SUFFIX "/**"
 
 /* Add "path:" terms for directory. */
@@ -570,9 +620,8 @@ _notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message)
 	directory = _notmuch_database_get_directory_path (ctx,
 							  message->notmuch,
 							  directory_id);
-	if (strlen (directory))
-	    _notmuch_message_gen_terms (message, "folder", directory);
 
+	_notmuch_message_add_folder_terms (message, directory);
 	_notmuch_message_add_path_terms (message, directory);
     }
 
@@ -610,9 +659,7 @@ _notmuch_message_add_filename (notmuch_message_t *message,
      * notmuch_directory_get_child_files() . */
     _notmuch_message_add_term (message, "file-direntry", direntry);
 
-    /* New terms allow user to search with folder: specification. */
-    _notmuch_message_gen_terms (message, "folder", directory);
-
+    _notmuch_message_add_folder_terms (message, directory);
     _notmuch_message_add_path_terms (message, directory);
 
     talloc_free (local);
@@ -637,8 +684,6 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
 				  const char *filename)
 {
     void *local = talloc_new (message);
-    const char *folder_prefix = _find_prefix ("folder");
-    char *zfolder_prefix = talloc_asprintf(local, "Z%s", folder_prefix);
     char *direntry;
     notmuch_private_status_t private_status;
     notmuch_status_t status;
@@ -659,10 +704,7 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
     /* Re-synchronize "folder:" and "path:" terms for this message. */
 
     /* Remove all "folder:" terms. */
-    _notmuch_message_remove_terms (message, folder_prefix);
-
-    /* Remove all "folder:" stemmed terms. */
-    _notmuch_message_remove_terms (message, zfolder_prefix);
+    _notmuch_message_remove_terms (message, _find_prefix ("folder"));
 
     /* Remove all "path:" terms. */
     _notmuch_message_remove_terms (message, _find_prefix ("path"));
@@ -675,6 +717,22 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
     return status;
 }
 
+/* Upgrade the "folder:" prefix from V1 to V2. */
+#define FOLDER_PREFIX_V1       "XFOLDER"
+#define ZFOLDER_PREFIX_V1      "Z" FOLDER_PREFIX_V1
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message)
+{
+    /* Remove all old "folder:" terms. */
+    _notmuch_message_remove_terms (message, FOLDER_PREFIX_V1);
+
+    /* Remove all old "folder:" stemmed terms. */
+    _notmuch_message_remove_terms (message, ZFOLDER_PREFIX_V1);
+
+    /* Add new boolean "folder:" and "path:" terms. */
+    _notmuch_message_add_directory_terms (message, message);
+}
+
 char *
 _notmuch_message_talloc_copy_data (notmuch_message_t *message)
 {
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index af185c7c5ba8..59eb2bc285a5 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -263,6 +263,9 @@ _notmuch_message_gen_terms (notmuch_message_t *message,
 void
 _notmuch_message_upgrade_filename_storage (notmuch_message_t *message);
 
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message);
+
 notmuch_status_t
 _notmuch_message_add_filename (notmuch_message_t *message,
 			       const char *filename);
diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh
index 5cc2ca8d388a..a7f63dd1750c 100755
--- a/test/T100-search-by-folder.sh
+++ b/test/T100-search-by-folder.sh
@@ -3,6 +3,7 @@ test_description='"notmuch search" by folder: (with variations)'
 . ./test-lib.sh
 
 add_message '[dir]=bad' '[subject]="To the bone"'
+add_message '[dir]=.' '[subject]="Top level"'
 add_message '[dir]=bad/news' '[subject]="Bears"'
 mkdir -p "${MAIL_DIR}/duplicate/bad/news"
 cp "$gen_msg_filename" "${MAIL_DIR}/duplicate/bad/news"
@@ -12,29 +13,46 @@ add_message '[dir]=things/favorite' '[subject]="Raindrops, whiskers, kettles"'
 add_message '[dir]=things/bad' '[subject]="Bites, stings, sad feelings"'
 
 test_begin_subtest "Single-world folder: specification (multiple results)"
-output=$(notmuch search folder:bad | notmuch_search_sanitize)
+output=$(notmuch search folder:bad folder:bad/news folder:things/bad | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)"
 
+test_begin_subtest "Top level folder"
+output=$(notmuch search folder:'""' | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Top level (inbox unread)"
+
 test_begin_subtest "Two-word path to narrow results to one"
 output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
 
+test_begin_subtest "Folder search with --output=files"
+output=$(notmuch search --output=files folder:bad/news | notmuch_search_files_sanitize)
+test_expect_equal "$output" "MAIL_DIR/bad/news/msg-003
+MAIL_DIR/duplicate/bad/news/msg-003"
+
 test_begin_subtest "After removing duplicate instance of matching path"
 rm -r "${MAIL_DIR}/bad/news"
 notmuch new
 output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
+test_expect_equal "$output" ""
+
+test_begin_subtest "Folder search with --output=files part #2"
+output=$(notmuch search --output=files folder:duplicate/bad/news | notmuch_search_files_sanitize)
+test_expect_equal "$output" "MAIL_DIR/duplicate/bad/news/msg-003"
+
+test_begin_subtest "After removing duplicate instance of matching path part #2"
+output=$(notmuch search folder:duplicate/bad/news | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
 
 test_begin_subtest "After rename, old path returns nothing"
 mv "${MAIL_DIR}/duplicate/bad/news" "${MAIL_DIR}/duplicate/bad/olds"
 notmuch new
-output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
+output=$(notmuch search folder:duplicate/bad/news | notmuch_search_sanitize)
 test_expect_equal "$output" ""
 
 test_begin_subtest "After rename, new path returns result"
-output=$(notmuch search folder:bad/olds | notmuch_search_sanitize)
+output=$(notmuch search folder:duplicate/bad/olds | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
 
 test_done
-- 
1.9.0

[PATCH v4 07/13] test: add tests for the new boolean folder: and path: prefixes

[Jani Nikula]

Additional tests for the boolean folder: and path: prefixes using the
full corpus.
---
 test/T100-search-by-folder.sh | 93 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 92 insertions(+), 1 deletion(-)

diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh
index a7f63dd1750c..583bdf5eb086 100755
--- a/test/T100-search-by-folder.sh
+++ b/test/T100-search-by-folder.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-test_description='"notmuch search" by folder: (with variations)'
+test_description='"notmuch search" by folder: and path: (with variations)'
 . ./test-lib.sh
 
 add_message '[dir]=bad' '[subject]="To the bone"'
@@ -55,4 +55,95 @@ test_begin_subtest "After rename, new path returns result"
 output=$(notmuch search folder:duplicate/bad/olds | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
 
+# folder: and path: searches with full corpus
+rm -rf $MAIL_DIR
+add_email_corpus
+
+# add some more dupes
+cp $MAIL_DIR/foo/new/03:2, $MAIL_DIR/new
+cp $MAIL_DIR/bar/baz/05:2, $MAIL_DIR/foo
+notmuch new >/dev/null
+
+test_begin_subtest "folder: search"
+output=$(notmuch search --output=files folder:foo | notmuch_search_files_sanitize | sort)
+# bar/baz/05:2, is a duplicate of foo/05:2,
+# new/03:2, is a duplicate of foo/new/03:2,
+test_expect_equal "$output" "MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/foo/05:2,
+MAIL_DIR/foo/06:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/new/10:2,
+MAIL_DIR/new/03:2,"
+
+test_begin_subtest "top level folder: search"
+output=$(notmuch search --output=files folder:'""' | notmuch_search_files_sanitize | sort)
+# bar/18:2, is a duplicate of cur/51:2,
+# foo/new/03:2, is a duplicate of new/03:2,
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/29:2,
+MAIL_DIR/cur/30:2,
+MAIL_DIR/cur/31:2,
+MAIL_DIR/cur/32:2,
+MAIL_DIR/cur/33:2,
+MAIL_DIR/cur/34:2,
+MAIL_DIR/cur/35:2,
+MAIL_DIR/cur/36:2,
+MAIL_DIR/cur/37:2,
+MAIL_DIR/cur/38:2,
+MAIL_DIR/cur/39:2,
+MAIL_DIR/cur/40:2,
+MAIL_DIR/cur/41:2,
+MAIL_DIR/cur/42:2,
+MAIL_DIR/cur/43:2,
+MAIL_DIR/cur/44:2,
+MAIL_DIR/cur/45:2,
+MAIL_DIR/cur/46:2,
+MAIL_DIR/cur/47:2,
+MAIL_DIR/cur/48:2,
+MAIL_DIR/cur/49:2,
+MAIL_DIR/cur/50:2,
+MAIL_DIR/cur/51:2,
+MAIL_DIR/cur/52:2,
+MAIL_DIR/cur/53:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/new/03:2,
+MAIL_DIR/new/04:2,"
+
+test_begin_subtest "path: search"
+output=$(notmuch search --output=files path:"bar" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/51:2,"
+
+test_begin_subtest "top level path: search"
+output=$(notmuch search --output=files path:'""' | notmuch_search_files_sanitize | sort)
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,"
+
+test_begin_subtest "recursive path: search"
+output=$(notmuch search --output=files path:"bar/**" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+# foo/05:2, is a duplicate of bar/baz/05:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/cur/51:2,
+MAIL_DIR/foo/05:2,"
+
 test_done
-- 
1.9.0

[PATCH v4 08/13] test: add database upgrade test from format version 1 to 2

[Jani Nikula]

Test the upgrade from probabilistic to boolean folder: terms, and
addition of path: terms.

The test depends on the pre-built test corpus and database tarball and
checksum file being in place. If it's not, the test is skipped. The
mechanism to fetch the test database will be added later.

At the time of writing, a working test database and checksum file is
available at

   http://notmuchmail.org/releases/test-databases/

It has been noted that some non-GNU environments make lack
sha256sum. We leave this portability issue for a followup patch.
---
 test/T530-upgrade.sh | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100755 test/T530-upgrade.sh

diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
new file mode 100755
index 000000000000..1d883986db83
--- /dev/null
+++ b/test/T530-upgrade.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+test_description="database upgrade"
+
+. ./test-lib.sh
+
+dbtarball=folders-v1.tar.xz
+
+# XXX: Accomplish the same with test lib helpers
+if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
+    test_subtest_missing_external_prereq_["${dbtarball}"]=t
+fi
+
+test_expect_success \
+    'database checksum' \
+    '( cd $TEST_DIRECTORY/test-databases &&
+       sha256sum --quiet --check --status ${dbtarball}.sha256 )'
+
+tar xf $TEST_DIRECTORY/test-databases/${dbtarball} -C ${MAIL_DIR} --strip-components=1
+
+test_begin_subtest "folder: search does not work with old database version"
+output=$(notmuch search folder:foo)
+test_expect_equal "$output" ""
+
+test_begin_subtest "path: search does not work with old database version"
+output=$(notmuch search path:foo)
+test_expect_equal "$output" ""
+
+test_begin_subtest "database upgrade from format version 1"
+output=$(notmuch new)
+test_expect_equal "$output" "\
+Welcome to a new version of notmuch! Your database will now be upgraded.
+Your notmuch database has now been upgraded to database format version 2.
+No new mail."
+
+test_begin_subtest "folder: no longer matches in the middle of path"
+output=$(notmuch search folder:baz)
+test_expect_equal "$output" ""
+
+test_begin_subtest "folder: search"
+output=$(notmuch search --output=files folder:foo | notmuch_search_files_sanitize | sort)
+# bar/baz/05:2, and new/03:2, are duplicates of foo/05:2, and
+# foo/new/03:2, respectively
+test_expect_equal "$output" "MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/foo/05:2,
+MAIL_DIR/foo/06:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/new/10:2,
+MAIL_DIR/new/03:2,"
+
+test_begin_subtest "top level folder: search"
+output=$(notmuch search --output=files folder:'""' | notmuch_search_files_sanitize | sort)
+# foo/new/03:2, is a duplicate of new/03:2,
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,
+MAIL_DIR/cur/29:2,
+MAIL_DIR/cur/30:2,
+MAIL_DIR/cur/31:2,
+MAIL_DIR/cur/32:2,
+MAIL_DIR/cur/33:2,
+MAIL_DIR/cur/34:2,
+MAIL_DIR/cur/35:2,
+MAIL_DIR/cur/36:2,
+MAIL_DIR/cur/37:2,
+MAIL_DIR/cur/38:2,
+MAIL_DIR/cur/39:2,
+MAIL_DIR/cur/40:2,
+MAIL_DIR/cur/41:2,
+MAIL_DIR/cur/42:2,
+MAIL_DIR/cur/43:2,
+MAIL_DIR/cur/44:2,
+MAIL_DIR/cur/45:2,
+MAIL_DIR/cur/46:2,
+MAIL_DIR/cur/47:2,
+MAIL_DIR/cur/48:2,
+MAIL_DIR/cur/49:2,
+MAIL_DIR/cur/50:2,
+MAIL_DIR/cur/52:2,
+MAIL_DIR/cur/53:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/new/03:2,
+MAIL_DIR/new/04:2,"
+
+test_begin_subtest "path: search"
+output=$(notmuch search --output=files path:"bar" | notmuch_search_files_sanitize | sort)
+# foo/05:2, is a duplicate of bar/baz/05:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,"
+
+test_begin_subtest "top level path: search"
+output=$(notmuch search --output=files path:'""' | notmuch_search_files_sanitize | sort)
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,"
+
+test_begin_subtest "recursive path: search"
+output=$(notmuch search --output=files path:"bar/**" | notmuch_search_files_sanitize | sort)
+# foo/05:2, is a duplicate of bar/baz/05:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/foo/05:2,"
+
+test_done
-- 
1.9.0

[PATCH v4 09/13] man: update man pages for folder: and path: search terms

[Jani Nikula]

Text from review by Austin Clements <amdragon@MIT.EDU>.
---
 man/man7/notmuch-search-terms.7 | 41 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 7 deletions(-)

diff --git a/man/man7/notmuch-search-terms.7 b/man/man7/notmuch-search-terms.7
index a768b630a4d1..a33c8d78c122 100644
--- a/man/man7/notmuch-search-terms.7
+++ b/man/man7/notmuch-search-terms.7
@@ -52,7 +52,9 @@ terms to match against specific portions of an email, (where
 
 	thread:<thread-id>
 
-	folder:<directory-path>
+	folder:<maildir-folder>
+
+	path:<directory-path> or path:<directory-path>/**
 
 	date:<since>..<until>
 
@@ -100,13 +102,38 @@ thread ID values can be seen in the first column of output from
 .B "notmuch search"
 
 The
+.B path:
+prefix searches for email messages that are in particular directories
+within the mail store. The directory must be specified relative to the
+top-level maildir (and without the leading slash). By default,
+.B path:
+matches messages in the specified directory only. The "/**" suffix can
+be used to match messages in the specified directory and all its
+subdirectories recursively.
+.B path:""
+matches messages in the root of the mail store and, likewise,
+.B path:**
+matches all messages.
+
+The
+.B folder:
+prefix searches for email messages by maildir or MH folder. For
+MH-style folders, this is equivalent to \fBpath:\fR. For maildir, this
+includes messages in the "new" and "cur" subdirectories. The exact
+syntax for maildir folders depends on your mail configuration. For
+maildir++, \fBfolder:""\fR matches the inbox folder (which is the root
+in maildir++), other folder names always start with ".", and nested
+folders are separated by "."s, such as \fBfolder:.classes.topology\fR.
+For "file system" maildir, the inbox is typically \fBfolder:INBOX\fR
+and nested folders are separated by slashes, such as
+\fBfolder:classes/topology\fR.
+
+Both
+.B path:
+and
 .B folder:
-prefix can be used to search for email message files that are
-contained within particular directories within the mail store. If the
-same email message has multiple message files associated with it, it's
-sufficient for a match that at least one of the files is contained
-within a matching directory. Only the directory components below the
-top-level mail database path are available to be searched.
+will find a message if \fIany\fR copy of that message is in the
+specific directory/folder.
 
 The
 .B date:
-- 
1.9.0

[PATCH v4 10/13] man: try to clarify the folder: and path: vs. --output=files confusion

[Jani Nikula]

---
 man/man1/notmuch-search.1 | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/man/man1/notmuch-search.1 b/man/man1/notmuch-search.1
index 55a81e79fce4..a2b1ae43f411 100644
--- a/man/man1/notmuch-search.1
+++ b/man/man1/notmuch-search.1
@@ -82,8 +82,14 @@ one per line (\-\-format=text), separated by null characters
 S-Expression list (\-\-format=sexp).
 
 Note that each message may have multiple filenames associated with it.
-All of them are included in the output, unless limited with the
-\-\-duplicate=N option.
+All of them are included in the output (unless limited with the
+\-\-duplicate=N option). This may be particularly confusing for
+.B folder:
+or
+.B path:
+searches in a specified directory, as the messages may have duplicates
+in other directories that are included in the output, although these
+files alone would not match the search.
 .RE
 .RS 4
 .TP 4
-- 
1.9.0

[PATCH v4 11/13] devel: add script to generate test databases

[Jani Nikula]

Add script to generate notmuch test databases using specified versions
of notmuch. This is useful for generating material for database
upgrade tests.

This reuses the test infrastructure to have a sandbox environment for
notmuch new etc.
---
 devel/gen-testdb.sh | 131 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)
 create mode 100755 devel/gen-testdb.sh

diff --git a/devel/gen-testdb.sh b/devel/gen-testdb.sh
new file mode 100755
index 000000000000..621b31e67fc2
--- /dev/null
+++ b/devel/gen-testdb.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+#
+# NAME
+#	gen-testdb.sh - generate test databases
+#
+# SYNOPSIS
+#	gen-testdb.sh -v NOTMUCH-VERSION [-c CORPUS-PATH] [-s TAR-SUFFIX]
+#
+# DESCRIPTION
+#	Generate a tarball containing the specified test corpus and
+#	the corresponding notmuch database, indexed using a specific
+#	version of notmuch, resulting in a specific version of the
+#	database.
+#
+#	The specific version of notmuch will be built on the fly.
+#	Therefore the script must be run within a git repository to be
+#	able to build the old versions of notmuch.
+#
+#	This script reuses the test infrastructure, and the script
+#	must be run from within the test directory.
+#
+#	The output tarballs, named database-<TAR-SUFFIX>.tar.gz, are
+#	placed in the test/test-databases directory.
+#
+# OPTIONS
+#	-v NOTMUCH-VERSION
+#		Notmuch version in terms of a git tag or commit to use
+#		for generating the database. Required.
+#
+#	-c CORPUS-PATH
+#		Path to a corpus to use for generating the
+#		database. Due to CWD changes within the test
+#		infrastructure, use absolute paths. Defaults to the
+#		test corpus.
+#
+#	-s TAR-SUFFIX
+#		Suffix for the tarball basename. Empty by default.
+#
+# EXAMPLE
+#
+#	Generate a database indexed with notmuch 0.17. Use the default
+#	test corpus. Name the tarball database-v1.tar.gz to reflect
+#	the fact that notmuch 0.17 used database version 1.
+#
+#	$ cd test
+#	$ ../devel/gen-testdb.sh -v 0.17 -s v1
+#
+# CAVEATS
+#	Test infrastructure options won't work.
+#
+#	Any existing databases with the same name will be overwritten.
+#
+#	It may not be possible to build old versions of notmuch with
+#	the set of dependencies that satisfy building the current
+#	version of notmuch.
+#
+# AUTHOR
+#	Jani Nikula <jani@nikula.org>
+#
+# LICENSE
+#	Same as notmuch test infrastructure (GPLv2+).
+#
+
+test_description="database generation abusing test infrastructure"
+
+# immediate exit on subtest failure; see test_failure_ in test-lib.sh
+immediate=t
+
+VERSION=
+CORPUS=
+SUFFIX=
+
+while getopts v:c:s: opt; do
+    case "$opt" in
+	v) VERSION="$OPTARG";;
+	c) CORPUS="$OPTARG";;
+	s) SUFFIX="-$OPTARG";;
+    esac
+done
+shift `expr $OPTIND - 1`
+
+. ./test-lib.sh
+
+SHORT_CORPUS=$(basename ${CORPUS:-database})
+DBNAME=${SHORT_CORPUS}${SUFFIX}
+TARBALLNAME=${DBNAME}.tar.xz
+
+CORPUS=${CORPUS:-${TEST_DIRECTORY}/corpus}
+
+test_expect_code 0 "notmuch version specified on the command line" \
+    "test -n ${VERSION}"
+
+test_expect_code 0 "the specified version ${VERSION} refers to a commit" \
+    "git show ${VERSION} >/dev/null 2>&1"
+
+BUILD_DIR="notmuch-${VERSION}"
+test_expect_code 0 "generate snapshot of notmuch version ${VERSION}" \
+    "git -C $TEST_DIRECTORY/.. archive --prefix=${BUILD_DIR}/ --format=tar ${VERSION} | tar x"
+
+# force version string
+git describe --match '[0-9.]*' ${VERSION} > ${BUILD_DIR}/version
+
+test_expect_code 0 "configure and build notmuch version ${VERSION}" \
+    "make -C ${BUILD_DIR}"
+
+# use the newly built notmuch
+export PATH=./${BUILD_DIR}:$PATH
+
+test_begin_subtest "verify the newly built notmuch version"
+test_expect_equal "`notmuch --version`" "notmuch `cat ${BUILD_DIR}/version`"
+
+# replace the existing mails, if any, with the specified corpus
+rm -rf ${MAIL_DIR}
+cp -a ${CORPUS} ${MAIL_DIR}
+
+test_expect_code 0 "index the corpus" \
+    "notmuch new"
+
+# wrap the resulting mail store and database in a tarball
+
+cp -a ${MAIL_DIR} ${TMP_DIRECTORY}/${DBNAME}
+tar Jcf ${TMP_DIRECTORY}/${TARBALLNAME} -C ${TMP_DIRECTORY} ${DBNAME}
+mkdir -p  ${TEST_DIRECTORY}/test-databases
+cp -a ${TMP_DIRECTORY}/${TARBALLNAME} ${TEST_DIRECTORY}/test-databases
+test_expect_code 0 "create the output tarball ${TARBALLNAME}" \
+    "test -f ${TEST_DIRECTORY}/test-databases/${TARBALLNAME}"
+
+# generate a checksum file
+test_expect_code 0 "compute checksum" \
+    "(cd ${TEST_DIRECTORY}/test-databases/ && sha256sum ${TARBALLNAME} > ${TARBALLNAME}.sha256)"
+test_done
-- 
1.9.0

[PATCH v4 12/13] test: commit folders-v1.tar.xz checksum, ignore actual databases

[Jani Nikula]

From: David Bremner <david@tethera.net>

The checksum file is used by the test infrastructure to verify the downloaded
test database is the one we had in mind.  Note that this test is
rather strict, and the the checksum file needs to be recommitted when
the database is regenerated.

add a pattern .gitignore to ignore the actual databases
---
 test/test-databases/.gitignore               | 1 +
 test/test-databases/folders-v1.tar.xz.sha256 | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 test/test-databases/.gitignore
 create mode 100644 test/test-databases/folders-v1.tar.xz.sha256

diff --git a/test/test-databases/.gitignore b/test/test-databases/.gitignore
new file mode 100644
index 000000000000..b5624b74f804
--- /dev/null
+++ b/test/test-databases/.gitignore
@@ -0,0 +1 @@
+*.tar.xz
diff --git a/test/test-databases/folders-v1.tar.xz.sha256 b/test/test-databases/folders-v1.tar.xz.sha256
new file mode 100644
index 000000000000..01ad79db454f
--- /dev/null
+++ b/test/test-databases/folders-v1.tar.xz.sha256
@@ -0,0 +1 @@
+ace8a61216756b90a421e23d03910e1228bcb910e197c35e51e29f2cf57b37d9  folders-v1.tar.xz
-- 
1.9.0

[PATCH v4 13/13] test: add machinery to download and verify databases

[Jani Nikula]

From: David Bremner <david@tethera.net>

Note that it is intentional that the checksum file is not
downloaded. The intent is to check those into git.
---
 Makefile                           |  3 ++-
 test/README                        |  8 ++++++++
 test/T530-upgrade.sh               |  2 +-
 test/test-databases/Makefile       |  7 +++++++
 test/test-databases/Makefile.local | 14 ++++++++++++++
 5 files changed, 32 insertions(+), 2 deletions(-)
 create mode 100644 test/test-databases/Makefile
 create mode 100644 test/test-databases/Makefile.local

diff --git a/Makefile b/Makefile
index 0428160b80c2..97084b1c1cd0 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,8 @@ all:
 # List all subdirectories here. Each contains its own Makefile.local.
 # Use of '=', without '+=', seems to be required for out-of-tree
 # builds to work.
-subdirs = compat completion emacs lib man parse-time-string performance-test util test
+subdirs = compat completion emacs lib man parse-time-string performance-test util test \
+	test/test-databases
 
 # We make all targets depend on the Makefiles themselves.
 global_deps = Makefile Makefile.config Makefile.local \
diff --git a/test/README b/test/README
index 79a9b1b2f9a1..81a1c82dcdbd 100644
--- a/test/README
+++ b/test/README
@@ -64,6 +64,14 @@ The following command-line options are available when running tests:
 	Pointing this argument at a tmpfs filesystem can improve the
 	speed of the test suite for some users.
 
+Certain tests require precomputed databases to complete. You can fetch these
+databases with
+
+	make download-test-databases
+
+If you do not download the test databases, the relevant tests will be
+skipped.
+
 When invoking the test suite via "make test" any of the above options
 can be specified as follows:
 
diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
index 1d883986db83..6cbbb860998a 100755
--- a/test/T530-upgrade.sh
+++ b/test/T530-upgrade.sh
@@ -7,7 +7,7 @@ dbtarball=folders-v1.tar.xz
 
 # XXX: Accomplish the same with test lib helpers
 if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
-    test_subtest_missing_external_prereq_["${dbtarball}"]=t
+    test_subtest_missing_external_prereq_["${dbtarball} - fetch with 'make download-test-databases'"]=t
 fi
 
 test_expect_success \
diff --git a/test/test-databases/Makefile b/test/test-databases/Makefile
new file mode 100644
index 000000000000..b250a8bea31d
--- /dev/null
+++ b/test/test-databases/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C ../.. all
+
+.DEFAULT:
+	$(MAKE) -C ../.. $@
diff --git a/test/test-databases/Makefile.local b/test/test-databases/Makefile.local
new file mode 100644
index 000000000000..e777ada71667
--- /dev/null
+++ b/test/test-databases/Makefile.local
@@ -0,0 +1,14 @@
+# -*- makefile -*-
+
+TEST_DATABASE_MIRROR=http://notmuchmail.org/releases/test-databases
+
+dir := test/test-databases
+
+test_databases := $(dir)/folders-v1.tar.xz
+
+%.tar.xz:
+	wget -nv -O $@ ${TEST_DATABASE_MIRROR}/$(notdir $@);
+
+download-test-databases: ${test_databases}
+
+DISTCLEAN := $(DISTCLEAN) ${test_databases}
-- 
1.9.0

[PATCH v5] test: add machinery to download and verify databases

[Jani Nikula]

From: David Bremner <david@tethera.net>

Note that it is intentional that the checksum file is not
downloaded. The intent is to check those into git.
---
 Makefile                           |  3 ++-
 test/README                        |  8 ++++++++
 test/T530-upgrade.sh               |  2 +-
 test/test-databases/Makefile       |  7 +++++++
 test/test-databases/Makefile.local | 14 ++++++++++++++
 5 files changed, 32 insertions(+), 2 deletions(-)
 create mode 100644 test/test-databases/Makefile
 create mode 100644 test/test-databases/Makefile.local

diff --git a/Makefile b/Makefile
index f53bec0ea401..061c55a1948a 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,8 @@ all:
 # List all subdirectories here. Each contains its own Makefile.local.
 # Use of '=', without '+=', seems to be required for out-of-tree
 # builds to work.
-subdirs = compat completion doc emacs lib parse-time-string performance-test util test
+subdirs = compat completion doc emacs lib parse-time-string	\
+	performance-test util test test/test-databases
 
 # We make all targets depend on the Makefiles themselves.
 global_deps = Makefile Makefile.config Makefile.local \
diff --git a/test/README b/test/README
index 79a9b1b2f9a1..81a1c82dcdbd 100644
--- a/test/README
+++ b/test/README
@@ -64,6 +64,14 @@ The following command-line options are available when running tests:
 	Pointing this argument at a tmpfs filesystem can improve the
 	speed of the test suite for some users.
 
+Certain tests require precomputed databases to complete. You can fetch these
+databases with
+
+	make download-test-databases
+
+If you do not download the test databases, the relevant tests will be
+skipped.
+
 When invoking the test suite via "make test" any of the above options
 can be specified as follows:
 
diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
index 1d883986db83..6cbbb860998a 100755
--- a/test/T530-upgrade.sh
+++ b/test/T530-upgrade.sh
@@ -7,7 +7,7 @@ dbtarball=folders-v1.tar.xz
 
 # XXX: Accomplish the same with test lib helpers
 if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
-    test_subtest_missing_external_prereq_["${dbtarball}"]=t
+    test_subtest_missing_external_prereq_["${dbtarball} - fetch with 'make download-test-databases'"]=t
 fi
 
 test_expect_success \
diff --git a/test/test-databases/Makefile b/test/test-databases/Makefile
new file mode 100644
index 000000000000..b250a8bea31d
--- /dev/null
+++ b/test/test-databases/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C ../.. all
+
+.DEFAULT:
+	$(MAKE) -C ../.. $@
diff --git a/test/test-databases/Makefile.local b/test/test-databases/Makefile.local
new file mode 100644
index 000000000000..e777ada71667
--- /dev/null
+++ b/test/test-databases/Makefile.local
@@ -0,0 +1,14 @@
+# -*- makefile -*-
+
+TEST_DATABASE_MIRROR=http://notmuchmail.org/releases/test-databases
+
+dir := test/test-databases
+
+test_databases := $(dir)/folders-v1.tar.xz
+
+%.tar.xz:
+	wget -nv -O $@ ${TEST_DATABASE_MIRROR}/$(notdir $@);
+
+download-test-databases: ${test_databases}
+
+DISTCLEAN := $(DISTCLEAN) ${test_databases}
-- 
1.9.0

[PATCH v5] man: update man pages for folder: and path: search terms

[Jani Nikula]

Text from review by Austin Clements <amdragon@MIT.EDU>.
---
 doc/man7/notmuch-search-terms.rst | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index 20bc6f1e1488..1acdaa0b38de 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -46,7 +46,9 @@ indicate user-supplied values):
 
 -  thread:<thread-id>
 
--  folder:<directory-path>
+-  folder:<maildir-folder>
+
+-  path:<directory-path> or path:<directory-path>/**
 
 -  date:<since>..<until>
 
@@ -77,12 +79,28 @@ generated internally by notmuch (and do not appear in email messages).
 These thread ID values can be seen in the first column of output from
 **notmuch search**
 
-The **folder:** prefix can be used to search for email message files
-that are contained within particular directories within the mail store.
-If the same email message has multiple message files associated with it,
-it's sufficient for a match that at least one of the files is contained
-within a matching directory. Only the directory components below the
-top-level mail database path are available to be searched.
+The **path:** prefix searches for email messages that are in
+particular directories within the mail store. The directory must be
+specified relative to the top-level maildir (and without the leading
+slash). By default, **path:** matches messages in the specified
+directory only. The "/\*\*" suffix can be used to match messages in
+the specified directory and all its subdirectories recursively.
+**path:""** matches messages in the root of the mail store and,
+likewise, **path:\*\*** matches all messages.
+
+The **folder:** prefix searches for email messages by maildir or MH
+folder. For MH-style folders, this is equivalent to **path:**. For
+maildir, this includes messages in the "new" and "cur"
+subdirectories. The exact syntax for maildir folders depends on your
+mail configuration. For maildir++, **folder:""** matches the inbox
+folder (which is the root in maildir++), other folder names always
+start with ".", and nested folders are separated by "."s, such as
+**folder:.classes.topology**. For "file system" maildir, the inbox is
+typically **folder:INBOX** and nested folders are separated by
+slashes, such as **folder:classes/topology**.
+
+Both **path:** and **folder:** will find a message if *any* copy of
+that message is in the specific directory/folder.
 
 The **date:** prefix can be used to restrict the results to only
 messages within a particular time range (based on the Date: header) with
-- 
1.9.0

[PATCH v5] man: try to clarify the folder: and path: vs. --output=files confusion

[Jani Nikula]

---
 doc/man1/notmuch-search.rst | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/doc/man1/notmuch-search.rst b/doc/man1/notmuch-search.rst
index 7ac6c68e68ac..90160f21e23c 100644
--- a/doc/man1/notmuch-search.rst
+++ b/doc/man1/notmuch-search.rst
@@ -64,8 +64,13 @@ Supported options for **search** include
             (--format=json), or as an S-Expression list (--format=sexp).
 
             Note that each message may have multiple filenames
-            associated with it. All of them are included in the output,
-            unless limited with the --duplicate=N option.
+            associated with it. All of them are included in the output
+            (unless limited with the --duplicate=N option). This may
+            be particularly confusing for **folder:** or **path:**
+            searches in a specified directory, as the messages may
+            have duplicates in other directories that are included in
+            the output, although these files alone would not match the
+            search.
 
         **tags**
             Output all tags that appear on any message matching the
-- 
1.9.0

Re: [PATCH v4 00/13] boolean folder: and path: searches

[Austin Clements]

v5 LGTM.

Quoth Jani Nikula on Mar 09 at 11:40 pm:
> This is v4 of id:1394313585-28422-1-git-send-email-david@tethera.net.

Re: [PATCH v4 00/13] boolean folder: and path: searches

[Tomi Ollila]

On Sun, Mar 09 2014, Jani Nikula <jani@nikula.org> wrote:

> This is v4 of id:1394313585-28422-1-git-send-email-david@tethera.net.


This series looks pushable to me.


Tomi

Re: [PATCH v4 08/13] test: add database upgrade test from format version 1 to 2

[David Bremner]

Jani Nikula <jani@nikula.org> writes:

> Test the upgrade from probabilistic to boolean folder: terms, and
> addition of path: terms.
>
> The test depends on the pre-built test corpus and database tarball and
> checksum file being in place. If it's not, the test is skipped. The
> mechanism to fetch the test database will be added later.
>
> At the time of writing, a working test database and checksum file is
> available at
>
>    http://notmuchmail.org/releases/test-databases/
>

Hi Jani;

This test has gotten out of sync with the corpus in the tree. It still
passes because the database tarball on notmuchmail.org is in the
previous (multi-corpus) style. In particular

% cd test
% ../devel/gen-testdb.sh -s v1 -v 0.17
% sed -i s/folders-v1/database-v1/ T530-upgrade.sh 
% ./T530-upgrade

yields several failures.

I think it would be better to update the test to match corpus currently
in the tree.

d

[PATCH v5] test: add database upgrade test from format version 1 to 2

[Jani Nikula]

Test the upgrade from probabilistic to boolean folder: terms, and
addition of path: terms.

The test depends on the pre-built test corpus and database tarball and
checksum file being in place. If it's not, the test is skipped. The
mechanism to fetch the test database will be added later.

At the time of writing, a working test database and checksum file is
available at

   http://notmuchmail.org/releases/test-databases/

It has been noted that some non-GNU environments make lack
sha256sum. We leave this portability issue for a followup patch.

---

v5 of this patch, sneakily after Austin's "v5 LGTM" mail. ;)

Fixed test to work with a database generated from the current
corpus. The rearrangement in patch 01 is slightly different from the
folders corpus in the earlier versions of the series (one more file,
two fewer dupes).

David, please generate a database, verify it passes the test for you
too, and upload to notmuchmail.org. Thanks.
---
 test/T530-upgrade.sh | 111 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100755 test/T530-upgrade.sh

diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
new file mode 100755
index 000000000000..02d40f13b8ac
--- /dev/null
+++ b/test/T530-upgrade.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+test_description="database upgrade"
+
+. ./test-lib.sh
+
+dbtarball=folders-v1.tar.xz
+
+# XXX: Accomplish the same with test lib helpers
+if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
+    test_subtest_missing_external_prereq_["${dbtarball}"]=t
+fi
+
+test_expect_success \
+    'database checksum' \
+    '( cd $TEST_DIRECTORY/test-databases &&
+       sha256sum --quiet --check --status ${dbtarball}.sha256 )'
+
+tar xf $TEST_DIRECTORY/test-databases/${dbtarball} -C ${MAIL_DIR} --strip-components=1
+
+test_begin_subtest "folder: search does not work with old database version"
+output=$(notmuch search folder:foo)
+test_expect_equal "$output" ""
+
+test_begin_subtest "path: search does not work with old database version"
+output=$(notmuch search path:foo)
+test_expect_equal "$output" ""
+
+test_begin_subtest "database upgrade from format version 1"
+output=$(notmuch new)
+test_expect_equal "$output" "\
+Welcome to a new version of notmuch! Your database will now be upgraded.
+Your notmuch database has now been upgraded to database format version 2.
+No new mail."
+
+test_begin_subtest "folder: no longer matches in the middle of path"
+output=$(notmuch search folder:baz)
+test_expect_equal "$output" ""
+
+test_begin_subtest "folder: search"
+output=$(notmuch search --output=files folder:foo | notmuch_search_files_sanitize | sort)
+test_expect_equal "$output" "MAIL_DIR/foo/06:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/new/10:2,"
+
+test_begin_subtest "top level folder: search"
+output=$(notmuch search --output=files folder:'""' | notmuch_search_files_sanitize | sort)
+# bar/18:2, is a duplicate of cur/51:2,
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/29:2,
+MAIL_DIR/cur/30:2,
+MAIL_DIR/cur/31:2,
+MAIL_DIR/cur/32:2,
+MAIL_DIR/cur/33:2,
+MAIL_DIR/cur/34:2,
+MAIL_DIR/cur/35:2,
+MAIL_DIR/cur/36:2,
+MAIL_DIR/cur/37:2,
+MAIL_DIR/cur/38:2,
+MAIL_DIR/cur/39:2,
+MAIL_DIR/cur/40:2,
+MAIL_DIR/cur/41:2,
+MAIL_DIR/cur/42:2,
+MAIL_DIR/cur/43:2,
+MAIL_DIR/cur/44:2,
+MAIL_DIR/cur/45:2,
+MAIL_DIR/cur/46:2,
+MAIL_DIR/cur/47:2,
+MAIL_DIR/cur/48:2,
+MAIL_DIR/cur/49:2,
+MAIL_DIR/cur/50:2,
+MAIL_DIR/cur/51:2,
+MAIL_DIR/cur/52:2,
+MAIL_DIR/cur/53:2,
+MAIL_DIR/new/04:2,"
+
+test_begin_subtest "path: search"
+output=$(notmuch search --output=files path:"bar" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/51:2,"
+
+test_begin_subtest "top level path: search"
+output=$(notmuch search --output=files path:'""' | notmuch_search_files_sanitize | sort)
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,"
+
+test_begin_subtest "recursive path: search"
+output=$(notmuch search --output=files path:"bar/**" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/cur/51:2,"
+
+test_done
-- 
1.9.0