[PATCH 1/3] lib/sexp: provide relative lastmod queries

[PATCH 1/3] lib/sexp: provide relative lastmod queries

[David Bremner]

Test the relatively trivial logic changes for the sexp query parser
first before refactoring that logic to share with the infix query
parser.
---
 doc/man7/notmuch-sexp-queries.rst |  3 +++
 lib/parse-sexp.cc                 |  6 ++++++
 test/T081-sexpr-search.sh         | 30 ++++++++++++++++++++++++++++++
 test/T570-revision-tracking.sh    |  8 ++++++++
 4 files changed, 47 insertions(+)

diff --git a/doc/man7/notmuch-sexp-queries.rst b/doc/man7/notmuch-sexp-queries.rst
index d28f40bb..422154c7 100644
--- a/doc/man7/notmuch-sexp-queries.rst
+++ b/doc/man7/notmuch-sexp-queries.rst
@@ -125,6 +125,9 @@ bounds. Either upper or lower bound may be specified as ``""`` or
 ``*`` to specify the lowest possible lower bound or highest possible
 upper bound.
 
+``lastmod`` ranges support negative arguments, interpreted relative to
+the most recent database revision (see :option:`count --lastmod`).
+
 .. _field-table:
 
 .. table:: Fields with supported modifiers
diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc
index 0f14d8b7..e9ef4268 100644
--- a/lib/parse-sexp.cc
+++ b/lib/parse-sexp.cc
@@ -575,6 +575,9 @@ _sexp_parse_range (notmuch_database_t *notmuch,  const _sexp_prefix_t *prefix,
 	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 	}
 
+	if (from_idx < 0)
+	    from_idx += notmuch_database_get_revision (notmuch, NULL);
+
 	try {
 	    if (EMPTY_STRING (to))
 		to_idx = LONG_MAX;
@@ -585,6 +588,9 @@ _sexp_parse_range (notmuch_database_t *notmuch,  const _sexp_prefix_t *prefix,
 	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 	}
 
+	if (to_idx < 0)
+	    to_idx += notmuch_database_get_revision (notmuch, NULL);
+
 	output = Xapian::Query (Xapian::Query::OP_VALUE_RANGE, NOTMUCH_VALUE_LAST_MOD,
 				Xapian::sortable_serialise (from_idx),
 				Xapian::sortable_serialise (to_idx));
diff --git a/test/T081-sexpr-search.sh b/test/T081-sexpr-search.sh
index ce6b11b6..0c7db9c2 100755
--- a/test/T081-sexpr-search.sh
+++ b/test/T081-sexpr-search.sh
@@ -934,6 +934,14 @@ notmuch search lastmod:$revision..$revision | notmuch_search_sanitize > EXPECTED
 notmuch search --query=sexp  "(and (lastmod $revision))" | notmuch_search_sanitize > OUTPUT
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "lastmod query, one argument (negative)"
+notmuch tag +4EFC743A.3060609@april.org id:4EFC743A.3060609@april.org
+revision=$(notmuch count --lastmod '*' | cut -f3)
+revision1=$((revision - 1))
+notmuch search lastmod:$revision1..$revision1 | notmuch_search_sanitize > EXPECTED
+notmuch search --query=sexp  "(lastmod -1)" | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file_nonempty EXPECTED OUTPUT
+
 test_begin_subtest "lastmod query, two arguments"
 notmuch tag +keithp from:keithp
 revision2=$(notmuch count --lastmod '*' | cut -f3)
@@ -941,16 +949,38 @@ notmuch search lastmod:$revision..$revision2 | notmuch_search_sanitize > EXPECTE
 notmuch search --query=sexp  "(and (lastmod $revision $revision2))" | notmuch_search_sanitize > OUTPUT
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest "lastmod query, two arguments, first negative"
+revdiff=$((revision2 - revision))
+notmuch search lastmod:$revision..$revision2 | notmuch_search_sanitize > EXPECTED
+notmuch search --query=sexp  "(lastmod -$revdiff $revision2)" | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "lastmod query, two arguments, second negative"
+revdiff=$((revision2 - revision))
+notmuch search lastmod:..$revision | notmuch_search_sanitize > EXPECTED
+notmuch search --query=sexp  "(lastmod 0 -$revdiff)" | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest "lastmod query, lower bound only"
 notmuch search lastmod:$revision.. | notmuch_search_sanitize > EXPECTED
 notmuch search --query=sexp  "(lastmod $revision \"\")" | notmuch_search_sanitize > OUTPUT
 test_expect_equal_file_nonempty EXPECTED OUTPUT
 
+test_begin_subtest "lastmod query, lower bound only (negative)"
+notmuch search lastmod:$revision.. | notmuch_search_sanitize > EXPECTED
+notmuch search --query=sexp  "(lastmod -$revdiff \"\")" | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file_nonempty EXPECTED OUTPUT
+
 test_begin_subtest "lastmod query, upper bound only"
 notmuch search lastmod:..$revision2 | notmuch_search_sanitize > EXPECTED
 notmuch search --query=sexp  "(lastmod \"\" $revision2)" | notmuch_search_sanitize > OUTPUT
 test_expect_equal_file_nonempty EXPECTED OUTPUT
 
+test_begin_subtest "lastmod query, upper bound only (negative)"
+notmuch search lastmod:..$revision | notmuch_search_sanitize > EXPECTED
+notmuch search --query=sexp  "(lastmod \"\" -$revdiff)" | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file_nonempty EXPECTED OUTPUT
+
 test_begin_subtest "lastmod query, lower bound only, using *"
 notmuch search lastmod:$revision.. | notmuch_search_sanitize > EXPECTED
 notmuch search --query=sexp  "(lastmod $revision *)" | notmuch_search_sanitize > OUTPUT
diff --git a/test/T570-revision-tracking.sh b/test/T570-revision-tracking.sh
index e1cc684d..aaa45468 100755
--- a/test/T570-revision-tracking.sh
+++ b/test/T570-revision-tracking.sh
@@ -95,4 +95,12 @@ subtotal=$(notmuch count lastmod:..$lastmod)
 result=$(($subtotal == $total-1))
 test_expect_equal 1 "$result"
 
+if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then
+    test_begin_subtest 'exclude one message using negative lastmod (sexp)'
+    total=$(notmuch count '*')
+    notmuch tag +${RANDOM} id:4EFC743A.3060609@april.org
+    count=$(notmuch count --query=sexp '(lastmod -1 *)')
+    test_expect_equal 1 "$count"
+fi
+
 test_done
-- 
2.35.1

[PATCH 2/3] lib: factor out lastmod range handling from sexp parser.

[David Bremner]

This will permit the re-use of the same logic in the infix query
parser. The location of the shared code in the infix side is for
consistency with the other shared parsing logic. It will make more
sense when a Xapian field processor is added for the lastmod prefix.
---
 lib/Makefile.local     |  3 +-
 lib/database-private.h |  6 ++++
 lib/lastmod-fp.cc      | 68 ++++++++++++++++++++++++++++++++++++++++++
 lib/parse-sexp.cc      | 37 ++++-------------------
 4 files changed, 82 insertions(+), 32 deletions(-)
 create mode 100644 lib/lastmod-fp.cc

diff --git a/lib/Makefile.local b/lib/Makefile.local
index 6d67a2a4..4e766305 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -65,7 +65,8 @@ libnotmuch_cxx_srcs =		\
 	$(dir)/open.cc		\
 	$(dir)/init.cc		\
 	$(dir)/parse-sexp.cc	\
-	$(dir)/sexp-fp.cc
+	$(dir)/sexp-fp.cc	\
+	$(dir)/lastmod-fp.cc
 
 libnotmuch_modules := $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o)
 
diff --git a/lib/database-private.h b/lib/database-private.h
index 419b9fe6..b9be4e22 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -381,5 +381,11 @@ _notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *q
 notmuch_status_t
 _notmuch_date_strings_to_query (Xapian::valueno slot, const std::string &from, const std::string &to,
 				Xapian::Query &output, std::string &msg);
+
+/* lastmod-fp.h */
+notmuch_status_t
+_notmuch_lastmod_strings_to_query (notmuch_database_t *notmuch,
+				   const std::string &from, const std::string &to,
+				   Xapian::Query &output, std::string &msg);
 #endif
 #endif
diff --git a/lib/lastmod-fp.cc b/lib/lastmod-fp.cc
new file mode 100644
index 00000000..5fdaf281
--- /dev/null
+++ b/lib/lastmod-fp.cc
@@ -0,0 +1,68 @@
+/* lastmod-fp.cc - lastmod range query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2022 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+
+notmuch_status_t
+_notmuch_lastmod_strings_to_query (notmuch_database_t *notmuch,
+				   const std::string &from, const std::string &to,
+				   Xapian::Query &output, std::string &msg)
+{
+    long from_idx = 0L, to_idx = LONG_MAX;
+    long current;
+    std::string str;
+
+    /* revision should not change, but for the avoidance of doubt,
+     * grab for both ends of range, if needed*/
+    current = notmuch_database_get_revision (notmuch, NULL);
+
+    try {
+	if (from.empty ())
+	    from_idx = 0L;
+	else
+	    from_idx = std::stol (from);
+    } catch (std::logic_error &e) {
+	msg = "bad 'from' revision: '" + from + "'";
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    if (from_idx < 0)
+	from_idx += current;
+
+    try {
+	if (EMPTY_STRING (to))
+	    to_idx = LONG_MAX;
+	else
+	    to_idx = std::stol (to);
+    } catch (std::logic_error &e) {
+	msg = "bad 'to' revision: '" + to + "'";
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    if (to_idx < 0)
+	to_idx += current;
+
+    output = Xapian::Query (Xapian::Query::OP_VALUE_RANGE, NOTMUCH_VALUE_LAST_MOD,
+			    Xapian::sortable_serialise (from_idx),
+			    Xapian::sortable_serialise (to_idx));
+    return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc
index e9ef4268..9cadbc13 100644
--- a/lib/parse-sexp.cc
+++ b/lib/parse-sexp.cc
@@ -563,38 +563,13 @@ _sexp_parse_range (notmuch_database_t *notmuch,  const _sexp_prefix_t *prefix,
     }
 
     if (strcmp (prefix->name, "lastmod") == 0) {
-	long from_idx, to_idx;
-
-	try {
-	    if (EMPTY_STRING (from))
-		from_idx = 0L;
-	    else
-		from_idx = std::stol (from);
-	} catch (std::logic_error &e) {
-	    _notmuch_database_log (notmuch, "bad 'from' revision: '%s'\n", from);
-	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
-	}
-
-	if (from_idx < 0)
-	    from_idx += notmuch_database_get_revision (notmuch, NULL);
-
-	try {
-	    if (EMPTY_STRING (to))
-		to_idx = LONG_MAX;
-	    else
-		to_idx = std::stol (to);
-	} catch (std::logic_error &e) {
-	    _notmuch_database_log (notmuch, "bad 'to' revision: '%s'\n", to);
-	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	notmuch_status_t status;
+	status = _notmuch_lastmod_strings_to_query (notmuch, from, to, output, msg);
+	if (status) {
+	    if (! msg.empty ())
+		_notmuch_database_log (notmuch, "%s\n", msg.c_str ());
 	}
-
-	if (to_idx < 0)
-	    to_idx += notmuch_database_get_revision (notmuch, NULL);
-
-	output = Xapian::Query (Xapian::Query::OP_VALUE_RANGE, NOTMUCH_VALUE_LAST_MOD,
-				Xapian::sortable_serialise (from_idx),
-				Xapian::sortable_serialise (to_idx));
-	return NOTMUCH_STATUS_SUCCESS;
+	return status;
     }
 
     _notmuch_database_log (notmuch, "unimplimented range prefix: '%s'\n", prefix->name);
-- 
2.35.1
\r

[PATCH 3/3] lib: add field processor for lastmod: prefix

[David Bremner]

By sharing the existing logic used by the sexp query parser, this
allows negative lastmod revisions to be interpreted as relative to the
most recent revision.
---
 doc/man7/notmuch-search-terms.rst |  9 +++----
 lib/lastmod-fp.cc                 | 15 ++++++++++++
 lib/lastmod-fp.h                  | 39 +++++++++++++++++++++++++++++++
 lib/open.cc                       |  4 ++--
 test/T570-revision-tracking.sh    | 19 +++++++++++++++
 5 files changed, 80 insertions(+), 6 deletions(-)
 create mode 100644 lib/lastmod-fp.h

diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
index 4f616b7e..acc1c967 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -153,10 +153,11 @@ date:<since>..<until> or date:<date>
 lastmod:<initial-revision>..<final-revision>
     The **lastmod:** prefix can be used to restrict the result by the
     database revision number of when messages were last modified (tags
-    were added/removed or filenames changed). This is usually used in
-    conjunction with the ``--uuid`` argument to
-    :any:`notmuch-search(1)` to find messages that have changed since
-    an earlier query.
+    were added/removed or filenames changed). Negative revisions are
+    interpreted relative to the most recent database revision (see
+    :option:`count --lastmod`). This is usually used in conjunction
+    with the ``--uuid`` argument to :any:`notmuch-search(1)` to find
+    messages that have changed since an earlier query.
 
 query:<name>
     The **query:** prefix allows queries to refer to previously saved
diff --git a/lib/lastmod-fp.cc b/lib/lastmod-fp.cc
index 5fdaf281..f85efd28 100644
--- a/lib/lastmod-fp.cc
+++ b/lib/lastmod-fp.cc
@@ -21,6 +21,7 @@
  */
 
 #include "database-private.h"
+#include "lastmod-fp.h"
 
 notmuch_status_t
 _notmuch_lastmod_strings_to_query (notmuch_database_t *notmuch,
@@ -66,3 +67,17 @@ _notmuch_lastmod_strings_to_query (notmuch_database_t *notmuch,
 			    Xapian::sortable_serialise (to_idx));
     return NOTMUCH_STATUS_SUCCESS;
 }
+
+Xapian::Query
+LastModRangeProcessor::operator() (const std::string &begin, const std::string &end)
+{
+
+    Xapian::Query output;
+    std::string msg;
+
+    if (_notmuch_lastmod_strings_to_query (notmuch, begin, end, output, msg))
+	throw Xapian::QueryParserError (msg);
+
+    return output;
+}
+
diff --git a/lib/lastmod-fp.h b/lib/lastmod-fp.h
new file mode 100644
index 00000000..448241f8
--- /dev/null
+++ b/lib/lastmod-fp.h
@@ -0,0 +1,39 @@
+/* lastmod-fp.h - database revision query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2022 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_LASTMOD_FP_H
+#define NOTMUCH_LASTMOD_FP_H
+
+#include <xapian.h>
+
+class LastModRangeProcessor : public Xapian::RangeProcessor {
+protected:
+    notmuch_database_t *notmuch;
+
+public:
+    LastModRangeProcessor (notmuch_database_t *notmuch_, const std::string prefix_)
+	:  Xapian::RangeProcessor(NOTMUCH_VALUE_LAST_MOD, prefix_, 0), notmuch(notmuch_) { }
+
+    Xapian::Query operator() (const std::string &begin, const std::string &end);
+};
+
+#endif /* NOTMUCH_LASTMOD_FP_H */
diff --git a/lib/open.cc b/lib/open.cc
index 30cfcf9e..02ed7285 100644
--- a/lib/open.cc
+++ b/lib/open.cc
@@ -3,6 +3,7 @@
 
 #include "database-private.h"
 #include "parse-time-vrp.h"
+#include "lastmod-fp.h"
 #include "path-util.h"
 
 #if HAVE_XAPIAN_DB_RETRY_LOCK
@@ -431,8 +432,7 @@ _finish_open (notmuch_database_t *notmuch,
 	notmuch->value_range_processor = new Xapian::NumberRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
 	notmuch->date_range_processor = new ParseTimeRangeProcessor (NOTMUCH_VALUE_TIMESTAMP,
 								     "date:");
-	notmuch->last_mod_range_processor = new Xapian::NumberRangeProcessor (NOTMUCH_VALUE_LAST_MOD,
-									      "lastmod:");
+	notmuch->last_mod_range_processor = new LastModRangeProcessor (notmuch, "lastmod:");
 	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
 	notmuch->query_parser->set_database (*notmuch->xapian_db);
 	notmuch->stemmer = new Xapian::Stem ("english");
diff --git a/test/T570-revision-tracking.sh b/test/T570-revision-tracking.sh
index aaa45468..a7480050 100755
--- a/test/T570-revision-tracking.sh
+++ b/test/T570-revision-tracking.sh
@@ -103,4 +103,23 @@ if [ $NOTMUCH_HAVE_SFSEXP -eq 1 ]; then
     test_expect_equal 1 "$count"
 fi
 
+test_begin_subtest 'exclude one message using negative lastmod'
+total=$(notmuch count '*')
+notmuch tag +${RANDOM} id:4EFC743A.3060609@april.org
+count=$(notmuch count lastmod:-1..)
+test_expect_equal 1 "$count"
+
+test_begin_subtest 'exclude one message using negative lastmod (second param)'
+total=$(notmuch count '*')
+notmuch tag +${RANDOM} id:4EFC743A.3060609@april.org
+count=$(notmuch count lastmod:..-1)
+test_expect_equal 51 "$count"
+
+test_begin_subtest 'negative lastmod (two parameters)'
+notmuch tag +${RANDOM} '*'
+before=$(notmuch count --lastmod '*' | cut -f3)
+notmuch tag +${RANDOM} id:4EFC743A.3060609@april.org
+count=$(notmuch count lastmod:-100..$before)
+test_expect_equal 51 "$count"
+
 test_done
-- 
2.35.1
\r

Re: [PATCH 1/3] lib/sexp: provide relative lastmod queries

[David Bremner]

David Bremner <david@tethera.net> writes:

> Test the relatively trivial logic changes for the sexp query parser
> first before refactoring that logic to share with the infix query
> parser.

series applied to master.

d