single argument date: queries

single argument date: queries

[David Bremner]

These rely on a feature called "Field Processors" that needs a pretty
new version of xapian, so it's optional. I haven't thought through all
the issues involved with having optional features in (lib)notmuch. I
guess it would make sense to have some API that tells you what
optional features are available. At least in this case there doesn't
seem to be any relationship with database features. We also need to
figure out how to document optional features; presumably it would be
helpful to tell people how to interrogate notmuch (CLI) to see if a
given feature is supported.

d

[PATCH 1/3] config: autodetect xapian-1.3

[David Bremner]

Mimic the handling of python2 versus python3. In particular if both
xapian-config and xapian-config-1.3 are found, use xapian-config
---
 configure | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index a79f6bd..eb6dbac 100755
--- a/configure
+++ b/configure
@@ -51,7 +51,7 @@ CPPFLAGS=${CPPFLAGS:-}
 CXXFLAGS_for_sh=${CXXFLAGS:-${CFLAGS}}
 CXXFLAGS=${CXXFLAGS:-\$(CFLAGS)}
 LDFLAGS=${LDFLAGS:-}
-XAPIAN_CONFIG=${XAPIAN_CONFIG:-xapian-config}
+XAPIAN_CONFIG=${XAPIAN_CONFIG:-}
 PYTHON=${PYTHON:-}
 
 # We don't allow the EMACS or GZIP Makefile variables inherit values
@@ -341,7 +341,7 @@ fi
 
 printf "Checking for Xapian development files... "
 have_xapian=0
-for xapian_config in ${XAPIAN_CONFIG}; do
+for xapian_config in ${XAPIAN_CONFIG} xapian-config xapian-config-1.3; do
     if ${xapian_config} --version > /dev/null 2>&1; then
 	xapian_version=$(${xapian_config} --version | sed -e 's/.* //')
 	printf "Yes (%s).\n" ${xapian_version}
-- 
2.7.0

[PATCH 2/3] configure: detect Xapian:FieldProcessor

[David Bremner]

Rather than check versions, it seems more robust to do a test compile.
---
 configure | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/configure b/configure
index eb6dbac..8feb23f 100755
--- a/configure
+++ b/configure
@@ -356,9 +356,10 @@ if [ ${have_xapian} = "0" ]; then
     errors=$((errors + 1))
 fi
 
-# Compaction is only supported on Xapian > 1.2.6
 have_xapian_compact=0
+have_xapian_field_processor=0
 if [ ${have_xapian} = "1" ]; then
+    # Compaction is only supported on Xapian > 1.2.6
     printf "Checking for Xapian compaction support... "
     case "${xapian_version}" in
 	0.*|1.[01].*|1.2.[0-5])
@@ -369,6 +370,25 @@ if [ ${have_xapian} = "1" ]; then
 	*)
 	    printf "Unknown version.\n" ;;
     esac
+
+    printf "Checking for Xapian FieldProcessor API... "
+    cat>_field_processor.cc<<EOF
+#include <xapian.h>
+class TitleFieldProcessor : public Xapian::FieldProcessor {
+    Xapian::Query operator()(const std::string & str) {
+        return Xapian::Query::MatchAll;
+    }
+};
+EOF
+    if ${CXX} ${CXXLAGS} ${xapian_cxxflags} -c _field_processor.cc -o _field_processor.o > /dev/null 2>&1
+    then
+	have_xapian_field_processor=1
+	printf "Yes.\n"
+    else
+	printf "No. (optional)\n"
+    fi
+
+    rm -f _field_processor.o _field_processor.cc
 fi
 
 
@@ -979,6 +999,9 @@ HAVE_D_TYPE = ${have_d_type}
 # Whether the Xapian version in use supports compaction
 HAVE_XAPIAN_COMPACT = ${have_xapian_compact}
 
+# Whether the Xapian version in use supports field processors
+HAVE_XAPIAN_FIELD_PROCESSOR = ${have_xapian_field_processor}
+
 # Whether the getpwuid_r function is standards-compliant
 # (if not, then notmuch will #define _POSIX_PTHREAD_SEMANTICS
 # to enable the standards-compliant version -- needed for Solaris)
@@ -1050,6 +1073,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)      \\
 		   -DSTD_GETPWUID=\$(STD_GETPWUID)                       \\
 		   -DSTD_ASCTIME=\$(STD_ASCTIME)                         \\
 		   -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT)	 \\
+		   -DHAVE_XAPIAN_FIELD_PROCESSOR=\$(HAVE_XAPIAN_PROCESSOR) \\
 		   -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 
 CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)    \\
@@ -1064,6 +1088,7 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)    \\
 		     -DSTD_GETPWUID=\$(STD_GETPWUID)                     \\
 		     -DSTD_ASCTIME=\$(STD_ASCTIME)                       \\
 		     -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT)       \\
+		     -DHAVE_XAPIAN_FIELD_PROCESSOR=\$(HAVE_XAPIAN_FIELD_PROCESSOR) \\
 		     -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 
 CONFIGURE_LDFLAGS =  \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS)
@@ -1077,6 +1102,9 @@ cat > sh.config <<EOF
 # Whether the Xapian version in use supports compaction
 NOTMUCH_HAVE_XAPIAN_COMPACT=${have_xapian_compact}
 
+# Whether the Xapian version in use supports field processors
+NOTMUCH_HAVE_XAPIAN_FIELD_PROCESSOR=${have_xapian_field_processor}
+
 # do we have man pages?
 NOTMUCH_HAVE_MAN=$((have_sphinx))
 
-- 
2.7.0

[PATCH] lib/cli: add library API / CLI for compile time options

[David Bremner]

This is intentionally low tech; if we have more than two options it may
make sense to build up what infrastructure is provided.
---

Here is attempt at answering some of the questions in
id:1458437904-17677-1-git-send-email-david@tethera.net

 lib/Makefile.local  |  1 +
 lib/notmuch.h       | 10 ++++++++++
 lib/options.c       | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 notmuch-config.c    | 20 ++++++++++++++++++++
 test/T030-config.sh |  6 ++++--
 test/T040-setup.sh  |  6 ++++--
 test/test-lib.sh    |  6 ++++++
 7 files changed, 95 insertions(+), 4 deletions(-)
 create mode 100644 lib/options.c

diff --git a/lib/Makefile.local b/lib/Makefile.local
index 3a07090..4ad0158 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -39,6 +39,7 @@ libnotmuch_c_srcs =		\
 	$(dir)/message-file.c	\
 	$(dir)/messages.c	\
 	$(dir)/sha1.c		\
+	$(dir)/options.c	\
 	$(dir)/tags.c
 
 libnotmuch_cxx_srcs =		\
diff --git a/lib/notmuch.h b/lib/notmuch.h
index cb46fc0..b29dd5f 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1838,6 +1838,16 @@ notmuch_filenames_move_to_next (notmuch_filenames_t *filenames);
 void
 notmuch_filenames_destroy (notmuch_filenames_t *filenames);
 
+typedef enum {
+    NOTMUCH_OPTION_COMPACT = 1,
+    NOTMUCH_OPTION_FIELD_PROCESSOR = 2
+} notmuch_option_t;
+
+notmuch_bool_t
+notmuch_options_present (notmuch_option_t mask);
+
+notmuch_bool_t
+notmuch_options_get (const char *name);
 /* @} */
 
 NOTMUCH_END_DECLS
diff --git a/lib/options.c b/lib/options.c
new file mode 100644
index 0000000..4e15d92
--- /dev/null
+++ b/lib/options.c
@@ -0,0 +1,50 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch.h"
+#include "notmuch-private.h"
+
+notmuch_bool_t
+notmuch_options_present (notmuch_option_t mask)
+{
+    notmuch_option_t present = 0;
+
+#if HAVE_XAPIAN_COMPACT
+    present |= NOTMUCH_OPTION_COMPACT;
+#endif
+
+#if HAVE_XAPIAN_COMPACT
+    present |= NOTMUCH_OPTION_FIELD_PROCESSOR;
+#endif
+
+    return (mask & present) != 0;
+
+}
+
+notmuch_bool_t
+notmuch_options_get (const char *name) {
+    if (STRNCMP_LITERAL (name, "compact") == 0) {
+	return notmuch_options_present (NOTMUCH_OPTION_COMPACT);
+    } else if (STRNCMP_LITERAL (name, "field_processor") == 0) {
+	return notmuch_options_present (NOTMUCH_OPTION_FIELD_PROCESSOR);
+    } else {
+	return FALSE;
+    }
+}
diff --git a/notmuch-config.c b/notmuch-config.c
index d252bb2..cfc549d 100644
--- a/notmuch-config.c
+++ b/notmuch-config.c
@@ -750,6 +750,8 @@ _item_split (char *item, char **group, char **key)
     return 0;
 }
 
+#define OPTION_PREFIX "options."
+
 static int
 notmuch_config_command_get (notmuch_config_t *config, char *item)
 {
@@ -773,6 +775,9 @@ notmuch_config_command_get (notmuch_config_t *config, char *item)
 	tags = notmuch_config_get_new_tags (config, &length);
 	for (i = 0; i < length; i++)
 	    printf ("%s\n", tags[i]);
+    } else if (STRNCMP_LITERAL (item, OPTION_PREFIX) == 0) {
+	printf ("%s\n",
+	       notmuch_options_get (item + strlen (OPTION_PREFIX)) ? "true" : "false");
     } else {
 	char **value;
 	size_t i, length;
@@ -804,6 +809,11 @@ notmuch_config_command_set (notmuch_config_t *config, char *item, int argc, char
 {
     char *group, *key;
 
+    if (STRNCMP_LITERAL (item, OPTION_PREFIX) == 0) {
+	fprintf (stderr, "Error: read only option: %s\n", item);
+	return 1;
+    }
+
     if (_item_split (item, &group, &key))
 	return 1;
 
@@ -830,6 +840,15 @@ notmuch_config_command_set (notmuch_config_t *config, char *item, int argc, char
     return notmuch_config_save (config);
 }
 
+static
+void
+_notmuch_config_list_options () {
+    printf("options.compact=%s\n",
+	   notmuch_options_present(NOTMUCH_OPTION_COMPACT) ? "true" : "false");
+    printf("options.field_processor=%s\n",
+	   notmuch_options_present(NOTMUCH_OPTION_FIELD_PROCESSOR) ? "true" : "false");
+}
+
 static int
 notmuch_config_command_list (notmuch_config_t *config)
 {
@@ -865,6 +884,7 @@ notmuch_config_command_list (notmuch_config_t *config)
 
     g_strfreev (groups);
 
+    _notmuch_config_list_options ();
     return 0;
 }
 
diff --git a/test/T030-config.sh b/test/T030-config.sh
index f404908..c37ba21 100755
--- a/test/T030-config.sh
+++ b/test/T030-config.sh
@@ -44,7 +44,7 @@ test_expect_equal "$(notmuch config get foo.nonexistent)" ""
 
 test_begin_subtest "List all items"
 notmuch config set database.path "/canonical/path"
-output=$(notmuch config list)
+output=$(notmuch config list | notmuch_options_sanitize)
 test_expect_equal "$output" "\
 database.path=/canonical/path
 user.name=Notmuch Test Suite
@@ -56,7 +56,9 @@ search.exclude_tags=
 maildir.synchronize_flags=true
 crypto.gpg_path=gpg
 foo.string=this is another string value
-foo.list=this;is another;list value;"
+foo.list=this;is another;list value;
+options.compact=something
+options.field_processor=something"
 
 test_begin_subtest "Top level --config=FILE option"
 cp "${NOTMUCH_CONFIG}" alt-config
diff --git a/test/T040-setup.sh b/test/T040-setup.sh
index cf0c00b..5db03a6 100755
--- a/test/T040-setup.sh
+++ b/test/T040-setup.sh
@@ -19,7 +19,7 @@ another.suite@example.com
 foo bar
 baz
 EOF
-output=$(notmuch --config=new-notmuch-config config list)
+output=$(notmuch --config=new-notmuch-config config list | notmuch_options_sanitize)
 test_expect_equal "$output" "\
 database.path=/path/to/maildir
 user.name=Test Suite
@@ -29,6 +29,8 @@ new.tags=foo;bar;
 new.ignore=
 search.exclude_tags=baz;
 maildir.synchronize_flags=true
-crypto.gpg_path=gpg"
+crypto.gpg_path=gpg
+options.compact=something
+options.field_processor=something"
 
 test_done
diff --git a/test/test-lib.sh b/test/test-lib.sh
index cc08a98..49e6f40 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -733,6 +733,12 @@ notmuch_uuid_sanitize ()
 {
     sed 's/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/UUID/g'
 }
+
+notmuch_options_sanitize ()
+{
+    sed 's/^options[.]\(.*\)=.*$/options.\1=something/'
+}
+
 # End of notmuch helper functions
 
 # Use test_set_prereq to tell that a particular prerequisite is available.
-- 
2.7.0

[PATCH] configure: check directly for xapian compaction API

[David Bremner]

This is consistent with the check for FieldProcessor, and probably a bit
more robust.
---

This is really an independent cleanup, but it will probably not apply
cleanly without the rest of the series because of code movement.

 configure | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/configure b/configure
index c48f7ce..03f28d5 100755
--- a/configure
+++ b/configure
@@ -359,17 +359,18 @@ fi
 have_xapian_compact=0
 have_xapian_field_processor=0
 if [ ${have_xapian} = "1" ]; then
-    # Compaction is only supported on Xapian > 1.2.6
     printf "Checking for Xapian compaction support... "
-    case "${xapian_version}" in
-	0.*|1.[01].*|1.2.[0-5])
-	    printf "No (only available with Xapian > 1.2.6).\n" ;;
-	[1-9]*.[0-9]*.[0-9]*)
-	    have_xapian_compact=1
-	    printf "Yes.\n" ;;
-	*)
-	    printf "Unknown version.\n" ;;
-    esac
+    cat>_compact.cc<<EOF
+#include <xapian.h>
+class TestCompactor : public Xapian::Compactor { };
+EOF
+    if ${CXX} ${CXXLAGS} ${xapian_cxxflags} -c _compact.cc -o _compact.o > /dev/null 2>&1
+    then
+	have_xapian_compact=1
+	printf "Yes.\n"
+    else
+	printf "No.\n"
+    fi
 
     printf "Checking for Xapian FieldProcessor API... "
     cat>_field_processor.cc<<EOF
-- 
2.7.0

[PATCH 3/3] lib: optionally support single argument date: queries

[David Bremner]

This relies on the FieldProcessor API, which is only present in xapian
>= 1.3.
---
 lib/database-private.h   |  4 ++++
 lib/database.cc          |  6 ++++++
 lib/parse-time-vrp.cc    | 21 +++++++++++++++++++++
 lib/parse-time-vrp.h     |  5 +++++
 test/T500-search-date.sh |  6 ++++++
 5 files changed, 42 insertions(+)

diff --git a/lib/database-private.h b/lib/database-private.h
index 3fb10f7..fb2c8ec 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -144,6 +144,7 @@ operator&=(_notmuch_features &a, _notmuch_features b)
     return a;
 }
 
+class DateFieldProcessor;
 struct _notmuch_database {
     notmuch_bool_t exception_reported;
 
@@ -176,6 +177,9 @@ struct _notmuch_database {
     Xapian::TermGenerator *term_gen;
     Xapian::ValueRangeProcessor *value_range_processor;
     Xapian::ValueRangeProcessor *date_range_processor;
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+    DateFieldProcessor *date_field_processor;
+#endif
     Xapian::ValueRangeProcessor *last_mod_range_processor;
 };
 
diff --git a/lib/database.cc b/lib/database.cc
index 3b342f1..0e3c90c 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -1000,6 +1000,12 @@ notmuch_database_open_verbose (const char *path,
 	notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
 	notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
 	notmuch->date_range_processor = new ParseTimeValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+	/* This currently relies on the query parser to pass anything
+	 * with a .. to the range processor */
+	notmuch->date_field_processor = new DateFieldProcessor();
+	notmuch->query_parser->add_boolean_prefix("date", notmuch->date_field_processor);
+#endif
 	notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
 
 	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
diff --git a/lib/parse-time-vrp.cc b/lib/parse-time-vrp.cc
index 03804cf..b15b77c 100644
--- a/lib/parse-time-vrp.cc
+++ b/lib/parse-time-vrp.cc
@@ -64,3 +64,24 @@ ParseTimeValueRangeProcessor::operator() (std::string &begin, std::string &end)
 
     return valno;
 }
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+/* XXX TODO: is throwing an exception the right thing to do here? */
+Xapian::Query DateFieldProcessor::operator()(const std::string & str) {
+    time_t from, to, now;
+
+    /* Use the same 'now' for begin and end. */
+    if (time (&now) == (time_t) -1)
+	throw Xapian::QueryParserError("Unable to get current time");
+
+    if (parse_time_string (str.c_str (), &from, &now, PARSE_TIME_ROUND_DOWN))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+
+    if (parse_time_string (str.c_str (), &to, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+
+    return Xapian::Query(Xapian::Query::OP_AND,
+			 Xapian::Query(Xapian::Query::OP_VALUE_GE, 0, Xapian::sortable_serialise ((double) from)),
+			 Xapian::Query(Xapian::Query::OP_VALUE_LE, 0, Xapian::sortable_serialise ((double) to)));
+}
+#endif
diff --git a/lib/parse-time-vrp.h b/lib/parse-time-vrp.h
index 094c4f8..3bd12bf 100644
--- a/lib/parse-time-vrp.h
+++ b/lib/parse-time-vrp.h
@@ -37,4 +37,9 @@ public:
     Xapian::valueno operator() (std::string &begin, std::string &end);
 };
 
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+class DateFieldProcessor : public Xapian::FieldProcessor {
+    Xapian::Query operator()(const std::string & str);
+};
+#endif
 #endif /* NOTMUCH_PARSE_TIME_VRP_H */
diff --git a/test/T500-search-date.sh b/test/T500-search-date.sh
index f5cea42..198a2e6 100755
--- a/test/T500-search-date.sh
+++ b/test/T500-search-date.sh
@@ -12,6 +12,12 @@ test_begin_subtest "Absolute date range with 'same' operator"
 output=$(notmuch search date:2010-12-16..! | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2010-12-16 [1/1] Olivier Berger; Essai accentué (inbox unread)"
 
+if [ "${NOTMUCH_HAVE_XAPIAN_FIELD_PROCESSOR}" = "1" ]; then
+    test_begin_subtest "Absolute date field"
+    output=$(notmuch search date:2010-12-16 | notmuch_search_sanitize)
+    test_expect_equal "$output" "thread:XXX   2010-12-16 [1/1] Olivier Berger; Essai accentué (inbox unread)"
+fi
+
 test_begin_subtest "Absolute time range with TZ"
 notmuch search date:18-Nov-2009_02:19:26-0800..2009-11-18_04:49:52-06:00 | notmuch_search_sanitize > OUTPUT
 cat <<EOF >EXPECTED
-- 
2.7.0