v6 of nondestructive excludes

v6 of nondestructive excludes

[David Bremner]

I decided to squash the first two patches
id:20170225160913.22844-1-david@tethera.net together to avoid temporarily marking a test broken.

Otherwise the net changes are pretty minimal since the last version.

diff --git a/lib/query.cc b/lib/query.cc
index 2c6a4ba6..59e9141a 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -137,13 +137,15 @@ _notmuch_query_ensure_parsed (notmuch_query_t *query)
 	query->parsed = TRUE;
 
     } catch (const Xapian::Error &error) {
-	_notmuch_database_log (query->notmuch,
-			       "A Xapian exception occured parsing query: %s\n",
-			       error.get_msg().c_str());
-	_notmuch_database_log_append (query->notmuch,
-			       "Query string was: %s\n",
-			       query->query_string);
-	query->notmuch->exception_reported = TRUE;
+	if (!query->notmuch->exception_reported) {
+	    _notmuch_database_log (query->notmuch,
+				   "A Xapian exception occurred parsing query: %s\n",
+				   error.get_msg ().c_str ());
+	    _notmuch_database_log_append (query->notmuch,
+					  "Query string was: %s\n",
+					  query->query_string);
+	    query->notmuch->exception_reported = TRUE;
+	}
 
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
diff --git a/test/T650-regexp-query.sh b/test/T650-regexp-query.sh
index a8039610..df48ab82 100755
--- a/test/T650-regexp-query.sh
+++ b/test/T650-regexp-query.sh
@@ -74,7 +74,7 @@ test_begin_subtest "regexp error reporting"
 notmuch search 'from:/unbalanced[/' 1>OUTPUT 2>&1
 cat <<EOF > EXPECTED
 notmuch search: A Xapian exception occurred
-A Xapian exception occurred performing query: Invalid regular expression
+A Xapian exception occurred parsing query: Invalid regular expression
 Query string was: from:/unbalanced[/
 EOF
 test_expect_equal_file EXPECTED OUTPUT

[PATCH 1/2] lib: centralize query parsing, store results.

[David Bremner]

The main goal is to prepare the way for non-destructive (or at least
less destructive) exclude tag handling. It does this by having a
pre-parsed query available for further processing. This also allows us
to provide slightly more precise errorr messages.
---
 lib/query.cc              | 65 +++++++++++++++++++++++++++++++++++++++++------
 test/T650-regexp-query.sh |  2 +-
 2 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/lib/query.cc b/lib/query.cc
index 4ccd8104..ba06bfeb 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -29,6 +29,8 @@ struct _notmuch_query {
     notmuch_sort_t sort;
     notmuch_string_list_t *exclude_terms;
     notmuch_exclude_t omit_excluded;
+    notmuch_bool_t parsed;
+    Xapian::Query xapian_query;
 };
 
 typedef struct _notmuch_mset_messages {
@@ -71,6 +73,13 @@ _debug_query (void)
     return (env && strcmp (env, "") != 0);
 }
 
+/* Explicit destructor call for placement new */
+static int
+_notmuch_query_destructor (notmuch_query_t *query) {
+    query->xapian_query.~Query();
+    return 0;
+}
+
 notmuch_query_t *
 notmuch_query_create (notmuch_database_t *notmuch,
 		      const char *query_string)
@@ -84,6 +93,11 @@ notmuch_query_create (notmuch_database_t *notmuch,
     if (unlikely (query == NULL))
 	return NULL;
 
+    new (&query->xapian_query) Xapian::Query ();
+    query->parsed = FALSE;
+
+    talloc_set_destructor (query, _notmuch_query_destructor);
+
     query->notmuch = notmuch;
 
     query->query_string = talloc_strdup (query, query_string);
@@ -97,6 +111,35 @@ notmuch_query_create (notmuch_database_t *notmuch,
     return query;
 }
 
+static notmuch_status_t
+_notmuch_query_ensure_parsed (notmuch_query_t *query)
+{
+    if (query->parsed)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    try {
+	query->xapian_query =
+	    query->notmuch->query_parser->
+		parse_query (query->query_string, NOTMUCH_QUERY_PARSER_FLAGS);
+
+	query->parsed = TRUE;
+
+    } catch (const Xapian::Error &error) {
+	if (!query->notmuch->exception_reported) {
+	    _notmuch_database_log (query->notmuch,
+				   "A Xapian exception occurred parsing query: %s\n",
+				   error.get_msg ().c_str ());
+	    _notmuch_database_log_append (query->notmuch,
+					  "Query string was: %s\n",
+					  query->query_string);
+	    query->notmuch->exception_reported = TRUE;
+	}
+
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 const char *
 notmuch_query_get_query_string (const notmuch_query_t *query)
 {
@@ -198,6 +241,11 @@ _notmuch_query_search_documents (notmuch_query_t *query,
     notmuch_database_t *notmuch = query->notmuch;
     const char *query_string = query->query_string;
     notmuch_mset_messages_t *messages;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
 
     messages = talloc (query, notmuch_mset_messages_t);
     if (unlikely (messages == NULL))
@@ -217,7 +265,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 	Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
 						   _find_prefix ("type"),
 						   type));
-	Xapian::Query string_query, final_query, exclude_query;
+	Xapian::Query final_query, exclude_query;
 	Xapian::MSet mset;
 	Xapian::MSetIterator iterator;
 
@@ -226,10 +274,8 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 	{
 	    final_query = mail_query;
 	} else {
-	    string_query = notmuch->query_parser->
-		parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
 	    final_query = Xapian::Query (Xapian::Query::OP_AND,
-					 mail_query, string_query);
+					 mail_query, query->xapian_query);
 	}
 	messages->base.excluded_doc_ids = NULL;
 
@@ -566,13 +612,18 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
     notmuch_database_t *notmuch = query->notmuch;
     const char *query_string = query->query_string;
     Xapian::doccount count = 0;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
 
     try {
 	Xapian::Enquire enquire (*notmuch->xapian_db);
 	Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
 						   _find_prefix ("type"),
 						   type));
-	Xapian::Query string_query, final_query, exclude_query;
+	Xapian::Query final_query, exclude_query;
 	Xapian::MSet mset;
 
 	if (strcmp (query_string, "") == 0 ||
@@ -580,10 +631,8 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
 	{
 	    final_query = mail_query;
 	} else {
-	    string_query = notmuch->query_parser->
-		parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
 	    final_query = Xapian::Query (Xapian::Query::OP_AND,
-					 mail_query, string_query);
+					 mail_query, query->xapian_query);
 	}
 
 	exclude_query = _notmuch_exclude_tags (query, final_query);
diff --git a/test/T650-regexp-query.sh b/test/T650-regexp-query.sh
index a8039610..df48ab82 100755
--- a/test/T650-regexp-query.sh
+++ b/test/T650-regexp-query.sh
@@ -74,7 +74,7 @@ test_begin_subtest "regexp error reporting"
 notmuch search 'from:/unbalanced[/' 1>OUTPUT 2>&1
 cat <<EOF > EXPECTED
 notmuch search: A Xapian exception occurred
-A Xapian exception occurred performing query: Invalid regular expression
+A Xapian exception occurred parsing query: Invalid regular expression
 Query string was: from:/unbalanced[/
 EOF
 test_expect_equal_file EXPECTED OUTPUT
-- 
2.11.0

[PATCH 2/2] lib: query make exclude handling non-destructive

[David Bremner]

We filter added exclude at add time, rather than modifying the query by
count search. As noted in the comments, there are several ignored
conditions here.
---
 lib/query.cc       | 55 ++++++++++++++++++++++++++++++++++++------------------
 test/T060-count.sh |  1 -
 2 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/lib/query.cc b/lib/query.cc
index ba06bfeb..59e9141a 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -31,6 +31,7 @@ struct _notmuch_query {
     notmuch_exclude_t omit_excluded;
     notmuch_bool_t parsed;
     Xapian::Query xapian_query;
+    std::set<std::string> terms;
 };
 
 typedef struct _notmuch_mset_messages {
@@ -77,6 +78,7 @@ _debug_query (void)
 static int
 _notmuch_query_destructor (notmuch_query_t *query) {
     query->xapian_query.~Query();
+    query->terms.~set<std::string>();
     return 0;
 }
 
@@ -94,6 +96,7 @@ notmuch_query_create (notmuch_database_t *notmuch,
 	return NULL;
 
     new (&query->xapian_query) Xapian::Query ();
+    new (&query->terms) std::set<std::string> ();
     query->parsed = FALSE;
 
     talloc_set_destructor (query, _notmuch_query_destructor);
@@ -122,6 +125,15 @@ _notmuch_query_ensure_parsed (notmuch_query_t *query)
 	    query->notmuch->query_parser->
 		parse_query (query->query_string, NOTMUCH_QUERY_PARSER_FLAGS);
 
+       /* Xapian doesn't support skip_to on terms from a query since
+	*  they are unordered, so cache a copy of all terms in
+	*  something searchable.
+	*/
+
+	for (Xapian::TermIterator t = query->xapian_query.get_terms_begin ();
+	     t != query->xapian_query.get_terms_end (); ++t)
+	    query->terms.insert (*t);
+
 	query->parsed = TRUE;
 
     } catch (const Xapian::Error &error) {
@@ -168,7 +180,25 @@ notmuch_query_get_sort (const notmuch_query_t *query)
 void
 notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag)
 {
-    char *term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
+    notmuch_status_t status;
+    char *term;
+
+    status = _notmuch_query_ensure_parsed (query);
+    /* The following is not ideal error handling, but to avoid
+     * breaking the ABI, we can live with it for now. In particular at
+     * least in the notmuch CLI, any syntax error in the query is
+     * caught in a later call to _notmuch_query_ensure_parsed with a
+     * better error path.
+     *
+     * TODO: add status return to this function.
+     */
+    if (status)
+	return;
+
+    term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
+    if (query->terms.count(term) != 0)
+	return; /* XXX report ignoring exclude? */
+
     _notmuch_string_list_append (query->exclude_terms, term);
 }
 
@@ -188,28 +218,17 @@ _notmuch_messages_destructor (notmuch_mset_messages_t *messages)
 }
 
 /* Return a query that matches messages with the excluded tags
- * registered with query.  Any tags that explicitly appear in xquery
- * will not be excluded, and will be removed from the list of exclude
- * tags.  The caller of this function has to combine the returned
+ * registered with query. The caller of this function has to combine the returned
  * query appropriately.*/
 static Xapian::Query
-_notmuch_exclude_tags (notmuch_query_t *query, Xapian::Query xquery)
+_notmuch_exclude_tags (notmuch_query_t *query)
 {
     Xapian::Query exclude_query = Xapian::Query::MatchNothing;
 
     for (notmuch_string_node_t *term = query->exclude_terms->head; term;
 	 term = term->next) {
-	Xapian::TermIterator it = xquery.get_terms_begin ();
-	Xapian::TermIterator end = xquery.get_terms_end ();
-	for (; it != end; it++) {
-	    if ((*it).compare (term->string) == 0)
-		break;
-	}
-	if (it == end)
-	    exclude_query = Xapian::Query (Xapian::Query::OP_OR,
-				    exclude_query, Xapian::Query (term->string));
-	else
-	    term->string = talloc_strdup (query, "");
+	exclude_query = Xapian::Query (Xapian::Query::OP_OR,
+				       exclude_query, Xapian::Query (term->string));
     }
     return exclude_query;
 }
@@ -280,7 +299,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 	messages->base.excluded_doc_ids = NULL;
 
 	if ((query->omit_excluded != NOTMUCH_EXCLUDE_FALSE) && (query->exclude_terms)) {
-	    exclude_query = _notmuch_exclude_tags (query, final_query);
+	    exclude_query = _notmuch_exclude_tags (query);
 
 	    if (query->omit_excluded == NOTMUCH_EXCLUDE_TRUE ||
 		query->omit_excluded == NOTMUCH_EXCLUDE_ALL)
@@ -635,7 +654,7 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
 					 mail_query, query->xapian_query);
 	}
 
-	exclude_query = _notmuch_exclude_tags (query, final_query);
+	exclude_query = _notmuch_exclude_tags (query);
 
 	final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
 					 final_query, exclude_query);
diff --git a/test/T060-count.sh b/test/T060-count.sh
index d27e1bab..4751440e 100755
--- a/test/T060-count.sh
+++ b/test/T060-count.sh
@@ -127,7 +127,6 @@ test_expect_equal_file EXPECTED OUTPUT.clean
 restore_database
 
 test_begin_subtest "count library function is non-destructive"
-test_subtest_known_broken
 cat<<EOF > EXPECTED
 1: 52 messages
 2: 52 messages
-- 
2.11.0

Re: [PATCH 1/2] lib: centralize query parsing, store results.

[David Bremner]

David Bremner <david@tethera.net> writes:

> The main goal is to prepare the way for non-destructive (or at least
> less destructive) exclude tag handling. It does this by having a
> pre-parsed query available for further processing. This also allows us
> to provide slightly more precise errorr messages.

series pushed to master, with the spelling errorr fixed.

d