unofficial mirror of notmuch@notmuchmail.org
 help / color / mirror / code / Atom feed
* v4 sexp query parser
@ 2021-08-12 17:06 David Bremner
  2021-08-12 17:06 ` [PATCH 01/31] configure: optional library sfsexp David Bremner
                   ` (30 more replies)
  0 siblings, 31 replies; 32+ messages in thread
From: David Bremner @ 2021-08-12 17:06 UTC (permalink / raw)
  To: notmuch

Two main user visible changes since v3.

1) I changed the option from

    --query-syntax=sexp

    to

    --query=sexp

This is arguably a bit more ambiguous (are we specifying a query
"sexp" to search for?) but considerably less painful to type. I
previously mentioned `--squery`, but it is a bit cryptic, and collides with completing other options that start with 's'.

2) The new form (macro (param1 param2) body). This is a bit of feature
creep, but I realized it is one option for dealing with requests like id:87k0nl8nvi.fsf@linux.ibm.com.

The series still only enables the new syntax in notmuch-search and
notmuch-address. I was pondering making the query syntax a global
option handled in the same was as --uuid, but I didn't get that far
yet.

Interdiff
---------

The change to the command line option introduces a bunch of noise to
the interdiff, which I've omitted here. A full diffstat is

doc/man1/notmuch-config.rst       |   5 +
doc/man7/notmuch-sexp-queries.rst |  48 ++++-
lib/parse-sexp.cc                 | 199 ++++++++++++++++++--
notmuch-config.c                  |   1 +
notmuch-search.c                  |   4 +-
test/T081-sexpr-search.sh         | 431 ++++++++++++++++++++++++++++++-------------
 test/T095-address.sh              |   2 +-

The core changes are as follows:

diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc
index e582e350..eea4d4da 100644
--- a/lib/parse-sexp.cc
+++ b/lib/parse-sexp.cc
@@ -7,6 +7,20 @@
 /* _sexp is used for file scope symbols to avoid clashing with
  * definitions from sexp.h */
 
+/* sexp_binding structs attach name to a sexp and a defining
+ * context. The latter allows lazy evaluation of parameters whose
+ * definition contains other parameters.  Lazy evaluation is needed
+ * because a primary goal of macros is to change the parent field for
+ * a sexp.
+ */
+
+typedef struct sexp_binding {
+    const char *name;
+    const sexp_t *sx;
+    const struct sexp_binding *context;
+    const struct sexp_binding *next;
+} _sexp_binding_t;
+
 typedef enum {
     SEXP_FLAG_NONE	= 0,
     SEXP_FLAG_FIELD	= 1 << 0,
@@ -99,12 +113,14 @@ static _sexp_prefix_t prefixes[] =
 
 static notmuch_status_t _sexp_to_xapian_query (notmuch_database_t *notmuch,
 					       const _sexp_prefix_t *parent,
+					       const _sexp_binding_t *env,
 					       const sexp_t *sx,
 					       Xapian::Query &output);
 
 static notmuch_status_t
 _sexp_combine_query (notmuch_database_t *notmuch,
 		     const _sexp_prefix_t *parent,
+		     const _sexp_binding_t *env,
 		     Xapian::Query::op operation,
 		     Xapian::Query left,
 		     const sexp_t *sx,
@@ -121,12 +137,13 @@ _sexp_combine_query (notmuch_database_t *notmuch,
 	return NOTMUCH_STATUS_SUCCESS;
     }
 
-    status = _sexp_to_xapian_query (notmuch, parent, sx, subquery);
+    status = _sexp_to_xapian_query (notmuch, parent, env, sx, subquery);
     if (status)
 	return status;
 
     return _sexp_combine_query (notmuch,
 				parent,
+				env,
 				operation,
 				Xapian::Query (operation, left, subquery),
 				sx->next, output);
@@ -165,6 +182,7 @@ _sexp_parse_phrase (std::string term_prefix, const char *phrase, Xapian::Query &
 static notmuch_status_t
 _sexp_parse_wildcard (notmuch_database_t *notmuch,
 		      const _sexp_prefix_t *parent,
+		      unused(const _sexp_binding_t *env),
 		      std::string match,
 		      Xapian::Query &output)
 {
@@ -201,6 +219,7 @@ _sexp_parse_one_term (notmuch_database_t *notmuch, std::string term_prefix, cons
 notmuch_status_t
 _sexp_parse_regex (notmuch_database_t *notmuch,
 		   const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent,
+		   unused(const _sexp_binding_t *env),
 		   std::string val, Xapian::Query &output)
 {
     if (! parent) {
@@ -225,7 +244,7 @@ _sexp_parse_regex (notmuch_database_t *notmuch,
 static notmuch_status_t
 _sexp_expand_query (notmuch_database_t *notmuch,
 		    const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent,
-		    const sexp_t *sx, Xapian::Query &output)
+		    unused(const _sexp_binding_t *env), const sexp_t *sx, Xapian::Query &output)
 {
     Xapian::Query subquery;
     notmuch_status_t status;
@@ -236,7 +255,8 @@ _sexp_expand_query (notmuch_database_t *notmuch,
 	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
     }
 
-    status = _sexp_combine_query (notmuch, NULL, prefix->xapian_op, prefix->initial, sx, subquery);
+    status = _sexp_combine_query (notmuch, NULL, NULL, prefix->xapian_op, prefix->initial, sx,
+				  subquery);
     if (status)
 	return status;
 
@@ -272,7 +292,7 @@ _sexp_parse_infix (notmuch_database_t *notmuch, const sexp_t *sx, Xapian::Query
 
 static notmuch_status_t
 _sexp_parse_header (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
-		    const sexp_t *sx, Xapian::Query &output)
+		    const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
 {
     _sexp_prefix_t user_prefix;
 
@@ -287,23 +307,164 @@ _sexp_parse_header (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
 
     parent = &user_prefix;
 
-    return _sexp_combine_query (notmuch, parent, Xapian::Query::OP_AND, Xapian::Query::MatchAll,
+    return _sexp_combine_query (notmuch, parent, env, Xapian::Query::OP_AND, Xapian::Query::MatchAll,
 				sx->list->next, output);
 }
 
+static _sexp_binding_t *
+_sexp_bind (void *ctx, const _sexp_binding_t *env, const char *name, const sexp_t *sx, const
+	    _sexp_binding_t *context)
+{
+    _sexp_binding_t *binding = talloc (ctx, _sexp_binding_t);
+
+    binding->name = talloc_strdup (ctx, name);
+    binding->sx = sx;
+    binding->context = context;
+    binding->next = env;
+    return binding;
+}
+
+static notmuch_status_t
+maybe_apply_macro (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		   const _sexp_binding_t *env, const sexp_t *sx, const sexp_t *args,
+		   Xapian::Query &output)
+{
+    const sexp_t *params, *param, *arg, *body;
+    void *local = talloc_new (notmuch);
+    _sexp_binding_t *new_env = NULL;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    if (sx->list->ty != SEXP_VALUE || strcmp (sx->list->val, "macro") != 0) {
+	status = NOTMUCH_STATUS_IGNORED;
+	goto DONE;
+    }
+
+    params = sx->list->next;
+
+    if (! params || (params->ty != SEXP_LIST)) {
+	_notmuch_database_log (notmuch, "missing (possibly empty) list of arguments to macro\n");
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    body = params->next;
+
+    if (! body) {
+	_notmuch_database_log (notmuch, "missing body of macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    for (param = params->list, arg = args;
+	 param && arg;
+	 param = param->next, arg = arg->next) {
+	if (param->ty != SEXP_VALUE || param->aty != SEXP_BASIC) {
+	    _notmuch_database_log (notmuch, "macro parameters must be unquoted atoms\n");
+	    status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	    goto DONE;
+	}
+	new_env = _sexp_bind (local, new_env, param->val, arg, env);
+    }
+
+    if (param && ! arg) {
+	_notmuch_database_log (notmuch, "too few arguments to macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    if (! param && arg) {
+	_notmuch_database_log (notmuch, "too many arguments to macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    status = _sexp_to_xapian_query (notmuch, parent, new_env, body, output);
+
+  DONE:
+    if (local)
+	talloc_free (local);
+
+    return status;
+}
+
+static notmuch_status_t
+maybe_saved_squery (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		    const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
+{
+    char *key;
+    char *expansion = NULL;
+    notmuch_status_t status;
+    sexp_t *saved_sexp;
+    void *local = talloc_new (notmuch);
+    char *buf;
+
+    key = talloc_asprintf (local, "squery.%s", sx->list->val);
+    if (! key) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    status = notmuch_database_get_config (notmuch, key, &expansion);
+    if (status)
+	goto DONE;
+    if (EMPTY_STRING (expansion)) {
+	status = NOTMUCH_STATUS_IGNORED;
+	goto DONE;
+    }
+
+    buf = talloc_strdup (local, expansion);
+    /* XXX TODO: free this memory */
+    saved_sexp = parse_sexp (buf, strlen (expansion));
+    if (! saved_sexp) {
+	_notmuch_database_log (notmuch, "invalid saved s-expression query: '%s'\n", expansion);
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    status = maybe_apply_macro (notmuch, parent, env, saved_sexp, sx->list->next, output);
+    if (status == NOTMUCH_STATUS_IGNORED)
+	status =  _sexp_to_xapian_query (notmuch, parent, env, saved_sexp, output);
+
+  DONE:
+    if (local)
+	talloc_free (local);
+
+    return status;
+}
+
+static notmuch_status_t
+_sexp_expand_param (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		    const _sexp_binding_t *env, const char *name,
+		    Xapian::Query &output)
+{
+    for (; env; env = env->next) {
+	if (strcmp (name, env->name) == 0) {
+	    return _sexp_to_xapian_query (notmuch, parent, env->context, env->sx,
+					  output);
+	}
+    }
+    _notmuch_database_log (notmuch, "undefined parameter %s\n", name);
+    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+}
+
 /* Here we expect the s-expression to be a proper list, with first
  * element defining and operation, or as a special case the empty
  * list */
 
 static notmuch_status_t
-_sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const sexp_t *sx,
-		       Xapian::Query &output)
+_sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		       const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
 {
+    notmuch_status_t status;
+
+    if (sx->ty == SEXP_VALUE && sx->aty == SEXP_BASIC && sx->val[0] == ',') {
+	return _sexp_expand_param (notmuch, parent, env, sx->val + 1, output);
+    }
+
     if (sx->ty == SEXP_VALUE) {
 	std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : "";
 
 	if (sx->aty == SEXP_BASIC && strcmp (sx->val, "*") == 0) {
-	    return _sexp_parse_wildcard (notmuch, parent, "", output);
+	    return _sexp_parse_wildcard (notmuch, parent, env, "", output);
 	}
 
 	if (parent && (parent->flags & SEXP_FLAG_BOOLEAN)) {
@@ -317,7 +478,6 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent
 	    Xapian::Query accumulator;
 	    for (_sexp_prefix_t *prefix = prefixes; prefix->name; prefix++) {
 		if (prefix->flags & SEXP_FLAG_FIELD) {
-		    notmuch_status_t status;
 		    Xapian::Query subquery;
 		    term_prefix = _notmuch_database_prefix (notmuch, prefix->name);
 		    status = _sexp_parse_one_term (notmuch, term_prefix, sx, subquery);
@@ -343,9 +503,18 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent
 	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
     }
 
+    status = maybe_saved_squery (notmuch, parent, env, sx, output);
+    if (status != NOTMUCH_STATUS_IGNORED)
+	return status;
+
     /* Check for user defined field */
     if (_notmuch_string_map_get (notmuch->user_prefix, sx->list->val)) {
-	return _sexp_parse_header (notmuch, parent, sx, output);
+	return _sexp_parse_header (notmuch, parent, env, sx, output);
+    }
+
+    if (strcmp (sx->list->val, "macro") == 0) {
+	_notmuch_database_log (notmuch, "macro definition not permitted here\n");
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
     }
 
     for (_sexp_prefix_t *prefix = prefixes; prefix && prefix->name; prefix++) {
@@ -381,17 +550,17 @@ _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent
 	    }
 
 	    if (prefix->xapian_op == Xapian::Query::OP_WILDCARD)
-		return _sexp_parse_wildcard (notmuch, parent, sx->list->next->val, output);
+		return _sexp_parse_wildcard (notmuch, parent, env, sx->list->next->val, output);
 
 	    if (prefix->flags & SEXP_FLAG_DO_REGEX) {
-		return _sexp_parse_regex (notmuch, prefix, parent, sx->list->next->val, output);
+		return _sexp_parse_regex (notmuch, prefix, parent, env, sx->list->next->val, output);
 	    }
 
 	    if (prefix->flags & SEXP_FLAG_DO_EXPAND) {
-		return _sexp_expand_query (notmuch, prefix, parent, sx->list->next, output);
+		return _sexp_expand_query (notmuch, prefix, parent, env, sx->list->next, output);
 	    }
 
-	    return _sexp_combine_query (notmuch, parent, prefix->xapian_op, prefix->initial,
+	    return _sexp_combine_query (notmuch, parent, env, prefix->xapian_op, prefix->initial,
 					sx->list->next, output);
 	}
     }
@@ -413,6 +582,6 @@ _notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *q
 	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
     }
 
-    return _sexp_to_xapian_query (notmuch, NULL, sx, output);
+    return _sexp_to_xapian_query (notmuch, NULL, NULL, sx, output);
 }
 #endif

^ permalink raw reply related	[flat|nested] 32+ messages in thread

end of thread, other threads:[~2021-08-12 17:10 UTC | newest]

Thread overview: 32+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-08-12 17:06 v4 sexp query parser David Bremner
2021-08-12 17:06 ` [PATCH 01/31] configure: optional library sfsexp David Bremner
2021-08-12 17:06 ` [PATCH 02/31] lib: split notmuch_query_create David Bremner
2021-08-12 17:07 ` [PATCH 03/31] lib: define notmuch_query_create_with_syntax David Bremner
2021-08-12 17:07 ` [PATCH 04/31] CLI/search+address: support sexpr queries David Bremner
2021-08-12 17:07 ` [PATCH 05/31] lib: add new status code for query syntax errors David Bremner
2021-08-12 17:07 ` [PATCH 06/31] lib/parse-sexp: parse single terms and the empty list David Bremner
2021-08-12 17:07 ` [PATCH 07/31] lib: leave stemmer object accessible David Bremner
2021-08-12 17:07 ` [PATCH 08/31] lib/parse-sexp: stem unquoted atoms David Bremner
2021-08-12 17:07 ` [PATCH 09/31] lib/parse-sexp: support and, not, and or David Bremner
2021-08-12 17:07 ` [PATCH 10/31] lib/parse-sexp: support subject field David Bremner
2021-08-12 17:07 ` [PATCH 11/31] util/unicode: allow calling from C++ David Bremner
2021-08-12 17:07 ` [PATCH 12/31] lib/parse-sexp: support phrase queries David Bremner
2021-08-12 17:07 ` [PATCH 13/31] lib/parse-sexp: add term prefix backed fields David Bremner
2021-08-12 17:07 ` [PATCH 14/31] lib/parse-sexp: 'starts-with' wildcard searches David Bremner
2021-08-12 17:07 ` [PATCH 15/31] lib/parse-sexp: add '*' as syntactic sugar for '(starts-with "")' David Bremner
2021-08-12 17:07 ` [PATCH 16/31] lib/parse-sexp: handle unprefixed terms David Bremner
2021-08-12 17:07 ` [PATCH 17/31] lib/query: generalize exclude handling to s-expression queries David Bremner
2021-08-12 17:07 ` [PATCH 18/31] lib: factor out query construction from regexp David Bremner
2021-08-12 17:07 ` [PATCH 19/31] lib/parse-sexp: support regular expressions David Bremner
2021-08-12 17:07 ` [PATCH 20/31] lib: generate actual Xapian query for "*" and "" David Bremner
2021-08-12 17:07 ` [PATCH 21/31] lib/query: factor out _notmuch_query_string_to_xapian_query David Bremner
2021-08-12 17:07 ` [PATCH 22/31] lib/thread-fp: factor out query expansion, rewrite in Xapian David Bremner
2021-08-12 17:07 ` [PATCH 23/31] lib/parse-sexp: expand queries David Bremner
2021-08-12 17:07 ` [PATCH 24/31] lib/parse-sexp: support infix subqueries David Bremner
2021-08-12 17:07 ` [PATCH 25/31] lib/parse-sexp: parse user headers David Bremner
2021-08-12 17:07 ` [PATCH 26/31] lib: factor out expansion of saved queries David Bremner
2021-08-12 17:07 ` [PATCH 27/31] lib/parse-sexp: handle " David Bremner
2021-08-12 17:07 ` [PATCH 28/31] CLI/config support saving s-expression queries David Bremner
2021-08-12 17:07 ` [PATCH 29/31] lib/parse-sexp: support saved " David Bremner
2021-08-12 17:07 ` [PATCH 30/31] lib/parse-sexp: thread environment argument through parser David Bremner
2021-08-12 17:07 ` [PATCH 31/31] lib/parse-sexp: apply macros David Bremner

Code repositories for project(s) associated with this public inbox

	https://yhetil.org/notmuch.git/

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).