#include "database-private.h" #if HAVE_SFSEXP #include "sexp.h" #include "unicode-util.h" /* _sexp is used for file scope symbols to avoid clashing with * definitions from sexp.h */ /* sexp_binding structs attach name to a sexp and a defining * context. The latter allows lazy evaluation of parameters whose * definition contains other parameters. Lazy evaluation is needed * because a primary goal of macros is to change the parent field for * a sexp. */ typedef struct sexp_binding { const char *name; const sexp_t *sx; const struct sexp_binding *context; const struct sexp_binding *next; } _sexp_binding_t; typedef enum { SEXP_FLAG_NONE = 0, SEXP_FLAG_FIELD = 1 << 0, SEXP_FLAG_BOOLEAN = 1 << 1, SEXP_FLAG_SINGLE = 1 << 2, SEXP_FLAG_WILDCARD = 1 << 3, SEXP_FLAG_REGEX = 1 << 4, SEXP_FLAG_DO_REGEX = 1 << 5, SEXP_FLAG_EXPAND = 1 << 6, SEXP_FLAG_DO_EXPAND = 1 << 7, SEXP_FLAG_ORPHAN = 1 << 8, SEXP_FLAG_RANGE = 1 << 9, } _sexp_flag_t; /* * define bitwise operators to hide casts */ inline _sexp_flag_t operator| (_sexp_flag_t a, _sexp_flag_t b) { return static_cast<_sexp_flag_t>( static_cast(a) | static_cast(b)); } inline _sexp_flag_t operator& (_sexp_flag_t a, _sexp_flag_t b) { return static_cast<_sexp_flag_t>( static_cast(a) & static_cast(b)); } typedef struct { const char *name; Xapian::Query::op xapian_op; Xapian::Query initial; _sexp_flag_t flags; } _sexp_prefix_t; static _sexp_prefix_t prefixes[] = { { "and", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_NONE }, { "attachment", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND }, { "body", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD }, { "date", Xapian::Query::OP_INVALID, Xapian::Query::MatchAll, SEXP_FLAG_RANGE }, { "from", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "folder", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "id", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX }, { "infix", Xapian::Query::OP_INVALID, Xapian::Query::MatchAll, SEXP_FLAG_SINGLE | SEXP_FLAG_ORPHAN }, { "is", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "matching", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_DO_EXPAND }, { "mid", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX }, { "mimetype", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND }, { "not", Xapian::Query::OP_AND_NOT, Xapian::Query::MatchAll, SEXP_FLAG_NONE }, { "of", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_DO_EXPAND }, { "or", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_NONE }, { "path", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX }, { "property", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "query", Xapian::Query::OP_INVALID, Xapian::Query::MatchNothing, SEXP_FLAG_SINGLE | SEXP_FLAG_ORPHAN }, { "regex", Xapian::Query::OP_INVALID, Xapian::Query::MatchAll, SEXP_FLAG_SINGLE | SEXP_FLAG_DO_REGEX }, { "rx", Xapian::Query::OP_INVALID, Xapian::Query::MatchAll, SEXP_FLAG_SINGLE | SEXP_FLAG_DO_REGEX }, { "starts-with", Xapian::Query::OP_WILDCARD, Xapian::Query::MatchAll, SEXP_FLAG_SINGLE }, { "subject", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "tag", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "thread", Xapian::Query::OP_OR, Xapian::Query::MatchNothing, SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND }, { "to", Xapian::Query::OP_AND, Xapian::Query::MatchAll, SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND }, { } }; static notmuch_status_t _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output); static notmuch_status_t _sexp_combine_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const _sexp_binding_t *env, Xapian::Query::op operation, Xapian::Query left, const sexp_t *sx, Xapian::Query &output) { Xapian::Query subquery; notmuch_status_t status; /* if we run out elements, return accumulator */ if (! sx) { output = left; return NOTMUCH_STATUS_SUCCESS; } status = _sexp_to_xapian_query (notmuch, parent, env, sx, subquery); if (status) return status; return _sexp_combine_query (notmuch, parent, env, operation, Xapian::Query (operation, left, subquery), sx->next, output); } static notmuch_status_t _sexp_parse_phrase (std::string term_prefix, const char *phrase, Xapian::Query &output) { Xapian::Utf8Iterator p (phrase); Xapian::Utf8Iterator end; std::vector terms; while (p != end) { Xapian::Utf8Iterator start; while (p != end && ! Xapian::Unicode::is_wordchar (*p)) p++; if (p == end) break; start = p; while (p != end && Xapian::Unicode::is_wordchar (*p)) p++; if (p != start) { std::string word (start, p); word = Xapian::Unicode::tolower (word); terms.push_back (term_prefix + word); } } output = Xapian::Query (Xapian::Query::OP_PHRASE, terms.begin (), terms.end ()); return NOTMUCH_STATUS_SUCCESS; } static notmuch_status_t _sexp_parse_wildcard (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, unused(const _sexp_binding_t *env), std::string match, Xapian::Query &output) { std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : ""; if (parent && ! (parent->flags & SEXP_FLAG_WILDCARD)) { _notmuch_database_log (notmuch, "'%s' does not support wildcard queries\n", parent->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } output = Xapian::Query (Xapian::Query::OP_WILDCARD, term_prefix + Xapian::Unicode::tolower (match)); return NOTMUCH_STATUS_SUCCESS; } static notmuch_status_t _sexp_parse_one_term (notmuch_database_t *notmuch, std::string term_prefix, const sexp_t *sx, Xapian::Query &output) { Xapian::Stem stem = *(notmuch->stemmer); if (sx->aty == SEXP_BASIC && unicode_word_utf8 (sx->val)) { std::string term = Xapian::Unicode::tolower (sx->val); output = Xapian::Query ("Z" + term_prefix + stem (term)); return NOTMUCH_STATUS_SUCCESS; } else { return _sexp_parse_phrase (term_prefix, sx->val, output); } } notmuch_status_t _sexp_parse_regex (notmuch_database_t *notmuch, const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent, unused(const _sexp_binding_t *env), std::string val, Xapian::Query &output) { if (! parent) { _notmuch_database_log (notmuch, "illegal '%s' outside field\n", prefix->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } if (! (parent->flags & SEXP_FLAG_REGEX)) { _notmuch_database_log (notmuch, "'%s' not supported in field '%s'\n", prefix->name, parent->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } std::string msg; /* ignored */ return _notmuch_regexp_to_query (notmuch, Xapian::BAD_VALUENO, parent->name, val, output, msg); } static notmuch_status_t _sexp_expand_query (notmuch_database_t *notmuch, const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent, unused(const _sexp_binding_t *env), const sexp_t *sx, Xapian::Query &output) { Xapian::Query subquery; notmuch_status_t status; std::string msg; if (! (parent->flags & SEXP_FLAG_EXPAND)) { _notmuch_database_log (notmuch, "'%s' unsupported inside '%s'\n", prefix->name, parent->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } status = _sexp_combine_query (notmuch, NULL, NULL, prefix->xapian_op, prefix->initial, sx, subquery); if (status) return status; status = _notmuch_query_expand (notmuch, parent->name, subquery, output, msg); if (status) { _notmuch_database_log (notmuch, "error expanding query %s\n", msg.c_str ()); } return status; } static notmuch_status_t _sexp_parse_infix (notmuch_database_t *notmuch, const sexp_t *sx, Xapian::Query &output) { try { output = notmuch->query_parser->parse_query (sx->val, NOTMUCH_QUERY_PARSER_FLAGS); } catch (const Xapian::QueryParserError &error) { _notmuch_database_log (notmuch, "Syntax error in infix query: %s\n", sx->val); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } catch (const Xapian::Error &error) { if (! notmuch->exception_reported) { _notmuch_database_log (notmuch, "A Xapian exception occurred parsing query: %s\n", error.get_msg ().c_str ()); _notmuch_database_log_append (notmuch, "Query string was: %s\n", sx->val); notmuch->exception_reported = true; return NOTMUCH_STATUS_XAPIAN_EXCEPTION; } } return NOTMUCH_STATUS_SUCCESS; } static notmuch_status_t _sexp_parse_header (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output) { _sexp_prefix_t user_prefix; user_prefix.name = sx->list->val; user_prefix.flags = SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD; if (parent) { _notmuch_database_log (notmuch, "nested field: '%s' inside '%s'\n", sx->list->val, parent->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } parent = &user_prefix; return _sexp_combine_query (notmuch, parent, env, Xapian::Query::OP_AND, Xapian::Query::MatchAll, sx->list->next, output); } static _sexp_binding_t * _sexp_bind (void *ctx, const _sexp_binding_t *env, const char *name, const sexp_t *sx, const _sexp_binding_t *context) { _sexp_binding_t *binding = talloc (ctx, _sexp_binding_t); binding->name = talloc_strdup (ctx, name); binding->sx = sx; binding->context = context; binding->next = env; return binding; } static notmuch_status_t maybe_apply_macro (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const _sexp_binding_t *env, const sexp_t *sx, const sexp_t *args, Xapian::Query &output) { const sexp_t *params, *param, *arg, *body; void *local = talloc_new (notmuch); _sexp_binding_t *new_env = NULL; notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; if (sx->list->ty != SEXP_VALUE || strcmp (sx->list->val, "macro") != 0) { status = NOTMUCH_STATUS_IGNORED; goto DONE; } params = sx->list->next; if (! params || (params->ty != SEXP_LIST)) { _notmuch_database_log (notmuch, "missing (possibly empty) list of arguments to macro\n"); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } body = params->next; if (! body) { _notmuch_database_log (notmuch, "missing body of macro\n"); status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX; goto DONE; } for (param = params->list, arg = args; param && arg; param = param->next, arg = arg->next) { if (param->ty != SEXP_VALUE || param->aty != SEXP_BASIC) { _notmuch_database_log (notmuch, "macro parameters must be unquoted atoms\n"); status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX; goto DONE; } new_env = _sexp_bind (local, new_env, param->val, arg, env); } if (param && ! arg) { _notmuch_database_log (notmuch, "too few arguments to macro\n"); status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX; goto DONE; } if (! param && arg) { _notmuch_database_log (notmuch, "too many arguments to macro\n"); status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX; goto DONE; } status = _sexp_to_xapian_query (notmuch, parent, new_env, body, output); DONE: if (local) talloc_free (local); return status; } static notmuch_status_t maybe_saved_squery (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output) { char *key; char *expansion = NULL; notmuch_status_t status; sexp_t *saved_sexp; void *local = talloc_new (notmuch); char *buf; key = talloc_asprintf (local, "squery.%s", sx->list->val); if (! key) { status = NOTMUCH_STATUS_OUT_OF_MEMORY; goto DONE; } status = notmuch_database_get_config (notmuch, key, &expansion); if (status) goto DONE; if (EMPTY_STRING (expansion)) { status = NOTMUCH_STATUS_IGNORED; goto DONE; } buf = talloc_strdup (local, expansion); /* XXX TODO: free this memory */ saved_sexp = parse_sexp (buf, strlen (expansion)); if (! saved_sexp) { _notmuch_database_log (notmuch, "invalid saved s-expression query: '%s'\n", expansion); status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX; goto DONE; } status = maybe_apply_macro (notmuch, parent, env, saved_sexp, sx->list->next, output); if (status == NOTMUCH_STATUS_IGNORED) status = _sexp_to_xapian_query (notmuch, parent, env, saved_sexp, output); DONE: if (local) talloc_free (local); return status; } static notmuch_status_t _sexp_expand_param (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const _sexp_binding_t *env, const char *name, Xapian::Query &output) { for (; env; env = env->next) { if (strcmp (name, env->name) == 0) { return _sexp_to_xapian_query (notmuch, parent, env->context, env->sx, output); } } _notmuch_database_log (notmuch, "undefined parameter %s\n", name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } static notmuch_status_t _sexp_parse_date (notmuch_database_t *notmuch, const sexp_t *sx, Xapian::Query &output) { /* empty date matches everything */ if (! sx) { output = Xapian::Query::MatchAll; return NOTMUCH_STATUS_SUCCESS; } _notmuch_database_log (notmuch, "unimplemented date query\n"); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } /* Here we expect the s-expression to be a proper list, with first * element defining and operation, or as a special case the empty * list */ static notmuch_status_t _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent, const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output) { notmuch_status_t status; if (sx->ty == SEXP_VALUE && sx->aty == SEXP_BASIC && sx->val[0] == ',') { return _sexp_expand_param (notmuch, parent, env, sx->val + 1, output); } if (sx->ty == SEXP_VALUE) { std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : ""; if (sx->aty == SEXP_BASIC && strcmp (sx->val, "*") == 0) { return _sexp_parse_wildcard (notmuch, parent, env, "", output); } if (parent && (parent->flags & SEXP_FLAG_BOOLEAN)) { output = Xapian::Query (term_prefix + sx->val); return NOTMUCH_STATUS_SUCCESS; } if (parent) { return _sexp_parse_one_term (notmuch, term_prefix, sx, output); } else { Xapian::Query accumulator; for (_sexp_prefix_t *prefix = prefixes; prefix->name; prefix++) { if (prefix->flags & SEXP_FLAG_FIELD) { Xapian::Query subquery; term_prefix = _notmuch_database_prefix (notmuch, prefix->name); status = _sexp_parse_one_term (notmuch, term_prefix, sx, subquery); if (status) return status; accumulator = Xapian::Query (Xapian::Query::OP_OR, accumulator, subquery); } } output = accumulator; return NOTMUCH_STATUS_SUCCESS; } } /* Empty list */ if (! sx->list) { output = Xapian::Query::MatchAll; return NOTMUCH_STATUS_SUCCESS; } if (sx->list->ty == SEXP_LIST) { _notmuch_database_log (notmuch, "unexpected list in field/operation position\n", sx->list->val); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } status = maybe_saved_squery (notmuch, parent, env, sx, output); if (status != NOTMUCH_STATUS_IGNORED) return status; /* Check for user defined field */ if (_notmuch_string_map_get (notmuch->user_prefix, sx->list->val)) { return _sexp_parse_header (notmuch, parent, env, sx, output); } if (strcmp (sx->list->val, "macro") == 0) { _notmuch_database_log (notmuch, "macro definition not permitted here\n"); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } for (_sexp_prefix_t *prefix = prefixes; prefix && prefix->name; prefix++) { if (strcmp (prefix->name, sx->list->val) == 0) { if (prefix->flags & (SEXP_FLAG_FIELD | SEXP_FLAG_RANGE)) { if (parent) { _notmuch_database_log (notmuch, "nested field: '%s' inside '%s'\n", prefix->name, parent->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } parent = prefix; } if (parent && (prefix->flags & SEXP_FLAG_ORPHAN)) { _notmuch_database_log (notmuch, "'%s' not supported inside '%s'\n", prefix->name, parent->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } if ((prefix->flags & SEXP_FLAG_SINGLE) && (! sx->list->next || sx->list->next->next || sx->list->next->ty != SEXP_VALUE)) { _notmuch_database_log (notmuch, "'%s' expects single atom as argument\n", prefix->name); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } if (strcmp (prefix->name, "date") == 0) { return _sexp_parse_date (notmuch, sx->list->next, output); } if (strcmp (prefix->name, "infix") == 0) { return _sexp_parse_infix (notmuch, sx->list->next, output); } if (strcmp (prefix->name, "query") == 0) { return _notmuch_query_name_to_query (notmuch, sx->list->next->val, output); } if (prefix->xapian_op == Xapian::Query::OP_WILDCARD) return _sexp_parse_wildcard (notmuch, parent, env, sx->list->next->val, output); if (prefix->flags & SEXP_FLAG_DO_REGEX) { return _sexp_parse_regex (notmuch, prefix, parent, env, sx->list->next->val, output); } if (prefix->flags & SEXP_FLAG_DO_EXPAND) { return _sexp_expand_query (notmuch, prefix, parent, env, sx->list->next, output); } return _sexp_combine_query (notmuch, parent, env, prefix->xapian_op, prefix->initial, sx->list->next, output); } } _notmuch_database_log (notmuch, "unknown prefix '%s'\n", sx->list->val); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } notmuch_status_t _notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *querystr, Xapian::Query &output) { const sexp_t *sx = NULL; char *buf = talloc_strdup (notmuch, querystr); sx = parse_sexp (buf, strlen (querystr)); if (! sx) { _notmuch_database_log (notmuch, "invalid s-expression: '%s'\n", querystr); return NOTMUCH_STATUS_BAD_QUERY_SYNTAX; } return _sexp_to_xapian_query (notmuch, NULL, NULL, sx, output); } #endif