From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id DA6A31FC11 for ; Mon, 8 Feb 2021 09:05:22 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 11/13] lei q: use git approxidate with d:, dt: and rt: ranges Date: Sun, 7 Feb 2021 23:05:19 -1000 Message-Id: <20210208090521.28909-12-e@80x24.org> In-Reply-To: <20210208090521.28909-1-e@80x24.org> References: <20210208090521.28909-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Instead of having --(sent|received)-(before|after)=s command-line switches, we'll just try to make sense of argv so it's usable within parenthesized statements and such. Given the negligible performance penalty with Inline::C process spawning, we'll probably wire this up to the WWW interface, too. "d:" is for mairix compatibility. I don't know if "dt:" and "rt:" will be too useful, but they exist because of IMAP (and JMAP). --- lib/PublicInbox/LeiQuery.pm | 12 +++---- lib/PublicInbox/Search.pm | 67 +++++++++++++++++++++++++++++++++++++ t/search.t | 44 ++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 8 deletions(-) diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index 9a6fa718..d637b1ae 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -34,9 +34,10 @@ sub lei_q { my @only = @{$opt->{only} // []}; # --local is enabled by default unless --only is used # we'll allow "--only $LOCATION --local" + my $sto = $self->_lei_store(1); + my $lse = $sto->search; if ($opt->{'local'} //= scalar(@only) ? 0 : 1) { - my $sto = $self->_lei_store(1); - $lxs->prepare_external($sto->search); + $lxs->prepare_external($lse); } if (@only) { for my $loc (@only) { @@ -107,12 +108,7 @@ no query allowed on command-line with --stdin PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self); return; } - # Consider spaces in argv to be for phrase search in Xapian. - # In other words, the users should need only care about - # normal shell quotes and not have to learn Xapian quoting. - $mset_opt{qstr} = join(' ', map {; - /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_ - } @argv); + $mset_opt{qstr} = $lse->query_argv_to_string($lse->git, \@argv); $lxs->do_query($self); } diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index dbae3bc5..f42d70e3 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -8,6 +8,7 @@ use strict; use parent qw(Exporter); our @EXPORT_OK = qw(retry_reopen int_val get_pct xap_terms); use List::Util qw(max); +use POSIX qw(strftime); # values for searching, changing the numeric value breaks # compatibility with old indices (so don't change them it) @@ -259,6 +260,72 @@ sub reopen { $self; # make chaining easier } +# Convert git "approxidate" ranges to something usable with our +# Xapian indices. At the moment, Xapian only offers a C++-only API +# and neither the SWIG nor XS bindings allow us to use custom code +# to parse dates (and libgit2 doesn't expose git__date_parse, either, +# so we're running git-rev-parse(1)). +sub date_range { + my ($git, $pfx, $range) = @_; + # are we inside a parenthesized statement? + my $end = $range =~ s/([\)\s]*)\z// ? $1 : ''; + my @r = split(/\.\./, $range, 2); + + # expand "d:20101002" => "d:20101002..20101003" and like + # n.b. git doesn't do YYYYMMDD w/o '-', it needs YYYY-MM-DD + if ($pfx eq 'd') { + if (!defined($r[1])) { + $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})\z/$1-$2-$3/; + $r[0] = $git->date_parse($r[0]); + $r[1] = $r[0] + 86400; + for my $x (@r) { + $x = strftime('%Y%m%d', gmtime($x)); + } + } else { + for my $x (@r) { + next if $x eq '' || $x =~ /\A[0-9]{8}\z/; + $x = strftime('%Y%m%d', + gmtime($git->date_parse($x))); + } + } + } elsif ($pfx eq 'dt') { + if (!defined($r[1])) { # git needs gaps and not /\d{14}/ + $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2}) + ([0-9]{2})([0-9]{2})([0-9]{2})\z + /$1-$2-$3 $4:$5:$6/x; + $r[0] = $git->date_parse($r[0]); + $r[1] = $r[0] + 86400; + for my $x (@r) { + $x = strftime('%Y%m%d%H%M%S', gmtime($x)); + } + } else { + for my $x (@r) { + next if $x eq '' || $x =~ /\A[0-9]{14}\z/; + $x = strftime('%Y%m%d%H%M%S', + gmtime($git->date_parse($x))); + } + } + } else { # "rt", let git interpret "YYYY", deal with Y10K later :P + for my $x (@r) { + next if $x eq '' || $x =~ /\A[0-9]{5,}\z/; + $x = $git->date_parse($x); + } + $r[1] //= $r[0] + 86400; + } + "$pfx:".join('..', @r).$end; +} + +sub query_argv_to_string { + my (undef, $git, $argv) = @_; + join(' ', map {; + if (s!\b(d|rt|dt):(.+)\z!date_range($git, $1, $2)!sge) { + $_; + } else { + /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_}) : $_ + } + } @$argv); +} + # read-only sub mset { my ($self, $query_string, $opts) = @_; diff --git a/t/search.t b/t/search.t index b2958c00..56c7db1c 100644 --- a/t/search.t +++ b/t/search.t @@ -9,6 +9,7 @@ require PublicInbox::SearchIdx; require PublicInbox::Inbox; require PublicInbox::InboxWritable; use PublicInbox::Eml; +use POSIX qw(strftime); my ($tmpdir, $for_destroy) = tmpdir(); my $git_dir = "$tmpdir/a.git"; my $ibx = PublicInbox::Inbox->new({ inboxdir => $git_dir }); @@ -534,4 +535,47 @@ $ibx->with_umask(sub { 'Subject search reaches inside message/rfc822'); }); +SKIP: { + local $ENV{TZ} = 'UTC'; + my $now = strftime('%H:%M:%S', gmtime(time)); + if ($now =~ /\A23:(?:59|60)/ || $now =~ /\A00:00:0[01]\z/) { + skip 'too close to midnight, time is tricky', 6; + } + my ($s, $g) = ($ibx->search, $ibx->git); + my $q = $s->query_argv_to_string($g, [qw(d:20101002 blah)]); + is($q, 'd:20101002..20101003 blah', 'YYYYMMDD expanded to range'); + $q = $s->query_argv_to_string($g, [qw(d:2010-10-02)]); + is($q, 'd:20101002..20101003', 'YYYY-MM-DD expanded to range'); + $q = $s->query_argv_to_string($g, [qw(rt:2010-10-02.. yy)]); + $q =~ /\Art:(\d+)\.\. yy/ or fail("rt: expansion failed: $q"); + is(strftime('%Y-%m-%d', gmtime($1//0)), '2010-10-02', 'rt: beg expand'); + $q = $s->query_argv_to_string($g, [qw(rt:..2010-10-02 zz)]); + $q =~ /\Art:\.\.(\d+) zz/ or fail("rt: expansion failed: $q"); + is(strftime('%Y-%m-%d', gmtime($1//0)), '2010-10-02', 'rt: end expand'); + $q = $s->query_argv_to_string($g, [qw(something dt:2010-10-02..)]); + like($q, qr/\Asomething dt:20101002\d{6}\.\./, 'dt: expansion'); + $q = $s->query_argv_to_string($g, [qw(x d:yesterday.. y)]); + is($q, strftime('x d:%Y%m%d.. y', gmtime(time - 86400)), + '"yesterday" handled'); + $q = $s->query_argv_to_string($g, [qw(x dt:20101002054123)]); + is($q, 'x dt:20101002054123..20101003054123', 'single dt: expanded'); + $q = $s->query_argv_to_string($g, [qw(x dt:2010-10-02T05:41:23Z)]); + is($q, 'x dt:20101002054123..20101003054123', 'ISO8601 dt: expanded'); + $q = $s->query_argv_to_string($g, [qw(rt:1970..1971)]); + $q =~ /\Art:(\d+)\.\.(\d+)\z/ or fail "YYYY rt: expansion: $q"; + my ($beg, $end) = ($1, $2); + is(strftime('%Y', gmtime($beg)), 1970, 'rt: starts at 1970'); + is(strftime('%Y', gmtime($end)), 1971, 'rt: ends at 1971'); + $q = $s->query_argv_to_string($g, [qw(rt:1970-01-01)]); + $q =~ /\Art:(\d+)\.\.(\d+)\z/ or fail "YYYY-MM-DD rt: expansion: $q"; + ($beg, $end) = ($1, $2); + is(strftime('%Y-%m-%d', gmtime($beg)), '1970-01-01', + 'rt: date-only w/o range'); + is(strftime('%Y-%m-%d', gmtime($end)), '1970-01-02', + 'rt: date-only auto-end'); + $q = $s->query_argv_to_string($g, [qw{OR (rt:1993-10-02)}]); + like($q, qr/\AOR \(rt:749\d{6}\.\.749\d{6}\)\z/, + 'trailing parentheses preserved'); +} + done_testing();