From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-3.9 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 3721920083 for ; Wed, 3 Feb 2021 08:11:45 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 11/11] lei q: support reading queries from stdin Date: Tue, 2 Feb 2021 22:11:43 -1000 Message-Id: <20210203081143.24424-12-e@80x24.org> In-Reply-To: <20210203081143.24424-1-e@80x24.org> References: <20210203081143.24424-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This will be useful on shared machines when a user doesn't want search queries visible to other users looking at the ps(1) output or similar. --- MANIFEST | 1 + lib/PublicInbox/InputPipe.pm | 37 ++++++++++++++++++++++++++++++++++ lib/PublicInbox/LEI.pm | 7 ++++--- lib/PublicInbox/LeiOverview.pm | 1 - lib/PublicInbox/LeiQuery.pm | 32 ++++++++++++++++++++++------- lib/PublicInbox/LeiXSearch.pm | 2 ++ t/lei.t | 19 +++++++++++++++++ 7 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 lib/PublicInbox/InputPipe.pm diff --git a/MANIFEST b/MANIFEST index bcb9d08e..6922f9b1 100644 --- a/MANIFEST +++ b/MANIFEST @@ -173,6 +173,7 @@ lib/PublicInbox/In2Tie.pm lib/PublicInbox/Inbox.pm lib/PublicInbox/InboxIdle.pm lib/PublicInbox/InboxWritable.pm +lib/PublicInbox/InputPipe.pm lib/PublicInbox/Isearch.pm lib/PublicInbox/KQNotify.pm lib/PublicInbox/LEI.pm diff --git a/lib/PublicInbox/InputPipe.pm b/lib/PublicInbox/InputPipe.pm new file mode 100644 index 00000000..a8bdf031 --- /dev/null +++ b/lib/PublicInbox/InputPipe.pm @@ -0,0 +1,37 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# for reading pipes and sockets off the DS event loop +package PublicInbox::InputPipe; +use strict; +use v5.10.1; +use parent qw(PublicInbox::DS); +use PublicInbox::Syscall qw(EPOLLIN EPOLLET); + +sub consume { + my ($in, $cb, @args) = @_; + my $self = bless { cb => $cb, sock => $in, args => \@args },__PACKAGE__; + if ($PublicInbox::DS::in_loop) { + eval { $self->SUPER::new($in, EPOLLIN|EPOLLET) }; + return $in->blocking(0) unless $@; # regular file sets $@ + } + event_step($self) while $self->{sock}; +} + +sub event_step { + my ($self) = @_; + my ($r, $rbuf); + while (($r = sysread($self->{sock}, $rbuf, 65536))) { + $self->{cb}->(@{$self->{args} // []}, $rbuf); + } + if (defined($r)) { # EOF + $self->{cb}->(@{$self->{args} // []}, ''); + } elsif ($!{EAGAIN}) { + return; + } else { + $self->{cb}->(@{$self->{args} // []}, undef) + } + $self->{sock}->blocking ? delete($self->{sock}) : $self->close +} + +1; diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 28dce0c5..49deed13 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -101,10 +101,10 @@ sub _config_path ($) { # TODO: generate shell completion + help using %CMD and %OPTDESC # command => [ positional_args, 1-line description, Getopt::Long option spec ] our %CMD = ( # sorted in order of importance/use: -'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw( +'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', qw( save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a sort|s=s reverse|r offset=i remote! local! external! pretty - include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g + include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g stdin| mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q received-after=s received-before=s sent-after=s sent-since=s), PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ], @@ -554,12 +554,13 @@ sub optparse ($$$) { } elsif ($var =~ /\A\[-?$POS_ARG\]\z/) { # one optional arg $i++; } elsif ($var =~ /\A.+?\|/) { # required FOO|--stdin + $inf = 1 if index($var, '...') > 0; my @or = split(/\|/, $var); my $ok; for my $o (@or) { if ($o =~ /\A--([a-z0-9\-]+)/) { $ok = defined($OPT->{$1}); - last; + last if $ok; } elsif (defined($argv->[$i])) { $ok = 1; $i++; diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index 88034ada..e33d63a2 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -81,7 +81,6 @@ sub new { my ($isatty, $seekable); if ($dst eq '/dev/stdout') { $isatty = -t $lei->{1}; - $lei->start_pager if $isatty; $opt->{pretty} //= $isatty; if (!$isatty && -f _) { my $fl = fcntl($lei->{1}, F_GETFL, 0) // diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index 8015ecec..4fe40400 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -12,6 +12,16 @@ sub prep_ext { # externals_each callback $lxs->prepare_external($loc) unless $exclude->{$loc}; } +sub qstr_add { # for --stdin + my ($self) = @_; # $_[1] = $rbuf + if (defined($_[1])) { + return eval { $self->{lxs}->do_query($self) } if $_[1] eq ''; + $self->{mset_opt}->{qstr} .= $_[1]; + } else { + $self->fail("error reading stdin: $!"); + } +} + # the main "lei q SEARCH_TERMS" method sub lei_q { my ($self, @argv) = @_; @@ -84,12 +94,6 @@ sub lei_q { my %mset_opt = map { $_ => $opt->{$_} } qw(thread limit offset); $mset_opt{asc} = $opt->{'reverse'} ? 1 : 0; $mset_opt{limit} //= 10000; - $mset_opt{qstr} = join(' ', map {; - # Consider spaces in argv to be for phrase search in Xapian. - # In other words, the users should need only care about - # normal shell quotes and not have to learn Xapian quoting. - /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_ - } @argv); if (defined(my $sort = $opt->{'sort'})) { if ($sort eq 'relevance') { $mset_opt{relevance} = 1; @@ -104,7 +108,21 @@ sub lei_q { # descending docid order $mset_opt{relevance} //= -2 if $opt->{thread}; $self->{mset_opt} = \%mset_opt; - $self->{ovv}->ovv_begin($self); + + if ($opt->{stdin}) { + return $self->fail(<<'') if @argv; +no query allowed on command-line with --stdin + + require PublicInbox::InputPipe; + PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self); + return; + } + # Consider spaces in argv to be for phrase search in Xapian. + # In other words, the users should need only care about + # normal shell quotes and not have to learn Xapian quoting. + $mset_opt{qstr} = join(' ', map {; + /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_ + } @argv); $lxs->do_query($self); } diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index d33064bb..965617b5 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -402,6 +402,8 @@ sub sigpipe_handler { # handles SIGPIPE from l2m/lxs workers sub do_query { my ($self, $lei) = @_; $lei->{1}->autoflush(1); + $lei->start_pager if -t $lei->{1}; + $lei->{ovv}->ovv_begin($lei); my ($au_done, $zpipe); my $l2m = $lei->{l2m}; if ($l2m) { diff --git a/t/lei.t b/t/lei.t index 03bbb078..01eed1da 100644 --- a/t/lei.t +++ b/t/lei.t @@ -275,6 +275,25 @@ my $test_external = sub { my $pretty = $json->decode($out); is_deeply($res, $pretty, '--pretty is identical after decode'); + { + open my $fh, '+>', undef or BAIL_OUT $!; + $fh->autoflush(1); + print $fh 's:use' or BAIL_OUT $!; + seek($fh, 0, SEEK_SET) or BAIL_OUT $!; + ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $fh }), + '--stdin on regular file works'); + like($out, qr/use boolean prefix/, '--stdin on regular file'); + } + { + pipe(my ($r, $w)) or BAIL_OUT $!; + print $w 's:use' or BAIL_OUT $!; + close $w or BAIL_OUT $!; + ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $r }), + '--stdin on pipe file works'); + like($out, qr/use boolean prefix/, '--stdin on pipe'); + } + ok(!$lei->(qw(q -q --stdin s:use)), "--stdin and argv don't mix"); + for my $fmt (qw(ldjson ndjson jsonl)) { $lei->('q', '-f', $fmt, 's:use boolean prefix'); is($out, $json->encode($pretty->[0])."\n", "-f $fmt");