From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id D9BE61F9F4 for ; Tue, 13 Apr 2021 10:54:46 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 4/5] lei q: start wiring up saved search Date: Tue, 13 Apr 2021 10:54:45 +0000 Message-Id: <20210413105446.7245-5-e@80x24.org> In-Reply-To: <20210413105446.7245-1-e@80x24.org> References: <20210413105446.7245-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This will have a over.sqlite3 for content-based deduplication. It may exhibit ibxish methods, so serving a read-only (or even R/W) IMAP or instance or displaying HTML isn't outside the realm of possibility. --- MANIFEST | 3 + lib/PublicInbox/LEI.pm | 4 +- lib/PublicInbox/LeiQuery.pm | 2 + lib/PublicInbox/LeiSavedSearch.pm | 134 ++++++++++++++++++++++++++++++ lib/PublicInbox/LeiToMail.pm | 10 ++- lib/PublicInbox/LeiXSearch.pm | 31 ++++++- t/lei-q-save.t | 12 +++ t/lei.t | 2 +- t/lei_saved_search.t | 10 +++ 9 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 lib/PublicInbox/LeiSavedSearch.pm create mode 100644 t/lei-q-save.t create mode 100644 t/lei_saved_search.t diff --git a/MANIFEST b/MANIFEST index 12247ad2..20615abc 100644 --- a/MANIFEST +++ b/MANIFEST @@ -201,6 +201,7 @@ lib/PublicInbox/LeiOverview.pm lib/PublicInbox/LeiP2q.pm lib/PublicInbox/LeiQuery.pm lib/PublicInbox/LeiRemote.pm +lib/PublicInbox/LeiSavedSearch.pm lib/PublicInbox/LeiSearch.pm lib/PublicInbox/LeiStore.pm lib/PublicInbox/LeiStoreErr.pm @@ -393,12 +394,14 @@ t/lei-mirror.t t/lei-p2q.t t/lei-q-kw.t t/lei-q-remote-import.t +t/lei-q-save.t t/lei-q-thread.t t/lei-tag.t t/lei.t t/lei_dedupe.t t/lei_external.t t/lei_overview.t +t/lei_saved_search.t t/lei_store.t t/lei_to_mail.t t/lei_xsearch.t diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 475af8f0..7292d0f2 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -131,7 +131,7 @@ our %CMD = ( # sorted in order of importance/use: 'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', 'stdin|', # /|\z/ must be first for lone dash @lxs_opt, - qw(save-as=s output|mfolder|o=s format|f=s dedupe|d=s threads|t+ + qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+ sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt, opt_dash('limit|n=i', '[0-9]+') ], @@ -249,7 +249,7 @@ my %OPTDESC = ( 'torsocks=s' => ['VAL|auto|no|yes', 'whether or not to wrap git and curl commands with torsocks'], 'no-torsocks' => 'alias for --torsocks=no', -'save-as=s' => ['NAME', 'save a search terms by given name'], +'save' => "save a search for `lei up'", 'import-remote!' => 'do not memoize remote messages into local store', 'type=s' => [ 'any|mid|git', 'disambiguate type' ], diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index 224eba69..8bca1020 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -53,6 +53,7 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin my ($self) = @_; # $_[1] = $rbuf if (defined($_[1])) { $_[1] eq '' and return eval { + $self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr}; $self->{lse}->query_approxidate($self->{lse}->git, $self->{mset_opt}->{qstr}); _start_query($self); @@ -142,6 +143,7 @@ no query allowed on command-line with --stdin PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self); return; } + $mset_opt{q_raw} = \@argv; $mset_opt{qstr} = $self->{lse}->query_argv_to_string($self->{lse}->git, \@argv); _start_query($self); diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm new file mode 100644 index 00000000..ab9f393b --- /dev/null +++ b/lib/PublicInbox/LeiSavedSearch.pm @@ -0,0 +1,134 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# pretends to be like LeiDedupe and also PublicInbox::Inbox +package PublicInbox::LeiSavedSearch; +use strict; +use v5.10.1; +use parent qw(PublicInbox::Lock); +use PublicInbox::OverIdx; +use PublicInbox::LeiSearch; +use PublicInbox::Config; +use PublicInbox::Spawn qw(run_die); +use PublicInbox::ContentHash qw(content_hash git_sha); +use PublicInbox::Eml; +use PublicInbox::Hval qw(to_filename); + +sub new { + my ($cls, $lei, $dir) = @_; + my $self = bless { ale => $lei->ale, -cfg => {} }, $cls; + if (defined $dir) { # updating existing saved search + my $f = $self->{'-f'} = "$dir/lei.saved-search"; + -f $f && -r _ or + return $lei->fail("$f non-existent or unreadable"); + $self->{-cfg} = PublicInbox::Config::git_config_dump($f); + my $q = $lei->{mset_opt}->{q_raw} = $self->{-cfg}->{'lei.q'} // + return $lei->fail("lei.q unset in $f"); + my $lse = $lei->{lse} // die 'BUG: {lse} missing'; + $lei->{mset_opt}->{qstr} = ref($q) ? + $lse->query_argv_to_string($lse->git, $q) : + $lse->query_approxidate($lse->git, $q); + } else { # new saved search + my $saved_dir = $lei->store_path . '/../saved-searches/'; + my (@name) = ($lei->{ovv}->{dst} =~ m{([\w\-\.]+)/*\z}); + push @name, to_filename($lei->{mset_opt}->{qstr}); + $dir = $saved_dir . join('-', @name); + require File::Path; + File::Path::make_path($dir); # raises on error + $self->{'-f'} = "$dir/lei.saved-search"; + my $q = $lei->{mset_opt}->{q_raw}; + if (ref $q) { + cfg_set($self, '--add', 'lei.q', $_) for @$q; + } else { + cfg_set($self, 'lei.q', $q); + } + } + bless $self->{-cfg}, 'PublicInbox::Config'; + $self->{lock_path} = "$self->{-f}.flock"; + $self->{-ovf} = "$dir/over.sqlite3"; + $self; +} + +sub description { $_[0]->{qstr} } # for WWW + +sub cfg_set { + my ($self, @args) = @_; + my $lk = $self->lock_for_scope; # git-config doesn't wait + run_die([qw(git config -f), $self->{'-f'}, @args]); +} + +# drop-in for LeiDedupe API +sub is_dup { + my ($self, $eml, $smsg) = @_; + my $oidx = $self->{oidx} // die 'BUG: no {oidx}'; + my $blob = $smsg ? $smsg->{blob} : undef; + return 1 if $blob && $oidx->blob_exists($blob); + my $lk = $self->lock_for_scope_fast; + if (my $xoids = PublicInbox::LeiSearch::xoids_for($self, $eml, 1)) { + for my $docid (values %$xoids) { + $oidx->add_xref3($docid, -1, $blob, '.'); + } + $oidx->commit_lazy; + 1; + } else { + # n.b. above xoids_for fills out eml->{-lei_fake_mid} if needed + unless ($smsg) { + $smsg = bless {}, 'PublicInbox::Smsg'; + $smsg->{bytes} = 0; + $smsg->populate($eml); + } + $oidx->begin_lazy; + $smsg->{num} = $oidx->adj_counter('eidx_docid', '+'); + $smsg->{blob} //= git_sha(1, $eml)->hexdigest; + $oidx->add_overview($eml, $smsg); + $oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.'); + $oidx->commit_lazy; + undef; + } +} + +sub prepare_dedupe { + my ($self) = @_; + $self->{oidx} //= do { + my $creat = !-f $self->{-ovf}; + my $lk = $self->lock_for_scope; # git-config doesn't wait + my $oidx = PublicInbox::OverIdx->new($self->{-ovf}); + $oidx->{-no_fsync} = 1; + $oidx->dbh; + if ($creat) { + $oidx->{dbh}->do('PRAGMA journal_mode = WAL'); + $oidx->eidx_prep; # for xref3 + } + $oidx + }; +} + +sub over { $_[0]->{oidx} } # for xoids_for + +sub git { $_[0]->{ale}->git } + +sub pause_dedupe { + my ($self) = @_; + $self->{ale}->git->cleanup; + my $oidx = delete($self->{oidx}) // return; + $oidx->commit_lazy; +} + +sub mm { undef } + +sub altid_map { {} } + +sub cloneurl { [] } +no warnings 'once'; +*nntp_url = \&cloneurl; +*base_url = \&PublicInbox::Inbox::base_url; +*smsg_eml = \&PublicInbox::Inbox::smsg_eml; +*smsg_by_mid = \&PublicInbox::Inbox::smsg_by_mid; +*msg_by_mid = \&PublicInbox::Inbox::msg_by_mid; +*modified = \&PublicInbox::Inbox::modified; +*recent = \&PublicInbox::Inbox::recent; +*max_git_epoch = *nntp_usable = *msg_by_path = \&mm; # undef +*isrch = *search = \&mm; # TODO +*DESTROY = \&pause_dedupe; + +1; diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index 7adbffe7..bd2b714a 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -9,7 +9,6 @@ use parent qw(PublicInbox::IPC); use PublicInbox::Eml; use PublicInbox::ProcessPipe; use PublicInbox::Spawn qw(spawn); -use PublicInbox::LeiDedupe; use PublicInbox::PktOp qw(pkt_do); use Symbol qw(gensym); use IO::Handle; # ->autoflush @@ -350,7 +349,11 @@ sub new { die "bad mail --format=$fmt\n"; } $self->{dst} = $dst; - $lei->{dedupe} = PublicInbox::LeiDedupe->new($lei); + my $dd_cls = 'PublicInbox::'. + ($lei->{opt}->{save} ? 'LeiSavedSearch' : 'LeiDedupe'); + eval "require $dd_cls"; + die "$dd_cls: $@" if $@; + $lei->{dedupe} = $dd_cls->new($lei); $self; } @@ -368,6 +371,7 @@ sub _pre_augment_maildir { sub _do_augment_maildir { my ($self, $lei) = @_; + return if defined($lei->{opt}->{save}); my $dst = $lei->{ovv}->{dst}; my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef; my $mdr = PublicInbox::MdirReader->new; @@ -398,6 +402,7 @@ sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb sub _do_augment_imap { my ($self, $lei) = @_; + return if defined($lei->{opt}->{save}); my $net = $lei->{net}; my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef; if ($lei->{opt}->{augment}) { @@ -468,6 +473,7 @@ sub _do_augment_mbox { my ($self, $lei) = @_; return unless $self->{seekable}; my $opt = $lei->{opt}; + return if defined($opt->{save}); my $out = $lei->{1}; my ($fmt, $dst) = @{$lei->{ovv}}{qw(fmt dst)}; return unless -s $out; diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index 9d367977..7c540c1c 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -149,22 +149,38 @@ sub query_one_mset { # for --threads and l2m w/o sort local $0 = "$0 query_one_mset"; my $lei = $self->{lei}; my ($srch, $over) = ($ibxish->search, $ibxish->over); - my $desc = $ibxish->{inboxdir} // $ibxish->{topdir}; - return warn("$desc not indexed by Xapian\n") unless ($srch && $over); - my $mo = { %{$lei->{mset_opt}} }; + my $dir = $ibxish->{inboxdir} // $ibxish->{topdir}; + return warn("$dir not indexed by Xapian\n") unless ($srch && $over); + my $mo = { %{$lei->{mset_opt}} }; # copy my $mset; my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei); my $can_kw = !!$ibxish->can('msg_keywords'); my $threads = $lei->{opt}->{threads} // 0; my $fl = $threads > 1 ? 1 : undef; + my $lss = $lei->{dedupe}; + $lss = undef unless $lss && $lss->can('cfg_set'); # saved search + my $maxk = "external.$dir.maxnum"; + my $stop_at = $lss ? $lss->{-cfg}->{$maxk} : undef; + if (defined $stop_at) { + die "$maxk=$stop_at has multiple values" if ref $stop_at; + my @e; + local $SIG{__WARN__} = sub { push @e, @_ }; + $stop_at += 0; + return warn("$maxk=$stop_at: @e") if @e; + } + my $first_ids; do { $mset = $srch->mset($mo->{qstr}, $mo); - mset_progress($lei, $desc, $mset->size, + mset_progress($lei, $dir, $mset->size, $mset->get_matches_estimated); wait_startq($lei); # wait for keyword updates my $ids = $srch->mset_to_artnums($mset, $mo); + @$ids = grep { $_ > $stop_at } @$ids if defined($stop_at); my $i = 0; if ($threads) { + # copy $ids if $lss since over->expand_thread + # shifts @{$ctx->{ids}} + $first_ids = [ @$ids ] if $lss; my $ctx = { ids => $ids }; my %n2item = map { ($ids->[$i++], $_) } $mset->items; while ($over->expand_thread($ctx)) { @@ -183,6 +199,7 @@ sub query_one_mset { # for --threads and l2m w/o sort @{$ctx->{xids}} = (); } } else { + $first_ids = $ids; my @items = $mset->items; for my $n (@$ids) { my $mitem = $items[$i++]; @@ -193,6 +210,12 @@ sub query_one_mset { # for --threads and l2m w/o sort } } } while (_mset_more($mset, $mo)); + if ($lss && scalar(@$first_ids)) { + undef $stop_at; + my $max = $first_ids->[0]; + $lss->cfg_set($maxk, $max); + undef $lss; + } undef $each_smsg; # may commit $lei->{ovv}->ovv_atexit_child($lei); } diff --git a/t/lei-q-save.t b/t/lei-q-save.t new file mode 100644 index 00000000..56f7cb37 --- /dev/null +++ b/t/lei-q-save.t @@ -0,0 +1,12 @@ +#!perl -w +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ +use strict; use v5.10.1; use PublicInbox::TestCommon; +test_lei(sub { + my $home = $ENV{HOME}; + lei_ok qw(import t/plack-qp.eml); + lei_ok qw(q --save z:0..), '-o', "$home/md/"; + my @s = glob("$home/.local/share/lei/saved-searches/md-*"); + is(scalar(@s), 1, 'got one saved search'); +}); +done_testing; diff --git a/t/lei.t b/t/lei.t index 2be9b4e8..6ade2f18 100644 --- a/t/lei.t +++ b/t/lei.t @@ -114,7 +114,7 @@ my $test_completion = sub { %out = map { $_ => 1 } split(/\s+/s, $lei_out); for my $sw (qw(-f --format -o --output --mfolder --augment -a --mua --no-local --local --verbose -v - --save-as --no-remote --remote --torsocks + --save --no-remote --remote --torsocks --reverse -r )) { ok($out{$sw}, "$sw offered as `lei q' completion"); } diff --git a/t/lei_saved_search.t b/t/lei_saved_search.t new file mode 100644 index 00000000..6d26cd2b --- /dev/null +++ b/t/lei_saved_search.t @@ -0,0 +1,10 @@ +#!perl -w +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ +use strict; +use v5.10.1; +use PublicInbox::TestCommon; +require_mods(qw(DBD::SQLite)); +use_ok 'PublicInbox::LeiSavedSearch'; + +done_testing;