6/6 is a fairly significant change, but I think it's worth doing by default to make things less surprising for eventual users of "lei export-kw" (I'm not sure if that's even a good name). Eric Wong (6): lei <q|up>: combine written/results into one line lei_input: common net_merge_all_done for lei <import|tag> lei_input: reject --mail-sync if using HTTP(S) for now lei: simplify workers_start API net_writer: use "FLAGS.SILENT" to set keywords lei <q|up>: writes to Maildirs and IMAP use mail-sync lib/PublicInbox/LEI.pm | 5 +++-- lib/PublicInbox/LeiBlob.pm | 2 +- lib/PublicInbox/LeiConvert.pm | 2 +- lib/PublicInbox/LeiImport.pm | 9 ++------- lib/PublicInbox/LeiInput.pm | 9 +++++++++ lib/PublicInbox/LeiLsSearch.pm | 2 +- lib/PublicInbox/LeiMirror.pm | 2 +- lib/PublicInbox/LeiP2q.pm | 2 +- lib/PublicInbox/LeiQuery.pm | 7 +++---- lib/PublicInbox/LeiStore.pm | 8 ++++---- lib/PublicInbox/LeiTag.pm | 9 ++------- lib/PublicInbox/LeiToMail.pm | 24 +++++++++++++++++------- lib/PublicInbox/LeiXSearch.pm | 9 +++++---- lib/PublicInbox/NetWriter.pm | 14 ++++++-------- t/lei-import-http.t | 5 +++++ xt/net_writer-imap.t | 9 +++++++-- 16 files changed, 68 insertions(+), 50 deletions(-)
Having multiple lines of output mean they can be interleaved in daemon mode. Put stats into one line to reduce screen real-estate size and improve readability. --- lib/PublicInbox/LeiXSearch.pm | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index d212a732..21b15025 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -359,9 +359,8 @@ sub query_done { # EOF callback for main daemon } my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef; $lei->{ovv}->ovv_end($lei); - my (@out, $start_mua); + my $start_mua; if ($l2m) { # close() calls LeiToMail reap_compress - @out = (" in $lei->{ovv}->{dst}"); if (my $out = delete $lei->{old_1}) { if (my $mbout = $lei->{1}) { close($mbout) or return $lei->fail(<<""); @@ -379,9 +378,11 @@ Error closing $lei->{ovv}->{dst}: $! } } if ($lei->{-progress}) { - $lei->qerr('# ', $lei->{-mset_total} // 0, " matches", @out); + my $tot = $lei->{-mset_total} // 0; my $nr = $lei->{-nr_write} // 0; - $lei->qerr("# $nr written to $lei->{ovv}->{dst}") if $l2m; + $lei->qerr($l2m ? + "# $nr written to $lei->{ovv}->{dst} ($tot matches)" : + "# $tot matches"); } $lei->start_mua if $start_mua; $lei->dclose;
I suspect there'll be more lei_input-only things in the future. --- lib/PublicInbox/LeiImport.pm | 7 +------ lib/PublicInbox/LeiInput.pm | 7 +++++++ lib/PublicInbox/LeiTag.pm | 7 +------ 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index 575cf125..394138b4 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -48,12 +48,6 @@ sub input_net_cb { # imap_each / nntp_each input_eml_cb($self, $eml, $vmd); } -sub net_merge_all_done { # callback used by LeiAuth - my ($self) = @_; - $self->wq_io_do('process_inputs'); - $self->wq_close(1); -} - sub lei_import { # the main "lei import" method my ($lei, @inputs) = @_; my $sto = $lei->_lei_store(1); @@ -99,6 +93,7 @@ sub _complete_import { no warnings 'once'; *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; +*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; # the following works even when LeiAuth is lazy-loaded *net_merge_all = \&PublicInbox::LeiAuth::net_merge_all; diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index 9bcc86e1..917f682b 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -329,6 +329,13 @@ sub input_only_atfork_child { undef; } +# alias this as "net_merge_all_done" to use as an LeiAuth callback +sub input_only_net_merge_all_done { + my ($self) = @_; + $self->wq_io_do('process_inputs'); + $self->wq_close(1); +} + # like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare # for update_xvmd -> update_vmd sub vmd_mod_extract { diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm index 2170e3f2..6025c93e 100644 --- a/lib/PublicInbox/LeiTag.pm +++ b/lib/PublicInbox/LeiTag.pm @@ -19,12 +19,6 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh sub input_mbox_cb { input_eml_cb($_[1], $_[0]) } -sub net_merge_all_done { # callback used by LeiAuth - my ($self) = @_; - $self->wq_io_do('process_inputs'); - $self->wq_close(1); -} - sub input_maildir_cb { # maildir_each_eml cb my ($f, $kw, $eml, $self) = @_; input_eml_cb($self, $eml); @@ -117,5 +111,6 @@ sub _complete_tag { no warnings 'once'; # the following works even when LeiAuth is lazy-loaded *net_merge_all = \&PublicInbox::LeiAuth::net_merge_all; +*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; 1;
I'm not sure how we'll distinguish JMAP vs read-only HTTPS, yet; but we'll focus on currently-supported stuff, first. --- lib/PublicInbox/LeiInput.pm | 2 ++ t/lei-import-http.t | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index 917f682b..85caac35 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -229,6 +229,8 @@ sub prepare_inputs { # returns undef on error } } } elsif ($input_path =~ m!\Ahttps?://!i) { + # TODO: how would we detect r/w JMAP? + push @{$sync->{no}}, $input if $sync; prepare_http_input($self, $lei, $input_path) or return; } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { my $ifmt = lc $1; diff --git a/t/lei-import-http.t b/t/lei-import-http.t index 35cbf369..6cb8a753 100644 --- a/t/lei-import-http.t +++ b/t/lei-import-http.t @@ -39,5 +39,10 @@ test_lei({ tmpdir => $tmpdir }, sub { $res = json_utf8->decode($lei_out); is($res->[0]->{'m'}, '20180720072141.GA15957@example', 'imported search result') or diag explain($res); + + ok(!lei(qw(import --mail-sync), "$url/x\@example.com/raw"), + '--mail-sync fails on HTTP'); + diag $lei_err; + }); done_testing;
In most cases, we just name the worker process based on the command. The only change is for LeiMirror vs "lei add-external --mirror", but I doubt it matters. --- lib/PublicInbox/LEI.pm | 3 ++- lib/PublicInbox/LeiBlob.pm | 2 +- lib/PublicInbox/LeiConvert.pm | 2 +- lib/PublicInbox/LeiImport.pm | 2 +- lib/PublicInbox/LeiLsSearch.pm | 2 +- lib/PublicInbox/LeiMirror.pm | 2 +- lib/PublicInbox/LeiP2q.pm | 2 +- lib/PublicInbox/LeiTag.pm | 2 +- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 5d701d5e..d5c6bd52 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -542,7 +542,7 @@ sub pkt_op_pair { } sub workers_start { - my ($lei, $wq, $ident, $jobs, $ops) = @_; + my ($lei, $wq, $jobs, $ops) = @_; $ops = { '!' => [ \&fail_handler, $lei ], '|' => [ \&sigpipe_handler, $lei ], @@ -552,6 +552,7 @@ sub workers_start { }; $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ]; my $end = $lei->pkt_op_pair; + my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker"; $wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei }); delete $lei->{pkt_op_p}; my $op_c = delete $lei->{pkt_op_c}; diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm index 0a957358..710430a2 100644 --- a/lib/PublicInbox/LeiBlob.pm +++ b/lib/PublicInbox/LeiBlob.pm @@ -151,7 +151,7 @@ EOM } require PublicInbox::SolverGit; my $self = bless { lxs => $lxs, oid_b => $blob }, __PACKAGE__; - my ($op_c, $ops) = $lei->workers_start($self, 'lei-blob', 1); + my ($op_c, $ops) = $lei->workers_start($self, 1); $lei->{wq1} = $self; $self->wq_io_do('do_solve_blob', []); $self->wq_close(1); diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm index 5b27ec2d..395a80f8 100644 --- a/lib/PublicInbox/LeiConvert.pm +++ b/lib/PublicInbox/LeiConvert.pm @@ -55,7 +55,7 @@ sub lei_convert { # the main "lei convert" method my $devfd = $lei->path_to_fd($ovv->{dst}) // return; $lei->{opt}->{augment} = 1 if $devfd < 0; $self->prepare_inputs($lei, \@inputs) or return; - my ($op_c, $ops) = $lei->workers_start($self, 'lei-convert', 1); + my ($op_c, $ops) = $lei->workers_start($self, 1); $lei->{wq1} = $self; $self->wq_io_do('process_inputs', []); $self->wq_close(1); diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index 394138b4..6a57df47 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -76,7 +76,7 @@ sub lei_import { # the main "lei import" method $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; $self->{-wq_nr_workers} = $j // 1; # locked $lei->{-eml_noisy} = 1; - (my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops); + (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); $lei->{wq1} = $self; $lei->{-err_type} = 'non-fatal'; net_merge_all_done($self) unless $lei->{auth}; diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm index a00e78fc..6cea6ae8 100644 --- a/lib/PublicInbox/LeiLsSearch.pm +++ b/lib/PublicInbox/LeiLsSearch.pm @@ -72,7 +72,7 @@ sub do_ls_search_long { sub bg_worker ($$$) { my ($lei, $pfx, $json) = @_; my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__; - my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1); + my ($op_c, $ops) = $lei->workers_start($self, 1); $lei->{wq1} = $self; $self->wq_io_do('do_ls_search_long', [], $pfx); $self->wq_close(1); diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index db97b98c..a37e1d5c 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -282,7 +282,7 @@ sub start { require PublicInbox::Inbox; require PublicInbox::Admin; require PublicInbox::InboxWritable; - my ($op, $ops) = $lei->workers_start($self, 'lei_mirror', 1); + my ($op, $ops) = $lei->workers_start($self, 1); $lei->{wq1} = $self; $self->wq_io_do('do_mirror', []); $self->wq_close(1); diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm index b4893489..f381a31c 100644 --- a/lib/PublicInbox/LeiP2q.pm +++ b/lib/PublicInbox/LeiP2q.pm @@ -188,7 +188,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point } else { $self->{input} = $input; } - my ($op, $ops) = $lei->workers_start($self, 'lei-p2q', 1); + my ($op, $ops) = $lei->workers_start($self, 1); $lei->{wq1} = $self; $self->wq_io_do('do_p2q', []); $self->wq_close(1); diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm index 6025c93e..c650e886 100644 --- a/lib/PublicInbox/LeiTag.pm +++ b/lib/PublicInbox/LeiTag.pm @@ -44,7 +44,7 @@ sub lei_tag { # the "lei tag" method $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; $self->{vmd_mod} = $vmd_mod; my $j = $self->{-wq_nr_workers} = 1; # locked for now - (my $op_c, $ops) = $lei->workers_start($self, 'lei-tag', $j, $ops); + (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); $lei->{wq1} = $self; $lei->{-err_type} = 'non-fatal'; net_merge_all_done($self) unless $lei->{auth};
Instead of "+FLAGS.SILENT" which merely adds to the keywords. We store all keywords together, so it's unlikely we will rely on the "+FLAGS.SILENT" or "-FLAGS.SILENT". --- lib/PublicInbox/NetWriter.pm | 14 ++++++-------- xt/net_writer-imap.t | 9 +++++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm index 49ac02a6..06d69f27 100644 --- a/lib/PublicInbox/NetWriter.pm +++ b/lib/PublicInbox/NetWriter.pm @@ -53,17 +53,15 @@ sub imap_delete_1 { $$delete_mic->delete_message($uid); } -sub imap_set_kw { - my ($self, $url, $uid, $kw) = @_; - my $mic = mic_for_folder($self, my $uri = $url) or return; - $mic->set_flag(kw2flags($kw), $uid); +sub imap_add_kw { + my ($self, $mic, $uid, $kw) = @_; + $mic->store($uid, '+FLAGS.SILENT', '('.kw2flags($kw).')'); $mic; # caller must ->expunge } -sub imap_unset_kw { - my ($self, $url, $uid, $kw) = @_; - my $mic = mic_for_folder($self, my $uri = $url) or return; - $mic->unset_flag(kw2flags($kw), $uid); +sub imap_set_kw { + my ($self, $mic, $uid, $kw) = @_; + $mic->store($uid, 'FLAGS.SILENT', '('.kw2flags($kw).')'); $mic; # caller must ->expunge } diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t index 007de35e..1298b958 100644 --- a/xt/net_writer-imap.t +++ b/xt/net_writer-imap.t @@ -157,8 +157,13 @@ test_lei(sub { lei_ok qw(import -F eml), $f, \'import local copy w/o keywords'; - $nwr->imap_set_kw($folder_uri, $uid[0], [ 'seen' ])->expunge - or BAIL_OUT "expunge $@"; + $mic = $nwr->mic_for_folder($folder_uri); + # dummy set to ensure second set_kw clobbers + $nwr->imap_set_kw($mic, $uid[0], [ qw(seen answered flagged) ] + )->expunge or BAIL_OUT "expunge $@"; + $nwr->imap_set_kw($mic, $uid[0], [ 'seen' ] + )->expunge or BAIL_OUT "expunge $@"; + $mic = undef; @res = (); $nwr->imap_each($folder_uri, $imap_slurp_all, \@res); is_deeply(\@res, [ [ ['seen'], $exp ] ], 'seen flag set') or
This will allow keyword updates from other folders to propagate to folders where search results may be duplicated. --- lib/PublicInbox/LEI.pm | 2 +- lib/PublicInbox/LeiQuery.pm | 7 +++---- lib/PublicInbox/LeiStore.pm | 8 ++++---- lib/PublicInbox/LeiToMail.pm | 24 +++++++++++++++++------- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index d5c6bd52..599cfab2 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -145,7 +145,7 @@ our %CMD = ( # sorted in order of importance/use: qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+ sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+ - color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ], + color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ], 'up' => [ 'OUTPUT|--all', 'update saved search', qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ], diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index efe328cc..1999a534 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -29,10 +29,9 @@ sub _start_query { # used by "lei q" and "lei up" return $self->fail("`$mj' writer jobs must be >= 1"); } my $l2m = $self->{l2m}; - if ($l2m && ($opt->{'import-remote'} //= 1) | - # we use \1 (a ref) to distinguish between - # user-supplied and default value - (($opt->{'import-before'} //= \1) ? 1 : 0)) { + # we use \1 (a ref) to distinguish between default vs. user-supplied + if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote + import-before))) { $self->_lei_store(1)->write_prepare($self); } $l2m and $l2m->{-wq_nr_workers} = $mj // do { diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index 8af740fd..29362b2e 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -193,15 +193,15 @@ sub remove_eml_vmd { \@docids; } -sub set_sync_info ($$$) { - my ($self, $oidhex, $sync_info) = @_; +sub set_sync_info { + my ($self, $oidhex, $folder, $id) = @_; ($self->{lms} //= do { require PublicInbox::LeiMailSync; my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3"; my $lms = PublicInbox::LeiMailSync->new($f); $lms->lms_begin; $lms; - })->set_src($oidhex, @$sync_info); + })->set_src($oidhex, $folder, $id); } sub add_eml { @@ -212,7 +212,7 @@ sub add_eml { my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg'; my $im_mark = $im->add($eml, undef, $smsg); if ($vmd && $vmd->{sync_info}) { - set_sync_info($self, $smsg->{blob}, $vmd->{sync_info}); + set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}}); } $im_mark or return; # duplicate blob returns undef diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index ab4de378..71acf952 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -242,7 +242,7 @@ sub _buf2maildir { my $kw = $smsg->{kw} // []; my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw)); my $rand = ''; # chosen by die roll :P - my ($tmp, $fh, $final, $ok); + my ($tmp, $fh, $base, $ok); my $common = $smsg->{blob} // _rand; if (defined(my $pct = $smsg->{pct})) { $common .= "=$pct" } do { @@ -257,11 +257,12 @@ sub _buf2maildir { $dst .= 'cur/'; $rand = ''; do { - $final = $dst.$rand.$common.':2,'.$sfx; - } while (!($ok = link($tmp, $final)) && $!{EEXIST} && + $base = $rand.$common.':2,'.$sfx + } while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} && ($rand = _rand.',')); - die "link($tmp, $final): $!" unless $ok; + die "link($tmp, $dst$base): $!" unless $ok; unlink($tmp) or warn "W: failed to unlink $tmp: $!\n"; + \$base; } else { my $err = "Error writing $smsg->{blob} to $dst: $!\n"; $_[0] = undef; # clobber dst @@ -276,13 +277,16 @@ sub _maildir_write_cb ($$) { $dedupe->prepare_dedupe if $dedupe; my $dst = $lei->{ovv}->{dst}; my $lse = $lei->{lse}; # may be undef + my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef; + my $out = $sto ? 'maildir:'.$lei->rel2abs($dst) : undef; sub { # for git_to_mail my ($bref, $smsg, $eml) = @_; $dst // return $lei->fail; # dst may be undef-ed in last run return if $dedupe && $dedupe->is_dup($eml // PublicInbox::Eml->new($$bref)); $lse->xsmsg_vmd($smsg) if $lse; - _buf2maildir($dst, $bref // \($eml->as_string), $smsg); + my $n = _buf2maildir($dst, $bref // \($eml->as_string), $smsg); + $sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto; ++$lei->{-nr_write}; } } @@ -291,21 +295,27 @@ sub _imap_write_cb ($$) { my ($self, $lei) = @_; my $dedupe = $lei->{dedupe}; $dedupe->prepare_dedupe if $dedupe; - my $imap_append = $lei->{net}->can('imap_append'); + my $append = $lei->{net}->can('imap_append'); my $mic = $lei->{net}->mic_get($self->{uri}); my $folder = $self->{uri}->mailbox; my $lse = $lei->{lse}; # may be undef + my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef; + my $out = $lei->{ovv}->{dst}; sub { # for git_to_mail my ($bref, $smsg, $eml) = @_; $mic // return $lei->fail; # mic may be undef-ed in last run return if $dedupe && $dedupe->is_dup($eml // PublicInbox::Eml->new($$bref)); $lse->xsmsg_vmd($smsg) if $lse; - eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) }; + my $uid = eval { $append->($mic, $folder, $bref, $smsg, $eml) }; if (my $err = $@) { undef $mic; die $err; } + # imap_append returns UID if IMAP server has UIDPLUS extension + ($sto && $uid =~ /\A[0-9]+\z/) and + $sto->ipc_do('set_sync_info', + $smsg->{blob}, $out, $uid + 0); ++$lei->{-nr_write}; } }