From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 3B9611F9FF for ; Sun, 21 Feb 2021 07:41:35 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 2/7] lei q: support IMAP/IMAPS --output destinations Date: Sun, 21 Feb 2021 07:41:29 +0000 Message-Id: <20210221074134.15084-3-e@80x24.org> In-Reply-To: <20210221074134.15084-1-e@80x24.org> References: <20210221074134.15084-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Augment (and dedupe) aren't parallel, yet, so its more sensitive to high-latency networks. --- lib/PublicInbox/LeiAuth.pm | 2 +- lib/PublicInbox/LeiOverview.pm | 7 +- lib/PublicInbox/LeiQuery.pm | 18 ++++- lib/PublicInbox/LeiToMail.pm | 56 +++++++++++++++- lib/PublicInbox/NetReader.pm | 7 +- lib/PublicInbox/NetWriter.pm | 12 ++++ xt/net_writer-imap.t | 118 ++++++++++++++++++++++++++++++--- 7 files changed, 202 insertions(+), 18 deletions(-) diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm index 7acb9900..bf0110ed 100644 --- a/lib/PublicInbox/LeiAuth.pm +++ b/lib/PublicInbox/LeiAuth.pm @@ -63,7 +63,7 @@ sub ipc_atfork_child { } sub new { - my ($cls, $nrd) = @_; + my ($cls, $nrd) = @_; # nrd may be NetReader or descendant (NetWriter) bless { nrd => $nrd }, $cls; } diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index 3169bae6..4db1d8c8 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -59,7 +59,12 @@ sub new { my $fmt = $opt->{$ofmt_key}; $fmt = lc($fmt) if defined $fmt; - if ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/ + if ($dst =~ m!\A([a-z0-9\+]+)://!is) { + defined($fmt) and return $lei->fail(<<""); +--$ofmt_key=$fmt invalid with URL $dst + + $fmt = lc $1; + } elsif ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/ my $ofmt = lc $1; $fmt //= $ofmt; return $lei->fail(<<"") if $fmt ne $ofmt; diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index f71beae6..eaf91f2e 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -11,14 +11,26 @@ sub prep_ext { # externals_each callback $lxs->prepare_external($loc) unless $exclude->{$loc}; } -sub qstr_add { # for --stdin +sub _start_query { + my ($self) = @_; + if (my $nwr = $self->{nwr}) { + require PublicInbox::LeiAuth; + my $auth = $self->{auth} = PublicInbox::LeiAuth->new($nwr); + my $lxs = $self->{lxs}; + $auth->auth_start($self, $lxs->can('do_query'), $lxs, $self); + } else { + $self->{lxs}->do_query($self); + } +} + +sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin my ($self) = @_; # $_[1] = $rbuf if (defined($_[1])) { $_[1] eq '' and return eval { my $lse = delete $self->{lse}; $lse->query_approxidate($lse->git, $self->{mset_opt}->{qstr}); - $self->{lxs}->do_query($self); + _start_query($self); }; $self->{mset_opt}->{qstr} .= $_[1]; } else { @@ -115,7 +127,7 @@ no query allowed on command-line with --stdin return; } $mset_opt{qstr} = $lse->query_argv_to_string($lse->git, \@argv); - $lxs->do_query($self); + _start_query($self); } # shell completion helper called by lei__complete diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index e89cca71..0e0b0a43 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -331,9 +331,31 @@ sub _maildir_write_cb ($$) { } } +sub _imap_write_cb ($$) { + my ($self, $lei) = @_; + my $dedupe = $lei->{dedupe}; + $dedupe->prepare_dedupe if $dedupe; + my $imap_append = $lei->{nwr}->can('imap_append'); + my $mic = $lei->{nwr}->mic_get($lei->{ovv}->{dst}); + my $folder = $self->{uri}->mailbox; + sub { # for git_to_mail + my ($bref, $smsg, $eml) = @_; + $mic // return $lei->fail; # dst may be undef-ed in last run + if ($dedupe) { + $eml //= PublicInbox::Eml->new($$bref); # copy bref + return if $dedupe->is_dup($eml, $smsg->{blob}); + } + eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) }; + if (my $err = $@) { + undef $mic; + die $err; + } + } +} + sub write_cb { # returns a callback for git_to_mail my ($self, $lei) = @_; - # _mbox_write_cb or _maildir_write_cb + # _mbox_write_cb, _maildir_write_cb or _imap_write_cb my $m = "_$self->{base_type}_write_cb"; $self->$m($lei); } @@ -360,6 +382,18 @@ sub new { "$dst exists and is not a writable file\n"; $self->can("eml2$fmt") or die "bad mbox format: $fmt\n"; $self->{base_type} = 'mbox'; + } elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support + require PublicInbox::NetWriter; + my $nwr = PublicInbox::NetWriter->new; + $nwr->add_url($dst); + $nwr->{quiet} = $lei->{opt}->{quiet}; + my $err = $nwr->errors($dst); + return $lei->fail($err) if $err; + require PublicInbox::URIimap; # TODO: URI cast early + $self->{uri} = PublicInbox::URIimap->new($dst); + $self->{uri}->mailbox or die "No mailbox: $dst"; + $lei->{nwr} = $nwr; + $self->{base_type} = 'imap'; } else { die "bad mail --format=$fmt\n"; } @@ -394,6 +428,26 @@ sub _do_augment_maildir { } } +sub _augment_imap { # PublicInbox::NetReader::imap_each cb + my ($url, $uid, $kw, $eml, $lei) = @_; + _augment($eml, $lei); +} + +sub _do_augment_imap { + my ($self, $lei) = @_; + my $dst = $lei->{ovv}->{dst}; + my $nwr = $lei->{nwr}; + if ($lei->{opt}->{augment}) { + my $dedupe = $lei->{dedupe}; + if ($dedupe && $dedupe->prepare_dedupe) { + $nwr->imap_each($dst, \&_augment_imap, $lei); + $dedupe->pause_dedupe; + } + } else { # clobber existing IMAP folder + $nwr->imap_delete_all($dst); + } +} + sub _pre_augment_mbox { my ($self, $lei) = @_; my $dst = $lei->{ovv}->{dst}; diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index 92d004bc..541094a0 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -422,8 +422,13 @@ sub _imap_fetch_all ($$$) { # uses cached auth info prepared by mic_for sub mic_get { my ($self, $sec) = @_; - my $mic_arg = $self->{mic_arg}->{$sec} or + my $mic_arg = $self->{mic_arg}->{$sec}; + unless ($mic_arg) { + my $uri = PublicInbox::URIimap->new($sec); + $sec = uri_section($uri); + $mic_arg = $self->{mic_arg}->{$sec} or die "BUG: no Mail::IMAPClient->new arg for $sec"; + } if (defined(my $cb_name = $mic_arg->{Authcallback})) { if (ref($cb_name) ne 'CODE') { $mic_arg->{Authcallback} = $self->can($cb_name); diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm index 6f0a0b94..89f8662e 100644 --- a/lib/PublicInbox/NetWriter.pm +++ b/lib/PublicInbox/NetWriter.pm @@ -23,4 +23,16 @@ sub imap_append { die "APPEND $folder: $@"; } +sub imap_delete_all { + my ($self, $url) = @_; + my $uri = PublicInbox::URIimap->new($url); + my $sec = $self->can('uri_section')->($uri); + local $0 = $uri->mailbox." $sec"; + my $mic = $self->mic_get($sec) or die "E: not connected: $@"; + $mic->select($uri->mailbox) or return; # non-existent + if ($mic->delete_message('1:*')) { + $mic->expunge; + } +} + 1; diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t index dfd765be..4832245a 100644 --- a/xt/net_writer-imap.t +++ b/xt/net_writer-imap.t @@ -7,6 +7,7 @@ use POSIX qw(strftime); use PublicInbox::OnDestroy; use PublicInbox::URIimap; use PublicInbox::Config; +use Fcntl qw(O_EXCL O_WRONLY O_CREAT); my $imap_url = $ENV{TEST_IMAP_WRITE_URL} or plan skip_all => 'TEST_IMAP_WRITE_URL unset'; my $uri = PublicInbox::URIimap->new($imap_url); @@ -19,30 +20,125 @@ my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!); my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)). "-$$-".sprintf('%x', int(rand(0xffffffff))); my $nwr = PublicInbox::NetWriter->new; -$imap_url .= '/' unless substr($imap_url, -1) eq '/'; +chop($imap_url) if substr($imap_url, -1) eq '/'; my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder"); is($folder_uri->mailbox, $folder, 'folder correct') or BAIL_OUT "BUG: bad $$uri"; $nwr->add_url($$folder_uri); is($nwr->errors, undef, 'no errors'); $nwr->{pi_cfg} = bless {}, 'PublicInbox::Config'; -my $mics = $nwr->imap_common_init; + +my $set_cred_helper = sub { + my ($f, $cred_set) = @_; + sysopen(my $fh, $f, O_CREAT|O_EXCL|O_WRONLY) or BAIL_OUT "open $f: $!"; + print $fh </dev/null`); +if ($cred_helper eq 'store') { + my $config = $ENV{XDG_CONFIG_HOME} // "$ENV{HOME}/.config"; + for my $f ("$ENV{HOME}/.git-credentials", "$config/git/credentials") { + next unless -f $f; + @cred_link = ($f, '/.git-credentials'); + last; + } + $cred_set = qq("$cred_helper"); +} elsif ($cred_helper =~ /\Acache(?:[ \t]|\z)/) { + my $cache = $ENV{XDG_CACHE_HOME} // "$ENV{HOME}/.cache"; + for my $d ("$ENV{HOME}/.git-credential-cache", + "$cache/git/credential") { + next unless -d $d; + @cred_link = ($d, '/.git-credential-cache'); + $cred_set = qq("$cred_helper"); + last; + } +} elsif (!$cred_helper) { # make the test less painful if no creds configured + ($tmpdir, $for_destroy) = tmpdir; + my $d = "$tmpdir/.git-credential-cache"; + mkdir($d, 0700) or BAIL_OUT $!; + $cred_set = "cache --timeout=60"; + @cred_link = ($d, '/.git-credential-cache'); +} else { + diag "credential.helper=$cred_helper will not be used for this test"; +} + +my $mics = do { + local $ENV{HOME} = $tmpdir // $ENV{HOME}; + if ($tmpdir && $cred_set) { + $set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) + } + $nwr->imap_common_init; +}; my $mic = (values %$mics)[0]; -my $cleanup = PublicInbox::OnDestroy->new(sub { +my $cleanup = PublicInbox::OnDestroy->new($$, sub { + my $mic = $nwr->mic_get($imap_url); $mic->delete($folder) or fail "delete $folder <$folder_uri>: $@"; + if ($tmpdir && -f "$tmpdir/.gitconfig") { + local $ENV{HOME} = $tmpdir; + system(qw(git credential-cache exit)); + } }); my $imap_append = $nwr->can('imap_append'); my $smsg = bless { kw => [ 'seen' ] }, 'PublicInbox::Smsg'; $imap_append->($mic, $folder, undef, $smsg, eml_load('t/plack-qp.eml')); -my @res; $nwr->{quiet} = 1; -$nwr->imap_each($$folder_uri, sub { - my ($u, $uid, $kw, $eml, $arg) = @_; - push @res, [ $kw, $eml ]; -}); -is(scalar(@res), 1, 'got appended message'); -is_deeply(\@res, [ [ [ 'seen' ], eml_load('t/plack-qp.eml') ] ], +my $imap_slurp_all = sub { + my ($u, $uid, $kw, $eml, $res) = @_; + push @$res, [ $kw, $eml ]; +}; +$nwr->imap_each($$folder_uri, $imap_slurp_all, my $res = []); +is(scalar(@$res), 1, 'got appended message'); +my $plack_qp_eml = eml_load('t/plack-qp.eml'); +is_deeply($res, [ [ [ 'seen' ], $plack_qp_eml ] ], 'uploaded message read back'); +$res = $mic = $mics = undef; + +test_lei(sub { + my ($ro_home, $cfg_path) = setup_public_inboxes; + my $cfg = PublicInbox::Config->new($cfg_path); + $cfg->each_inbox(sub { + my ($ibx) = @_; + lei_ok qw(add-external -q), $ibx->{inboxdir} or BAIL_OUT; + }); + + # cred_link[0] may be on a different (hopefully encrypted) FS, + # we only symlink to it here, so we don't copy any sensitive data + # into the temporary directory + if (@cred_link && !symlink($cred_link[0], $ENV{HOME}.$cred_link[1])) { + diag "symlink @cred_link: $! (non-fatal)"; + $cred_set = undef; + } + $set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) if $cred_set; + + lei_ok qw(q f:qp@example.com -o), $$folder_uri; + $nwr->imap_each($$folder_uri, $imap_slurp_all, my $res = []); + is(scalar(@$res), 1, 'got one deduped result') or diag explain($res); + is_deeply($res->[0]->[1], $plack_qp_eml, + 'lei q wrote expected result'); + + lei_ok qw(q f:matz -a -o), $$folder_uri; + $nwr->imap_each($$folder_uri, $imap_slurp_all, my $aug = []); + is(scalar(@$aug), 2, '2 results after augment') or diag explain($aug); + my $exp = $res->[0]->[1]->as_string; + is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1, + 'original remains after augment'); + $exp = eml_load('t/iso-2202-jp.eml')->as_string; + is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1, + 'new result shown after augment'); + + lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string; + $nwr->imap_each($$folder_uri, $imap_slurp_all, my $empty = []); + is(scalar(@$empty), 0, 'no results w/o augment'); + +}); -undef $cleanup; +undef $cleanup; # remove temporary folder done_testing;