* [PATCH 3/4] lei up: support --all for IMAP folders
2021-09-07 11:32 [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
2021-09-07 11:32 ` [PATCH 2/4] lei: dump and clear log at exit Eric Wong
@ 2021-09-07 11:32 ` Eric Wong
2021-09-07 11:32 ` [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
Since "lei up" is expected to be a heavily-used command,
better support for IMAP seems like a reasonable idea.
This is inefficient since we waste an IMAP(S) TCP connection
since it dies when an auth-only LeiUp worker process dies, but
it's better than not working at all, right now.
---
Documentation/lei-up.pod | 15 ++++-
lib/PublicInbox/LEI.pm | 19 ++++---
lib/PublicInbox/LeiAuth.pm | 15 +++--
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 104 ++++++++++++++++++++++-------------
xt/net_writer-imap.t | 1 +
6 files changed, 102 insertions(+), 55 deletions(-)
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index cea0f619..ca4cf4fe 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -6,15 +6,24 @@ lei-up - update a saved search
lei up [OPTIONS] OUTPUT
-lei up [OPTIONS] --all=TYPE
+lei up [OPTIONS] --all[=<local|remote>]
=head1 DESCRIPTION
-Update the saved search at C<OUTPUT> or all saved searches of C<TYPE>
-(currently C<local> is the only supported value).
+Update the saved search at C<OUTPUT> or all saved searches.
=head1 OPTIONS
+=over
+
+=item --all[=<local|remote>]
+
+C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
+C<--all=local> only updates local mailboxes, C<--all=remote> only
+updates remote mailboxes (currently C<imap://> and C<imaps://>).
+
+=back
+
The following options, described in L<lei-q(1)>, are supported.
=over
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bd44cfae..a258722e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -605,16 +605,19 @@ sub incr {
$self->{counters}->{$field} += $nr;
}
+sub pkt_ops {
+ my ($lei, $ops) = @_;
+ $ops->{'!'} = [ \&fail_handler, $lei ];
+ $ops->{'|'} = [ \&sigpipe_handler, $lei ];
+ $ops->{x_it} = [ \&x_it, $lei ];
+ $ops->{child_error} = [ \&child_error, $lei ];
+ $ops->{incr} = [ \&incr, $lei ];
+ $ops;
+}
+
sub workers_start {
my ($lei, $wq, $jobs, $ops, $flds) = @_;
- $ops = {
- '!' => [ \&fail_handler, $lei ],
- '|' => [ \&sigpipe_handler, $lei ],
- 'x_it' => [ \&x_it, $lei ],
- 'child_error' => [ \&child_error, $lei ],
- 'incr' => [ \&incr, $lei ],
- ($ops ? %$ops : ()),
- };
+ $ops = pkt_ops($lei, { ($ops ? %$ops : ()) });
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm
index 465a2758..9b09cecf 100644
--- a/lib/PublicInbox/LeiAuth.pm
+++ b/lib/PublicInbox/LeiAuth.pm
@@ -30,10 +30,16 @@ sub do_auth_atfork { # used by IPC WQ workers
return if $wq->{-wq_worker_nr} != 0; # only first worker calls this
my $lei = $wq->{lei};
my $net = $lei->{net};
+ if ($net->{-auth_done}) { # from previous worker... (ugly)
+ $lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
+ $lei->fail("pkt_do net_merge_continue: $!");
+ return;
+ }
eval { # fill auth info (may prompt user or read netrc)
my $mics = $net->imap_common_init($lei);
my $nn = $net->nntp_common_init($lei);
# broadcast successful auth info to lei-daemon:
+ $net->{-auth_done} = 1;
$lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
die "pkt_do net_merge_continue: $!";
$net->{mics_cached} = $mics if $mics;
@@ -51,14 +57,15 @@ sub net_merge_all { # called in wq worker via wq_broadcast
# called by top-level lei-daemon when first worker is done with auth
# passes updated net auth info to current workers
sub net_merge_continue {
- my ($wq, $net_new) = @_;
+ my ($wq, $lei, $net_new) = @_;
+ $wq->{-net_new} = $net_new; # for "lei up"
$wq->wq_broadcast('PublicInbox::LeiAuth::net_merge_all', $net_new);
- $wq->net_merge_all_done; # defined per-WQ
+ $wq->net_merge_all_done($lei); # defined per-WQ
}
sub op_merge { # prepares PktOp->pair ops
- my ($self, $ops, $wq) = @_;
- $ops->{net_merge_continue} = [ \&net_merge_continue, $wq ];
+ my ($self, $ops, $wq, $lei) = @_;
+ $ops->{net_merge_continue} = [ \&net_merge_continue, $wq, $lei ];
}
sub new { bless \(my $x), __PACKAGE__ }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 2c7a92de..01f08384 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -396,7 +396,8 @@ sub new {
} elsif ($fmt =~ /\Aimaps?\z/) {
require PublicInbox::NetWriter;
require PublicInbox::URIimap;
- my $net = PublicInbox::NetWriter->new;
+ # {net} may exist from "lei up" for auth
+ my $net = $lei->{net} // PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
$net->add_url($$uri);
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index a39d6047..30358e9d 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -5,8 +5,14 @@
package PublicInbox::LeiUp;
use strict;
use v5.10.1;
+# n.b. we use LeiInput to setup IMAP auth
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiSavedSearch;
-use parent qw(PublicInbox::IPC);
+use PublicInbox::DS;
+use PublicInbox::PktOp;
+use PublicInbox::LeiFinmsg;
+use PublicInbox::LEI;
+my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
sub up1 ($$) {
my ($lei, $out) = @_;
@@ -48,75 +54,95 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
- require PublicInbox::LeiFinmsg;
- $lei->{fmsg} //= PublicInbox::LeiFinmsg->new($lei->{2});
my $l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} };
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
+ delete $l->{sock}; # do not close
+ $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ local $PublicInbox::LEI::current_lei = $l;
$l->{1} = $fh;
eval {
$l->qerr("# updating $out");
up1($l, $out);
- $l->qerr("# $out done");
};
- $l->child_error(0, $@) if $@;
+ $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
+}
+
+sub redispatch_all ($$) {
+ my ($self, $lei) = @_;
+ # re-dispatch into our event loop w/o creating an extra fork-level
+ $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei->{2});
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
+ PublicInbox::DS::requeue(sub {
+ up1_redispatch($lei, $o, $op_p);
+ });
+ }
+ $lei->event_step_init;
+ $lei->pkt_ops($op_c->{ops} = { '' => [$lei->can('dclose'), $lei] });
}
sub lei_up {
my ($lei, @outs) = @_;
- $lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
- my @local;
- if (defined $opt->{all}) {
+ my $self = bless { -mail_sync => 1 }, __PACKAGE__;
+ $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
+ if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
length($opt->{mua}//'') and return
$lei->fail('--all and --mua= are incompatible');
-
- # supporting IMAP outputs is more involved due to
- # git-credential prompts. TODO: add this in 1.8
- $opt->{all} eq 'local' or return
- $lei->fail('only --all=local works at the moment');
- my @all = PublicInbox::LeiSavedSearch::list($lei);
- @local = grep(!m!\Aimaps?://!i, @all);
+ @outs = PublicInbox::LeiSavedSearch::list($lei);
+ if ($all eq 'local') {
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq 'remote') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq '') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } else {
+ $lei->fail("only --all=$all not understood");
+ }
} else {
- @local = @outs;
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
- if (scalar(@outs) > 1) {
- length($opt->{mua}//'') and return $lei->fail(<<EOM);
+ ((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
+ length($opt->{mua} // '')) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
- # TODO:
- return $lei->fail(<<EOM) if grep(m!\Aimaps?://!i, @outs);
-multiple destinations only supported for local outputs (FIXME)
-EOM
+ if ($self->{remote}) { # setup lei->{auth}
+ $self->prepare_inputs($lei, $self->{remote}) or return;
}
- if (scalar(@local) > 1) {
- $lei->_lei_store->write_prepare($lei); # share early
- # daemon mode, re-dispatch into our event loop w/o
- # creating an extra fork-level
- require PublicInbox::DS;
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@local) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
- }
- $lei->event_step_init;
- $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
+ if ($lei->{auth}) { # start auth worker
+ require PublicInbox::NetWriter;
+ bless $lei->{net}, 'PublicInbox::NetWriter';
+ $lei->{auth}->op_merge(my $ops = {}, $self, $lei);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
+ $lei->{wq1} = $self;
+ $lei->wait_wq_events($op_c, $ops);
+ # net_merge_all_done will fire when auth is done
} else {
- up1($lei, $local[0]);
+ redispatch_all($self, $lei); # see below
}
}
+# called in top-level lei-daemon when LeiAuth is done
+sub net_merge_all_done {
+ my ($self, $lei) = @_;
+ $lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
+ $self->wq_close(1);
+ redispatch_all($self, $lei);
+}
+
sub _complete_up {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
}
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+
1;
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index aeed3144..0a4cea68 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -242,6 +242,7 @@ EOM
lei_ok qw(q m:testmessage --no-external -o), $folder_url;
lei_ok qw(up), $folder_url;
+ lei_ok qw(up --all=remote);
});
undef $cleanup; # remove temporary folder
^ permalink raw reply related [flat|nested] 5+ messages in thread