From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id AF9381FA18 for ; Fri, 21 May 2021 10:28:33 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Date: Fri, 21 May 2021 10:28:32 +0000 Message-Id: <20210521102832.10784-9-e@80x24.org> In-Reply-To: <20210521102832.10784-1-e@80x24.org> References: <20210521102832.10784-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Just having UIDVALIDITY in the URI isn't enough, since a single lei user may have multiple IMAP logins on the same server. This leads to compatibility problems and forces a reimport for the few users already using this lei functionality, but it's not stable nor released, yet. --- lib/PublicInbox/NetReader.pm | 42 ++++++++++++++++++++++-------------- t/lei-import-imap.t | 9 +++++--- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index fd0d1682..a532b218 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -58,12 +58,10 @@ sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback # mic_for may prompt the user and store auth info, prepares mic_get sub mic_for ($$$$) { # mic = Mail::IMAPClient - my ($self, $url, $mic_args, $lei) = @_; - require PublicInbox::URIimap; - my $uri = PublicInbox::URIimap->new($url); + my ($self, $uri, $mic_args, $lei) = @_; require PublicInbox::GitCredential; my $cred = bless { - url => $url, + url => "$uri", protocol => $uri->scheme, host => $uri->host, username => $uri->user, @@ -83,13 +81,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient }; require PublicInbox::IMAPClient; my $mic = mic_new($self, $mic_arg, $sec, $uri) or - die "E: <$url> new: $@\n"; + die "E: <$uri> new: $@\n"; # default to using STARTTLS if it's available, but allow # it to be disabled since I usually connect to localhost if (!$mic_arg->{Ssl} && !defined($mic_arg->{Starttls}) && $mic->has_capability('STARTTLS') && $mic->can('starttls')) { - $mic->starttls or die "E: <$url> STARTTLS: $@\n"; + $mic->starttls or die "E: <$uri> STARTTLS: $@\n"; } # do we even need credentials? @@ -111,8 +109,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient if ($mic->login && $mic->IsAuthenticated) { # success! keep IMAPClient->new arg in case we get disconnected $self->{mic_arg}->{$sec} = $mic_arg; + if ($cred) { + $uri->user($cred->{username}) if !defined($uri->user); + } elsif ($mic_arg->{Authmechanism} eq 'ANONYMOUS') { + $uri->auth('ANONYMOUS') if !defined($uri->auth); + } } else { - $err = "E: <$url> LOGIN: $@\n"; + $err = "E: <$uri> LOGIN: $@\n"; if ($cred && defined($cred->{password})) { $err =~ s/\Q$cred->{password}\E/*******/g; } @@ -304,15 +307,16 @@ sub imap_common_init ($;$) { # make sure we can connect and cache the credentials in memory $self->{mic_arg} = {}; # schema://authority => IMAPClient->new args my $mics = {}; # schema://authority => IMAPClient obj - for my $uri (@{$self->{imap_order}}) { - my $sec = uri_section($uri); + for my $orig_uri (@{$self->{imap_order}}) { + my $sec = uri_section($orig_uri); + my $uri = PublicInbox::URIimap->new("$sec/"); my $mic = $mics->{$sec} //= - mic_for($self, "$sec/", $mic_args, $lei) // + mic_for($self, $uri, $mic_args, $lei) // die "Unable to continue\n"; next unless $self->isa('PublicInbox::NetWriter'); - my $dst = $uri->mailbox // next; + my $dst = $orig_uri->mailbox // next; next if $mic->exists($dst); # already exists - $mic->create($dst) or die "CREATE $dst failed <$uri>: $@"; + $mic->create($dst) or die "CREATE $dst failed <$orig_uri>: $@"; } $mics; } @@ -419,12 +423,18 @@ sub run_commit_cb ($) { $cb->(@args); } -sub _itrk_last ($$;$) { - my ($self, $uri, $r_uidval) = @_; +sub itrk_last ($$;$$) { + my ($self, $uri, $r_uidval, $mic) = @_; return (undef, undef, $r_uidval) unless $self->{incremental}; my ($itrk, $l_uid, $l_uidval); if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0 $uri->uidvalidity($r_uidval) if defined $r_uidval; + if ($mic) { + my $auth = $mic->Authmechanism // ''; + $uri->auth($auth) if $auth eq 'ANONYMOUS'; + my $user = $mic->User; + $uri->user($user) if defined($user); + } my $x; $l_uid = ($lms && ($x = $lms->location_stats($$uri))) ? $x->{'uid.max'} : undef; @@ -459,7 +469,7 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval) EOF my $uri = $orig_uri->clone; - my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval); + my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic); return < $tmpdir }, sub { lei_ok('import', $url); lei_ok 'ls-mail-sync'; - like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-mail-sync'); + like($lei_out, qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E + /t\.v2\.0;UIDVALIDITY=\d+\n\z!x, 'ls-mail-sync'); chomp(my $u = $lei_out); lei_ok('import', $u, \'UIDVALIDITY match in URL'); + $url = $u; $u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s; ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected'); @@ -33,7 +35,7 @@ test_lei({ tmpdir => $tmpdir }, sub { my $inspect = json_utf8->decode($lei_out); my @k = keys %$inspect; is(scalar(@k), 1, 'one URL resolved'); - like($k[0], qr!\A\Q$url\E;UIDVALIDITY=\d+\z!, 'inspect URL matches'); + is($k[0], $url, 'inspect URL matches'); my $stats = $inspect->{$k[0]}; is_deeply([ sort keys %$stats ], [ qw(uid.count uid.max uid.min) ], 'keys match'); @@ -55,7 +57,8 @@ test_lei({ tmpdir => $tmpdir }, sub { my $x = json_utf8->decode($lei_out); is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect'); is(ref($x->{'mail-sync'}), 'HASH', 'sync in inspect'); - is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect'); + is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect') + or diag explain($x); my $psgi_attach = 'cfa3622cbeffc9bd6b0fc66c4d60d420ba74f60d'; lei_ok('blob', $psgi_attach);