[PATCH 0/8] lei: export-kw, IMAP import incompatibility

[PATCH 0/8] lei: export-kw, IMAP import incompatibility

[Eric Wong]

"lei export-kw" is a new command.  I'm not sure exactly how
it'll be used but it's probably more of a plumbing command,
for now.  My brain hurts thinking about synchronization
and merge/conflict resolution when it comes to propagating
keywords assignments/clearing.

(I frequently mark messages as Unread in my MUA so I know to
reread them in the future, and I suspect it's a common thing).

mail_sync.sqlite3 now tracks AUTH=ANONYMOUS or username in the
folder name to account for lei(Unix) users having multiple IMAP
accounts on the same host with the same folders+UIDVALIDITY.

"lei import imap(s)://" users will waste a bit of bandwidth
resyncing as a result.

Eric Wong (8):
  treewide: favor open(..., '+<&=', $fd)
  lei: drop EOFpipe in favor of PktOp
  lei tag: support tagging index-only messages
  lei_input: fix canonicalization of Maildirs for sync
  lei index: support command-line options
  lei export-kw: new command to export keywords to Maildirs
  uri_imap: support uid/auth/user as full accessors
  lei import: store IMAP user+auth in mail_sync folder URI

 MANIFEST                       |   2 +
 examples/unsubscribe.milter    |   3 +-
 lib/PublicInbox/DS.pm          |   3 +-
 lib/PublicInbox/Daemon.pm      |   2 +-
 lib/PublicInbox/LEI.pm         |  18 +++-
 lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++
 lib/PublicInbox/LeiInput.pm    |   3 +-
 lib/PublicInbox/LeiMailSync.pm |  10 ++
 lib/PublicInbox/LeiOverview.pm |   2 +-
 lib/PublicInbox/LeiSearch.pm   |  22 +++-
 lib/PublicInbox/LeiTag.pm      |  10 +-
 lib/PublicInbox/LeiToMail.pm   |  12 ++-
 lib/PublicInbox/MdirReader.pm  |  14 +++
 lib/PublicInbox/NetReader.pm   |  42 +++++---
 lib/PublicInbox/Sigfd.pm       |   3 +-
 lib/PublicInbox/URIimap.pm     |  82 +++++++++++----
 t/epoll.t                      |   7 +-
 t/lei-export-kw.t              |  35 +++++++
 t/lei-import-imap.t            |   9 +-
 t/lei-index.t                  |  12 ++-
 t/mdir_reader.t                |   5 +
 t/uri_imap.t                   |  60 ++++++++---
 22 files changed, 461 insertions(+), 75 deletions(-)
 create mode 100644 lib/PublicInbox/LeiExportKw.pm
 create mode 100644 t/lei-export-kw.t

[PATCH 1/8] treewide: favor open(..., '+<&=', $fd)

[Eric Wong]

Cut down on unnecessary imports of IO::Handle and
method lookup + dispatch overhead.
---
 examples/unsubscribe.milter | 3 +--
 lib/PublicInbox/DS.pm       | 3 +--
 lib/PublicInbox/Daemon.pm   | 2 +-
 lib/PublicInbox/Sigfd.pm    | 3 +--
 t/epoll.t                   | 7 +++++--
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/examples/unsubscribe.milter b/examples/unsubscribe.milter
index 7b126e30..608524cb 100644
--- a/examples/unsubscribe.milter
+++ b/examples/unsubscribe.milter
@@ -2,7 +2,6 @@
 # Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
-use warnings;
 use Sendmail::PMilter qw(:all);
 use IO::Socket;
 use Crypt::CBC;
@@ -128,7 +127,7 @@ my $fds = $ENV{LISTEN_FDS};
 if ($fds && (($ENV{LISTEN_PID} || 0) == $$)) {
 	die "$0 can only listen on one FD\n" if $fds != 1;
 	my $start_fd = 3;
-	my $s = IO::Socket->new_from_fd($start_fd, 'r') or
+	open(my $s, '<&=', $start_fd) or
 		die "inherited bad FD from LISTEN_FDS: $!\n";
 	$milter->set_socket($s);
 } else {
diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index 3cddfd18..7a4dfed0 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -25,7 +25,6 @@ use v5.10.1;
 use parent qw(Exporter);
 use bytes;
 use POSIX qw(WNOHANG sigprocmask SIG_SETMASK);
-use IO::Handle qw();
 use Fcntl qw(SEEK_SET :DEFAULT O_APPEND);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use Scalar::Util qw(blessed);
@@ -135,7 +134,7 @@ sub add_timer ($$;@) {
 sub set_cloexec ($) {
     my ($fd) = @_;
 
-    $_io = IO::Handle->new_from_fd($fd, 'r+') or return;
+    open($_io, '+<&=', $fd) or return;
     defined(my $fl = fcntl($_io, F_GETFD, 0)) or return;
     fcntl($_io, F_SETFD, $fl | FD_CLOEXEC);
 }
diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index b5f97d81..727311a4 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -367,7 +367,7 @@ sub inherit ($) {
 	my $end = $fds + 2; # LISTEN_FDS_START - 1
 	my @rv = ();
 	foreach my $fd (3..$end) {
-		my $s = IO::Handle->new_from_fd($fd, 'r');
+		open(my $s, '<&=', $fd) or warn "fdopen fd=$fd: $!";
 		if (my $k = sockname($s)) {
 			my $prev_was_blocking = $s->blocking(0);
 			warn <<"" if $prev_was_blocking;
diff --git a/lib/PublicInbox/Sigfd.pm b/lib/PublicInbox/Sigfd.pm
index a4d1b3bb..d91ea0e7 100644
--- a/lib/PublicInbox/Sigfd.pm
+++ b/lib/PublicInbox/Sigfd.pm
@@ -8,7 +8,6 @@ use strict;
 use parent qw(PublicInbox::DS);
 use PublicInbox::Syscall qw(signalfd EPOLLIN EPOLLET SFD_NONBLOCK);
 use POSIX ();
-use IO::Handle ();
 
 # returns a coderef to unblock signals if neither signalfd or kqueue
 # are available.
@@ -27,7 +26,7 @@ sub new {
 	my $io;
 	my $fd = signalfd(-1, [keys %signo], $flags);
 	if (defined $fd && $fd >= 0) {
-		$io = IO::Handle->new_from_fd($fd, 'r+');
+		open($io, '+<&=', $fd) or die "open: $!";
 	} elsif (eval { require PublicInbox::DSKQXS }) {
 		$io = PublicInbox::DSKQXS->signalfd([keys %signo], $flags);
 	} else {
diff --git a/t/epoll.t b/t/epoll.t
index f2a68904..f346b387 100644
--- a/t/epoll.t
+++ b/t/epoll.t
@@ -1,11 +1,14 @@
+#!perl -w
+# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
+use v5.10.1;
 use Test::More;
-use IO::Handle;
 use PublicInbox::Syscall qw(:epoll);
 plan skip_all => 'not Linux' if $^O ne 'linux';
 my $epfd = epoll_create();
 ok($epfd >= 0, 'epoll_create');
-my $hnd = IO::Handle->new_from_fd($epfd, 'r+'); # close on exit
+open(my $hnd, '+<&=', $epfd); # for autoclose
 
 pipe(my ($r, $w)) or die "pipe: $!";
 is(epoll_ctl($epfd, EPOLL_CTL_ADD, fileno($w), EPOLLOUT), 0,

[PATCH 2/8] lei: drop EOFpipe in favor of PktOp

[Eric Wong]

lei already uses PktOp and SOCK_SEQPACKET throughout; whereas
EOFpipe had one single use in lei.  Since PktOp is a strict
superset of EOFpipe functionality, we may be able to get rid of
EOFpipe entirely.

However, lei is considered a portability canary and I'm not sure
if the stable public-inbox-* code can drop EOFpipe just yet.
---
 lib/PublicInbox/LEI.pm         | 9 ++++-----
 lib/PublicInbox/LeiOverview.pm | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 98e79a76..d7768426 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1151,7 +1151,7 @@ sub lazy_start {
 (Socket::MsgHdr || Inline::C) missing/unconfigured (narg=$narg);
 
 	require PublicInbox::Listener;
-	require PublicInbox::EOFpipe;
+	require PublicInbox::PktOp;
 	(-p STDOUT) or die "E: stdout must be a pipe\n";
 	open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
 	STDIN->autoflush(1);
@@ -1165,13 +1165,12 @@ sub lazy_start {
 	my $exit_code;
 	my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
 	local $quit = do {
-		pipe(my ($eof_r, $eof_w)) or die "pipe: $!";
-		PublicInbox::EOFpipe->new($eof_r, \&noop, undef);
+		my (undef, $eof_p) = PublicInbox::PktOp->pair;
 		sub {
 			$exit_code //= shift;
 			my $lis = $pil or exit($exit_code);
-			# closing eof_w triggers \&noop wakeup
-			$listener = $eof_w = $pil = $path = undef;
+			# closing eof_p triggers \&noop wakeup
+			$listener = $eof_p = $pil = $path = undef;
 			$lis->close; # DS::close
 			PublicInbox::DS->SetLoopTimeout(1000);
 		};
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index bfb8b143..28891460 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -119,7 +119,7 @@ sub ovv_begin {
 	} # TODO HTML/Atom/...
 }
 
-# called once by parent (via PublicInbox::EOFpipe)
+# called once by parent (via PublicInbox::PktOp  '' => query_done)
 sub ovv_end {
 	my ($self, $lei) = @_;
 	if ($self->{fmt} eq 'json') {

[PATCH 3/8] lei tag: support tagging index-only messages

[Eric Wong]

This will make some of our tests faster and allow users to try
more features of lei without high storage requirements.
---
 lib/PublicInbox/LeiSearch.pm |  8 ++++++--
 lib/PublicInbox/LeiTag.pm    | 10 ++++++++--
 lib/PublicInbox/LeiToMail.pm |  4 +++-
 t/lei-index.t                | 12 +++++++++++-
 4 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index c2b12146..fb19229f 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -63,7 +63,9 @@ sub content_key ($) {
 }
 
 sub _cmp_1st { # git->cat_async callback
-	my ($bref, $oid, $type, $size, $cmp) = @_; # cmp: [chash, xoids, smsg]
+	my ($bref, $oid, $type, $size, $cmp) = @_;
+	# cmp: [chash, xoids, smsg, lms]
+	$bref //= $cmp->[3] ? $cmp->[3]->local_blob($oid, 1) : undef;
 	if ($bref && content_hash(PublicInbox::Eml->new($bref)) eq $cmp->[0]) {
 		$cmp->[1]->{$oid} = $cmp->[2]->{num};
 	}
@@ -78,6 +80,8 @@ sub xoids_for {
 	my @overs = ($self->over // $self->overs_all);
 	my $git = $self->git;
 	my $xoids = {};
+	# no lms when used via {ale}:
+	my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir});
 	for my $mid (@$mids) {
 		for my $o (@overs) {
 			my ($id, $prev);
@@ -85,7 +89,7 @@ sub xoids_for {
 				next if $cur->{bytes} == 0 ||
 					$xoids->{$cur->{blob}};
 				$git->cat_async($cur->{blob}, \&_cmp_1st,
-						[ $chash, $xoids, $cur ]);
+						[$chash, $xoids, $cur, $lms]);
 				if ($min && scalar(keys %$xoids) >= $min) {
 					$git->cat_async_wait;
 					return $xoids;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index c650e886..b6abd533 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -9,7 +9,8 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
 
 sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
 	my ($self, $eml) = @_;
-	if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) {
+	if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
+			$self->{lei}->{ale}->xoids_for($eml)) {
 		$self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
 						$self->{vmd_mod});
 	} else {
@@ -17,7 +18,11 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
 	}
 }
 
-sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
+sub input_mbox_cb {
+	my ($eml, $self) = @_;
+	$eml->header_set($_) for (qw(X-Status Status));
+	input_eml_cb($self, $eml);
+}
 
 sub input_maildir_cb { # maildir_each_eml cb
 	my ($f, $kw, $eml, $self) = @_;
@@ -60,6 +65,7 @@ sub note_missing {
 sub ipc_atfork_child {
 	my ($self) = @_;
 	PublicInbox::LeiInput::input_only_atfork_child($self);
+	$self->{lse} = $self->{lei}->{sto}->search;
 	# this goes out-of-scope at worker process exit:
 	PublicInbox::OnDestroy->new($$, \&note_missing, $self);
 }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index da3a95d2..0cbdff8b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -650,7 +650,9 @@ sub ipc_atfork_child {
 	my ($self) = @_;
 	my $lei = $self->{lei};
 	$lei->_lei_atfork_child;
-	$self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse};
+	if (my $lse = $lei->{lse}) {
+		$self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
+	}
 	$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
 	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$self->SUPER::ipc_atfork_child;
diff --git a/t/lei-index.t b/t/lei-index.t
index b7dafb71..9a45d885 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -40,7 +40,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	my $res_a = json_utf8->decode($lei_out);
 	my $blob = $res_a->[0]->{'blob'};
 	like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example');
-	lei_ok('blob', $blob);
+	lei_ok(qw(-C / blob), $blob);
 	is($lei_out, $expect, 'got expected blob via Maildir');
 	lei_ok(qw(q mid:qp@example.com -f text));
 	like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
@@ -58,6 +58,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	my $res_b = json_utf8->decode($lei_out);
 	is_deeply($res_b, $res_a, 'no extra DB entries');
 
+	# ensure tag works on index-only messages:
+	lei_ok(qw(tag +kw:seen t/utf8.eml));
+	lei_ok(qw(q mid:testmessage@example.com));
+	is_deeply(json_utf8->decode($lei_out)->[0]->{kw},
+		['seen'], 'seen kw can be set on index-only message');
+
+	lei_ok(qw(q z:0.. -o), "$tmpdir/all-results") for (1..2);
+	is_deeply([xqx($all_obj)], \@objs,
+		'no new objects after 2x q to trigger implicit import');
+
 	lei_ok('index', "nntp://$nntp_host_port/t.v2");
 	lei_ok('index', "imap://$imap_host_port/t.v2.0");
 	is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');

[PATCH 4/8] lei_input: fix canonicalization of Maildirs for sync

[Eric Wong]

This is needed for the upcoming "lei export-kw"
---
 lib/PublicInbox/LeiInput.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index cfdd3628..4ff7a379 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -250,7 +250,8 @@ sub prepare_inputs { # returns undef on error
 				require PublicInbox::MdirReader;
 				$ifmt eq 'maildir' or return
 					$lei->fail("$ifmt not supported");
-				$input = $lei->abs_path($input) if $sync;
+				$sync and $input = 'maildir:'.
+						$lei->abs_path($input_path);
 			} else {
 				return $lei->fail("Unable to handle $input");
 			}

[PATCH 5/8] lei index: support command-line options

[Eric Wong]

This mostly takes after "lei import", and at least --quiet needs
to be supported.
---
 lib/PublicInbox/LEI.pm | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d7768426..15680fe3 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -233,6 +233,11 @@ our %CMD = ( # sorted in order of importance/use:
 'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
 	qw(prune), @c_opt ],
 
+'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
+	qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
+	verbose|v+ incremental!),
+	 PublicInbox::LeiQuery::curl_opt(), # mainly for --proxy=
+	 @c_opt ],
 'import' => [ 'LOCATION...|--stdin',
 	'one-time import/update from URL or filesystem',
 	qw(stdin| offset=i recursive|r exclude=s include|I=s

[PATCH 6/8] lei export-kw: new command to export keywords to Maildirs

[Eric Wong]

IMAP will eventually be supported.
---
 MANIFEST                       |   2 +
 lib/PublicInbox/LEI.pm         |   4 +
 lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++
 lib/PublicInbox/LeiMailSync.pm |  10 ++
 lib/PublicInbox/LeiSearch.pm   |  14 +++
 lib/PublicInbox/LeiToMail.pm   |   8 +-
 lib/PublicInbox/MdirReader.pm  |  14 +++
 t/lei-export-kw.t              |  35 +++++++
 t/mdir_reader.t                |   5 +
 9 files changed, 270 insertions(+), 2 deletions(-)
 create mode 100644 lib/PublicInbox/LeiExportKw.pm
 create mode 100644 t/lei-export-kw.t

diff --git a/MANIFEST b/MANIFEST
index 684128aa..2d1ad5c3 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -202,6 +202,7 @@ lib/PublicInbox/LeiConvert.pm
 lib/PublicInbox/LeiCurl.pm
 lib/PublicInbox/LeiDedupe.pm
 lib/PublicInbox/LeiEditSearch.pm
+lib/PublicInbox/LeiExportKw.pm
 lib/PublicInbox/LeiExternal.pm
 lib/PublicInbox/LeiForgetSearch.pm
 lib/PublicInbox/LeiHelp.pm
@@ -408,6 +409,7 @@ t/iso-2202-jp.eml
 t/kqnotify.t
 t/lei-convert.t
 t/lei-daemon.t
+t/lei-export-kw.t
 t/lei-externals.t
 t/lei-import-http.t
 t/lei-import-imap.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 15680fe3..628908b5 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -243,6 +243,10 @@ our %CMD = ( # sorted in order of importance/use:
 	qw(stdin| offset=i recursive|r exclude=s include|I=s
 	lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
 	qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+
+'export-kw' => [ 'LOCATION...|--all',
+	'one-time export of keywords of sync sources',
+	qw(all:s mode=s), @c_opt ],
 'convert' => [ 'LOCATION...|--stdin',
 	'one-time conversion from URL or filesystem to another format',
 	qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
new file mode 100644
index 00000000..db4f7441
--- /dev/null
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -0,0 +1,180 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# front-end for the "lei export-kw" sub-command
+package PublicInbox::LeiExportKw;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use Errno qw(EEXIST ENOENT);
+
+sub export_kw_md { # LeiMailSync->each_src callback
+	my ($oidbin, $id, $self, $mdir) = @_;
+	my $oidhex = unpack('H*', $oidbin);
+	my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return;
+	my $bn = $$id;
+	my ($md_kw, $unknown, @try);
+	if ($bn =~ s/:2,([a-zA-Z]*)\z//) {
+		($md_kw, $unknown) = PublicInbox::MdirReader::flags2kw($1);
+		@try = qw(cur new);
+	} else {
+		$unknown = [];
+		@try = qw(new cur);
+	}
+	if ($self->{-merge_kw} && $md_kw) { # merging keywords is the default
+		@$sto_kw{keys %$md_kw} = values(%$md_kw);
+	}
+	$bn .= ':2,'.
+		PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown);
+	my $dst = "$mdir/cur/$bn";
+	my @fail;
+	for my $d (@try) {
+		my $src = "$mdir/$d/$$id";
+		next if $src eq $dst;
+
+		# we use link(2) + unlink(2) since rename(2) may
+		# inadvertently clobber if the "uniquefilename" part wasn't
+		# actually unique.
+		if (link($src, $dst)) { # success
+			# unlink(2) may ENOENT from parallel invocation,
+			# ignore it, but not other serious errors
+			if (!unlink($src) and $! != ENOENT) {
+				$self->{lei}->child_error(1,
+							"E: unlink($src): $!");
+			}
+			$self->{lms}->mv_src("maildir:$mdir",
+						$oidbin, $id, $bn) or die;
+			return; # success anyways if link(2) worked
+		}
+		if ($! == ENOENT && !-e $src) { # some other process moved it
+			$self->{lms}->clear_src("maildir:$mdir", $id);
+			next;
+		}
+		push @fail, $src if $! != EEXIST;
+	}
+	return unless @fail;
+	# both tries failed
+	my $e = $!;
+	my $orig = '['.join('|', @fail).']';
+	$self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
+}
+
+# overrides PublicInbox::LeiInput::input_path_url
+sub input_path_url {
+	my ($self, $input, @args) = @_;
+	my $lms = $self->{lms} //= $self->{lse}->lms;
+	$lms->lms_begin;
+	if ($input =~ s/\Amaildir://i) {
+		require PublicInbox::LeiToMail; # kw2suffix
+		$lms->each_src("maildir:$input", \&export_kw_md, $self, $input);
+	}
+	$lms->lms_commit;
+}
+
+sub lei_export_kw {
+	my ($lei, @folders) = @_;
+	my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
+lei/store uninitialized, see lei-import(1)
+EOM
+	my $lse = $sto->search;
+	my $lms = $lse->lms or return $lei->fail(<<EOM);
+lei mail_sync uninitialized, see lei-import(1)
+EOM
+	my $opt = $lei->{opt};
+	my $all = $opt->{all};
+	my @all = $lms->folders;
+	if (defined $all) { # --all=<local|remote>
+		my %x = map { $_ => $_ } split(/,/, $all);
+		my @ok = grep(defined, delete(@x{qw(local remote), ''}));
+		my @no = keys %x;
+		if (@no) {
+			@no = (join(',', @no));
+			return $lei->fail(<<EOM);
+--all=@no not accepted (must be `local' and/or `remote')
+EOM
+		}
+		my (%seen, @inc);
+		for my $ok (@ok) {
+			if ($ok eq 'local') {
+				@inc = grep(!m!\A[a-z0-9\+]+://!i, @all);
+			} elsif ($ok eq 'remote') {
+				@inc = grep(m!\A[a-z0-9\+]+://!i, @all);
+			} elsif ($ok ne '') {
+				return $lei->fail("--all=$all not understood");
+			} else {
+				@inc = @all;
+			}
+			for (@inc) {
+				push(@folders, $_) unless $seen{$_}++;
+			}
+		}
+		return $lei->fail(<<EOM) if !@folders;
+no --mail-sync folders known to lei
+EOM
+	} else {
+		my %all = map { $_ => 1 } @all;
+		my @no;
+		for (@folders) {
+			next if $all{$_}; # ok
+			if (-d "$_/new" && -d "$_/cur") {
+				my $d = 'maildir:'.$lei->rel2abs($_);
+				push(@no, $_) unless $all{$d};
+				$_ = $d;
+			} else {
+				push @no, $_;
+			}
+		}
+		my $no = join("\n\t", @no);
+		return $lei->fail(<<EOF) if @no;
+No sync information for: $no
+Run `lei ls-mail-sync' to display valid choices
+EOF
+	}
+	my $self = bless { lse => $lse }, __PACKAGE__;
+	$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+	$self->prepare_inputs($lei, \@folders) or return;
+	my $j = $opt->{jobs} // scalar(@{$self->{inputs}}) || 1;
+	if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
+		return $lei->fail("cannot export to read-only folders: @ro");
+	}
+	if (my $net = $lei->{net}) {
+		require PublicInbox::NetWriter;
+		bless $net, 'PublicInbox::NetWriter';
+	}
+	undef $lms;
+	my $m = $opt->{mode} // 'merge';
+	if ($m eq 'merge') { # default
+		$self->{-merge_kw} = 1;
+	} elsif ($m eq 'set') {
+	} else {
+		return $lei->fail(<<EOM);
+--mode=$m not supported (`set' or `merge')
+EOM
+	}
+	my $ops = {};
+	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+	$self->{-wq_nr_workers} = $j // 1; # locked
+	(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+	$lei->{wq1} = $self;
+	$lei->{-err_type} = 'non-fatal';
+	net_merge_all_done($self) unless $lei->{auth};
+	$op_c->op_wait_event($ops); # calls net_merge_all_done if $lei->{auth}
+}
+
+sub _complete_export_kw {
+	my ($lei, @argv) = @_;
+	my $sto = $lei->_lei_store or return;
+	my $lms = $sto->search->lms or return;
+	my $match_cb = $lei->complete_url_prepare(\@argv);
+	map { $match_cb->($_) } $lms->folders;
+}
+
+no warnings 'once';
+
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
+# the following works even when LeiAuth is lazy-loaded
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 3bada42d..32e17c65 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -138,6 +138,16 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ?
 	$sth->execute($fid, $id);
 }
 
+# Maildir-only
+sub mv_src {
+	my ($self, $folder, $oidbin, $id, $newbn) = @_;
+	my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+	my $sth = $self->{dbh}->prepare_cached(<<'');
+UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
+
+	$sth->execute($newbn, $fid, $oidbin, $$id);
+}
+
 # read-only, iterates every oidbin + UID or name for a given folder
 sub each_src {
 	my ($self, $folder, $cb, @args) = @_;
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index fb19229f..9297d060 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -27,6 +27,20 @@ sub msg_keywords {
 	wantarray ? sort(keys(%$kw)) : $kw;
 }
 
+# returns undef if blob is unknown
+sub oid_keywords {
+	my ($self, $oidhex) = @_;
+	my @num = $self->over->blob_exists($oidhex) or return;
+	my $xdb = $self->xdb; # set {nshard};
+	my %kw;
+	for my $num (@num) { # there should only be one...
+		my $doc = $xdb->get_document(num2docid($self, $num));
+		my $x = xap_terms('K', $doc);
+		%kw = (%kw, %$x);
+	}
+	\%kw;
+}
+
 # lookup keywords+labels for external messages
 sub xsmsg_vmd {
 	my ($self, $smsg, $want_label) = @_;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 0cbdff8b..96a1f881 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -243,10 +243,14 @@ sub _rand () {
 	sprintf('%x,%x,%x,%x', rand(0xffffffff), time, $$, ++$seq);
 }
 
+sub kw2suffix ($;@) {
+	my $kw = shift;
+	join('', sort(map { $kw2char{$_} // () } @$kw, @_));
+}
+
 sub _buf2maildir {
 	my ($dst, $buf, $smsg) = @_;
 	my $kw = $smsg->{kw} // [];
-	my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw));
 	my $rand = ''; # chosen by die roll :P
 	my ($tmp, $fh, $base, $ok);
 	my $common = $smsg->{blob} // _rand;
@@ -263,7 +267,7 @@ sub _buf2maildir {
 		$dst .= 'cur/';
 		$rand = '';
 		do {
-			$base = $rand.$common.':2,'.$sfx
+			$base = $rand.$common.':2,'.kw2suffix($kw);
 		} while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} &&
 			($rand = _rand.','));
 		die "link($tmp, $dst$base): $!" unless $ok;
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index 7a0641fb..304be63d 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -86,4 +86,18 @@ sub maildir_each_eml {
 
 sub new { bless {}, __PACKAGE__ }
 
+sub flags2kw ($) {
+	my @unknown;
+	my %kw;
+	for (split(//, $_[0])) {
+		my $k = $c2kw{$_};
+		if (defined($k)) {
+			$kw{$k} = 1;
+		} else {
+			push @unknown, $_;
+		}
+	}
+	(\%kw, \@unknown);
+}
+
 1;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
new file mode 100644
index 00000000..9531949a
--- /dev/null
+++ b/t/lei-export-kw.t
@@ -0,0 +1,35 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Copy qw(cp);
+use File::Path qw(make_path);
+require_mods(qw(lei -imapd Mail::IMAPClient));
+my ($tmpdir, $for_destroy) = tmpdir;
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my $expect = eml_load('t/data/0001.patch');
+test_lei({ tmpdir => $tmpdir }, sub {
+	my $home = $ENV{HOME};
+	my $md = "$home/md";
+	make_path("$md/new", "$md/cur", "$md/tmp");
+	cp('t/data/0001.patch', "$md/new/y") or xbail "cp $md $!";
+	cp('t/data/message_embed.eml', "$md/cur/x:2,S") or xbail "cp $md $!";
+	lei_ok qw(index -q), $md;
+	lei_ok qw(tag t/data/0001.patch +kw:seen);
+	lei_ok qw(export-kw --all=local);
+	ok(!-e "$md/new/y", 'original gone');
+	is_deeply(eml_load("$md/cur/y:2,S"), $expect,
+		"`seen' kw exported");
+
+	lei_ok qw(tag t/data/0001.patch +kw:answered);
+	lei_ok qw(export-kw --all=local);
+	ok(!-e "$md/cur/y:2,S", 'seen-only file gone');
+	is_deeply(eml_load("$md/cur/y:2,RS"), $expect, "`R' added");
+
+	lei_ok qw(tag t/data/0001.patch -kw:answered -kw:seen);
+	lei_ok qw(export-kw --mode=set --all=local);
+	ok(!-e "$md/cur/y:2,RS", 'seen+answered file gone');
+	is_deeply(eml_load("$md/cur/y:2,"), $expect, 'no keywords left');
+});
+
+done_testing;
diff --git a/t/mdir_reader.t b/t/mdir_reader.t
index 51b38af4..c927e1a7 100644
--- a/t/mdir_reader.t
+++ b/t/mdir_reader.t
@@ -19,4 +19,9 @@ is(maildir_path_flags('/path/to/foo:2,'), '', 'no flags in path');
 use_ok 'PublicInbox::InboxWritable', qw(eml_from_path);
 is(eml_from_path('.'), undef, 'eml_from_path fails on directory');
 
+is_deeply([PublicInbox::MdirReader::flags2kw('S')], [{ 'seen' => 1 }, []],
+	"`seen' kw set from flag");
+is_deeply([PublicInbox::MdirReader::flags2kw('Su')], [{ 'seen' => 1 }, ['u']],
+	'unknown flag ignored');
+
 done_testing;

[PATCH 7/8] uri_imap: support uid/auth/user as full accessors

[Eric Wong]

We will need this for mail synchronization
---
 lib/PublicInbox/URIimap.pm | 82 ++++++++++++++++++++++++++++++--------
 t/uri_imap.t               | 60 +++++++++++++++++++++-------
 2 files changed, 110 insertions(+), 32 deletions(-)

diff --git a/lib/PublicInbox/URIimap.pm b/lib/PublicInbox/URIimap.pm
index f6244137..a309fde0 100644
--- a/lib/PublicInbox/URIimap.pm
+++ b/lib/PublicInbox/URIimap.pm
@@ -12,11 +12,14 @@
 # RFC 2192 also describes ";TYPE=<list_type>"
 package PublicInbox::URIimap;
 use strict;
+use v5.10.1;
 use URI::Split qw(uri_split uri_join); # part of URI
-use URI::Escape qw(uri_unescape);
+use URI::Escape qw(uri_unescape uri_escape);
 use overload '""' => \&as_string;
 
 my %default_ports = (imap => 143, imaps => 993);
+# for enc-auth-type and enc-user in RFC 5092
+my $achar = qr/[A-Za-z0-9%\-_\.\!\$'\(\)\+\,\&\=\*]+/;
 
 sub new {
 	my ($class, $url) = @_;
@@ -86,14 +89,15 @@ sub uidvalidity { # read/write
 	$path =~ m!\A[^;/]+;UIDVALIDITY=([1-9][0-9]*)\b!i ? ($1 + 0) : undef;
 }
 
-sub iuid {
+sub uid {
 	my ($self, $val) = @_;
 	my ($scheme, $auth, $path, $query, $frag) = uri_split($$self);
-	if (defined $val) {
-		if ($path =~ s!/;UID=[^;/]*\b!/;UID=$val!i) {
-			# s// already changed it
-		} else { # both s// failed, so just append
-			$path .= ";UID=$val";
+	if (scalar(@_) == 2) {
+		if (!defined $val) {
+			$path =~ s!/;UID=[^;/]*\b!!i;
+		} else {
+			$path =~ s!/;UID=[^;/]*\b!/;UID=$val!i or
+				$path .= ";UID=$val";
 		}
 		$$self = uri_join($scheme, $auth, $path, $query);
 	}
@@ -114,12 +118,34 @@ sub authority {
 }
 
 sub user {
-	my ($self) = @_;
-	my (undef, $auth) = uri_split($$self);
-	$auth =~ s/@.*\z// or return undef; # drop host:port
-	$auth =~ s/;.*\z//; # drop ;AUTH=...
-	$auth =~ s/:.*\z//; # drop password
-	uri_unescape($auth);
+	my ($self, $val) = @_;
+	my ($scheme, $auth, $path, $query) = uri_split($$self);
+	my $at_host_port;
+	$auth =~ s/(@.*)\z// and $at_host_port = $1; # stash host:port for now
+	if (scalar(@_) == 2) { # set, this clobbers password, too
+		if (defined $val) {
+			my $uval = uri_escape($val);
+			if (defined($at_host_port)) {
+				$auth =~ s!\A.*?(;AUTH=$achar).*!$uval$1!ix
+					or $auth = $uval;
+			} else {
+				substr($auth, 0, 0) = "$uval@";
+			}
+		} elsif (defined($at_host_port)) { # clobber
+			$auth =~ s!\A.*?(;AUTH=$achar).*!$1!i or $auth = '';
+			if ($at_host_port && $auth eq '') {
+				$at_host_port =~ s/\A\@//;
+			}
+		}
+		$at_host_port //= '';
+		$$self = uri_join($scheme, $auth.$at_host_port, $path, $query);
+		$val;
+	} else { # read-only
+		$at_host_port // return undef; # explicit undef for scalar
+		$auth =~ s/;.*\z//; # drop ;AUTH=...
+		$auth =~ s/:.*\z//; # drop password
+		$auth eq '' ? undef : uri_unescape($auth);
+	}
 }
 
 sub password {
@@ -131,10 +157,32 @@ sub password {
 }
 
 sub auth {
-	my ($self) = @_;
-	my (undef, $auth) = uri_split($$self);
-	$auth =~ s/@.*\z//; # drop host:port
-	$auth =~ /;AUTH=(.+)\z/i ? uri_unescape($1) : undef;
+	my ($self, $val) = @_;
+	my ($scheme, $auth, $path, $query) = uri_split($$self);
+	my $at_host_port;
+	$auth =~ s/(@.*)\z// and $at_host_port = $1; # stash host:port for now
+	if (scalar(@_) == 2) {
+		if (defined $val) {
+			my $uval = uri_escape($val);
+			if ($auth =~ s!;AUTH=$achar!;AUTH=$uval!ix) {
+				# replaced existing
+			} elsif (defined($at_host_port)) {
+				$auth .= ";AUTH=$uval";
+			} else {
+				substr($auth, 0, 0) = ";AUTH=$uval@";
+			}
+		} else { # clobber
+			$auth =~ s!;AUTH=$achar!!i;
+			if ($at_host_port && $auth eq '') {
+				$at_host_port =~ s/\A\@//;
+			}
+		}
+		$at_host_port //= '';
+		$$self = uri_join($scheme, $auth.$at_host_port, $path, $query);
+		$val;
+	} else { # read-only
+		$auth =~ /;AUTH=(.+)\z/i ? uri_unescape($1) : undef;
+	}
 }
 
 sub scheme {
diff --git a/t/uri_imap.t b/t/uri_imap.t
index ed24fc1b..14f0f346 100644
--- a/t/uri_imap.t
+++ b/t/uri_imap.t
@@ -2,7 +2,7 @@
 # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
-use Test::More;
+use v5.10.1;
 use PublicInbox::TestCommon;
 require_mods 'URI::Split';
 use_ok 'PublicInbox::URIimap';
@@ -69,36 +69,66 @@ $uri = PublicInbox::URIimap->new('imap://0/mmm;UIDVALIDITY=21');
 is($uri->uidvalidity, 21, 'multi-digit UIDVALIDITY');
 $uri = PublicInbox::URIimap->new('imap://0/mmm;UIDVALIDITY=bogus');
 is($uri->uidvalidity, undef, 'bogus UIDVALIDITY');
-is($uri->uidvalidity(2), 2, 'iuid set');
+is($uri->uidvalidity(2), 2, 'uid set');
 is($$uri, 'imap://0/mmm;UIDVALIDITY=2', 'bogus uidvalidity replaced');
-is($uri->uidvalidity(13), 13, 'iuid set');
+is($uri->uidvalidity(13), 13, 'uid set');
 is($$uri, 'imap://0/mmm;UIDVALIDITY=13', 'valid uidvalidity replaced');
 
 $uri = PublicInbox::URIimap->new('imap://0/mmm');
-is($uri->uidvalidity(2), 2, 'iuid set');
+is($uri->uidvalidity(2), 2, 'uid set');
 is($$uri, 'imap://0/mmm;UIDVALIDITY=2', 'uidvalidity appended');
-is($uri->iuid, undef, 'no iuid');
+is($uri->uid, undef, 'no uid');
 
 is(PublicInbox::URIimap->new('imap://0/x;uidvalidity=1')->canonical->as_string,
 	'imap://0/x;UIDVALIDITY=1', 'capitalized UIDVALIDITY');
 
 $uri = PublicInbox::URIimap->new('imap://0/mmm/;uid=8');
 is($uri->canonical->as_string, 'imap://0/mmm/;UID=8', 'canonicalized UID');
-is($uri->mailbox, 'mmm', 'mailbox works with iuid');
-is($uri->iuid, 8, 'iuid extracted');
-is($uri->iuid(9), 9, 'iuid set');
-is($$uri, 'imap://0/mmm/;UID=9', 'correct iuid when stringified');
-is($uri->uidvalidity(1), 1, 'set uidvalidity with iuid');
+is($uri->mailbox, 'mmm', 'mailbox works with uid');
+is($uri->uid, 8, 'uid extracted');
+is($uri->uid(9), 9, 'uid set');
+is($$uri, 'imap://0/mmm/;UID=9', 'correct uid when stringified');
+is($uri->uidvalidity(1), 1, 'set uidvalidity with uid');
 is($$uri, 'imap://0/mmm;UIDVALIDITY=1/;UID=9',
-	'uidvalidity added with iuid');
-is($uri->uidvalidity(4), 4, 'set uidvalidity with iuid');
+	'uidvalidity added with uid');
+is($uri->uidvalidity(4), 4, 'set uidvalidity with uid');
 is($$uri, 'imap://0/mmm;UIDVALIDITY=4/;UID=9',
-	'uidvalidity replaced with iuid');
-is($uri->iuid(3), 3, 'iuid set with uidvalidity');
-is($$uri, 'imap://0/mmm;UIDVALIDITY=4/;UID=3', 'iuid replaced properly');
+	'uidvalidity replaced with uid');
+is($uri->uid(3), 3, 'uid set with uidvalidity');
+is($$uri, 'imap://0/mmm;UIDVALIDITY=4/;UID=3', 'uid replaced properly');
 
 my $lc = lc($$uri);
 is(PublicInbox::URIimap->new($lc)->canonical->as_string, "$$uri",
 	'canonical uppercased both params');
 
+is($uri->uid(undef), undef, 'uid can be clobbered');
+is($$uri, 'imap://0/mmm;UIDVALIDITY=4', 'uid dropped');
+
+$uri->auth('ANONYMOUS');
+is($$uri, 'imap://;AUTH=ANONYMOUS@0/mmm;UIDVALIDITY=4', 'AUTH= set');
+is($uri->user, undef, 'user is undef w/ AUTH=');
+is($uri->password, undef, 'password is undef w/ AUTH=');
+
+$uri->user('foo');
+is($$uri, 'imap://foo;AUTH=ANONYMOUS@0/mmm;UIDVALIDITY=4', 'user set w/AUTH');
+is($uri->password, undef, 'password is undef w/ AUTH= & user');
+$uri->auth(undef);
+is($$uri, 'imap://foo@0/mmm;UIDVALIDITY=4', 'user remains set w/o auth');
+is($uri->password, undef, 'password is undef w/ user only');
+
+$uri->user('bar');
+is($$uri, 'imap://bar@0/mmm;UIDVALIDITY=4', 'user set w/o AUTH');
+$uri->auth('NTML');
+is($$uri, 'imap://bar;AUTH=NTML@0/mmm;UIDVALIDITY=4', 'auth set w/user');
+$uri->auth(undef);
+$uri->user(undef);
+is($$uri, 'imap://0/mmm;UIDVALIDITY=4', 'auth and user both cleared');
+is($uri->user, undef, 'user is undef');
+is($uri->auth, undef, 'auth is undef');
+is($uri->password, undef, 'password is undef');
+$uri = PublicInbox::URIimap->new('imap://[::1]:36281/');
+my $cred = bless { username => $uri->user, password => $uri->password };
+is($cred->{username}, undef, 'user is undef in array context');
+is($cred->{password}, undef, 'password is undef in array context');
+
 done_testing;

[PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI

[Eric Wong]

Just having UIDVALIDITY in the URI isn't enough, since a single
lei user may have multiple IMAP logins on the same server.

This leads to compatibility problems and forces a reimport for
the few users already using this lei functionality, but it's not
stable nor released, yet.
---
 lib/PublicInbox/NetReader.pm | 42 ++++++++++++++++++++++--------------
 t/lei-import-imap.t          |  9 +++++---
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index fd0d1682..a532b218 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -58,12 +58,10 @@ sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
 
 # mic_for may prompt the user and store auth info, prepares mic_get
 sub mic_for ($$$$) { # mic = Mail::IMAPClient
-	my ($self, $url, $mic_args, $lei) = @_;
-	require PublicInbox::URIimap;
-	my $uri = PublicInbox::URIimap->new($url);
+	my ($self, $uri, $mic_args, $lei) = @_;
 	require PublicInbox::GitCredential;
 	my $cred = bless {
-		url => $url,
+		url => "$uri",
 		protocol => $uri->scheme,
 		host => $uri->host,
 		username => $uri->user,
@@ -83,13 +81,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
 	};
 	require PublicInbox::IMAPClient;
 	my $mic = mic_new($self, $mic_arg, $sec, $uri) or
-			die "E: <$url> new: $@\n";
+			die "E: <$uri> new: $@\n";
 	# default to using STARTTLS if it's available, but allow
 	# it to be disabled since I usually connect to localhost
 	if (!$mic_arg->{Ssl} && !defined($mic_arg->{Starttls}) &&
 			$mic->has_capability('STARTTLS') &&
 			$mic->can('starttls')) {
-		$mic->starttls or die "E: <$url> STARTTLS: $@\n";
+		$mic->starttls or die "E: <$uri> STARTTLS: $@\n";
 	}
 
 	# do we even need credentials?
@@ -111,8 +109,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
 	if ($mic->login && $mic->IsAuthenticated) {
 		# success! keep IMAPClient->new arg in case we get disconnected
 		$self->{mic_arg}->{$sec} = $mic_arg;
+		if ($cred) {
+			$uri->user($cred->{username}) if !defined($uri->user);
+		} elsif ($mic_arg->{Authmechanism} eq 'ANONYMOUS') {
+			$uri->auth('ANONYMOUS') if !defined($uri->auth);
+		}
 	} else {
-		$err = "E: <$url> LOGIN: $@\n";
+		$err = "E: <$uri> LOGIN: $@\n";
 		if ($cred && defined($cred->{password})) {
 			$err =~ s/\Q$cred->{password}\E/*******/g;
 		}
@@ -304,15 +307,16 @@ sub imap_common_init ($;$) {
 	# make sure we can connect and cache the credentials in memory
 	$self->{mic_arg} = {}; # schema://authority => IMAPClient->new args
 	my $mics = {}; # schema://authority => IMAPClient obj
-	for my $uri (@{$self->{imap_order}}) {
-		my $sec = uri_section($uri);
+	for my $orig_uri (@{$self->{imap_order}}) {
+		my $sec = uri_section($orig_uri);
+		my $uri = PublicInbox::URIimap->new("$sec/");
 		my $mic = $mics->{$sec} //=
-				mic_for($self, "$sec/", $mic_args, $lei) //
+				mic_for($self, $uri, $mic_args, $lei) //
 				die "Unable to continue\n";
 		next unless $self->isa('PublicInbox::NetWriter');
-		my $dst = $uri->mailbox // next;
+		my $dst = $orig_uri->mailbox // next;
 		next if $mic->exists($dst); # already exists
-		$mic->create($dst) or die "CREATE $dst failed <$uri>: $@";
+		$mic->create($dst) or die "CREATE $dst failed <$orig_uri>: $@";
 	}
 	$mics;
 }
@@ -419,12 +423,18 @@ sub run_commit_cb ($) {
 	$cb->(@args);
 }
 
-sub _itrk_last ($$;$) {
-	my ($self, $uri, $r_uidval) = @_;
+sub itrk_last ($$;$$) {
+	my ($self, $uri, $r_uidval, $mic) = @_;
 	return (undef, undef, $r_uidval) unless $self->{incremental};
 	my ($itrk, $l_uid, $l_uidval);
 	if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0
 		$uri->uidvalidity($r_uidval) if defined $r_uidval;
+		if ($mic) {
+			my $auth = $mic->Authmechanism // '';
+			$uri->auth($auth) if $auth eq 'ANONYMOUS';
+			my $user = $mic->User;
+			$uri->user($user) if defined($user);
+		}
 		my $x;
 		$l_uid = ($lms && ($x = $lms->location_stats($$uri))) ?
 				$x->{'uid.max'} : undef;
@@ -459,7 +469,7 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
 EOF
 
 	my $uri = $orig_uri->clone;
-	my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval);
+	my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
 	return <<EOF if $l_uidval != $r_uidval;
 E: $uri UIDVALIDITY mismatch
 E: local=$l_uidval != remote=$r_uidval
@@ -612,7 +622,7 @@ sub _nntp_fetch_all ($$$) {
 	# IMAPTracker is also used for tracking NNTP, UID == article number
 	# LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
 	# expensive.  So we assume newsgroups don't change:
-	my ($itrk, $l_art) = _itrk_last($self, $uri);
+	my ($itrk, $l_art) = itrk_last($self, $uri);
 
 	# allow users to specify articles to refetch
 	# cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index fd15ef4f..d424ebb1 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -23,9 +23,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
 
 	lei_ok('import', $url);
 	lei_ok 'ls-mail-sync';
-	like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-mail-sync');
+	like($lei_out, qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E
+			/t\.v2\.0;UIDVALIDITY=\d+\n\z!x, 'ls-mail-sync');
 	chomp(my $u = $lei_out);
 	lei_ok('import', $u, \'UIDVALIDITY match in URL');
+	$url = $u;
 	$u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s;
 	ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected');
 
@@ -33,7 +35,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	my $inspect = json_utf8->decode($lei_out);
 	my @k = keys %$inspect;
 	is(scalar(@k), 1, 'one URL resolved');
-	like($k[0], qr!\A\Q$url\E;UIDVALIDITY=\d+\z!, 'inspect URL matches');
+	is($k[0], $url, 'inspect URL matches');
 	my $stats = $inspect->{$k[0]};
 	is_deeply([ sort keys %$stats ],
 		[ qw(uid.count uid.max uid.min) ], 'keys match');
@@ -55,7 +57,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	my $x = json_utf8->decode($lei_out);
 	is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect');
 	is(ref($x->{'mail-sync'}), 'HASH', 'sync in inspect');
-	is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect');
+	is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect')
+		or diag explain($x);
 
 	my $psgi_attach = 'cfa3622cbeffc9bd6b0fc66c4d60d420ba74f60d';
 	lei_ok('blob', $psgi_attach);