"lei export-kw" is a new command. I'm not sure exactly how it'll be used but it's probably more of a plumbing command, for now. My brain hurts thinking about synchronization and merge/conflict resolution when it comes to propagating keywords assignments/clearing. (I frequently mark messages as Unread in my MUA so I know to reread them in the future, and I suspect it's a common thing). mail_sync.sqlite3 now tracks AUTH=ANONYMOUS or username in the folder name to account for lei(Unix) users having multiple IMAP accounts on the same host with the same folders+UIDVALIDITY. "lei import imap(s)://" users will waste a bit of bandwidth resyncing as a result. Eric Wong (8): treewide: favor open(..., '+<&=', $fd) lei: drop EOFpipe in favor of PktOp lei tag: support tagging index-only messages lei_input: fix canonicalization of Maildirs for sync lei index: support command-line options lei export-kw: new command to export keywords to Maildirs uri_imap: support uid/auth/user as full accessors lei import: store IMAP user+auth in mail_sync folder URI MANIFEST | 2 + examples/unsubscribe.milter | 3 +- lib/PublicInbox/DS.pm | 3 +- lib/PublicInbox/Daemon.pm | 2 +- lib/PublicInbox/LEI.pm | 18 +++- lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++ lib/PublicInbox/LeiInput.pm | 3 +- lib/PublicInbox/LeiMailSync.pm | 10 ++ lib/PublicInbox/LeiOverview.pm | 2 +- lib/PublicInbox/LeiSearch.pm | 22 +++- lib/PublicInbox/LeiTag.pm | 10 +- lib/PublicInbox/LeiToMail.pm | 12 ++- lib/PublicInbox/MdirReader.pm | 14 +++ lib/PublicInbox/NetReader.pm | 42 +++++--- lib/PublicInbox/Sigfd.pm | 3 +- lib/PublicInbox/URIimap.pm | 82 +++++++++++---- t/epoll.t | 7 +- t/lei-export-kw.t | 35 +++++++ t/lei-import-imap.t | 9 +- t/lei-index.t | 12 ++- t/mdir_reader.t | 5 + t/uri_imap.t | 60 ++++++++--- 22 files changed, 461 insertions(+), 75 deletions(-) create mode 100644 lib/PublicInbox/LeiExportKw.pm create mode 100644 t/lei-export-kw.t
Cut down on unnecessary imports of IO::Handle and method lookup + dispatch overhead. --- examples/unsubscribe.milter | 3 +-- lib/PublicInbox/DS.pm | 3 +-- lib/PublicInbox/Daemon.pm | 2 +- lib/PublicInbox/Sigfd.pm | 3 +-- t/epoll.t | 7 +++++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/unsubscribe.milter b/examples/unsubscribe.milter index 7b126e30..608524cb 100644 --- a/examples/unsubscribe.milter +++ b/examples/unsubscribe.milter @@ -2,7 +2,6 @@ # Copyright (C) 2016-2021 all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> use strict; -use warnings; use Sendmail::PMilter qw(:all); use IO::Socket; use Crypt::CBC; @@ -128,7 +127,7 @@ my $fds = $ENV{LISTEN_FDS}; if ($fds && (($ENV{LISTEN_PID} || 0) == $$)) { die "$0 can only listen on one FD\n" if $fds != 1; my $start_fd = 3; - my $s = IO::Socket->new_from_fd($start_fd, 'r') or + open(my $s, '<&=', $start_fd) or die "inherited bad FD from LISTEN_FDS: $!\n"; $milter->set_socket($s); } else { diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm index 3cddfd18..7a4dfed0 100644 --- a/lib/PublicInbox/DS.pm +++ b/lib/PublicInbox/DS.pm @@ -25,7 +25,6 @@ use v5.10.1; use parent qw(Exporter); use bytes; use POSIX qw(WNOHANG sigprocmask SIG_SETMASK); -use IO::Handle qw(); use Fcntl qw(SEEK_SET :DEFAULT O_APPEND); use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC); use Scalar::Util qw(blessed); @@ -135,7 +134,7 @@ sub add_timer ($$;@) { sub set_cloexec ($) { my ($fd) = @_; - $_io = IO::Handle->new_from_fd($fd, 'r+') or return; + open($_io, '+<&=', $fd) or return; defined(my $fl = fcntl($_io, F_GETFD, 0)) or return; fcntl($_io, F_SETFD, $fl | FD_CLOEXEC); } diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm index b5f97d81..727311a4 100644 --- a/lib/PublicInbox/Daemon.pm +++ b/lib/PublicInbox/Daemon.pm @@ -367,7 +367,7 @@ sub inherit ($) { my $end = $fds + 2; # LISTEN_FDS_START - 1 my @rv = (); foreach my $fd (3..$end) { - my $s = IO::Handle->new_from_fd($fd, 'r'); + open(my $s, '<&=', $fd) or warn "fdopen fd=$fd: $!"; if (my $k = sockname($s)) { my $prev_was_blocking = $s->blocking(0); warn <<"" if $prev_was_blocking; diff --git a/lib/PublicInbox/Sigfd.pm b/lib/PublicInbox/Sigfd.pm index a4d1b3bb..d91ea0e7 100644 --- a/lib/PublicInbox/Sigfd.pm +++ b/lib/PublicInbox/Sigfd.pm @@ -8,7 +8,6 @@ use strict; use parent qw(PublicInbox::DS); use PublicInbox::Syscall qw(signalfd EPOLLIN EPOLLET SFD_NONBLOCK); use POSIX (); -use IO::Handle (); # returns a coderef to unblock signals if neither signalfd or kqueue # are available. @@ -27,7 +26,7 @@ sub new { my $io; my $fd = signalfd(-1, [keys %signo], $flags); if (defined $fd && $fd >= 0) { - $io = IO::Handle->new_from_fd($fd, 'r+'); + open($io, '+<&=', $fd) or die "open: $!"; } elsif (eval { require PublicInbox::DSKQXS }) { $io = PublicInbox::DSKQXS->signalfd([keys %signo], $flags); } else { diff --git a/t/epoll.t b/t/epoll.t index f2a68904..f346b387 100644 --- a/t/epoll.t +++ b/t/epoll.t @@ -1,11 +1,14 @@ +#!perl -w +# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> use strict; +use v5.10.1; use Test::More; -use IO::Handle; use PublicInbox::Syscall qw(:epoll); plan skip_all => 'not Linux' if $^O ne 'linux'; my $epfd = epoll_create(); ok($epfd >= 0, 'epoll_create'); -my $hnd = IO::Handle->new_from_fd($epfd, 'r+'); # close on exit +open(my $hnd, '+<&=', $epfd); # for autoclose pipe(my ($r, $w)) or die "pipe: $!"; is(epoll_ctl($epfd, EPOLL_CTL_ADD, fileno($w), EPOLLOUT), 0,
lei already uses PktOp and SOCK_SEQPACKET throughout; whereas EOFpipe had one single use in lei. Since PktOp is a strict superset of EOFpipe functionality, we may be able to get rid of EOFpipe entirely. However, lei is considered a portability canary and I'm not sure if the stable public-inbox-* code can drop EOFpipe just yet. --- lib/PublicInbox/LEI.pm | 9 ++++----- lib/PublicInbox/LeiOverview.pm | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 98e79a76..d7768426 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -1151,7 +1151,7 @@ sub lazy_start { (Socket::MsgHdr || Inline::C) missing/unconfigured (narg=$narg); require PublicInbox::Listener; - require PublicInbox::EOFpipe; + require PublicInbox::PktOp; (-p STDOUT) or die "E: stdout must be a pipe\n"; open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!"; STDIN->autoflush(1); @@ -1165,13 +1165,12 @@ sub lazy_start { my $exit_code; my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch); local $quit = do { - pipe(my ($eof_r, $eof_w)) or die "pipe: $!"; - PublicInbox::EOFpipe->new($eof_r, \&noop, undef); + my (undef, $eof_p) = PublicInbox::PktOp->pair; sub { $exit_code //= shift; my $lis = $pil or exit($exit_code); - # closing eof_w triggers \&noop wakeup - $listener = $eof_w = $pil = $path = undef; + # closing eof_p triggers \&noop wakeup + $listener = $eof_p = $pil = $path = undef; $lis->close; # DS::close PublicInbox::DS->SetLoopTimeout(1000); }; diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index bfb8b143..28891460 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -119,7 +119,7 @@ sub ovv_begin { } # TODO HTML/Atom/... } -# called once by parent (via PublicInbox::EOFpipe) +# called once by parent (via PublicInbox::PktOp '' => query_done) sub ovv_end { my ($self, $lei) = @_; if ($self->{fmt} eq 'json') {
This will make some of our tests faster and allow users to try more features of lei without high storage requirements. --- lib/PublicInbox/LeiSearch.pm | 8 ++++++-- lib/PublicInbox/LeiTag.pm | 10 ++++++++-- lib/PublicInbox/LeiToMail.pm | 4 +++- t/lei-index.t | 12 +++++++++++- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm index c2b12146..fb19229f 100644 --- a/lib/PublicInbox/LeiSearch.pm +++ b/lib/PublicInbox/LeiSearch.pm @@ -63,7 +63,9 @@ sub content_key ($) { } sub _cmp_1st { # git->cat_async callback - my ($bref, $oid, $type, $size, $cmp) = @_; # cmp: [chash, xoids, smsg] + my ($bref, $oid, $type, $size, $cmp) = @_; + # cmp: [chash, xoids, smsg, lms] + $bref //= $cmp->[3] ? $cmp->[3]->local_blob($oid, 1) : undef; if ($bref && content_hash(PublicInbox::Eml->new($bref)) eq $cmp->[0]) { $cmp->[1]->{$oid} = $cmp->[2]->{num}; } @@ -78,6 +80,8 @@ sub xoids_for { my @overs = ($self->over // $self->overs_all); my $git = $self->git; my $xoids = {}; + # no lms when used via {ale}: + my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir}); for my $mid (@$mids) { for my $o (@overs) { my ($id, $prev); @@ -85,7 +89,7 @@ sub xoids_for { next if $cur->{bytes} == 0 || $xoids->{$cur->{blob}}; $git->cat_async($cur->{blob}, \&_cmp_1st, - [ $chash, $xoids, $cur ]); + [$chash, $xoids, $cur, $lms]); if ($min && scalar(keys %$xoids) >= $min) { $git->cat_async_wait; return $xoids; diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm index c650e886..b6abd533 100644 --- a/lib/PublicInbox/LeiTag.pm +++ b/lib/PublicInbox/LeiTag.pm @@ -9,7 +9,8 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput); sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh my ($self, $eml) = @_; - if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) { + if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync + $self->{lei}->{ale}->xoids_for($eml)) { $self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml, $self->{vmd_mod}); } else { @@ -17,7 +18,11 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh } } -sub input_mbox_cb { input_eml_cb($_[1], $_[0]) } +sub input_mbox_cb { + my ($eml, $self) = @_; + $eml->header_set($_) for (qw(X-Status Status)); + input_eml_cb($self, $eml); +} sub input_maildir_cb { # maildir_each_eml cb my ($f, $kw, $eml, $self) = @_; @@ -60,6 +65,7 @@ sub note_missing { sub ipc_atfork_child { my ($self) = @_; PublicInbox::LeiInput::input_only_atfork_child($self); + $self->{lse} = $self->{lei}->{sto}->search; # this goes out-of-scope at worker process exit: PublicInbox::OnDestroy->new($$, \¬e_missing, $self); } diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index da3a95d2..0cbdff8b 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -650,7 +650,9 @@ sub ipc_atfork_child { my ($self) = @_; my $lei = $self->{lei}; $lei->_lei_atfork_child; - $self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse}; + if (my $lse = $lei->{lse}) { + $self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms; + } $lei->{auth}->do_auth_atfork($self) if $lei->{auth}; $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb(); $self->SUPER::ipc_atfork_child; diff --git a/t/lei-index.t b/t/lei-index.t index b7dafb71..9a45d885 100644 --- a/t/lei-index.t +++ b/t/lei-index.t @@ -40,7 +40,7 @@ test_lei({ tmpdir => $tmpdir }, sub { my $res_a = json_utf8->decode($lei_out); my $blob = $res_a->[0]->{'blob'}; like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example'); - lei_ok('blob', $blob); + lei_ok(qw(-C / blob), $blob); is($lei_out, $expect, 'got expected blob via Maildir'); lei_ok(qw(q mid:qp@example.com -f text)); like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback'); @@ -58,6 +58,16 @@ test_lei({ tmpdir => $tmpdir }, sub { my $res_b = json_utf8->decode($lei_out); is_deeply($res_b, $res_a, 'no extra DB entries'); + # ensure tag works on index-only messages: + lei_ok(qw(tag +kw:seen t/utf8.eml)); + lei_ok(qw(q mid:testmessage@example.com)); + is_deeply(json_utf8->decode($lei_out)->[0]->{kw}, + ['seen'], 'seen kw can be set on index-only message'); + + lei_ok(qw(q z:0.. -o), "$tmpdir/all-results") for (1..2); + is_deeply([xqx($all_obj)], \@objs, + 'no new objects after 2x q to trigger implicit import'); + lei_ok('index', "nntp://$nntp_host_port/t.v2"); lei_ok('index', "imap://$imap_host_port/t.v2.0"); is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
This is needed for the upcoming "lei export-kw" --- lib/PublicInbox/LeiInput.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index cfdd3628..4ff7a379 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -250,7 +250,8 @@ sub prepare_inputs { # returns undef on error require PublicInbox::MdirReader; $ifmt eq 'maildir' or return $lei->fail("$ifmt not supported"); - $input = $lei->abs_path($input) if $sync; + $sync and $input = 'maildir:'. + $lei->abs_path($input_path); } else { return $lei->fail("Unable to handle $input"); }
This mostly takes after "lei import", and at least --quiet needs to be supported. --- lib/PublicInbox/LEI.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index d7768426..15680fe3 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -233,6 +233,11 @@ our %CMD = ( # sorted in order of importance/use: 'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch', qw(prune), @c_opt ], +'index' => [ 'LOCATION...', 'one-time index from URL or filesystem', + qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s + verbose|v+ incremental!), + PublicInbox::LeiQuery::curl_opt(), # mainly for --proxy= + @c_opt ], 'import' => [ 'LOCATION...|--stdin', 'one-time import/update from URL or filesystem', qw(stdin| offset=i recursive|r exclude=s include|I=s
IMAP will eventually be supported. --- MANIFEST | 2 + lib/PublicInbox/LEI.pm | 4 + lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++ lib/PublicInbox/LeiMailSync.pm | 10 ++ lib/PublicInbox/LeiSearch.pm | 14 +++ lib/PublicInbox/LeiToMail.pm | 8 +- lib/PublicInbox/MdirReader.pm | 14 +++ t/lei-export-kw.t | 35 +++++++ t/mdir_reader.t | 5 + 9 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 lib/PublicInbox/LeiExportKw.pm create mode 100644 t/lei-export-kw.t diff --git a/MANIFEST b/MANIFEST index 684128aa..2d1ad5c3 100644 --- a/MANIFEST +++ b/MANIFEST @@ -202,6 +202,7 @@ lib/PublicInbox/LeiConvert.pm lib/PublicInbox/LeiCurl.pm lib/PublicInbox/LeiDedupe.pm lib/PublicInbox/LeiEditSearch.pm +lib/PublicInbox/LeiExportKw.pm lib/PublicInbox/LeiExternal.pm lib/PublicInbox/LeiForgetSearch.pm lib/PublicInbox/LeiHelp.pm @@ -408,6 +409,7 @@ t/iso-2202-jp.eml t/kqnotify.t t/lei-convert.t t/lei-daemon.t +t/lei-export-kw.t t/lei-externals.t t/lei-import-http.t t/lei-import-imap.t diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 15680fe3..628908b5 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -243,6 +243,10 @@ our %CMD = ( # sorted in order of importance/use: qw(stdin| offset=i recursive|r exclude=s include|I=s lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!), qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ], + +'export-kw' => [ 'LOCATION...|--all', + 'one-time export of keywords of sync sources', + qw(all:s mode=s), @c_opt ], 'convert' => [ 'LOCATION...|--stdin', 'one-time conversion from URL or filesystem to another format', qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!), diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm new file mode 100644 index 00000000..db4f7441 --- /dev/null +++ b/lib/PublicInbox/LeiExportKw.pm @@ -0,0 +1,180 @@ +# Copyright (C) 2021 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> + +# front-end for the "lei export-kw" sub-command +package PublicInbox::LeiExportKw; +use strict; +use v5.10.1; +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); +use Errno qw(EEXIST ENOENT); + +sub export_kw_md { # LeiMailSync->each_src callback + my ($oidbin, $id, $self, $mdir) = @_; + my $oidhex = unpack('H*', $oidbin); + my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return; + my $bn = $$id; + my ($md_kw, $unknown, @try); + if ($bn =~ s/:2,([a-zA-Z]*)\z//) { + ($md_kw, $unknown) = PublicInbox::MdirReader::flags2kw($1); + @try = qw(cur new); + } else { + $unknown = []; + @try = qw(new cur); + } + if ($self->{-merge_kw} && $md_kw) { # merging keywords is the default + @$sto_kw{keys %$md_kw} = values(%$md_kw); + } + $bn .= ':2,'. + PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown); + my $dst = "$mdir/cur/$bn"; + my @fail; + for my $d (@try) { + my $src = "$mdir/$d/$$id"; + next if $src eq $dst; + + # we use link(2) + unlink(2) since rename(2) may + # inadvertently clobber if the "uniquefilename" part wasn't + # actually unique. + if (link($src, $dst)) { # success + # unlink(2) may ENOENT from parallel invocation, + # ignore it, but not other serious errors + if (!unlink($src) and $! != ENOENT) { + $self->{lei}->child_error(1, + "E: unlink($src): $!"); + } + $self->{lms}->mv_src("maildir:$mdir", + $oidbin, $id, $bn) or die; + return; # success anyways if link(2) worked + } + if ($! == ENOENT && !-e $src) { # some other process moved it + $self->{lms}->clear_src("maildir:$mdir", $id); + next; + } + push @fail, $src if $! != EEXIST; + } + return unless @fail; + # both tries failed + my $e = $!; + my $orig = '['.join('|', @fail).']'; + $self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e"); +} + +# overrides PublicInbox::LeiInput::input_path_url +sub input_path_url { + my ($self, $input, @args) = @_; + my $lms = $self->{lms} //= $self->{lse}->lms; + $lms->lms_begin; + if ($input =~ s/\Amaildir://i) { + require PublicInbox::LeiToMail; # kw2suffix + $lms->each_src("maildir:$input", \&export_kw_md, $self, $input); + } + $lms->lms_commit; +} + +sub lei_export_kw { + my ($lei, @folders) = @_; + my $sto = $lei->_lei_store or return $lei->fail(<<EOM); +lei/store uninitialized, see lei-import(1) +EOM + my $lse = $sto->search; + my $lms = $lse->lms or return $lei->fail(<<EOM); +lei mail_sync uninitialized, see lei-import(1) +EOM + my $opt = $lei->{opt}; + my $all = $opt->{all}; + my @all = $lms->folders; + if (defined $all) { # --all=<local|remote> + my %x = map { $_ => $_ } split(/,/, $all); + my @ok = grep(defined, delete(@x{qw(local remote), ''})); + my @no = keys %x; + if (@no) { + @no = (join(',', @no)); + return $lei->fail(<<EOM); +--all=@no not accepted (must be `local' and/or `remote') +EOM + } + my (%seen, @inc); + for my $ok (@ok) { + if ($ok eq 'local') { + @inc = grep(!m!\A[a-z0-9\+]+://!i, @all); + } elsif ($ok eq 'remote') { + @inc = grep(m!\A[a-z0-9\+]+://!i, @all); + } elsif ($ok ne '') { + return $lei->fail("--all=$all not understood"); + } else { + @inc = @all; + } + for (@inc) { + push(@folders, $_) unless $seen{$_}++; + } + } + return $lei->fail(<<EOM) if !@folders; +no --mail-sync folders known to lei +EOM + } else { + my %all = map { $_ => 1 } @all; + my @no; + for (@folders) { + next if $all{$_}; # ok + if (-d "$_/new" && -d "$_/cur") { + my $d = 'maildir:'.$lei->rel2abs($_); + push(@no, $_) unless $all{$d}; + $_ = $d; + } else { + push @no, $_; + } + } + my $no = join("\n\t", @no); + return $lei->fail(<<EOF) if @no; +No sync information for: $no +Run `lei ls-mail-sync' to display valid choices +EOF + } + my $self = bless { lse => $lse }, __PACKAGE__; + $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs + $self->prepare_inputs($lei, \@folders) or return; + my $j = $opt->{jobs} // scalar(@{$self->{inputs}}) || 1; + if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) { + return $lei->fail("cannot export to read-only folders: @ro"); + } + if (my $net = $lei->{net}) { + require PublicInbox::NetWriter; + bless $net, 'PublicInbox::NetWriter'; + } + undef $lms; + my $m = $opt->{mode} // 'merge'; + if ($m eq 'merge') { # default + $self->{-merge_kw} = 1; + } elsif ($m eq 'set') { + } else { + return $lei->fail(<<EOM); +--mode=$m not supported (`set' or `merge') +EOM + } + my $ops = {}; + $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; + $self->{-wq_nr_workers} = $j // 1; # locked + (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); + $lei->{wq1} = $self; + $lei->{-err_type} = 'non-fatal'; + net_merge_all_done($self) unless $lei->{auth}; + $op_c->op_wait_event($ops); # calls net_merge_all_done if $lei->{auth} +} + +sub _complete_export_kw { + my ($lei, @argv) = @_; + my $sto = $lei->_lei_store or return; + my $lms = $sto->search->lms or return; + my $match_cb = $lei->complete_url_prepare(\@argv); + map { $match_cb->($_) } $lms->folders; +} + +no warnings 'once'; + +*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; +*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; + +# the following works even when LeiAuth is lazy-loaded +*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all; + +1; diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm index 3bada42d..32e17c65 100644 --- a/lib/PublicInbox/LeiMailSync.pm +++ b/lib/PublicInbox/LeiMailSync.pm @@ -138,6 +138,16 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ? $sth->execute($fid, $id); } +# Maildir-only +sub mv_src { + my ($self, $folder, $oidbin, $id, $newbn) = @_; + my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1); + my $sth = $self->{dbh}->prepare_cached(<<''); +UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ? + + $sth->execute($newbn, $fid, $oidbin, $$id); +} + # read-only, iterates every oidbin + UID or name for a given folder sub each_src { my ($self, $folder, $cb, @args) = @_; diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm index fb19229f..9297d060 100644 --- a/lib/PublicInbox/LeiSearch.pm +++ b/lib/PublicInbox/LeiSearch.pm @@ -27,6 +27,20 @@ sub msg_keywords { wantarray ? sort(keys(%$kw)) : $kw; } +# returns undef if blob is unknown +sub oid_keywords { + my ($self, $oidhex) = @_; + my @num = $self->over->blob_exists($oidhex) or return; + my $xdb = $self->xdb; # set {nshard}; + my %kw; + for my $num (@num) { # there should only be one... + my $doc = $xdb->get_document(num2docid($self, $num)); + my $x = xap_terms('K', $doc); + %kw = (%kw, %$x); + } + \%kw; +} + # lookup keywords+labels for external messages sub xsmsg_vmd { my ($self, $smsg, $want_label) = @_; diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index 0cbdff8b..96a1f881 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -243,10 +243,14 @@ sub _rand () { sprintf('%x,%x,%x,%x', rand(0xffffffff), time, $$, ++$seq); } +sub kw2suffix ($;@) { + my $kw = shift; + join('', sort(map { $kw2char{$_} // () } @$kw, @_)); +} + sub _buf2maildir { my ($dst, $buf, $smsg) = @_; my $kw = $smsg->{kw} // []; - my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw)); my $rand = ''; # chosen by die roll :P my ($tmp, $fh, $base, $ok); my $common = $smsg->{blob} // _rand; @@ -263,7 +267,7 @@ sub _buf2maildir { $dst .= 'cur/'; $rand = ''; do { - $base = $rand.$common.':2,'.$sfx + $base = $rand.$common.':2,'.kw2suffix($kw); } while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} && ($rand = _rand.',')); die "link($tmp, $dst$base): $!" unless $ok; diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm index 7a0641fb..304be63d 100644 --- a/lib/PublicInbox/MdirReader.pm +++ b/lib/PublicInbox/MdirReader.pm @@ -86,4 +86,18 @@ sub maildir_each_eml { sub new { bless {}, __PACKAGE__ } +sub flags2kw ($) { + my @unknown; + my %kw; + for (split(//, $_[0])) { + my $k = $c2kw{$_}; + if (defined($k)) { + $kw{$k} = 1; + } else { + push @unknown, $_; + } + } + (\%kw, \@unknown); +} + 1; diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t new file mode 100644 index 00000000..9531949a --- /dev/null +++ b/t/lei-export-kw.t @@ -0,0 +1,35 @@ +#!perl -w +# Copyright (C) 2021 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; use v5.10.1; use PublicInbox::TestCommon; +use File::Copy qw(cp); +use File::Path qw(make_path); +require_mods(qw(lei -imapd Mail::IMAPClient)); +my ($tmpdir, $for_destroy) = tmpdir; +my ($ro_home, $cfg_path) = setup_public_inboxes; +my $expect = eml_load('t/data/0001.patch'); +test_lei({ tmpdir => $tmpdir }, sub { + my $home = $ENV{HOME}; + my $md = "$home/md"; + make_path("$md/new", "$md/cur", "$md/tmp"); + cp('t/data/0001.patch', "$md/new/y") or xbail "cp $md $!"; + cp('t/data/message_embed.eml', "$md/cur/x:2,S") or xbail "cp $md $!"; + lei_ok qw(index -q), $md; + lei_ok qw(tag t/data/0001.patch +kw:seen); + lei_ok qw(export-kw --all=local); + ok(!-e "$md/new/y", 'original gone'); + is_deeply(eml_load("$md/cur/y:2,S"), $expect, + "`seen' kw exported"); + + lei_ok qw(tag t/data/0001.patch +kw:answered); + lei_ok qw(export-kw --all=local); + ok(!-e "$md/cur/y:2,S", 'seen-only file gone'); + is_deeply(eml_load("$md/cur/y:2,RS"), $expect, "`R' added"); + + lei_ok qw(tag t/data/0001.patch -kw:answered -kw:seen); + lei_ok qw(export-kw --mode=set --all=local); + ok(!-e "$md/cur/y:2,RS", 'seen+answered file gone'); + is_deeply(eml_load("$md/cur/y:2,"), $expect, 'no keywords left'); +}); + +done_testing; diff --git a/t/mdir_reader.t b/t/mdir_reader.t index 51b38af4..c927e1a7 100644 --- a/t/mdir_reader.t +++ b/t/mdir_reader.t @@ -19,4 +19,9 @@ is(maildir_path_flags('/path/to/foo:2,'), '', 'no flags in path'); use_ok 'PublicInbox::InboxWritable', qw(eml_from_path); is(eml_from_path('.'), undef, 'eml_from_path fails on directory'); +is_deeply([PublicInbox::MdirReader::flags2kw('S')], [{ 'seen' => 1 }, []], + "`seen' kw set from flag"); +is_deeply([PublicInbox::MdirReader::flags2kw('Su')], [{ 'seen' => 1 }, ['u']], + 'unknown flag ignored'); + done_testing;
We will need this for mail synchronization --- lib/PublicInbox/URIimap.pm | 82 ++++++++++++++++++++++++++++++-------- t/uri_imap.t | 60 +++++++++++++++++++++------- 2 files changed, 110 insertions(+), 32 deletions(-) diff --git a/lib/PublicInbox/URIimap.pm b/lib/PublicInbox/URIimap.pm index f6244137..a309fde0 100644 --- a/lib/PublicInbox/URIimap.pm +++ b/lib/PublicInbox/URIimap.pm @@ -12,11 +12,14 @@ # RFC 2192 also describes ";TYPE=<list_type>" package PublicInbox::URIimap; use strict; +use v5.10.1; use URI::Split qw(uri_split uri_join); # part of URI -use URI::Escape qw(uri_unescape); +use URI::Escape qw(uri_unescape uri_escape); use overload '""' => \&as_string; my %default_ports = (imap => 143, imaps => 993); +# for enc-auth-type and enc-user in RFC 5092 +my $achar = qr/[A-Za-z0-9%\-_\.\!\$'\(\)\+\,\&\=\*]+/; sub new { my ($class, $url) = @_; @@ -86,14 +89,15 @@ sub uidvalidity { # read/write $path =~ m!\A[^;/]+;UIDVALIDITY=([1-9][0-9]*)\b!i ? ($1 + 0) : undef; } -sub iuid { +sub uid { my ($self, $val) = @_; my ($scheme, $auth, $path, $query, $frag) = uri_split($$self); - if (defined $val) { - if ($path =~ s!/;UID=[^;/]*\b!/;UID=$val!i) { - # s// already changed it - } else { # both s// failed, so just append - $path .= ";UID=$val"; + if (scalar(@_) == 2) { + if (!defined $val) { + $path =~ s!/;UID=[^;/]*\b!!i; + } else { + $path =~ s!/;UID=[^;/]*\b!/;UID=$val!i or + $path .= ";UID=$val"; } $$self = uri_join($scheme, $auth, $path, $query); } @@ -114,12 +118,34 @@ sub authority { } sub user { - my ($self) = @_; - my (undef, $auth) = uri_split($$self); - $auth =~ s/@.*\z// or return undef; # drop host:port - $auth =~ s/;.*\z//; # drop ;AUTH=... - $auth =~ s/:.*\z//; # drop password - uri_unescape($auth); + my ($self, $val) = @_; + my ($scheme, $auth, $path, $query) = uri_split($$self); + my $at_host_port; + $auth =~ s/(@.*)\z// and $at_host_port = $1; # stash host:port for now + if (scalar(@_) == 2) { # set, this clobbers password, too + if (defined $val) { + my $uval = uri_escape($val); + if (defined($at_host_port)) { + $auth =~ s!\A.*?(;AUTH=$achar).*!$uval$1!ix + or $auth = $uval; + } else { + substr($auth, 0, 0) = "$uval@"; + } + } elsif (defined($at_host_port)) { # clobber + $auth =~ s!\A.*?(;AUTH=$achar).*!$1!i or $auth = ''; + if ($at_host_port && $auth eq '') { + $at_host_port =~ s/\A\@//; + } + } + $at_host_port //= ''; + $$self = uri_join($scheme, $auth.$at_host_port, $path, $query); + $val; + } else { # read-only + $at_host_port // return undef; # explicit undef for scalar + $auth =~ s/;.*\z//; # drop ;AUTH=... + $auth =~ s/:.*\z//; # drop password + $auth eq '' ? undef : uri_unescape($auth); + } } sub password { @@ -131,10 +157,32 @@ sub password { } sub auth { - my ($self) = @_; - my (undef, $auth) = uri_split($$self); - $auth =~ s/@.*\z//; # drop host:port - $auth =~ /;AUTH=(.+)\z/i ? uri_unescape($1) : undef; + my ($self, $val) = @_; + my ($scheme, $auth, $path, $query) = uri_split($$self); + my $at_host_port; + $auth =~ s/(@.*)\z// and $at_host_port = $1; # stash host:port for now + if (scalar(@_) == 2) { + if (defined $val) { + my $uval = uri_escape($val); + if ($auth =~ s!;AUTH=$achar!;AUTH=$uval!ix) { + # replaced existing + } elsif (defined($at_host_port)) { + $auth .= ";AUTH=$uval"; + } else { + substr($auth, 0, 0) = ";AUTH=$uval@"; + } + } else { # clobber + $auth =~ s!;AUTH=$achar!!i; + if ($at_host_port && $auth eq '') { + $at_host_port =~ s/\A\@//; + } + } + $at_host_port //= ''; + $$self = uri_join($scheme, $auth.$at_host_port, $path, $query); + $val; + } else { # read-only + $auth =~ /;AUTH=(.+)\z/i ? uri_unescape($1) : undef; + } } sub scheme { diff --git a/t/uri_imap.t b/t/uri_imap.t index ed24fc1b..14f0f346 100644 --- a/t/uri_imap.t +++ b/t/uri_imap.t @@ -2,7 +2,7 @@ # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> use strict; -use Test::More; +use v5.10.1; use PublicInbox::TestCommon; require_mods 'URI::Split'; use_ok 'PublicInbox::URIimap'; @@ -69,36 +69,66 @@ $uri = PublicInbox::URIimap->new('imap://0/mmm;UIDVALIDITY=21'); is($uri->uidvalidity, 21, 'multi-digit UIDVALIDITY'); $uri = PublicInbox::URIimap->new('imap://0/mmm;UIDVALIDITY=bogus'); is($uri->uidvalidity, undef, 'bogus UIDVALIDITY'); -is($uri->uidvalidity(2), 2, 'iuid set'); +is($uri->uidvalidity(2), 2, 'uid set'); is($$uri, 'imap://0/mmm;UIDVALIDITY=2', 'bogus uidvalidity replaced'); -is($uri->uidvalidity(13), 13, 'iuid set'); +is($uri->uidvalidity(13), 13, 'uid set'); is($$uri, 'imap://0/mmm;UIDVALIDITY=13', 'valid uidvalidity replaced'); $uri = PublicInbox::URIimap->new('imap://0/mmm'); -is($uri->uidvalidity(2), 2, 'iuid set'); +is($uri->uidvalidity(2), 2, 'uid set'); is($$uri, 'imap://0/mmm;UIDVALIDITY=2', 'uidvalidity appended'); -is($uri->iuid, undef, 'no iuid'); +is($uri->uid, undef, 'no uid'); is(PublicInbox::URIimap->new('imap://0/x;uidvalidity=1')->canonical->as_string, 'imap://0/x;UIDVALIDITY=1', 'capitalized UIDVALIDITY'); $uri = PublicInbox::URIimap->new('imap://0/mmm/;uid=8'); is($uri->canonical->as_string, 'imap://0/mmm/;UID=8', 'canonicalized UID'); -is($uri->mailbox, 'mmm', 'mailbox works with iuid'); -is($uri->iuid, 8, 'iuid extracted'); -is($uri->iuid(9), 9, 'iuid set'); -is($$uri, 'imap://0/mmm/;UID=9', 'correct iuid when stringified'); -is($uri->uidvalidity(1), 1, 'set uidvalidity with iuid'); +is($uri->mailbox, 'mmm', 'mailbox works with uid'); +is($uri->uid, 8, 'uid extracted'); +is($uri->uid(9), 9, 'uid set'); +is($$uri, 'imap://0/mmm/;UID=9', 'correct uid when stringified'); +is($uri->uidvalidity(1), 1, 'set uidvalidity with uid'); is($$uri, 'imap://0/mmm;UIDVALIDITY=1/;UID=9', - 'uidvalidity added with iuid'); -is($uri->uidvalidity(4), 4, 'set uidvalidity with iuid'); + 'uidvalidity added with uid'); +is($uri->uidvalidity(4), 4, 'set uidvalidity with uid'); is($$uri, 'imap://0/mmm;UIDVALIDITY=4/;UID=9', - 'uidvalidity replaced with iuid'); -is($uri->iuid(3), 3, 'iuid set with uidvalidity'); -is($$uri, 'imap://0/mmm;UIDVALIDITY=4/;UID=3', 'iuid replaced properly'); + 'uidvalidity replaced with uid'); +is($uri->uid(3), 3, 'uid set with uidvalidity'); +is($$uri, 'imap://0/mmm;UIDVALIDITY=4/;UID=3', 'uid replaced properly'); my $lc = lc($$uri); is(PublicInbox::URIimap->new($lc)->canonical->as_string, "$$uri", 'canonical uppercased both params'); +is($uri->uid(undef), undef, 'uid can be clobbered'); +is($$uri, 'imap://0/mmm;UIDVALIDITY=4', 'uid dropped'); + +$uri->auth('ANONYMOUS'); +is($$uri, 'imap://;AUTH=ANONYMOUS@0/mmm;UIDVALIDITY=4', 'AUTH= set'); +is($uri->user, undef, 'user is undef w/ AUTH='); +is($uri->password, undef, 'password is undef w/ AUTH='); + +$uri->user('foo'); +is($$uri, 'imap://foo;AUTH=ANONYMOUS@0/mmm;UIDVALIDITY=4', 'user set w/AUTH'); +is($uri->password, undef, 'password is undef w/ AUTH= & user'); +$uri->auth(undef); +is($$uri, 'imap://foo@0/mmm;UIDVALIDITY=4', 'user remains set w/o auth'); +is($uri->password, undef, 'password is undef w/ user only'); + +$uri->user('bar'); +is($$uri, 'imap://bar@0/mmm;UIDVALIDITY=4', 'user set w/o AUTH'); +$uri->auth('NTML'); +is($$uri, 'imap://bar;AUTH=NTML@0/mmm;UIDVALIDITY=4', 'auth set w/user'); +$uri->auth(undef); +$uri->user(undef); +is($$uri, 'imap://0/mmm;UIDVALIDITY=4', 'auth and user both cleared'); +is($uri->user, undef, 'user is undef'); +is($uri->auth, undef, 'auth is undef'); +is($uri->password, undef, 'password is undef'); +$uri = PublicInbox::URIimap->new('imap://[::1]:36281/'); +my $cred = bless { username => $uri->user, password => $uri->password }; +is($cred->{username}, undef, 'user is undef in array context'); +is($cred->{password}, undef, 'password is undef in array context'); + done_testing;
Just having UIDVALIDITY in the URI isn't enough, since a single lei user may have multiple IMAP logins on the same server. This leads to compatibility problems and forces a reimport for the few users already using this lei functionality, but it's not stable nor released, yet. --- lib/PublicInbox/NetReader.pm | 42 ++++++++++++++++++++++-------------- t/lei-import-imap.t | 9 +++++--- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index fd0d1682..a532b218 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -58,12 +58,10 @@ sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback # mic_for may prompt the user and store auth info, prepares mic_get sub mic_for ($$$$) { # mic = Mail::IMAPClient - my ($self, $url, $mic_args, $lei) = @_; - require PublicInbox::URIimap; - my $uri = PublicInbox::URIimap->new($url); + my ($self, $uri, $mic_args, $lei) = @_; require PublicInbox::GitCredential; my $cred = bless { - url => $url, + url => "$uri", protocol => $uri->scheme, host => $uri->host, username => $uri->user, @@ -83,13 +81,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient }; require PublicInbox::IMAPClient; my $mic = mic_new($self, $mic_arg, $sec, $uri) or - die "E: <$url> new: $@\n"; + die "E: <$uri> new: $@\n"; # default to using STARTTLS if it's available, but allow # it to be disabled since I usually connect to localhost if (!$mic_arg->{Ssl} && !defined($mic_arg->{Starttls}) && $mic->has_capability('STARTTLS') && $mic->can('starttls')) { - $mic->starttls or die "E: <$url> STARTTLS: $@\n"; + $mic->starttls or die "E: <$uri> STARTTLS: $@\n"; } # do we even need credentials? @@ -111,8 +109,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient if ($mic->login && $mic->IsAuthenticated) { # success! keep IMAPClient->new arg in case we get disconnected $self->{mic_arg}->{$sec} = $mic_arg; + if ($cred) { + $uri->user($cred->{username}) if !defined($uri->user); + } elsif ($mic_arg->{Authmechanism} eq 'ANONYMOUS') { + $uri->auth('ANONYMOUS') if !defined($uri->auth); + } } else { - $err = "E: <$url> LOGIN: $@\n"; + $err = "E: <$uri> LOGIN: $@\n"; if ($cred && defined($cred->{password})) { $err =~ s/\Q$cred->{password}\E/*******/g; } @@ -304,15 +307,16 @@ sub imap_common_init ($;$) { # make sure we can connect and cache the credentials in memory $self->{mic_arg} = {}; # schema://authority => IMAPClient->new args my $mics = {}; # schema://authority => IMAPClient obj - for my $uri (@{$self->{imap_order}}) { - my $sec = uri_section($uri); + for my $orig_uri (@{$self->{imap_order}}) { + my $sec = uri_section($orig_uri); + my $uri = PublicInbox::URIimap->new("$sec/"); my $mic = $mics->{$sec} //= - mic_for($self, "$sec/", $mic_args, $lei) // + mic_for($self, $uri, $mic_args, $lei) // die "Unable to continue\n"; next unless $self->isa('PublicInbox::NetWriter'); - my $dst = $uri->mailbox // next; + my $dst = $orig_uri->mailbox // next; next if $mic->exists($dst); # already exists - $mic->create($dst) or die "CREATE $dst failed <$uri>: $@"; + $mic->create($dst) or die "CREATE $dst failed <$orig_uri>: $@"; } $mics; } @@ -419,12 +423,18 @@ sub run_commit_cb ($) { $cb->(@args); } -sub _itrk_last ($$;$) { - my ($self, $uri, $r_uidval) = @_; +sub itrk_last ($$;$$) { + my ($self, $uri, $r_uidval, $mic) = @_; return (undef, undef, $r_uidval) unless $self->{incremental}; my ($itrk, $l_uid, $l_uidval); if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0 $uri->uidvalidity($r_uidval) if defined $r_uidval; + if ($mic) { + my $auth = $mic->Authmechanism // ''; + $uri->auth($auth) if $auth eq 'ANONYMOUS'; + my $user = $mic->User; + $uri->user($user) if defined($user); + } my $x; $l_uid = ($lms && ($x = $lms->location_stats($$uri))) ? $x->{'uid.max'} : undef; @@ -459,7 +469,7 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval) EOF my $uri = $orig_uri->clone; - my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval); + my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic); return <<EOF if $l_uidval != $r_uidval; E: $uri UIDVALIDITY mismatch E: local=$l_uidval != remote=$r_uidval @@ -612,7 +622,7 @@ sub _nntp_fetch_all ($$$) { # IMAPTracker is also used for tracking NNTP, UID == article number # LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's # expensive. So we assume newsgroups don't change: - my ($itrk, $l_art) = _itrk_last($self, $uri); + my ($itrk, $l_art) = itrk_last($self, $uri); # allow users to specify articles to refetch # cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t index fd15ef4f..d424ebb1 100644 --- a/t/lei-import-imap.t +++ b/t/lei-import-imap.t @@ -23,9 +23,11 @@ test_lei({ tmpdir => $tmpdir }, sub { lei_ok('import', $url); lei_ok 'ls-mail-sync'; - like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-mail-sync'); + like($lei_out, qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E + /t\.v2\.0;UIDVALIDITY=\d+\n\z!x, 'ls-mail-sync'); chomp(my $u = $lei_out); lei_ok('import', $u, \'UIDVALIDITY match in URL'); + $url = $u; $u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s; ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected'); @@ -33,7 +35,7 @@ test_lei({ tmpdir => $tmpdir }, sub { my $inspect = json_utf8->decode($lei_out); my @k = keys %$inspect; is(scalar(@k), 1, 'one URL resolved'); - like($k[0], qr!\A\Q$url\E;UIDVALIDITY=\d+\z!, 'inspect URL matches'); + is($k[0], $url, 'inspect URL matches'); my $stats = $inspect->{$k[0]}; is_deeply([ sort keys %$stats ], [ qw(uid.count uid.max uid.min) ], 'keys match'); @@ -55,7 +57,8 @@ test_lei({ tmpdir => $tmpdir }, sub { my $x = json_utf8->decode($lei_out); is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect'); is(ref($x->{'mail-sync'}), 'HASH', 'sync in inspect'); - is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect'); + is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect') + or diag explain($x); my $psgi_attach = 'cfa3622cbeffc9bd6b0fc66c4d60d420ba74f60d'; lei_ok('blob', $psgi_attach);