[PATCH 2/7] lei q: support IMAP/IMAPS --output destinations

[Eric Wong <e@80x24.org>]

Augment (and dedupe) aren't parallel, yet, so its more sensitive to
high-latency networks.
---
 lib/PublicInbox/LeiAuth.pm     |   2 +-
 lib/PublicInbox/LeiOverview.pm |   7 +-
 lib/PublicInbox/LeiQuery.pm    |  18 ++++-
 lib/PublicInbox/LeiToMail.pm   |  56 +++++++++++++++-
 lib/PublicInbox/NetReader.pm   |   7 +-
 lib/PublicInbox/NetWriter.pm   |  12 ++++
 xt/net_writer-imap.t           | 118 ++++++++++++++++++++++++++++++---
 7 files changed, 202 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm
index 7acb9900..bf0110ed 100644
--- a/lib/PublicInbox/LeiAuth.pm
+++ b/lib/PublicInbox/LeiAuth.pm
@@ -63,7 +63,7 @@ sub ipc_atfork_child {
 }
 
 sub new {
-	my ($cls, $nrd) = @_;
+	my ($cls, $nrd) = @_; # nrd may be NetReader or descendant (NetWriter)
 	bless { nrd => $nrd }, $cls;
 }
 
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 3169bae6..4db1d8c8 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -59,7 +59,12 @@ sub new {
 
 	my $fmt = $opt->{$ofmt_key};
 	$fmt = lc($fmt) if defined $fmt;
-	if ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/
+	if ($dst =~ m!\A([a-z0-9\+]+)://!is) {
+		defined($fmt) and return $lei->fail(<<"");
+--$ofmt_key=$fmt invalid with URL $dst
+
+		$fmt = lc $1;
+	} elsif ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/
 		my $ofmt = lc $1;
 		$fmt //= $ofmt;
 		return $lei->fail(<<"") if $fmt ne $ofmt;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index f71beae6..eaf91f2e 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -11,14 +11,26 @@ sub prep_ext { # externals_each callback
 	$lxs->prepare_external($loc) unless $exclude->{$loc};
 }
 
-sub qstr_add { # for --stdin
+sub _start_query {
+	my ($self) = @_;
+	if (my $nwr = $self->{nwr}) {
+		require PublicInbox::LeiAuth;
+		my $auth = $self->{auth} = PublicInbox::LeiAuth->new($nwr);
+		my $lxs = $self->{lxs};
+		$auth->auth_start($self, $lxs->can('do_query'), $lxs, $self);
+	} else {
+		$self->{lxs}->do_query($self);
+	}
+}
+
+sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
 	my ($self) = @_; # $_[1] = $rbuf
 	if (defined($_[1])) {
 		$_[1] eq '' and return eval {
 			my $lse = delete $self->{lse};
 			$lse->query_approxidate($lse->git,
 						$self->{mset_opt}->{qstr});
-			$self->{lxs}->do_query($self);
+			_start_query($self);
 		};
 		$self->{mset_opt}->{qstr} .= $_[1];
 	} else {
@@ -115,7 +127,7 @@ no query allowed on command-line with --stdin
 		return;
 	}
 	$mset_opt{qstr} = $lse->query_argv_to_string($lse->git, \@argv);
-	$lxs->do_query($self);
+	_start_query($self);
 }
 
 # shell completion helper called by lei__complete
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index e89cca71..0e0b0a43 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -331,9 +331,31 @@ sub _maildir_write_cb ($$) {
 	}
 }
 
+sub _imap_write_cb ($$) {
+	my ($self, $lei) = @_;
+	my $dedupe = $lei->{dedupe};
+	$dedupe->prepare_dedupe if $dedupe;
+	my $imap_append = $lei->{nwr}->can('imap_append');
+	my $mic = $lei->{nwr}->mic_get($lei->{ovv}->{dst});
+	my $folder = $self->{uri}->mailbox;
+	sub { # for git_to_mail
+		my ($bref, $smsg, $eml) = @_;
+		$mic // return $lei->fail; # dst may be undef-ed in last run
+		if ($dedupe) {
+			$eml //= PublicInbox::Eml->new($$bref); # copy bref
+			return if $dedupe->is_dup($eml, $smsg->{blob});
+		}
+		eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
+		if (my $err = $@) {
+			undef $mic;
+			die $err;
+		}
+	}
+}
+
 sub write_cb { # returns a callback for git_to_mail
 	my ($self, $lei) = @_;
-	# _mbox_write_cb or _maildir_write_cb
+	# _mbox_write_cb, _maildir_write_cb or _imap_write_cb
 	my $m = "_$self->{base_type}_write_cb";
 	$self->$m($lei);
 }
@@ -360,6 +382,18 @@ sub new {
 			"$dst exists and is not a writable file\n";
 		$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
 		$self->{base_type} = 'mbox';
+	} elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support
+		require PublicInbox::NetWriter;
+		my $nwr = PublicInbox::NetWriter->new;
+		$nwr->add_url($dst);
+		$nwr->{quiet} = $lei->{opt}->{quiet};
+		my $err = $nwr->errors($dst);
+		return $lei->fail($err) if $err;
+		require PublicInbox::URIimap; # TODO: URI cast early
+		$self->{uri} = PublicInbox::URIimap->new($dst);
+		$self->{uri}->mailbox or die "No mailbox: $dst";
+		$lei->{nwr} = $nwr;
+		$self->{base_type} = 'imap';
 	} else {
 		die "bad mail --format=$fmt\n";
 	}
@@ -394,6 +428,26 @@ sub _do_augment_maildir {
 	}
 }
 
+sub _augment_imap { # PublicInbox::NetReader::imap_each cb
+	my ($url, $uid, $kw, $eml, $lei) = @_;
+	_augment($eml, $lei);
+}
+
+sub _do_augment_imap {
+	my ($self, $lei) = @_;
+	my $dst = $lei->{ovv}->{dst};
+	my $nwr = $lei->{nwr};
+	if ($lei->{opt}->{augment}) {
+		my $dedupe = $lei->{dedupe};
+		if ($dedupe && $dedupe->prepare_dedupe) {
+			$nwr->imap_each($dst, \&_augment_imap, $lei);
+			$dedupe->pause_dedupe;
+		}
+	} else { # clobber existing IMAP folder
+		$nwr->imap_delete_all($dst);
+	}
+}
+
 sub _pre_augment_mbox {
 	my ($self, $lei) = @_;
 	my $dst = $lei->{ovv}->{dst};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 92d004bc..541094a0 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -422,8 +422,13 @@ sub _imap_fetch_all ($$$) {
 # uses cached auth info prepared by mic_for
 sub mic_get {
 	my ($self, $sec) = @_;
-	my $mic_arg = $self->{mic_arg}->{$sec} or
+	my $mic_arg = $self->{mic_arg}->{$sec};
+	unless ($mic_arg) {
+		my $uri = PublicInbox::URIimap->new($sec);
+		$sec = uri_section($uri);
+		$mic_arg = $self->{mic_arg}->{$sec} or
 			die "BUG: no Mail::IMAPClient->new arg for $sec";
+	}
 	if (defined(my $cb_name = $mic_arg->{Authcallback})) {
 		if (ref($cb_name) ne 'CODE') {
 			$mic_arg->{Authcallback} = $self->can($cb_name);
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 6f0a0b94..89f8662e 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -23,4 +23,16 @@ sub imap_append {
 		die "APPEND $folder: $@";
 }
 
+sub imap_delete_all {
+	my ($self, $url) = @_;
+	my $uri = PublicInbox::URIimap->new($url);
+	my $sec = $self->can('uri_section')->($uri);
+	local $0 = $uri->mailbox." $sec";
+	my $mic = $self->mic_get($sec) or die "E: not connected: $@";
+	$mic->select($uri->mailbox) or return; # non-existent
+	if ($mic->delete_message('1:*')) {
+		$mic->expunge;
+	}
+}
+
 1;
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index dfd765be..4832245a 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -7,6 +7,7 @@ use POSIX qw(strftime);
 use PublicInbox::OnDestroy;
 use PublicInbox::URIimap;
 use PublicInbox::Config;
+use Fcntl qw(O_EXCL O_WRONLY O_CREAT);
 my $imap_url = $ENV{TEST_IMAP_WRITE_URL} or
 	plan skip_all => 'TEST_IMAP_WRITE_URL unset';
 my $uri = PublicInbox::URIimap->new($imap_url);
@@ -19,30 +20,125 @@ my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
 my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)).
 		"-$$-".sprintf('%x', int(rand(0xffffffff)));
 my $nwr = PublicInbox::NetWriter->new;
-$imap_url .= '/' unless substr($imap_url, -1) eq '/';
+chop($imap_url) if substr($imap_url, -1) eq '/';
 my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder");
 is($folder_uri->mailbox, $folder, 'folder correct') or
 		BAIL_OUT "BUG: bad $$uri";
 $nwr->add_url($$folder_uri);
 is($nwr->errors, undef, 'no errors');
 $nwr->{pi_cfg} = bless {}, 'PublicInbox::Config';
-my $mics = $nwr->imap_common_init;
+
+my $set_cred_helper = sub {
+	my ($f, $cred_set) = @_;
+	sysopen(my $fh, $f, O_CREAT|O_EXCL|O_WRONLY) or BAIL_OUT "open $f: $!";
+	print $fh <<EOF or BAIL_OUT "print $f: $!";
+[credential]
+	helper = $cred_set
+EOF
+	close $fh or BAIL_OUT "close $f: $!";
+};
+
+# allow testers with git-credential-store configured to reuse
+# stored credentials inside test_lei(sub {...}) when $ENV{HOME}
+# is overridden and localized.
+my ($cred_set, @cred_link, $tmpdir, $for_destroy);
+chomp(my $cred_helper = `git config credential.helper 2>/dev/null`);
+if ($cred_helper eq 'store') {
+	my $config = $ENV{XDG_CONFIG_HOME} // "$ENV{HOME}/.config";
+	for my $f ("$ENV{HOME}/.git-credentials", "$config/git/credentials") {
+		next unless -f $f;
+		@cred_link = ($f, '/.git-credentials');
+		last;
+	}
+	$cred_set = qq("$cred_helper");
+} elsif ($cred_helper =~ /\Acache(?:[ \t]|\z)/) {
+	my $cache = $ENV{XDG_CACHE_HOME} // "$ENV{HOME}/.cache";
+	for my $d ("$ENV{HOME}/.git-credential-cache",
+			"$cache/git/credential") {
+		next unless -d $d;
+		@cred_link = ($d, '/.git-credential-cache');
+		$cred_set = qq("$cred_helper");
+		last;
+	}
+} elsif (!$cred_helper) { # make the test less painful if no creds configured
+	($tmpdir, $for_destroy) = tmpdir;
+	my $d = "$tmpdir/.git-credential-cache";
+	mkdir($d, 0700) or BAIL_OUT $!;
+	$cred_set = "cache --timeout=60";
+	@cred_link = ($d, '/.git-credential-cache');
+} else {
+	diag "credential.helper=$cred_helper will not be used for this test";
+}
+
+my $mics = do {
+	local $ENV{HOME} = $tmpdir // $ENV{HOME};
+	if ($tmpdir && $cred_set) {
+		$set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set)
+	}
+	$nwr->imap_common_init;
+};
 my $mic = (values %$mics)[0];
-my $cleanup = PublicInbox::OnDestroy->new(sub {
+my $cleanup = PublicInbox::OnDestroy->new($$, sub {
+	my $mic = $nwr->mic_get($imap_url);
 	$mic->delete($folder) or fail "delete $folder <$folder_uri>: $@";
+	if ($tmpdir && -f "$tmpdir/.gitconfig") {
+		local $ENV{HOME} = $tmpdir;
+		system(qw(git credential-cache exit));
+	}
 });
 my $imap_append = $nwr->can('imap_append');
 my $smsg = bless { kw => [ 'seen' ] }, 'PublicInbox::Smsg';
 $imap_append->($mic, $folder, undef, $smsg, eml_load('t/plack-qp.eml'));
-my @res;
 $nwr->{quiet} = 1;
-$nwr->imap_each($$folder_uri, sub {
-	my ($u, $uid, $kw, $eml, $arg) = @_;
-	push @res, [ $kw, $eml ];
-});
-is(scalar(@res), 1, 'got appended message');
-is_deeply(\@res, [ [ [ 'seen' ], eml_load('t/plack-qp.eml') ] ],
+my $imap_slurp_all = sub {
+	my ($u, $uid, $kw, $eml, $res) = @_;
+	push @$res, [ $kw, $eml ];
+};
+$nwr->imap_each($$folder_uri, $imap_slurp_all, my $res = []);
+is(scalar(@$res), 1, 'got appended message');
+my $plack_qp_eml = eml_load('t/plack-qp.eml');
+is_deeply($res, [ [ [ 'seen' ], $plack_qp_eml ] ],
 	'uploaded message read back');
+$res = $mic = $mics = undef;
+
+test_lei(sub {
+	my ($ro_home, $cfg_path) = setup_public_inboxes;
+	my $cfg = PublicInbox::Config->new($cfg_path);
+	$cfg->each_inbox(sub {
+		my ($ibx) = @_;
+		lei_ok qw(add-external -q), $ibx->{inboxdir} or BAIL_OUT;
+	});
+
+	# cred_link[0] may be on a different (hopefully encrypted) FS,
+	# we only symlink to it here, so we don't copy any sensitive data
+	# into the temporary directory
+	if (@cred_link && !symlink($cred_link[0], $ENV{HOME}.$cred_link[1])) {
+		diag "symlink @cred_link: $! (non-fatal)";
+		$cred_set = undef;
+	}
+	$set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) if $cred_set;
+
+	lei_ok qw(q f:qp@example.com -o), $$folder_uri;
+	$nwr->imap_each($$folder_uri, $imap_slurp_all, my $res = []);
+	is(scalar(@$res), 1, 'got one deduped result') or diag explain($res);
+	is_deeply($res->[0]->[1], $plack_qp_eml,
+			'lei q wrote expected result');
+
+	lei_ok qw(q f:matz -a -o), $$folder_uri;
+	$nwr->imap_each($$folder_uri, $imap_slurp_all, my $aug = []);
+	is(scalar(@$aug), 2, '2 results after augment') or diag explain($aug);
+	my $exp = $res->[0]->[1]->as_string;
+	is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1,
+			'original remains after augment');
+	$exp = eml_load('t/iso-2202-jp.eml')->as_string;
+	is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1,
+			'new result shown after augment');
+
+	lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string;
+	$nwr->imap_each($$folder_uri, $imap_slurp_all, my $empty = []);
+	is(scalar(@$empty), 0, 'no results w/o augment');
+
+});
 
-undef $cleanup;
+undef $cleanup; # remove temporary folder
 done_testing;