[PATCH 5/6] net_writer: start implementing IMAP write support

[Eric Wong <e@80x24.org>]

Requiring TEST_IMAP_WRITE_URL to be set to a writable IMAP
server URL isn't ideal, but it works for now until we have time
to setup a mock dovecot/cyrus/etc... instance for testing.
---
 MANIFEST                     |  2 ++
 lib/PublicInbox/NetReader.pm | 42 +++++++++++++++++++------------
 lib/PublicInbox/NetWriter.pm | 26 +++++++++++++++++++
 xt/net_writer-imap.t         | 48 ++++++++++++++++++++++++++++++++++++
 4 files changed, 102 insertions(+), 16 deletions(-)
 create mode 100644 lib/PublicInbox/NetWriter.pm
 create mode 100644 xt/net_writer-imap.t

diff --git a/MANIFEST b/MANIFEST
index 3d9ad616..21e37678 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -213,6 +213,7 @@ lib/PublicInbox/NNTP.pm
 lib/PublicInbox/NNTPD.pm
 lib/PublicInbox/NNTPdeflate.pm
 lib/PublicInbox/NetReader.pm
+lib/PublicInbox/NetWriter.pm
 lib/PublicInbox/NewsWWW.pm
 lib/PublicInbox/OnDestroy.pm
 lib/PublicInbox/Over.pm
@@ -471,6 +472,7 @@ xt/lei-sigpipe.t
 xt/mem-imapd-tls.t
 xt/mem-msgview.t
 xt/msgtime_cmp.t
+xt/net_writer-imap.t
 xt/nntpd-validate.t
 xt/perf-msgview.t
 xt/perf-nntpd.t
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 22ba4be7..92d004bc 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -8,6 +8,8 @@ use v5.10.1;
 use parent qw(Exporter PublicInbox::IPC);
 use PublicInbox::Eml;
 
+our %IMAPflags2kw = map {; "\\\u$_" => $_ } qw(seen answered flagged draft);
+
 # TODO: trim this down, this is huge
 our @EXPORT = qw(uri_new uri_scheme uri_section
 		mic_for nn_new nn_for
@@ -33,6 +35,7 @@ sub uri_section ($) {
 
 sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
 
+# mic_for may prompt the user and store auth info, prepares mic_get
 sub mic_for { # mic = Mail::IMAPClient
 	my ($self, $url, $mic_args, $lei) = @_;
 	require PublicInbox::URIimap;
@@ -286,7 +289,12 @@ sub imap_common_init ($;$) {
 	for my $url (@{$self->{imap_order}}) {
 		my $uri = PublicInbox::URIimap->new($url);
 		my $sec = uri_section($uri);
-		$mics->{$sec} //= mic_for($self, $url, $mic_args, $lei);
+		$mics->{$sec} //= mic_for($self, "$sec/", $mic_args, $lei);
+		next unless $self->isa('PublicInbox::NetWriter');
+		my $dst = $uri->mailbox // next;
+		my $mic = $mics->{$sec};
+		next if $mic->exists($dst); # already exists
+		$mic->create($dst) or die "CREATE $dst failed <$url>: $@";
 	}
 	$mics;
 }
@@ -312,13 +320,6 @@ sub errors {
 	undef;
 }
 
-my %IMAPflags2kw = (
-	'\Seen' => 'seen',
-	'\Answered' => 'answered',
-	'\Flagged' => 'flagged',
-	'\Draft' => 'draft',
-);
-
 sub _imap_do_msg ($$$$$) {
 	my ($self, $url, $uid, $raw, $flags) = @_;
 	# our target audience expects LF-only, save storage
@@ -418,25 +419,34 @@ sub _imap_fetch_all ($$$) {
 	$err;
 }
 
+# uses cached auth info prepared by mic_for
+sub mic_get {
+	my ($self, $sec) = @_;
+	my $mic_arg = $self->{mic_arg}->{$sec} or
+			die "BUG: no Mail::IMAPClient->new arg for $sec";
+	if (defined(my $cb_name = $mic_arg->{Authcallback})) {
+		if (ref($cb_name) ne 'CODE') {
+			$mic_arg->{Authcallback} = $self->can($cb_name);
+		}
+	}
+	my $mic = PublicInbox::IMAPClient->new(%$mic_arg);
+	$mic && $mic->IsConnected ? $mic : undef;
+}
+
 sub imap_each {
 	my ($self, $url, $eml_cb, @args) = @_;
 	my $uri = PublicInbox::URIimap->new($url);
 	my $sec = uri_section($uri);
-	my $mic_arg = $self->{mic_arg}->{$sec} or
-			die "BUG: no Mail::IMAPClient->new arg for $sec";
 	local $0 = $uri->mailbox." $sec";
-	my $cb_name = $mic_arg->{Authcallback};
-	if (ref($cb_name) ne 'CODE') {
-		$mic_arg->{Authcallback} = $self->can($cb_name);
-	}
-	my $mic = PublicInbox::IMAPClient->new(%$mic_arg, Debug => 0);
+	my $mic = mic_get($self, $sec);
 	my $err;
-	if ($mic && $mic->IsConnected) {
+	if ($mic) {
 		local $self->{eml_each} = [ $eml_cb, @args ];
 		$err = _imap_fetch_all($self, $mic, $url);
 	} else {
 		$err = "E: not connected: $!";
 	}
+	warn $err if $err;
 	$mic;
 }
 
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
new file mode 100644
index 00000000..6f0a0b94
--- /dev/null
+++ b/lib/PublicInbox/NetWriter.pm
@@ -0,0 +1,26 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# common writer code for IMAP (and later, JMAP)
+package PublicInbox::NetWriter;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::NetReader);
+use PublicInbox::Smsg;
+use PublicInbox::MsgTime qw(msg_timestamp);
+
+my %IMAPkw2flags;
+@IMAPkw2flags{values %PublicInbox::NetReader::IMAPflags2kw} =
+				keys %PublicInbox::NetReader::IMAPflags2kw;
+
+sub imap_append {
+	my ($mic, $folder, $bref, $smsg, $eml) = @_;
+	$bref //= \($eml->as_string);
+	$smsg //= bless { }, 'PublicInbox::Smsg';
+	$smsg->{ts} //= msg_timestamp($eml // PublicInbox::Eml->new($$bref));
+	my @f = map { $IMAPkw2flags{$_} } @{$smsg->{kw}};
+	$mic->append_string($folder, $$bref, "@f", $smsg->internaldate) or
+		die "APPEND $folder: $@";
+}
+
+1;
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
new file mode 100644
index 00000000..ea812f16
--- /dev/null
+++ b/xt/net_writer-imap.t
@@ -0,0 +1,48 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use Sys::Hostname qw(hostname);
+use POSIX qw(strftime);
+use PublicInbox::OnDestroy;
+use PublicInbox::URIimap;
+use PublicInbox::Config;
+my $imap_url = $ENV{TEST_IMAP_WRITE_URL} or
+	plan skip_all => 'TEST_IMAP_WRITE_URL unset';
+my $uri = PublicInbox::URIimap->new($imap_url);
+defined($uri->path) and
+	plan skip_all => "$imap_url should not be a mailbox (just host:port)";
+require_mods('Mail::IMAPClient');
+require_ok 'PublicInbox::NetWriter';
+my $host = (split(/\./, hostname))[0];
+my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
+my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)).
+		"-$$-".sprintf('%x', int(rand(0xffffffff)));
+my $nwr = PublicInbox::NetWriter->new;
+$imap_url .= '/' unless substr($imap_url, -1) eq '/';
+my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder");
+is($folder_uri->mailbox, $folder, 'folder correct') or
+		BAIL_OUT "BUG: bad $$uri";
+$nwr->add_url($$folder_uri);
+is($nwr->errors, undef, 'no errors');
+$nwr->{pi_cfg} = bless {}, 'PublicInbox::Config';
+my $mics = $nwr->imap_common_init;
+my $mic = (values %$mics)[0];
+my $cleanup = PublicInbox::OnDestroy->new(sub {
+	$mic->delete($folder) or fail "delete $folder <$$folder_uri>: $@";
+});
+my $imap_append = $nwr->can('imap_append');
+my $smsg = bless { kw => [ 'seen' ] }, 'PublicInbox::Smsg';
+$imap_append->($mic, $folder, undef, $smsg, eml_load('t/plack-qp.eml'));
+my @res;
+$nwr->{quiet} = 1;
+$nwr->imap_each($$folder_uri, sub {
+	my ($u, $uid, $kw, $eml, $arg) = @_;
+	push @res, [ $kw, $eml ];
+});
+is(scalar(@res), 1, 'got appended message');
+is_deeply(\@res, [ [ [ 'seen' ], eml_load('t/plack-qp.eml') ] ],
+	'uploaded message read back');
+
+undef $cleanup;
+done_testing;