unofficial mirror of meta@public-inbox.org
 help / color / mirror / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/3] lei export-kw: support exporting keywords to IMAP
Date: Sun, 23 May 2021 01:38:27 +0000	[thread overview]
Message-ID: <20210523013828.4888-3-e@80x24.org> (raw)
In-Reply-To: <20210523013828.4888-1-e@80x24.org>

We support writing to IMAP stores in other places (just like
Maildir), and it's actually less complex for us to write to
IMAP.  Neither usability nor performance is ideal, but usability
will be addressed in the next commit to relax CLI argument
checking.

Performance is poor due to the synchronous Mail::IMAPClient
API and will need to be addressed with pipelining sometime
further in the future.
---
 lib/PublicInbox/LeiExportKw.pm | 31 +++++++++++++++++++++++--------
 lib/PublicInbox/LeiToMail.pm   |  9 +++++----
 lib/PublicInbox/NetWriter.pm   | 10 ++++++++++
 xt/net_writer-imap.t           | 18 +++++++++++++-----
 4 files changed, 51 insertions(+), 17 deletions(-)

diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index db4f7441..5ad33959 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -59,15 +59,28 @@ sub export_kw_md { # LeiMailSync->each_src callback
 	$self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
 }
 
+sub export_kw_imap { # LeiMailSync->each_src callback
+	my ($oidbin, $id, $self, $mic) = @_;
+	my $oidhex = unpack('H*', $oidbin);
+	my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return;
+	$self->{imap_mod_kw}->($self->{nwr}, $mic, $id, [ keys %$sto_kw ]);
+}
+
 # overrides PublicInbox::LeiInput::input_path_url
 sub input_path_url {
 	my ($self, $input, @args) = @_;
 	my $lms = $self->{lms} //= $self->{lse}->lms;
 	$lms->lms_begin;
-	if ($input =~ s/\Amaildir://i) {
+	if ($input =~ /\Amaildir:(.+)/i) {
+		my $mdir = $1;
 		require PublicInbox::LeiToMail; # kw2suffix
-		$lms->each_src("maildir:$input", \&export_kw_md, $self, $input);
-	}
+		$lms->each_src($input, \&export_kw_md, $self, $mdir);
+	} elsif ($input =~ m!\Aimaps?://!) {
+		my $uri = PublicInbox::URIimap->new($input);
+		my $mic = $self->{nwr}->mic_for_folder($uri);
+		$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
+		$mic->expunge;
+	} else { die "BUG: $input not supported" }
 	$lms->lms_commit;
 }
 
@@ -137,11 +150,6 @@ EOF
 	if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
 		return $lei->fail("cannot export to read-only folders: @ro");
 	}
-	if (my $net = $lei->{net}) {
-		require PublicInbox::NetWriter;
-		bless $net, 'PublicInbox::NetWriter';
-	}
-	undef $lms;
 	my $m = $opt->{mode} // 'merge';
 	if ($m eq 'merge') { # default
 		$self->{-merge_kw} = 1;
@@ -151,6 +159,13 @@ EOF
 --mode=$m not supported (`set' or `merge')
 EOM
 	}
+	if (my $net = $lei->{net}) {
+		require PublicInbox::NetWriter;
+		$self->{nwr} = bless $net, 'PublicInbox::NetWriter';
+		$self->{imap_mod_kw} = $net->can($self->{-merge_kw} ?
+					'imap_add_kw' : 'imap_set_kw');
+	}
+	undef $lms;
 	my $ops = {};
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
 	$self->{-wq_nr_workers} = $j // 1; # locked
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9d4c856..f3c03969 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -307,11 +307,12 @@ sub _imap_write_cb ($$) {
 	my $dedupe = $lei->{dedupe};
 	$dedupe->prepare_dedupe if $dedupe;
 	my $append = $lei->{net}->can('imap_append');
-	my $mic = $lei->{net}->mic_get($self->{uri});
-	my $folder = $self->{uri}->mailbox;
+	my $uri = $self->{uri};
+	my $mic = $lei->{net}->mic_get($uri);
+	my $folder = $uri->mailbox;
+	$uri->uidvalidity($mic->uidvalidity($folder));
 	my $lse = $lei->{lse}; # may be undef
 	my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
-	my $out = $lei->{ovv}->{dst};
 	sub { # for git_to_mail
 		my ($bref, $smsg, $eml) = @_;
 		$mic // return $lei->fail; # mic may be undef-ed in last run
@@ -327,7 +328,7 @@ sub _imap_write_cb ($$) {
 		# imap_append returns UID if IMAP server has UIDPLUS extension
 		($sto && $uid =~ /\A[0-9]+\z/) and
 			$sto->ipc_do('set_sync_info',
-					$smsg->{blob}, $out, $uid + 0);
+					$smsg->{blob}, $$uri, $uid + 0);
 		++$lei->{-nr_write};
 	}
 }
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 2032a1fd..8ec7f85c 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,10 +26,20 @@ sub imap_append {
 		die "APPEND $folder: $@";
 }
 
+# updates $uri with UIDVALIDITY
 sub mic_for_folder {
 	my ($self, $uri) = @_;
 	my $mic = $self->mic_get($uri) or die "E: not connected: $@";
 	$mic->select($uri->mailbox) or return;
+	my $uidval;
+	for ($mic->Results) {
+		/^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
+		$uidval = $1;
+		last;
+	}
+	$uidval //= $mic->uidvalidity($uri->mailbox) or
+		die "E: failed to get uidvalidity from <$uri>: $@";
+	$uri->uidvalidity($uidval);
 	$mic;
 }
 
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 1298b958..0e6d4831 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -157,12 +157,20 @@ test_lei(sub {
 
 	lei_ok qw(import -F eml), $f, \'import local copy w/o keywords';
 
+	lei_ok 'ls-mail-sync'; diag $lei_out;
+	lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3
+	lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f;
+	lei_ok 'ls-mail-sync'; diag $lei_out;
+	chomp(my $uri_val = $lei_out);
+	lei_ok 'export-kw', $uri_val;
 	$mic = $nwr->mic_for_folder($folder_uri);
-	# dummy set to ensure second set_kw clobbers
-	$nwr->imap_set_kw($mic, $uid[0], [ qw(seen answered flagged) ]
-			)->expunge or BAIL_OUT "expunge $@";
-	$nwr->imap_set_kw($mic, $uid[0], [ 'seen' ]
-			)->expunge or BAIL_OUT "expunge $@";
+	my $flags = $mic->flags($uid[0]);
+	is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ],
+		'IMAP flags set by export-kw') or diag explain($flags);
+
+	# ensure this imap_set_kw clobbers
+	$nwr->imap_set_kw($mic, $uid[0], [ 'seen' ])->expunge or
+		BAIL_OUT "expunge $@";
 	$mic = undef;
 	@res = ();
 	$nwr->imap_each($folder_uri, $imap_slurp_all, \@res);

  parent reply	other threads:[~2021-05-23  1:38 UTC|newest]

Thread overview: 4+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-05-23  1:38 [PATCH 0/3] lei export-kw: IMAP support Eric Wong
2021-05-23  1:38 ` [PATCH 1/3] net_reader|net_writer: pass URI refs deeper into callbacks Eric Wong
2021-05-23  1:38 ` Eric Wong [this message]
2021-05-23  1:38 ` [PATCH 3/3] lei export-kw: relax IMAP URL matching Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210523013828.4888-3-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).