[PATCH] lei: handle a single IMAP message in most places

[PATCH] lei: handle a single IMAP message in most places

[Eric Wong]

"lei import" can now import a single IMAP message via
<imaps://example.com/MAILBOX/;UID=$UID>

Likewise, "lei inspect" can show the blob information for UID
URLs and "lei lcat" can display the blob without network access
if imported.

"lei lcat" also gets rid of some unused code and supports
"blob:$OIDHEX" syntax as described in the comments (and used by
our "text" output format).
---
 lib/PublicInbox/LeiInspect.pm  | 24 +++++++++++++++++++++--
 lib/PublicInbox/LeiLcat.pm     | 35 +++++++++++++++++++++-------------
 lib/PublicInbox/LeiMailSync.pm | 23 ++++++++++++++++++++++
 lib/PublicInbox/LeiToMail.pm   |  8 +++++++-
 lib/PublicInbox/NetReader.pm   |  9 +++++++--
 t/lei-import-imap.t            | 14 ++++++++++++++
 6 files changed, 95 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 46b9197f..7205979e 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -24,6 +24,19 @@ sub inspect_blob ($$) {
 	$ent;
 }
 
+sub inspect_imap_uid ($$) {
+	my ($lei, $uid_uri) = @_;
+	my $ent = {};
+	my $lse = $lei->{lse} or return $ent;
+	my $lms = $lse->lms or return $ent;
+	my $oidhex = $lms->imap_oid($lei, $uid_uri);
+	if (ref(my $err = $oidhex)) { # art2folder error
+		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+	}
+	$ent->{$$uid_uri} = $oidhex;
+	$ent;
+}
+
 sub inspect_sync_folder ($$) {
 	my ($lei, $folder) = @_;
 	my $ent = {};
@@ -49,8 +62,15 @@ sub inspect1 ($$$) {
 	my $ent;
 	if ($item =~ /\Ablob:(.+)/) {
 		$ent = inspect_blob($lei, $1);
-	} elsif ($item =~ m!\Aimaps?://!i ||
-			$item =~ m!\A(?:maildir|mh):!i || -d $item) {
+	} elsif ($item =~ m!\Aimaps?://!i) {
+		require PublicInbox::URIimap;
+		my $uri = PublicInbox::URIimap->new($item);
+		if (defined($uri->uid)) {
+			$ent = inspect_imap_uid($lei, $uri);
+		} else {
+			$ent = inspect_sync_folder($lei, $item);
+		}
+	} elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
 		$ent = inspect_sync_folder($lei, $item);
 	} else { # TODO: more things
 		return $lei->fail("$item not understood");
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 87729acf..e5d8e805 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -9,27 +9,33 @@ use strict;
 use v5.10.1;
 use PublicInbox::LeiViewText;
 use URI::Escape qw(uri_unescape);
-use URI;
 use PublicInbox::MID qw($MID_EXTRACT);
 
-sub lcat_redispatch {
-	my ($lei, $out, $op_p) = @_;
-	my $l = bless { %$lei }, ref($lei);
-	delete $l->{sock};
-	$l->{''} = $op_p; # daemon only
-	eval {
-		$l->qerr("# updating $out");
-		up1($l, $out);
-		$l->qerr("# $out done");
-	};
-	$l->err($@) if $@;
+sub lcat_imap_uid_uri ($$) {
+	my ($lei, $uid_uri) = @_;
+	my $lms = $lei->{lse}->lms or return;
+	my $oidhex = $lms->imap_oid($lei, $uid_uri);
+	if (ref(my $err = $oidhex)) { # art2folder error
+		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+	}
+	push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
+	'""'; # blank query
 }
 
 sub extract_1 ($$) {
 	my ($lei, $x) = @_;
-	if ($x =~ m!\b([a-z]+?://\S+)!i) {
+	if ($x =~ m!\b(imaps?://[^>]+)!i) {
+		my $u = $1;
+		require PublicInbox::URIimap;
+		$u = PublicInbox::URIimap->new($u);
+		if (!defined($u->uid)) {
+			$lei->qerr("# no UID= in $u");
+		}
+		lcat_imap_uid_uri($lei, $u);
+	} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
 		my $u = $1;
 		$u =~ s/[\>\]\)\,\.\;]+\z//;
+		require URI;
 		$u = URI->new($u);
 		my $p = $u->path;
 		my $term;
@@ -57,6 +63,9 @@ sub extract_1 ($$) {
 		$1;
 	} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
 		"mid:$1";
+	} elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
+		push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+		'""'; # blank query
 	} else {
 		undef;
 	}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index b3e5e035..eb2b0c0f 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -356,6 +356,29 @@ sub forget_folder {
 	$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
 }
 
+sub imap_oid {
+	my ($self, $lei, $uid_uri) = @_;
+	my $mailbox_uri = $uid_uri->clone;
+	$mailbox_uri->uid(undef);
+	warn "$uid_uri  $mailbox_uri";
+	my $folders = [ $$mailbox_uri ];
+	if (my $err = $self->arg2folder($lei, $folders)) {
+		if ($err->{fail}) {
+			$lei->qerr("# no sync information for $mailbox_uri");
+			return;
+		}
+		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+	}
+	my $fid = $self->{fmap}->{$folders->[0]} //=
+		_fid_for($self, $folders->[0]) // return;
+	my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
+EOM
+	$sth->execute($fid, $uid_uri->uid);
+	my ($oidbin) = $sth->fetchrow_array;
+	$oidbin ? unpack('H*', $oidbin) : undef;
+}
+
 sub DESTROY {
 	my ($self) = @_;
 	my $dbh = $self->{dbh} or return;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ad6b9439..b3aec50b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -702,8 +702,14 @@ sub write_mail { # via ->wq_io_do
 
 sub wq_atexit_child {
 	my ($self) = @_;
-	delete $self->{wcb};
 	my $lei = $self->{lei};
+	if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
+		for my $oid (@{$lei->{lcat_blob}}) {
+			my $smsg = { blob => $oid, pct => 100 };
+			write_mail($self, $smsg);
+		}
+	}
+	delete $self->{wcb};
 	$lei->{ale}->git->async_wait_all;
 	my $nr = delete($lei->{-nr_write}) or return;
 	return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 73b8b1cd..76d2fe62 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -469,7 +469,10 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
 EOF
 
 	my $uri = $orig_uri->clone;
+	my $single_uid = $uri->uid;
 	my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
+	$itrk = $l_uid = undef if defined($single_uid);
+
 	return <<EOF if $l_uidval != $r_uidval;
 E: $uri UIDVALIDITY mismatch
 E: local=$l_uidval != remote=$r_uidval
@@ -499,7 +502,9 @@ EOF
 		# I wish "UID FETCH $START:*" could work, but:
 		# 1) servers do not need to return results in any order
 		# 2) Mail::IMAPClient doesn't offer a streaming API
-		unless ($uids = $mic->search("UID $l_uid:*")) {
+		if (defined $single_uid) {
+			$uids = [ $single_uid ];
+		} elsif (!($uids = $mic->search("UID $l_uid:*"))) {
 			return if $!{EINTR} && $self->{quit};
 			return "E: $uri UID SEARCH $l_uid:* error: $!";
 		}
@@ -541,7 +546,7 @@ EOF
 		}
 		run_commit_cb($self);
 		$itrk->update_last($r_uidval, $last_uid) if $itrk;
-	} until ($err || $self->{quit});
+	} until ($err || $self->{quit} || defined($single_uid));
 	$err;
 }
 
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 5283cc23..2f928a2e 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -75,5 +75,19 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	lei_ok 'forget-mail-sync', $url;
 	lei_ok 'ls-mail-sync';
 	unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
+	my $uid_url = "$url/;UID=".$stats->{'uid.max'};
+	lei_ok 'import', $uid_url;
+	lei_ok 'inspect', $uid_url;
+	$lei_out =~ /([a-f0-9]{40,})/ or
+		xbail 'inspect missed blob with UID URL';
+	my $blob = $1;
+	lei_ok 'lcat', $uid_url;
+	like $lei_out, qr/^Subject: /sm,
+		'lcat shows mail text with UID URL';
+	like $lei_out, qr/\bblob:$blob\b/, 'lcat showed blob';
+	my $orig = $lei_out;
+	lei_ok 'lcat', "blob:$blob";
+	is($lei_out, $orig, 'lcat understands blob:...');
 });
+
 done_testing;

[PATCH 0/6] lei: odds and ends and a resend

[Eric Wong]

"lei: handle a single IMAP message in most places" didn't actually
apply, since I was still running 3/6 in this series.  I've been
having trouble reproducing the uncommitted transaction error
messages I've been seeing from lms, but hopefully 3/6 can
eventually find it.

1/6 and 2/6 were somethings I noticed randomly.  Been looking
into FUSE support for exposing lei/store as a Maildir, too.

Eric Wong (6):
  viewdiff: make $UNSAFE a variable
  viewdiff: escape '{' and '}' for regexp
  lei_mail_sync: debug code for uncommitted txn
  lei: mark reorder-and-rewrite-local-history as a TODO item
  lei: handle a single IMAP message in most places
  lei: add TODO item for FUSE mount

 lib/PublicInbox/LEI.pm         | 13 ++++++-------
 lib/PublicInbox/LeiInspect.pm  | 24 ++++++++++++++++++++++--
 lib/PublicInbox/LeiLcat.pm     | 33 ++++++++++++++++++++-------------
 lib/PublicInbox/LeiMailSync.pm | 33 +++++++++++++++++++++++++++++++++
 lib/PublicInbox/LeiToMail.pm   |  8 +++++++-
 lib/PublicInbox/LeiXSearch.pm  |  3 +++
 lib/PublicInbox/NetReader.pm   |  9 +++++++--
 lib/PublicInbox/ViewDiff.pm    | 11 +++++------
 t/lei-import-imap.t            | 15 +++++++++++++++
 9 files changed, 118 insertions(+), 31 deletions(-)

[PATCH 1/6] viewdiff: make $UNSAFE a variable

[Eric Wong]

There's no sense in using a constant here since it
gets copied into the uri_escape_utf8 function anyways.
Furthermore, inlined constants still leave behind a
subroutine and subs cost several KB of memory.

Finally, add a comment as to why it's different than the default
escape, since I just spent a minute wondering that.
---
 lib/PublicInbox/ViewDiff.pm | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index d2c5fabe..05acc242 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -15,8 +15,7 @@ use URI::Escape qw(uri_escape_utf8);
 use PublicInbox::Hval qw(ascii_html to_attr);
 use PublicInbox::Git qw(git_unquote);
 
-sub UNSAFE () { "^A-Za-z0-9\-\._~/" }
-
+my $UNSAFE = "^A-Za-z0-9\-\._~/"; # '/' + $URI::Escape::Unsafe{RFC3986}
 my $OID_NULL = '0{7,}';
 my $OID_BLOB = '[a-f0-9]{7,}';
 my $LF = qr!\n!;
@@ -124,14 +123,14 @@ sub diff_header ($$$) {
 	$pa = (split(m'/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
 	$pb = (split(m'/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
 	if ($pa eq $pb && $pb ne '/dev/null') {
-		$dctx->{Q} = "?b=".uri_escape_utf8($pb, UNSAFE);
+		$dctx->{Q} = "?b=".uri_escape_utf8($pb, $UNSAFE);
 	} else {
 		my @q;
 		if ($pb ne '/dev/null') {
-			push @q, 'b='.uri_escape_utf8($pb, UNSAFE);
+			push @q, 'b='.uri_escape_utf8($pb, $UNSAFE);
 		}
 		if ($pa ne '/dev/null') {
-			push @q, 'a='.uri_escape_utf8($pa, UNSAFE);
+			push @q, 'a='.uri_escape_utf8($pa, $UNSAFE);
 		}
 		$dctx->{Q} = '?'.join('&', @q);
 	}

[PATCH 2/6] viewdiff: escape '{' and '}' for regexp

[Eric Wong]

Perl 5 doesn't warn on this, yet, but it warns on unescaped
'(' and ')' nowadays, so it's conceivable Perl could start
warning on this in the future.  So future-proof our code and
reduce reader confusion.
---
 lib/PublicInbox/ViewDiff.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index 05acc242..f492b697 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -77,7 +77,7 @@ sub anchor0 ($$$$) {
 	# which works well in practice. If projects put "=>", or trailing
 	# spaces in filenames, oh well :P
 	$fn =~ s/$DIFFSTAT_COMMENT//;
-	$fn =~ s/{(?:.+) => (.+)}/$1/ or $fn =~ s/.* => (.+)/$1/;
+	$fn =~ s/\{(?:.+) => (.+)\}/$1/ or $fn =~ s/.* => (.+)/$1/;
 	$fn = git_unquote($fn);
 
 	# long filenames will require us to check in anchor1()

[PATCH 3/6] lei_mail_sync: debug code for uncommitted txn

[Eric Wong]

I'm not 100% sure why, but "lei up" seems to cause uncommitted
transaction errors.  LeiToMail calls sto->set_sync_info, but
LeiXSearch should call sto->done and lms_commit, so I'm not
sure where the uncommited transaction is coming from...
---
 lib/PublicInbox/LeiMailSync.pm | 33 +++++++++++++++++++++++++++++++++
 lib/PublicInbox/LeiXSearch.pm  |  3 +++
 2 files changed, 36 insertions(+)

diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index d9c30580..2c071f5a 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -356,4 +356,37 @@ sub forget_folder {
 	$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
 }
 
+sub imap_oid {
+	my ($self, $lei, $uid_uri) = @_;
+	my $mailbox_uri = $uid_uri->clone;
+	$mailbox_uri->uid(undef);
+	warn "$uid_uri  $mailbox_uri";
+	my $folders = [ $$mailbox_uri ];
+	if (my $err = $self->arg2folder($lei, $folders)) {
+		if ($err->{fail}) {
+			$lei->qerr("# no sync information for $mailbox_uri");
+			return;
+		}
+		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+	}
+	my $fid = $self->{fmap}->{$folders->[0]} //=
+		_fid_for($self, $folders->[0]) // return;
+	my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
+EOM
+	$sth->execute($fid, $uid_uri->uid);
+	my ($oidbin) = $sth->fetchrow_array;
+	$oidbin ? unpack('H*', $oidbin) : undef;
+}
+
+# FIXME: something with "lei <up|q>" is causing uncommitted transaction
+# warnings, not sure what...
+sub DESTROY {
+	my ($self) = @_;
+	my $dbh = $self->{dbh} or return;
+	return if $dbh->{ReadOnly};
+	use Carp;
+	warn "BUG $$ $0 $self {dbh} UNCOMMITTED ", Carp::longmess();
+}
+
 1;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 3482082d..e2a8e8e3 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -357,6 +357,9 @@ sub query_done { # EOF callback for main daemon
 	if (my $lxs = delete $lei->{lxs}) {
 		$lxs->wq_wait_old(\&xsearch_done_wait, $lei);
 	}
+	if ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) {
+		warn "BUG: {sto} missing with --mail-sync";
+	}
 	my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
 	$lei->{ovv}->ovv_end($lei);
 	my $start_mua;

[PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item

[Eric Wong]

This is low priority, for now.
---
 lib/PublicInbox/LEI.pm | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7acc05bf..ad5f06be 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -274,13 +274,11 @@ our %CMD = ( # sorted in order of importance/use:
 'daemon-pid' => [ '', 'show the PID of the lei-daemon' ],
 'help' => [ '[SUBCOMMAND]', 'show help' ],
 
-# XXX do we need this?
-# 'git' => [ '[ANYTHING...]', 'git(1) wrapper', pass_through('git') ],
-
-'reorder-local-store-and-break-history' => [ '[REFNAME]',
-	'rewrite git history in an attempt to improve compression',
-	qw(gc!), @c_opt ],
-
+# TODO
+#'reorder-local-store-and-break-history' => [ '[REFNAME]',
+#	'rewrite git history in an attempt to improve compression',
+#	qw(gc!), @c_opt ],
+#
 # internal commands are prefixed with '_'
 '_complete' => [ '[...]', 'internal shell completion helper',
 		pass_through('everything') ],

[PATCH 5/6] lei: handle a single IMAP message in most places

[Eric Wong]

"lei import" can now import a single IMAP message via
<imaps://example.com/MAILBOX/;UID=$UID>

Likewise, "lei inspect" can show the blob information for UID
URLs and "lei lcat" can display the blob without network access
if imported.

"lei lcat" also gets rid of some unused code and supports
"blob:$OIDHEX" syntax as described in the comments (and used by
our "text" output format).

v2: enforce UID in URL, fail without
---
 lib/PublicInbox/LeiInspect.pm | 24 ++++++++++++++++++++++--
 lib/PublicInbox/LeiLcat.pm    | 33 ++++++++++++++++++++-------------
 lib/PublicInbox/LeiToMail.pm  |  8 +++++++-
 lib/PublicInbox/NetReader.pm  |  9 +++++++--
 t/lei-import-imap.t           | 15 +++++++++++++++
 5 files changed, 71 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 46b9197f..7205979e 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -24,6 +24,19 @@ sub inspect_blob ($$) {
 	$ent;
 }
 
+sub inspect_imap_uid ($$) {
+	my ($lei, $uid_uri) = @_;
+	my $ent = {};
+	my $lse = $lei->{lse} or return $ent;
+	my $lms = $lse->lms or return $ent;
+	my $oidhex = $lms->imap_oid($lei, $uid_uri);
+	if (ref(my $err = $oidhex)) { # art2folder error
+		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+	}
+	$ent->{$$uid_uri} = $oidhex;
+	$ent;
+}
+
 sub inspect_sync_folder ($$) {
 	my ($lei, $folder) = @_;
 	my $ent = {};
@@ -49,8 +62,15 @@ sub inspect1 ($$$) {
 	my $ent;
 	if ($item =~ /\Ablob:(.+)/) {
 		$ent = inspect_blob($lei, $1);
-	} elsif ($item =~ m!\Aimaps?://!i ||
-			$item =~ m!\A(?:maildir|mh):!i || -d $item) {
+	} elsif ($item =~ m!\Aimaps?://!i) {
+		require PublicInbox::URIimap;
+		my $uri = PublicInbox::URIimap->new($item);
+		if (defined($uri->uid)) {
+			$ent = inspect_imap_uid($lei, $uri);
+		} else {
+			$ent = inspect_sync_folder($lei, $item);
+		}
+	} elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
 		$ent = inspect_sync_folder($lei, $item);
 	} else { # TODO: more things
 		return $lei->fail("$item not understood");
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 87729acf..c4712662 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -9,27 +9,31 @@ use strict;
 use v5.10.1;
 use PublicInbox::LeiViewText;
 use URI::Escape qw(uri_unescape);
-use URI;
 use PublicInbox::MID qw($MID_EXTRACT);
 
-sub lcat_redispatch {
-	my ($lei, $out, $op_p) = @_;
-	my $l = bless { %$lei }, ref($lei);
-	delete $l->{sock};
-	$l->{''} = $op_p; # daemon only
-	eval {
-		$l->qerr("# updating $out");
-		up1($l, $out);
-		$l->qerr("# $out done");
-	};
-	$l->err($@) if $@;
+sub lcat_imap_uid_uri ($$) {
+	my ($lei, $uid_uri) = @_;
+	my $lms = $lei->{lse}->lms or return;
+	my $oidhex = $lms->imap_oid($lei, $uid_uri);
+	if (ref(my $err = $oidhex)) { # art2folder error
+		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+	}
+	push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
 }
 
 sub extract_1 ($$) {
 	my ($lei, $x) = @_;
-	if ($x =~ m!\b([a-z]+?://\S+)!i) {
+	if ($x =~ m!\b(imaps?://[^>]+)!i) {
+		my $u = $1;
+		require PublicInbox::URIimap;
+		$u = PublicInbox::URIimap->new($u);
+		defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
+				$lei->fail("# no UID= in $u");
+		'""'; # blank query, using {lcat_blob}
+	} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
 		my $u = $1;
 		$u =~ s/[\>\]\)\,\.\;]+\z//;
+		require URI;
 		$u = URI->new($u);
 		my $p = $u->path;
 		my $term;
@@ -57,6 +61,9 @@ sub extract_1 ($$) {
 		$1;
 	} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
 		"mid:$1";
+	} elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
+		push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+		'""'; # blank query
 	} else {
 		undef;
 	}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ad6b9439..b3aec50b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -702,8 +702,14 @@ sub write_mail { # via ->wq_io_do
 
 sub wq_atexit_child {
 	my ($self) = @_;
-	delete $self->{wcb};
 	my $lei = $self->{lei};
+	if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
+		for my $oid (@{$lei->{lcat_blob}}) {
+			my $smsg = { blob => $oid, pct => 100 };
+			write_mail($self, $smsg);
+		}
+	}
+	delete $self->{wcb};
 	$lei->{ale}->git->async_wait_all;
 	my $nr = delete($lei->{-nr_write}) or return;
 	return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 73b8b1cd..76d2fe62 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -469,7 +469,10 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
 EOF
 
 	my $uri = $orig_uri->clone;
+	my $single_uid = $uri->uid;
 	my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
+	$itrk = $l_uid = undef if defined($single_uid);
+
 	return <<EOF if $l_uidval != $r_uidval;
 E: $uri UIDVALIDITY mismatch
 E: local=$l_uidval != remote=$r_uidval
@@ -499,7 +502,9 @@ EOF
 		# I wish "UID FETCH $START:*" could work, but:
 		# 1) servers do not need to return results in any order
 		# 2) Mail::IMAPClient doesn't offer a streaming API
-		unless ($uids = $mic->search("UID $l_uid:*")) {
+		if (defined $single_uid) {
+			$uids = [ $single_uid ];
+		} elsif (!($uids = $mic->search("UID $l_uid:*"))) {
 			return if $!{EINTR} && $self->{quit};
 			return "E: $uri UID SEARCH $l_uid:* error: $!";
 		}
@@ -541,7 +546,7 @@ EOF
 		}
 		run_commit_cb($self);
 		$itrk->update_last($r_uidval, $last_uid) if $itrk;
-	} until ($err || $self->{quit});
+	} until ($err || $self->{quit} || defined($single_uid));
 	$err;
 }
 
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 5283cc23..59d481d5 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -75,5 +75,20 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	lei_ok 'forget-mail-sync', $url;
 	lei_ok 'ls-mail-sync';
 	unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
+	my $uid_url = "$url/;UID=".$stats->{'uid.max'};
+	lei_ok 'import', $uid_url;
+	lei_ok 'inspect', $uid_url;
+	$lei_out =~ /([a-f0-9]{40,})/ or
+		xbail 'inspect missed blob with UID URL';
+	my $blob = $1;
+	lei_ok 'lcat', $uid_url;
+	like $lei_out, qr/^Subject: /sm,
+		'lcat shows mail text with UID URL';
+	like $lei_out, qr/\bblob:$blob\b/, 'lcat showed blob';
+	my $orig = $lei_out;
+	lei_ok 'lcat', "blob:$blob";
+	is($lei_out, $orig, 'lcat understands blob:...');
+	ok(!lei('lcat', $url), "lcat doesn't work on IMAP URL w/o UID");
 });
+
 done_testing;

[PATCH 6/6] lei: add TODO item for FUSE mount

[Eric Wong]

It seems possible and natural to allow browsing lei/store as
a Maildir (as well as read-write JMAP/IMAP store).
---
 lib/PublicInbox/LEI.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ad5f06be..07378ca7 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -278,6 +278,7 @@ our %CMD = ( # sorted in order of importance/use:
 #'reorder-local-store-and-break-history' => [ '[REFNAME]',
 #	'rewrite git history in an attempt to improve compression',
 #	qw(gc!), @c_opt ],
+#'fuse-mount' => [ 'PATHNAME', 'expose lei/store as Maildir(s)', @c_opt ],
 #
 # internal commands are prefixed with '_'
 '_complete' => [ '[...]', 'internal shell completion helper',

Re: [PATCH 3/6] lei_mail_sync: debug code for uncommitted txn

[Eric Wong]

Eric Wong <e@80x24.org> wrote:
> --- a/lib/PublicInbox/LeiMailSync.pm
> +++ b/lib/PublicInbox/LeiMailSync.pm
> @@ -356,4 +356,37 @@ sub forget_folder {
>  	$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
>  }
>  
> +sub imap_oid {

Oops, that was leftover from a sloppy rebase on my part;
should've been in 5/6

Re: [PATCH 5/6] lei: handle a single IMAP message in most places

[Eric Wong]

Eric Wong <e@80x24.org> wrote:
> v2: enforce UID in URL, fail without

Pushed v3 as commit 9b3cd5e254fafa08c774a24f85c2b2eac12a9de5

v3: fix error reporting (s/fail/child_error/)

Using ->fail was causing errors to cascade elsewhere, not 100%
sure why...

interdiff:

diff -u b/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
--- b/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -28,7 +28,7 @@
 		require PublicInbox::URIimap;
 		$u = PublicInbox::URIimap->new($u);
 		defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
-				$lei->fail("# no UID= in $u");
+				$lei->child_error(1 << 8, "# no UID= in $u");
 		'""'; # blank query, using {lcat_blob}
 	} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
 		my $u = $1;