[PATCH 0/5] lei ls-mail-source-related things

[PATCH 0/5] lei ls-mail-source-related things

[Eric Wong]

A couple of minor usability things that'll make "lei import"
easier for bash completion users.

Eric Wong (5):
  lei: stop pager early on exit
  lei ls-mail-source: write through to URL folder cache
  t/lei-import-http: quiet unnecessary diag message
  lei import: use url_folder_cache for completion
  net_reader: canonicalize URL args on add_url

 lib/PublicInbox/LEI.pm             | 10 ++++++-
 lib/PublicInbox/LeiImport.pm       | 11 +++++---
 lib/PublicInbox/LeiIndex.pm        |  2 +-
 lib/PublicInbox/LeiLsMailSource.pm | 42 ++++++++++++++++++++++++++++--
 lib/PublicInbox/NetReader.pm       |  2 ++
 lib/PublicInbox/SharedKV.pm        | 13 +++++++--
 t/lei-import-http.t                |  2 --
 7 files changed, 71 insertions(+), 11 deletions(-)

[PATCH 1/5] lei: stop pager early on exit

[Eric Wong]

This is necessary when using "ls-mail-source" on an unreachable
IMAP server.
---
 lib/PublicInbox/LEI.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 833d9c4d..122045ee 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -446,6 +446,7 @@ sub x_it ($$) {
 	# make sure client sees stdout before exit
 	$self->{1}->autoflush(1) if $self->{1};
 	dump_and_clear_log();
+	stop_pager($self);
 	if ($self->{pkt_op_p}) { # to top lei-daemon
 		$self->{pkt_op_p}->pkt_do('x_it', $code);
 	} elsif ($self->{sock}) { # to lei(1) client

[PATCH 2/5] lei ls-mail-source: write through to URL folder cache

[Eric Wong]

We'll be able to use this for shell completion for
lei import, lcat, tag, etc..

This also adds --url support for scripting purposes.
---
 lib/PublicInbox/LEI.pm             |  9 ++++++-
 lib/PublicInbox/LeiLsMailSource.pm | 42 ++++++++++++++++++++++++++++--
 lib/PublicInbox/SharedKV.pm        | 13 +++++++--
 3 files changed, 59 insertions(+), 5 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 122045ee..50a7fdad 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -116,6 +116,12 @@ sub cache_dir ($) {
 		.'/lei');
 }
 
+sub url_folder_cache {
+	my ($self) = @_;
+	require PublicInbox::SharedKV; # URI => updated_at_sec_
+	PublicInbox::SharedKV->new(cache_dir($self).'/uri_folder');
+}
+
 sub ale {
 	my ($self) = @_;
 	$self->{ale} //= do {
@@ -197,7 +203,7 @@ our %CMD = ( # sorted in order of importance/use:
 'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
 		qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
 'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
-		qw(z|0 ascii l), @c_opt ],
+		qw(z|0 ascii l url), @c_opt ],
 'forget-external' => [ 'LOCATION...|--prune',
 	'exclude further results from a publicinbox|extindex',
 	qw(prune), @c_opt ],
@@ -384,6 +390,7 @@ my %OPTDESC = (
 			'listing output format' ],
 'l	ls-search' => 'long listing format',
 'l	ls-mail-source' => 'long listing format',
+'url	ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
 'format|f=s	ls-external' => $ls_format,
 
 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index e95c0b34..cadc61ed 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -21,19 +21,27 @@ sub input_path_url { # overrides LeiInput version
 	} elsif ($self->{lei}->{opt}->{z}) {
 		$ORS = "\0";
 	}
+	my @f;
 	if ($url =~ m!\Aimaps?://!i) {
 		my $uri = PublicInbox::URIimap->new($url);
+		my $sec = $lei->{net}->can('uri_section')->($uri);
 		my $mic = $lei->{net}->mic_get($uri);
 		my $l = $mic->folders_hash($uri->path); # server-side filter
+		@f = map { "$sec/$_->{name}" } @$l;
 		if ($json) {
+			$_->{url} = "$sec/$_->{name}" for @$l;
 			$lei->puts($json->encode($l));
 		} else {
+			if ($self->{lei}->{opt}->{url}) {
+				$_->{name} = "$sec/$_->{name}" for @$l;
+			}
 			$lei->out(join($ORS, (map { $_->{name} } @$l), ''));
 		}
 	} elsif ($url =~ m!\A(?:nntps?|s?news)://!i) {
 		my $uri = PublicInbox::URInntps->new($url);
 		my $nn = $lei->{net}->nn_get($uri);
 		my $l = $nn->newsgroups($uri->group); # name => description
+		my $sec = $lei->{net}->can('uri_section')->($uri);
 		if ($json) {
 			my $all = $nn->list;
 			my @x;
@@ -46,15 +54,30 @@ sub input_path_url { # overrides LeiInput version
 				$desc =~ s/\r\z//;
 
 				my ($hwm, $lwm, $status) = @{$all->{$ng}};
-				push @x, { name => $ng, lwm => $lwm + 0,
+				push @x, { name => $ng, url => "$sec/$ng",
+					lwm => $lwm + 0,
 					hwm => $hwm + 0, status => $status,
 					description => $desc };
 			}
+			@f = map { "$sec/$_" } keys %$all;
 			$lei->puts($json->encode(\@x));
 		} else {
-			$lei->out(join($ORS, sort(keys %$l), ''));
+			@f = map { "$sec/$_" } keys %$l;
+			if ($self->{lei}->{opt}->{url}) {
+				$lei->out(join($ORS, sort(@f), ''));
+			} else {
+				$lei->out(join($ORS, sort(keys %$l), ''));
+			}
 		}
 	} else { die "BUG: $url not supported" }
+	if (@f) {
+		my $fc = $lei->url_folder_cache;
+		my $lk = $fc->lock_for_scope;
+		$fc->dbh->begin_work;
+		my $now = time;
+		$fc->set($_, $now) for @f;
+		$fc->dbh->commit;
+	}
 }
 
 sub lei_ls_mail_source {
@@ -62,6 +85,7 @@ sub lei_ls_mail_source {
 	$url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return
 		$lei->fail('only NNTP and IMAP URLs supported');
 	my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
+	$self->{cfg} = $lei->_lei_cfg; # may be undef
 	$self->prepare_inputs($lei, [ $url ]) or return;
 	$lei->start_pager if -t $lei->{1};
 	my $ops = {};
@@ -74,6 +98,20 @@ sub lei_ls_mail_source {
 	$lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
 }
 
+sub _complete_ls_mail_source {
+	my ($lei, @argv) = @_;
+	my $match_cb = $lei->complete_url_prepare(\@argv);
+	my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
+	my %f = map { $_ => 1 } @m;
+	my $sto = $lei->_lei_store;
+	if (my $lms = $sto ? $sto->search->lms : undef) {
+		@m = map { $match_cb->($_) } grep(
+			m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
+		@f{@m} = @m;
+	}
+	keys %f;
+}
+
 no warnings 'once';
 *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
 *net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index d65c3158..8347b195 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -11,7 +11,7 @@ use parent qw(PublicInbox::Lock);
 use File::Temp qw(tempdir);
 use DBI ();
 use PublicInbox::Spawn;
-use File::Path qw(rmtree);
+use File::Path qw(rmtree make_path);
 
 sub dbh {
 	my ($self, $lock) = @_;
@@ -46,8 +46,8 @@ CREATE TABLE IF NOT EXISTS kv (
 sub new {
 	my ($cls, $dir, $base, $opt) = @_;
 	my $self = bless { opt => $opt }, $cls;
+	make_path($dir) if defined($dir) && !-d $dir;
 	$dir //= $self->{"tmp$$.$self"} = tempdir("skv.$$-XXXX", TMPDIR => 1);
-	-d $dir or mkdir($dir) or die "mkdir($dir): $!";
 	$base //= '';
 	my $f = $self->{filename} = "$dir/$base.sqlite3";
 	$self->{lock_path} = $opt->{lock_path} // "$dir/$base.flock";
@@ -83,6 +83,15 @@ SELECT k,v FROM kv
 	$sth
 }
 
+sub keys {
+	my ($self) = @_;
+	my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
+SELECT k FROM kv
+
+	$sth->execute;
+	map { $_->[0] } @{$sth->fetchall_arrayref};
+}
+
 sub delete_by_val {
 	my ($self, $val, $lock) = @_;
 	$lock //= $self->lock_for_scope_fast;

[PATCH 3/5] t/lei-import-http: quiet unnecessary diag message

[Eric Wong]

Leftover while writing the test.
---
 t/lei-import-http.t | 2 --
 1 file changed, 2 deletions(-)

diff --git a/t/lei-import-http.t b/t/lei-import-http.t
index 6cb8a753..2104c778 100644
--- a/t/lei-import-http.t
+++ b/t/lei-import-http.t
@@ -42,7 +42,5 @@ test_lei({ tmpdir => $tmpdir }, sub {
 
 	ok(!lei(qw(import --mail-sync), "$url/x\@example.com/raw"),
 		'--mail-sync fails on HTTP');
-	diag $lei_err;
-
 });
 done_testing;

[PATCH 4/5] lei import: use url_folder_cache for completion

[Eric Wong]

And fix "lei index" completion while we're at it.
---
 lib/PublicInbox/LeiImport.pm | 11 ++++++++---
 lib/PublicInbox/LeiIndex.pm  |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 08794f71..7580e37e 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -118,10 +118,15 @@ sub lei_import { # the main "lei import" method
 
 sub _complete_import {
 	my ($lei, @argv) = @_;
-	my $sto = $lei->_lei_store or return;
-	my $lms = $sto->search->lms or return;
 	my $match_cb = $lei->complete_url_prepare(\@argv);
-	map { $match_cb->($_) } $lms->folders;
+	my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
+	my %f = map { $_ => 1 } @m;
+	my $sto = $lei->_lei_store;
+	if (my $lms = $sto ? $sto->search->lms : undef) {
+		@m = map { $match_cb->($_) } $lms->folders;
+		@f{@m} = @m;
+	}
+	keys %f;
 }
 
 no warnings 'once';
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 4be0c649..5b545998 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -39,7 +39,7 @@ for my $m (qw(pmdir_cb input_net_cb)) {
 	*$m = PublicInbox::LeiImport->can($m);
 }
 
-*_complete_import = \&PublicInbox::LeiImport::_complete_import;
+*_complete_index = \&PublicInbox::LeiImport::_complete_import;
 *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
 *net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
 

[PATCH 5/5] net_reader: canonicalize URL args on add_url

[Eric Wong]

This fixes cases when users specify an IMAP or NNTP URL
with standard port numbers explicitly.

In other words, this allows users to use
"lei ls-mail-source nntps://public-inbox.org:563/" and
"lei ls-mail-source imaps://public-inbox.org:993/"
without hitting "BUG:" errors.
---
 lib/PublicInbox/NetReader.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index bcc6cbf0..30784199 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -371,8 +371,10 @@ sub add_url {
 	my ($self, $arg, $ls_ok) = @_;
 	my $uri;
 	if ($uri = imap_uri($arg, $ls_ok)) {
+		$_[1] = $$uri; # canonicalized
 		push @{$self->{imap_order}}, $uri;
 	} elsif ($uri = nntp_uri($arg, $ls_ok)) {
+		$_[1] = $$uri; # canonicalized
 		push @{$self->{nntp_order}}, $uri;
 	} else {
 		push @{$self->{unsupported_url}}, $arg;