From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 1D46A1F5AE for ; Fri, 11 Jun 2021 09:42:41 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] lei ls-mail-source: list IMAP folders and NNTP groups Date: Fri, 11 Jun 2021 09:42:40 +0000 Message-Id: <20210611094240.8519-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: While other tools can provide the same functionality, having integration with git-credential is convenient, here. Caching and completion will be implemented separately. --- MANIFEST | 1 + lib/PublicInbox/LEI.pm | 3 ++ lib/PublicInbox/LeiInput.pm | 2 +- lib/PublicInbox/LeiLsMailSource.pm | 82 ++++++++++++++++++++++++++++++ lib/PublicInbox/NetReader.pm | 15 +++--- t/lei-import-imap.t | 6 +++ t/lei-import-nntp.t | 7 +++ 7 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 lib/PublicInbox/LeiLsMailSource.pm diff --git a/MANIFEST b/MANIFEST index 3d4c6cbd..d4b3e75d 100644 --- a/MANIFEST +++ b/MANIFEST @@ -215,6 +215,7 @@ lib/PublicInbox/LeiInput.pm lib/PublicInbox/LeiInspect.pm lib/PublicInbox/LeiLcat.pm lib/PublicInbox/LeiLsLabel.pm +lib/PublicInbox/LeiLsMailSource.pm lib/PublicInbox/LeiLsMailSync.pm lib/PublicInbox/LeiLsSearch.pm lib/PublicInbox/LeiMailSync.pm diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index d34997fd..833d9c4d 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -196,6 +196,8 @@ our %CMD = ( # sorted in order of importance/use: 'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ], 'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders', qw(z|0 globoff|g invert-match|v local remote), @c_opt ], +'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders', + qw(z|0 ascii l), @c_opt ], 'forget-external' => [ 'LOCATION...|--prune', 'exclude further results from a publicinbox|extindex', qw(prune), @c_opt ], @@ -381,6 +383,7 @@ my %OPTDESC = ( 'format|f=s ls-search' => ['OUT|json|jsonl|concatjson', 'listing output format' ], 'l ls-search' => 'long listing format', +'l ls-mail-source' => 'long listing format', 'format|f=s ls-external' => $ls_format, 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ], diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index 24211bf0..92d67715 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -229,7 +229,7 @@ sub prepare_inputs { # returns undef on error if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { require PublicInbox::NetReader; $net //= PublicInbox::NetReader->new; - $net->add_url($input); + $net->add_url($input, $self->{-ls_ok}); push @{$sync->{ok}}, $input if $sync; } elsif ($input_path =~ m!\Ahttps?://!i) { # mboxrd.gz # TODO: how would we detect r/w JMAP? diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm new file mode 100644 index 00000000..e95c0b34 --- /dev/null +++ b/lib/PublicInbox/LeiLsMailSource.pm @@ -0,0 +1,82 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# command for listing NNTP groups and IMAP folders, +# handy for users with git-credential-helper configured +# TODO: list JMAP labels +package PublicInbox::LeiLsMailSource; +use strict; +use v5.10.1; +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); + +sub input_path_url { # overrides LeiInput version + my ($self, $url) = @_; + # TODO: support ndjson and other JSONs we support elsewhere + my $json; + my $lei = $self->{lei}; + my $ORS = "\n"; + if ($self->{lei}->{opt}->{l}) { + $json = ref(PublicInbox::Config->json)->new->utf8->canonical; + $json->ascii(1) if $lei->{opt}->{ascii}; + } elsif ($self->{lei}->{opt}->{z}) { + $ORS = "\0"; + } + if ($url =~ m!\Aimaps?://!i) { + my $uri = PublicInbox::URIimap->new($url); + my $mic = $lei->{net}->mic_get($uri); + my $l = $mic->folders_hash($uri->path); # server-side filter + if ($json) { + $lei->puts($json->encode($l)); + } else { + $lei->out(join($ORS, (map { $_->{name} } @$l), '')); + } + } elsif ($url =~ m!\A(?:nntps?|s?news)://!i) { + my $uri = PublicInbox::URInntps->new($url); + my $nn = $lei->{net}->nn_get($uri); + my $l = $nn->newsgroups($uri->group); # name => description + if ($json) { + my $all = $nn->list; + my @x; + for my $ng (sort keys %$l) { + my $desc = $l->{$ng}; + +# we need to drop CR ourselves iff using IO::Socket::SSL since +# Net::Cmd::getline doesn't get used by Net::NNTP if TLS is in play, noted in: +# + $desc =~ s/\r\z//; + + my ($hwm, $lwm, $status) = @{$all->{$ng}}; + push @x, { name => $ng, lwm => $lwm + 0, + hwm => $hwm + 0, status => $status, + description => $desc }; + } + $lei->puts($json->encode(\@x)); + } else { + $lei->out(join($ORS, sort(keys %$l), '')); + } + } else { die "BUG: $url not supported" } +} + +sub lei_ls_mail_source { + my ($lei, $url, $pfx) = @_; + $url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return + $lei->fail('only NNTP and IMAP URLs supported'); + my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__; + $self->prepare_inputs($lei, [ $url ]) or return; + $lei->start_pager if -t $lei->{1}; + my $ops = {}; + $lei->{auth}->op_merge($ops, $self); + my $j = $self->{-wq_nr_workers} = 1; # locked + (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); + $lei->{wq1} = $self; + $lei->{-err_type} = 'non-fatal'; + net_merge_all_done($self) unless $lei->{auth}; + $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth} +} + +no warnings 'once'; +*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child; +*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done; +*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all; + +1; diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index 2795a9d4..bcc6cbf0 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -236,18 +236,19 @@ W: see https://rt.cpan.org/Ticket/Display.html?id=129967 for updates } sub imap_uri { - my ($url) = @_; + my ($url, $ls_ok) = @_; require PublicInbox::URIimap; my $uri = PublicInbox::URIimap->new($url); - $uri ? $uri->canonical : undef; + $uri && ($ls_ok || $uri->mailbox) ? $uri->canonical : undef; } my %IS_NNTP = (news => 1, snews => 1, nntp => 1, nntps => 1); sub nntp_uri { - my ($url) = @_; + my ($url, $ls_ok) = @_; require PublicInbox::URInntps; my $uri = PublicInbox::URInntps->new($url); - $uri && $IS_NNTP{$uri->scheme} && $uri->group ? $uri->canonical : undef; + $uri && $IS_NNTP{$uri->scheme} && ($ls_ok || $uri->group) ? + $uri->canonical : undef; } sub cfg_intvl ($$$) { @@ -367,11 +368,11 @@ sub nntp_common_init ($;$) { } sub add_url { - my ($self, $arg) = @_; + my ($self, $arg, $ls_ok) = @_; my $uri; - if ($uri = imap_uri($arg)) { + if ($uri = imap_uri($arg, $ls_ok)) { push @{$self->{imap_order}}, $uri; - } elsif ($uri = nntp_uri($arg)) { + } elsif ($uri = nntp_uri($arg, $ls_ok)) { push @{$self->{nntp_order}}, $uri; } else { push @{$self->{unsupported_url}}, $arg; diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t index 34fd6cf9..12f6fad0 100644 --- a/t/lei-import-imap.t +++ b/t/lei-import-imap.t @@ -14,6 +14,12 @@ undef $sock; test_lei({ tmpdir => $tmpdir }, sub { my $url = "imap://$host_port/t.v2.0"; my $url_orig = $url; + lei_ok(qw(ls-mail-source), "imap://$host_port/"); + like($lei_out, qr/^t\.v2\.0$/ms, 'shows mailbox'); + lei_ok(qw(ls-mail-source), $url); + is($lei_out, "t.v2.0\n", 'shows only mailbox with filter'); + lei_ok(qw(ls-mail-source -l), "imap://$host_port/"); + is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON'); lei_ok(qw(q z:1..)); my $out = json_utf8->decode($lei_out); diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t index 662da309..f2c35406 100644 --- a/t/lei-import-nntp.t +++ b/t/lei-import-nntp.t @@ -17,6 +17,13 @@ test_lei({ tmpdir => $tmpdir }, sub { my $out = json_utf8->decode($lei_out); is_deeply($out, [ undef ], 'nothing imported, yet'); my $url = "nntp://$host_port/t.v2"; + lei_ok(qw(ls-mail-source), "nntp://$host_port/"); + like($lei_out, qr/^t\.v2$/ms, 'shows newsgroup'); + lei_ok(qw(ls-mail-source), $url); + is($lei_out, "t.v2\n", 'shows only newsgroup with filter'); + lei_ok(qw(ls-mail-source -l), "nntp://$host_port/"); + is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON'); + lei_ok('import', $url); lei_ok(qw(q z:1..)); $out = json_utf8->decode($lei_out);