* [PATCH] lei_input: support PublicInbox::WWW mboxrd URLs
@ 2021-04-26 8:27 Eric Wong
0 siblings, 0 replies; 2+ messages in thread
From: Eric Wong @ 2021-04-26 8:27 UTC (permalink / raw)
To: meta
This gives "lei import", "lei tag", and similar commands
the ability to use URLs recognized by our PSGI frontend
directly.
This is more convenient than an equivalent shell pipeline
since "set -o pipefail" is not portable and errors may be
lost.
---
MANIFEST | 1 +
lib/PublicInbox/LeiInput.pm | 76 +++++++++++++++++++++++++++++++++++++
t/lei-import-http.t | 43 +++++++++++++++++++++
3 files changed, 120 insertions(+)
create mode 100644 t/lei-import-http.t
diff --git a/MANIFEST b/MANIFEST
index 79d393c5..ce824fcf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -394,6 +394,7 @@ t/kqnotify.t
t/lei-convert.t
t/lei-daemon.t
t/lei-externals.t
+t/lei-import-http.t
t/lei-import-imap.t
t/lei-import-maildir.t
t/lei-import-nntp.t
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index d11d23d4..785e607d 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiInput;
use strict;
use v5.10.1;
use PublicInbox::DS;
+use PublicInbox::Spawn qw(which popen_rd);
# JMAP RFC 8621 4.1.1
# https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -77,6 +78,32 @@ error reading $name: $!
}
}
+# handles mboxrd endpoints described in Documentation/design_notes.txt
+sub handle_http_input ($$@) {
+ my ($self, $url, @args) = @_;
+ my $lei = $self->{lei} or die 'BUG: {lei} missing';
+ my $curl_opt = delete $self->{"-curl-$url"} or
+ die("BUG: $url curl options not prepared");
+ my $uri = pop @$curl_opt;
+ my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
+ push @$curl, '-s', @$curl_opt;
+ my $cmd = $curl->for_uri($lei, $uri);
+ $lei->qerr("# $cmd");
+ my $rdr = { 2 => $lei->{2}, pgid => 0 };
+ my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+ grep(/\A--compressed\z/, @$curl) or
+ $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+ eval {
+ PublicInbox::MboxReader->mboxrd($fh,
+ $self->can('input_mbox_cb'),
+ $self, @args);
+ };
+ my $err = $@;
+ waitpid($pid, 0);
+ $? || $err and
+ $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+}
+
sub input_path_url {
my ($self, $input, @args) = @_;
my $lei = $self->{lei};
@@ -92,6 +119,9 @@ sub input_path_url {
$self->can('input_net_cb'),
$self, @args);
return;
+ } elsif ($input =~ m!\Ahttps?://!i) {
+ handle_http_input($self, $input, @args);
+ return;
}
if ($input =~ s!\A([a-z0-9]+):!!i) {
$ifmt = lc($1);
@@ -129,6 +159,50 @@ EOM
}
}
+sub bad_http ($$;$) {
+ my ($lei, $url, $alt) = @_;
+ my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
+ $lei->fail("E: <$url> not recognized, $x");
+}
+
+sub prepare_http_input ($$$) {
+ my ($self, $lei, $url) = @_;
+ require URI;
+ require PublicInbox::MboxReader;
+ require PublicInbox::LeiCurl;
+ require IO::Uncompress::Gunzip;
+ $self->{curl} //= which('curl') or
+ return $lei->fail("curl missing for <$url>");
+ my $uri = URI->new($url);
+ my $path = $uri->path;
+ my %qf = $uri->query_form;
+ my @curl_opt;
+ if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) {
+ # OK
+ } elsif ($path =~ m!/raw\z!) {
+ push @curl_opt, '--compressed';
+ # convert search query to mboxrd request since they require POST
+ # this is only intended for PublicInbox::WWW, and will false-positive
+ # on many other search engines... oh well
+ } elsif (defined $qf{'q'}) {
+ $qf{x} = 'm';
+ $uri->query_form(\%qf);
+ push @curl_opt, '-d', '';
+ $$uri ne $url and $lei->qerr(<<"");
+# <$url> rewritten to <$$uri> with HTTP POST
+
+ # try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/
+ } elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! ||
+ $path =~ s!/t\.atom\z!/t.mbox.gz! ||
+ $path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) {
+ $uri->path($path);
+ return bad_http($lei, $url, $$uri);
+ } else {
+ return bad_http($lei, $url);
+ }
+ $self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input
+}
+
sub prepare_inputs { # returns undef on error
my ($self, $lei, $inputs) = @_;
my $in_fmt = $lei->{opt}->{'in-format'};
@@ -156,6 +230,8 @@ sub prepare_inputs { # returns undef on error
push @{$sync->{no}}, $input;
}
}
+ } elsif ($input_path =~ m!\Ahttps?://!i) {
+ prepare_http_input($self, $lei, $input_path) or return;
} elsif ($input_path =~ s/\A([a-z0-9]+)://is) {
my $ifmt = lc $1;
if (($in_fmt // $ifmt) ne $ifmt) {
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
new file mode 100644
index 00000000..35cbf369
--- /dev/null
+++ b/t/lei-import-http.t
@@ -0,0 +1,43 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use PublicInbox::Spawn qw(which);
+require_mods(qw(lei -httpd));
+which('curl') or plan skip_all => "curl required for $0";
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my ($tmpdir, $for_destroy) = tmpdir;
+my $sock = tcp_server;
+my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/1", "--stderr=$tmpdir/2" ];
+my $env = { PI_CONFIG => $cfg_path };
+my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-httpd $?");
+my $host_port = tcp_host_port($sock);
+undef $sock;
+test_lei({ tmpdir => $tmpdir }, sub {
+ my $url = "http://$host_port/t2";
+ for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) {
+ ok(!lei('import', "$url/$p"), "/$p fails properly");
+ }
+ for my $p (qw(/ /T/ /t/ /t.atom)) {
+ ok(!lei('import', "$url/m\@example$p"), "/$p fails");
+ like($lei_err, qr/did you mean/, "gave hint for $p");
+ }
+ lei_ok 'import', "$url/testmessage\@example.com/raw";
+ lei_ok 'q', 'm:testmessage@example.com';
+ my $res = json_utf8->decode($lei_out);
+ is($res->[0]->{'m'}, 'testmessage@example.com', 'imported raw')
+ or diag explain($res);
+
+ lei_ok 'import', "$url/qp\@example.com/t.mbox.gz";
+ lei_ok 'q', 'm:qp@example.com';
+ $res = json_utf8->decode($lei_out);
+ is($res->[0]->{'m'}, 'qp@example.com', 'imported t.mbox.gz')
+ or diag explain($res);
+
+ lei_ok 'import', "$url/?q=s:boolean";
+ lei_ok 'q', 'm:20180720072141.GA15957@example';
+ $res = json_utf8->decode($lei_out);
+ is($res->[0]->{'m'}, '20180720072141.GA15957@example',
+ 'imported search result') or diag explain($res);
+});
+done_testing;
^ permalink raw reply related [flat|nested] 2+ messages in thread* [PATCH] lei_input: support PublicInbox::WWW mboxrd URLs
@ 2021-04-26 8:30 Eric Wong
0 siblings, 0 replies; 2+ messages in thread
From: Eric Wong @ 2021-04-26 8:30 UTC (permalink / raw)
To: meta
This gives "lei import", "lei tag", and similar commands
the ability to use URLs recognized by our PSGI frontend
directly.
This is more convenient than an equivalent shell pipeline
since "set -o pipefail" is not portable and errors may be
lost.
---
MANIFEST | 1 +
lib/PublicInbox/LeiInput.pm | 76 +++++++++++++++++++++++++++++++++++++
t/lei-import-http.t | 43 +++++++++++++++++++++
3 files changed, 120 insertions(+)
create mode 100644 t/lei-import-http.t
diff --git a/MANIFEST b/MANIFEST
index 79d393c5..ce824fcf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -394,6 +394,7 @@ t/kqnotify.t
t/lei-convert.t
t/lei-daemon.t
t/lei-externals.t
+t/lei-import-http.t
t/lei-import-imap.t
t/lei-import-maildir.t
t/lei-import-nntp.t
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index d11d23d4..785e607d 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiInput;
use strict;
use v5.10.1;
use PublicInbox::DS;
+use PublicInbox::Spawn qw(which popen_rd);
# JMAP RFC 8621 4.1.1
# https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -77,6 +78,32 @@ error reading $name: $!
}
}
+# handles mboxrd endpoints described in Documentation/design_notes.txt
+sub handle_http_input ($$@) {
+ my ($self, $url, @args) = @_;
+ my $lei = $self->{lei} or die 'BUG: {lei} missing';
+ my $curl_opt = delete $self->{"-curl-$url"} or
+ die("BUG: $url curl options not prepared");
+ my $uri = pop @$curl_opt;
+ my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
+ push @$curl, '-s', @$curl_opt;
+ my $cmd = $curl->for_uri($lei, $uri);
+ $lei->qerr("# $cmd");
+ my $rdr = { 2 => $lei->{2}, pgid => 0 };
+ my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+ grep(/\A--compressed\z/, @$curl) or
+ $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+ eval {
+ PublicInbox::MboxReader->mboxrd($fh,
+ $self->can('input_mbox_cb'),
+ $self, @args);
+ };
+ my $err = $@;
+ waitpid($pid, 0);
+ $? || $err and
+ $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+}
+
sub input_path_url {
my ($self, $input, @args) = @_;
my $lei = $self->{lei};
@@ -92,6 +119,9 @@ sub input_path_url {
$self->can('input_net_cb'),
$self, @args);
return;
+ } elsif ($input =~ m!\Ahttps?://!i) {
+ handle_http_input($self, $input, @args);
+ return;
}
if ($input =~ s!\A([a-z0-9]+):!!i) {
$ifmt = lc($1);
@@ -129,6 +159,50 @@ EOM
}
}
+sub bad_http ($$;$) {
+ my ($lei, $url, $alt) = @_;
+ my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
+ $lei->fail("E: <$url> not recognized, $x");
+}
+
+sub prepare_http_input ($$$) {
+ my ($self, $lei, $url) = @_;
+ require URI;
+ require PublicInbox::MboxReader;
+ require PublicInbox::LeiCurl;
+ require IO::Uncompress::Gunzip;
+ $self->{curl} //= which('curl') or
+ return $lei->fail("curl missing for <$url>");
+ my $uri = URI->new($url);
+ my $path = $uri->path;
+ my %qf = $uri->query_form;
+ my @curl_opt;
+ if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) {
+ # OK
+ } elsif ($path =~ m!/raw\z!) {
+ push @curl_opt, '--compressed';
+ # convert search query to mboxrd request since they require POST
+ # this is only intended for PublicInbox::WWW, and will false-positive
+ # on many other search engines... oh well
+ } elsif (defined $qf{'q'}) {
+ $qf{x} = 'm';
+ $uri->query_form(\%qf);
+ push @curl_opt, '-d', '';
+ $$uri ne $url and $lei->qerr(<<"");
+# <$url> rewritten to <$$uri> with HTTP POST
+
+ # try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/
+ } elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! ||
+ $path =~ s!/t\.atom\z!/t.mbox.gz! ||
+ $path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) {
+ $uri->path($path);
+ return bad_http($lei, $url, $$uri);
+ } else {
+ return bad_http($lei, $url);
+ }
+ $self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input
+}
+
sub prepare_inputs { # returns undef on error
my ($self, $lei, $inputs) = @_;
my $in_fmt = $lei->{opt}->{'in-format'};
@@ -156,6 +230,8 @@ sub prepare_inputs { # returns undef on error
push @{$sync->{no}}, $input;
}
}
+ } elsif ($input_path =~ m!\Ahttps?://!i) {
+ prepare_http_input($self, $lei, $input_path) or return;
} elsif ($input_path =~ s/\A([a-z0-9]+)://is) {
my $ifmt = lc $1;
if (($in_fmt // $ifmt) ne $ifmt) {
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
new file mode 100644
index 00000000..35cbf369
--- /dev/null
+++ b/t/lei-import-http.t
@@ -0,0 +1,43 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use PublicInbox::Spawn qw(which);
+require_mods(qw(lei -httpd));
+which('curl') or plan skip_all => "curl required for $0";
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my ($tmpdir, $for_destroy) = tmpdir;
+my $sock = tcp_server;
+my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/1", "--stderr=$tmpdir/2" ];
+my $env = { PI_CONFIG => $cfg_path };
+my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-httpd $?");
+my $host_port = tcp_host_port($sock);
+undef $sock;
+test_lei({ tmpdir => $tmpdir }, sub {
+ my $url = "http://$host_port/t2";
+ for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) {
+ ok(!lei('import', "$url/$p"), "/$p fails properly");
+ }
+ for my $p (qw(/ /T/ /t/ /t.atom)) {
+ ok(!lei('import', "$url/m\@example$p"), "/$p fails");
+ like($lei_err, qr/did you mean/, "gave hint for $p");
+ }
+ lei_ok 'import', "$url/testmessage\@example.com/raw";
+ lei_ok 'q', 'm:testmessage@example.com';
+ my $res = json_utf8->decode($lei_out);
+ is($res->[0]->{'m'}, 'testmessage@example.com', 'imported raw')
+ or diag explain($res);
+
+ lei_ok 'import', "$url/qp\@example.com/t.mbox.gz";
+ lei_ok 'q', 'm:qp@example.com';
+ $res = json_utf8->decode($lei_out);
+ is($res->[0]->{'m'}, 'qp@example.com', 'imported t.mbox.gz')
+ or diag explain($res);
+
+ lei_ok 'import', "$url/?q=s:boolean";
+ lei_ok 'q', 'm:20180720072141.GA15957@example';
+ $res = json_utf8->decode($lei_out);
+ is($res->[0]->{'m'}, '20180720072141.GA15957@example',
+ 'imported search result') or diag explain($res);
+});
+done_testing;
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2021-04-26 8:30 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-04-26 8:27 [PATCH] lei_input: support PublicInbox::WWW mboxrd URLs Eric Wong
-- strict thread matches above, loose matches on Subject: below --
2021-04-26 8:30 Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).