[PATCH] wwwtext: support $INBOX_URL/_/text/config/raw

[PATCH] wwwtext: support $INBOX_URL/_/text/config/raw

[Eric Wong]

This returns a git-config(1)-compatible file to make it easier
to get started on mirroring an existing public-inbox.  Omitting
the "raw" from the URL works, as well, but I'm not sure if
it's very useful.
---
 lib/PublicInbox/WwwStream.pm |  7 +++-
 lib/PublicInbox/WwwText.pm   | 75 +++++++++++++++++++++++++++++++++---
 t/psgi_text.t                |  7 ++++
 3 files changed, 82 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index e0823c8d..7399b0ad 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -127,10 +127,15 @@ sub _html_end {
 		$addrs
 	public-inbox-index $dir
 EOF
+	} else { # v1
+		$urls .= "\n";
 	}
+
+	my $cfg_link = ($ctx->{-upfx} // '').'_/text/config/raw';
+	$urls .= qq(\nExample <a\nhref="$cfg_link">config snippet</a> for mirrors\n);
 	my @nntp = map { qq(<a\nhref="$_">$_</a>) } @{$ibx->nntp_url};
 	if (@nntp) {
-		$urls .= "\n\n";
+		$urls .= "\n";
 		$urls .= @nntp == 1 ? 'Newsgroup' : 'Newsgroups are';
 		$urls .= ' available over NNTP:';
 		$urls .= "\n\t" . join("\n\t", @nntp) . "\n";
diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm
index 975ebddb..491f7e63 100644
--- a/lib/PublicInbox/WwwText.pm
+++ b/lib/PublicInbox/WwwText.pm
@@ -9,6 +9,7 @@ use bytes (); # only for bytes::length
 use PublicInbox::Linkify;
 use PublicInbox::WwwStream;
 use PublicInbox::Hval qw(ascii_html);
+use URI::Escape qw(uri_escape_utf8);
 our $QP_URL = 'https://xapian.org/docs/queryparser.html';
 our $WIKI_URL = 'https://en.wikipedia.org/wiki';
 my $hl = eval {
@@ -29,14 +30,14 @@ sub get_text {
 	my $have_tslash = ($key =~ s!/\z!!) if !$raw;
 
 	my $txt = '';
-	if (!_default_text($ctx, $key, \$txt)) {
+	my $hdr = [ 'Content-Type', 'text/plain', 'Content-Length', undef ];
+	if (!_default_text($ctx, $key, $hdr, \$txt)) {
 		$code = 404;
 		$txt = "404 Not Found ($key)\n";
 	}
 	if ($raw) {
-		return [ $code, [ 'Content-Type', 'text/plain',
-				  'Content-Length', bytes::length($txt) ],
-			[ $txt ] ]
+		$hdr->[3] = bytes::length($txt);
+		return [ $code, $hdr, [ $txt ] ]
 	}
 
 	# enforce trailing slash for "wget -r" compatibility
@@ -123,9 +124,71 @@ EOF
 	$$txt .= PublicInbox::UserContent::sample($ibx, $env) . "```\n";
 }
 
-sub _default_text ($$$) {
-	my ($ctx, $key, $txt) = @_;
+# git-config section names are quoted in the config file, so escape them
+sub dq_escape ($) {
+	my ($name) = @_;
+	$name =~ s/\\/\\\\/g;
+	$name =~ s/"/\\"/g;
+	$name;
+}
+
+sub URI_PATH () { '^A-Za-z0-9\-\._~/' }
+
+# n.b. this is a perfect candidate for memoization
+sub inbox_config ($$$) {
+	my ($ctx, $hdr, $txt) = @_;
+	my $ibx = $ctx->{-inbox};
+	push @$hdr, 'Content-Disposition', 'inline; filename=inbox.config';
+	my $name = dq_escape($ibx->{name});
+	$$txt .= <<EOS;
+; example public-inbox config snippet for "$name"
+; see public-inbox-config(5) manpage for more details:
+; https://public-inbox.org/public-inbox-config.html
+[publicinbox "$name"]
+	mainrepo = /path/to/top-level-inbox
+EOS
+	for my $k (qw(address)) {
+		defined(my $v = $ibx->{$k}) or next;
+		$$txt .= "\t$k = $_\n" for @$v;
+	}
+
+	for my $k (qw(filter infourl newsgroup obfuscate replyto watchheader)) {
+		defined(my $v = $ibx->{$k}) or next;
+		$$txt .= "\t$k = $v\n";
+	}
+	$$txt .= "\tnntpmirror = $_\n" for (@{$ibx->nntp_url});
+
+	# note: this doesn't preserve cgitrc layout, since we parse cgitrc
+	# and drop the original structure
+	if (defined(my $cr = $ibx->{coderepo})) {
+		$$txt .= "\tcoderepo = $_\n" for @$cr;
+
+		my $pi_config = $ctx->{www}->{pi_config};
+		for my $cr_name (@$cr) {
+			my $url = $pi_config->{"coderepo.$cr_name.cgiturl"};
+			my $path = "/path/to/$cr_name";
+			$cr_name = dq_escape($cr_name);
+
+			$$txt .= qq([coderepo "$cr_name"]\n);
+			if (defined($url)) {
+				my $cpath = $path;
+				if ($path !~ m![a-z0-9_/\.\-]!i) {
+					$cpath = dq_escape($cpath);
+				}
+				$$txt .= qq(\t; git clone $url "$cpath"\n);
+			}
+			$$txt .= "\tdir = $path\n";
+			$$txt .= "\tcgiturl = https://example.com/";
+			$$txt .= uri_escape_utf8($cr_name, URI_PATH)."\n";
+		}
+	}
+	1;
+}
+
+sub _default_text ($$$$) {
+	my ($ctx, $key, $hdr, $txt) = @_;
 	return _colors_help($ctx, $txt) if $key eq 'color';
+	return inbox_config($ctx, $hdr, $txt) if $key eq 'config';
 	return if $key ne 'help'; # TODO more keys?
 
 	my $ibx = $ctx->{-inbox};
diff --git a/t/psgi_text.t b/t/psgi_text.t
index 9979e7b3..bdc1ebfd 100644
--- a/t/psgi_text.t
+++ b/t/psgi_text.t
@@ -34,6 +34,13 @@ test_psgi(sub { $www->call(@_) }, sub {
 	$res = $cb->(GET('/test/_/text/help/'));
 	like($res->content, qr!<title>public-inbox help.*</title>!,
 		'default help');
+	$res = $cb->(GET('/test/_/text/config/raw'));
+	my $f = "$tmpdir/cfg";
+	open my $fh, '>', $f or die;
+	print $fh $res->content or die;
+	close $fh or die;
+	my $cfg = PublicInbox::Config->new($f);
+	is($cfg->{"$cfgpfx.address"}, $addr, 'got expected address in config');
 });
 
 done_testing();
-- 
EW

Re: [PATCH] wwwtext: support $INBOX_URL/_/text/config/raw

[Konstantin Ryabitsev]

On Fri, Sep 27, 2019 at 10:48:25AM +0000, Eric Wong wrote:
>This returns a git-config(1)-compatible file to make it easier
>to get started on mirroring an existing public-inbox.  Omitting
>the "raw" from the URL works, as well, but I'm not sure if
>it's very useful.

This is now live on lore.kernel.org. Theoretically, this provides 
everything needed to recreate mirrors via grokmirror, once the 
post-update hook is changed to incorporate this feature.

Quick question that occurred to me -- when there are history-rewriting 
changes on the master mirror (GDPR-related message removals, etc), how 
well will that propagate to other mirrors? Would they need to be fully 
reindexed?

-K

Re: [PATCH] wwwtext: support $INBOX_URL/_/text/config/raw

[Eric Wong]

Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Fri, Sep 27, 2019 at 10:48:25AM +0000, Eric Wong wrote:
> > This returns a git-config(1)-compatible file to make it easier
> > to get started on mirroring an existing public-inbox.  Omitting
> > the "raw" from the URL works, as well, but I'm not sure if
> > it's very useful.
> 
> This is now live on lore.kernel.org. Theoretically, this provides everything
> needed to recreate mirrors via grokmirror, once the post-update hook is
> changed to incorporate this feature.

Cool!  I'm starting to form a better idea of client-side stuff I want
to do...

> Quick question that occurred to me -- when there are history-rewriting
> changes on the master mirror (GDPR-related message removals, etc), how well
> will that propagate to other mirrors? Would they need to be fully reindexed?

public-inbox-index should pick up changes and only reindex the
history starting with the rewrite.  t/v2mirror.t tests the purge
case, but purge is a special case of replace (which edit uses).