[PATCH 0/5] www: some coderepo stuff

[PATCH 0/5] www: some coderepo stuff

[Eric Wong]

5/5 is clearly incomplete, but it's nicer than broken
links before in $CODEREPO/$OID/s/ endpoints.

Deployed to https://yhbt.net/lore/git.git/<COMMIT_OID>/s/#related

Eric Wong (5):
  config: remove {-cgitrc_unparsed} field
  www_coderepo: wire up snapshots from summary
  www_coderepo: update blurb on the goal/purpose of this
  www: cgit: fix fallback to WwwCoderepo on array responses
  www_coderepo: allow searching one extindex|inbox

 Documentation/public-inbox-config.pod |  4 +++
 lib/PublicInbox/Config.pm             |  9 ++++-
 lib/PublicInbox/GitHTTPBackend.pm     |  6 +++-
 lib/PublicInbox/Inbox.pm              |  3 +-
 lib/PublicInbox/RepoSnapshot.pm       |  4 ++-
 lib/PublicInbox/ViewVCS.pm            | 31 ++++++++++++++---
 lib/PublicInbox/WwwCoderepo.pm        | 50 +++++++++++++++++++++++----
 t/solver_git.t                        |  6 ++++
 8 files changed, 99 insertions(+), 14 deletions(-)

[PATCH 1/5] config: remove {-cgitrc_unparsed} field

[Eric Wong]

This field has been unneeded since commit 6890430df808
(cgit: fix fallout from lazy coderepo loading, 2021-03-18)
---
 lib/PublicInbox/Config.pm      | 2 +-
 lib/PublicInbox/WwwCoderepo.pm | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 42bd9438..5cdf182e 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -46,7 +46,6 @@ sub new {
 	$self->{-no_obfuscate} = {};
 	$self->{-limiters} = {};
 	$self->{-code_repos} = {}; # nick => PublicInbox::Git object
-	$self->{-cgitrc_unparsed} = $self->{'publicinbox.cgitrc'};
 
 	if (my $no = delete $self->{'publicinbox.noobfuscate'}) {
 		$no = _array($no);
@@ -277,6 +276,7 @@ sub scan_projects_coderepo ($$$) {
 
 sub parse_cgitrc {
 	my ($self, $cgitrc, $nesting) = @_;
+	$cgitrc //= $self->{'publicinbox.cgitrc'};
 	if ($nesting == 0) {
 		# defaults:
 		my %s = map { $_ => 1 } qw(/cgit.css /cgit.png
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index 6c119b28..d491bba2 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -23,9 +23,9 @@ sub prepare_coderepos {
 	my $pi_cfg = $self->{pi_cfg};
 
 	# TODO: support gitweb and other repository viewers?
-	if (defined(my $cgitrc = $pi_cfg->{-cgitrc_unparsed})) {
-		$pi_cfg->parse_cgitrc($cgitrc, 0);
-	}
+	defined($pi_cfg->{'publicinbox.cgitrc'}) and
+		$pi_cfg->parse_cgitrc(undef, 0);
+
 	my $code_repos = $pi_cfg->{-code_repos};
 	for my $k (grep(/\Acoderepo\.(?:.+)\.dir\z/, keys %$pi_cfg)) {
 		$k = substr($k, length('coderepo.'), -length('.dir'));

[PATCH 2/5] www_coderepo: wire up snapshots from summary

[Eric Wong]

This also ensures we won't waste CPU cycles on snapshots
which aren't configured if somebody attempts them by
guessing URLs.
---
 Documentation/public-inbox-config.pod |  4 ++++
 lib/PublicInbox/Config.pm             |  2 ++
 lib/PublicInbox/RepoSnapshot.pm       |  4 +++-
 lib/PublicInbox/WwwCoderepo.pm        | 25 +++++++++++++++++++++++--
 t/solver_git.t                        |  6 ++++++
 5 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
index e926a27b..d175d2d7 100644
--- a/Documentation/public-inbox-config.pod
+++ b/Documentation/public-inbox-config.pod
@@ -265,6 +265,10 @@ The URL of the cgit instance associated with the coderepo.
 
 Default: none
 
+=item coderepo.snapshots
+
+See C<snapshots> in L<cgitrc(5)>
+
 =item publicinbox.cgitrc
 
 A path to a L<cgitrc(5)> file.  "repo.url" directives in the cgitrc
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 5cdf182e..a430cd5c 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -325,6 +325,8 @@ sub parse_cgitrc {
 		} elsif (m!\A(?:css|favicon|logo|repo\.logo)=(/.+)\z!) {
 			# absolute paths for static files via PublicInbox::Cgit
 			$self->{-cgit_static}->{$1} = 1;
+		} elsif (s!\Asnapshots=\s*!!) {
+			$self->{'coderepo.snapshots'} = $_;
 		}
 	}
 	cgit_repo_merge($self, $repo->{dir}, $repo) if $repo;
diff --git a/lib/PublicInbox/RepoSnapshot.pm b/lib/PublicInbox/RepoSnapshot.pm
index 460340e6..826392a8 100644
--- a/lib/PublicInbox/RepoSnapshot.pm
+++ b/lib/PublicInbox/RepoSnapshot.pm
@@ -72,10 +72,12 @@ sub ver_check { # git->check_async callback
 sub srv {
 	my ($ctx, $fn) = @_;
 	return if $fn =~ /["\s]/s;
-	$fn =~ s/\.($SUFFIX)\z//o or return;
+	my $fmt = $ctx->{wcr}->{snapshots}; # TODO per-repo snapshots
+	$fn =~ s/\.($SUFFIX)\z//o and $fmt->{$1} or return;
 	$ctx->{snap_fmt} = $1;
 	my $pfx = $ctx->{git}->local_nick // return;
 	$pfx =~ s/(?:\.git)?\z/-/;
+	($pfx) = ($pfx =~ m!([^/]+)\z!);
 	substr($fn, 0, length($pfx)) eq $pfx or return;
 	$ctx->{snap_pfx} = $fn;
 	my $v = $ctx->{snap_ver} = substr($fn, length($pfx), length($fn));
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index d491bba2..01ed562b 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -12,6 +12,7 @@ use PublicInbox::Git;
 use PublicInbox::GitAsyncCat;
 use PublicInbox::WwwStream;
 use PublicInbox::Hval qw(ascii_html);
+use PublicInbox::RepoSnapshot;
 
 my $EACH_REF = "git for-each-ref --sort=-creatordate --format='%(HEAD)%00".
 	join('%00', map { "%($_)" }
@@ -40,6 +41,11 @@ sub new {
 	my ($cls, $pi_cfg) = @_;
 	my $self = bless { pi_cfg => $pi_cfg }, $cls;
 	prepare_coderepos($self);
+	$self->{snapshots} = do {
+		my $s = $pi_cfg->{'coderepo.snapshots'} // '';
+		$s eq 'all' ? \%PublicInbox::RepoSnapshot::FMT_TYPES :
+			+{ map { $_ => 1 } split(/\s+/, $s) };
+	};
 	$self->{$_} = 10 for qw(summary_branches summary_tags);
 	$self->{$_} = 10 for qw(summary_log);
 	$self;
@@ -111,13 +117,28 @@ EOM
 		"%(refname:short) %(subject) (%(creatordate:short))'\n";
 	@r = split(/^/sm, shift(@x) // '');
 	$last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_tags};
+	my @s = sort keys %{$ctx->{wcr}->{snapshots}};
+	my $n;
+	if (@s) {
+		$n = $ctx->{git}->local_nick // die "BUG: $ctx->{git_dir} nick";
+		$n =~ s/\.git\z/-/;
+		($n) = ($n =~ m!([^/]+)\z!);
+		$n = ascii_html($n);
+	}
 	for (@r) {
 		my (undef, $oid, $ref, $s, $cd) = split(/\0/);
 		utf8::decode($_) for ($ref, $s);
 		chomp $cd;
 		my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
 		print $zfh "<a\nhref=./$oid/s/>", ascii_html($ref),
-			"</a>$align ", ascii_html($s), " ($cd)\n";
+			"</a>$align ", ascii_html($s), " ($cd)";
+		if (@s) {
+			my $v = $ref;
+			$v =~ s/\A[vV]//;
+			print $zfh "\t",  join(' ', map {
+				qq{<a href="snapshot/$n$v.$_">$_</a>} } @s);
+		}
+		print $zfh "\n";
 	}
 	print $zfh "# no tags yet...\n" if !@r;
 	print $zfh "...\n" if $last;
@@ -186,7 +207,7 @@ sub srv { # endpoint called by PublicInbox::WWW
 	# snapshots:
 	if ($path_info =~ m!\A/(.+?)/snapshot/([^/]+)\z! and
 			($ctx->{git} = $self->{"\0$1"})) {
-		require PublicInbox::RepoSnapshot;
+		$ctx->{wcr} = $self;
 		return PublicInbox::RepoSnapshot::srv($ctx, $2) // r(404);
 	}
 
diff --git a/t/solver_git.t b/t/solver_git.t
index 71b9554a..d8942747 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -247,6 +247,8 @@ SKIP: {
 	my $cfgpath = "$tmpdir/httpd-config";
 	open my $cfgfh, '>', $cfgpath or die;
 	print $cfgfh <<EOF or die;
+[coderepo]
+	snapshots = tar.gz
 [publicinbox "$name"]
 	address = $ibx->{-primary_address}
 	inboxdir = $ibx->{inboxdir}
@@ -351,6 +353,10 @@ EOF
 		$fn = 'public-inbox-1.0.2.tar.gz';
 		$res = $cb->(GET("/public-inbox/snapshot/$fn"));
 		is($res->code, 404, '404 on non-existent tag');
+
+		$fn = 'public-inbox-1.0.0.tar.bz2';
+		$res = $cb->(GET("/public-inbox/snapshot/$fn"));
+		is($res->code, 404, '404 on unconfigured snapshot format');
 	};
 	test_psgi(sub { $www->call(@_) }, $client);
 	my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };

[PATCH 3/5] www_coderepo: update blurb on the goal/purpose of this

[Eric Wong]

I think putting too much functionality in web services leads
to ignorance of local/offline tools, so this web UI will give
hints here and there for web users.  Things like diff options
can get expensive and become cache-unfriendly on the web server,
so promoting local tools can reduce overall network traffic
and server load.
---
 lib/PublicInbox/WwwCoderepo.pm | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index 01ed562b..2184e89d 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -1,7 +1,10 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
-# Standalone code repository viewer for users w/o cgit
+# Standalone code repository viewer for users w/o cgit.
+# This isn't intended to replicate all of cgit, but merely to be a
+# "good enough" viewer with search support and some UI hints to encourage
+# cloning + command-line usage.
 package PublicInbox::WwwCoderepo;
 use v5.12;
 use File::Temp 0.19 (); # newdir

[PATCH 4/5] www: cgit: fix fallback to WwwCoderepo on array responses

[Eric Wong]

For fast PSGI responses which don't require returning a coderef,
just reuse qspawn.wcb directly on the arrayref to avoid an undef
$wcb from firing in psgi_return_init_cb.

I only noticed this because the ViewVCS search form is broken
for /$CODEREPO/$OID/s/ endpoints at the moment.
---
 lib/PublicInbox/GitHTTPBackend.pm | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index 61a13560..3aae5454 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -155,7 +155,11 @@ sub parse_cgi_headers {
 		delete $ctx->{env}->{'qspawn.wcb'};
 		$ctx->{env}->{'plack.skip-deflater'} = 1; # prevent 2x gzip
 		my $res = $ctx->{www}->coderepo->srv(\%ctx);
-		$res->(delete $ctx{env}->{'qspawn.wcb'}) if ref($res) eq 'CODE';
+		if (ref($res) eq 'CODE') {
+			$res->(delete $ctx{env}->{'qspawn.wcb'});
+		} else { # ref($res) eq 'ARRAY'
+			$ctx->{env}->{'qspawn.wcb'} = $ctx{env}->{'qspawn.wcb'};
+		}
 		$res; # non ARRAY ref for ->psgi_return_init_cb
 	} else {
 		[ $code, \@h ]

[PATCH 5/5] www_coderepo: allow searching one extindex|inbox

[Eric Wong]

I'm not sure how to best make a UI for one coderepo to many
inboxes/extindices, yet; but at least allow a simple 1:1
mapping, for now.  This ensures /$CODEREPO/$OID/s/ can work
as effectively as /$INBOX/$OID/s/ when looking for emails
associated with a git commit.
---
 lib/PublicInbox/Config.pm      |  5 +++++
 lib/PublicInbox/Inbox.pm       |  3 ++-
 lib/PublicInbox/ViewVCS.pm     | 31 +++++++++++++++++++++++++++----
 lib/PublicInbox/WwwCoderepo.pm | 14 ++++++++++++++
 4 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index a430cd5c..c27928de 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -404,6 +404,11 @@ sub repo_objs {
 			push @repo_objs, $repo if $repo;
 		}
 		if (scalar @repo_objs) {
+			require Scalar::Util;
+			for (@repo_objs) {
+				push @{$_->{-ibxs}}, $ibxish;
+				Scalar::Util::weaken($_->{-ibxs}->[-1]);
+			}
 			$ibxish->{-repo_objs} = \@repo_objs;
 		} else {
 			delete $ibxish->{coderepo};
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 3532bb58..cb98d2ad 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -205,7 +205,8 @@ sub base_url {
 		$url .= '/' if $url !~ m!/\z!;
 		return $url .= $self->{name} . '/';
 	}
-	# called from a non-PSGI environment (e.g. NNTP/POP3):
+	# called from a non-PSGI environment or cross-inbox environment
+	# where multiple inboxes can have different domains
 	my $url = $self->{url} // return undef;
 	$url = $url->[0] // return undef;
 	# expand protocol-relative URLs to HTTPS if we're
diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 6ada03e6..72b79ab7 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -25,7 +25,7 @@ use PublicInbox::ViewDiff qw(flush_diff uri_escape_path);
 use PublicInbox::View;
 use PublicInbox::Eml;
 use Text::Wrap qw(wrap);
-use PublicInbox::Hval qw(ascii_html to_filename);
+use PublicInbox::Hval qw(ascii_html to_filename prurl);
 my $hl = eval {
 	require PublicInbox::HlMod;
 	PublicInbox::HlMod->new;
@@ -152,6 +152,25 @@ sub show_commit_start { # ->psgi_qx callback
 	}
 }
 
+sub ibx_url_for {
+	my ($ctx) = @_;
+	$ctx->{ibx} and return; # just fall back to $upfx
+	$ctx->{git} or return; # /$CODEREPO/$OID/s/ to (eidx|ibx)
+	if (my $ALL = $ctx->{www}->{pi_cfg}->ALL) {
+		$ALL->base_url // $ALL->base_url($ctx->{env});
+	} elsif (my $ibxs = $ctx->{git}->{-ibxs}) {
+		for my $ibx (@$ibxs) {
+			if ($ibx->isrch) {
+				return defined($ibx->{url}) ?
+					prurl($ctx->{env}, $ibx->{url}) :
+					"../../../$ibx->{name}/";
+			}
+		}
+	} else {
+		undef;
+	}
+}
+
 sub cmt_finalize {
 	my ($ctx) = @_;
 	$ctx->{-linkify} //= PublicInbox::Linkify->new;
@@ -227,12 +246,16 @@ EOM
 			$q = wrap('', '', $q);
 			my $rows = ($q =~ tr/\n/\n/) + 1;
 			$q = ascii_html($q);
+			my $ibx_url = ibx_url_for($ctx);
+			my $alt = $ibx_url ? ' '.ascii_html($ibx_url) : '';
+			$ibx_url = ascii_html($ibx_url) if defined $ibx_url;
+			$ibx_url //= $upfx;
 			print $zfh <<EOM;
-<hr><form action=$upfx
+<hr><form action="$ibx_url"
 id=related><pre>find related emails, including ancestors/descendants/conflicts
 <textarea name=q cols=${\PublicInbox::View::COLS} rows=$rows>$q</textarea>
-<input type=submit value=search
-/>\t(<a href=${upfx}_/text/help/>help</a>)</pre></form>
+<input type=submit value="search$alt"
+/>\t(<a href="${ibx_url}_/text/help/">help</a>)</pre></form>
 EOM
 		}
 	}
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index 2184e89d..99df39ef 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -35,6 +35,20 @@ sub prepare_coderepos {
 		$k = substr($k, length('coderepo.'), -length('.dir'));
 		$code_repos->{$k} //= $pi_cfg->fill_code_repo($k);
 	}
+
+	# associate inboxes and extindices with coderepos for search:
+	for my $k (grep(/\Apublicinbox\.(?:.+)\.coderepo\z/, keys %$pi_cfg)) {
+		$k = substr($k, length('publicinbox.'), -length('.coderepo'));
+		my $ibx = $pi_cfg->lookup_name($k) // next;
+		$pi_cfg->repo_objs($ibx);
+		push @{$self->{-strong}}, $ibx; # strengthen {-ibxs} weakref
+	}
+	for my $k (grep(/\Aextindex\.(?:.+)\.coderepo\z/, keys %$pi_cfg)) {
+		$k = substr($k, length('extindex.'), -length('.coderepo'));
+		my $eidx = $pi_cfg->lookup_ei($k) // next;
+		$pi_cfg->repo_objs($eidx);
+		push @{$self->{-strong}}, $eidx; # strengthen {-ibxs} weakref
+	}
 	while (my ($nick, $repo) = each %$code_repos) {
 		$self->{"\0$nick"} = $repo;
 	}