From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 52A111F526 for ; Tue, 4 Oct 2022 19:12:41 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1664910761; bh=ordmtYLDts3TdzRjqm/Aa89HD0LOrSLQdEXvD5TPQRI=; h=From:To:Subject:Date:In-Reply-To:References:From; b=po1/VGNTyA5iYHzBxo5wkUrX5e2My6czhrz17VoWa6Fmo2a5C2s5SF9S+R/jt33mX 90/vog5/TCe7oTrZKdNxOWGLQaz3j00LqyCSL/a50jwwVb4l0myL4+KeMweZG4rC4K XXGRfU32LbxWqJ9tB+7H5FP+ArJUbaytihX5MMH4= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 05/10] www_coderepo: an alternative to cgit Date: Tue, 4 Oct 2022 19:12:35 +0000 Message-Id: <20221004191240.1056304-6-e@80x24.org> In-Reply-To: <20221004191240.1056304-1-e@80x24.org> References: <20221004191240.1056304-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This will allow it to easily map a single coderepo to multiple inboxes (or multiple coderepos to any number of inboxes). For now, this is just a summary, but $REPO/$OID/s/ support will be added, along with archive downloads. Indexing of coderepos will probably be supported via -extindex, only. --- MANIFEST | 1 + lib/PublicInbox/Cgit.pm | 14 +-- lib/PublicInbox/Config.pm | 2 +- lib/PublicInbox/Git.pm | 14 ++- lib/PublicInbox/GitAsyncCat.pm | 17 +-- lib/PublicInbox/WWW.pm | 12 ++- lib/PublicInbox/WwwCoderepo.pm | 185 +++++++++++++++++++++++++++++++++ lib/PublicInbox/WwwStream.pm | 16 ++- t/solver_git.t | 33 +++++- 9 files changed, 266 insertions(+), 28 deletions(-) create mode 100644 lib/PublicInbox/WwwCoderepo.pm diff --git a/MANIFEST b/MANIFEST index 35382d2d..cf6d97e1 100644 --- a/MANIFEST +++ b/MANIFEST @@ -342,6 +342,7 @@ lib/PublicInbox/Watch.pm lib/PublicInbox/WwwAltId.pm lib/PublicInbox/WwwAtomStream.pm lib/PublicInbox/WwwAttach.pm +lib/PublicInbox/WwwCoderepo.pm lib/PublicInbox/WwwHighlight.pm lib/PublicInbox/WwwListing.pm lib/PublicInbox/WwwStatic.pm diff --git a/lib/PublicInbox/Cgit.pm b/lib/PublicInbox/Cgit.pm index a63f8902..1112d9f8 100644 --- a/lib/PublicInbox/Cgit.pm +++ b/lib/PublicInbox/Cgit.pm @@ -7,6 +7,7 @@ package PublicInbox::Cgit; use v5.12; +use parent qw(PublicInbox::WwwCoderepo); use PublicInbox::GitHTTPBackend; use PublicInbox::Git; # not bothering with Exporter for a one-off @@ -52,10 +53,6 @@ sub locate_cgit ($) { sub new { my ($class, $pi_cfg) = @_; my ($cgit_bin, $cgit_data) = locate_cgit($pi_cfg); - # TODO: support gitweb and other repository viewers? - if (defined(my $cgitrc = $pi_cfg->{-cgitrc_unparsed})) { - $pi_cfg->parse_cgitrc($cgitrc, 0); - } my $self = bless { cmd => [ $cgit_bin ], cgit_data => $cgit_data, @@ -63,14 +60,7 @@ sub new { }, $class; # some cgit repos may not be mapped to inboxes, so ensure those exist: - my $code_repos = $pi_cfg->{-code_repos}; - for my $k (grep(/\Acoderepo\.(?:.+)\.dir\z/, keys %$pi_cfg)) { - $k = substr($k, length('coderepo.'), -length('.dir')); - $code_repos->{$k} //= $pi_cfg->fill_code_repo($k); - } - while (my ($nick, $repo) = each %$code_repos) { - $self->{"\0$nick"} = $repo; - } + PublicInbox::WwwCoderepo::prepare_coderepos($self); my $s = join('|', map { quotemeta } keys %{$pi_cfg->{-cgit_static}}); $self->{static} = qr/\A($s)\z/; $self; diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index 1b5d87e2..42bd9438 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -343,7 +343,7 @@ sub fill_code_repo { $git->{cgit_url} = $cgits = _array($cgits); $self->{"$pfx.cgiturl"} = $cgits; } - + $git->{nick} = $nick; $git; } diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm index 2f0bb6a0..395add1f 100644 --- a/lib/PublicInbox/Git.pm +++ b/lib/PublicInbox/Git.pm @@ -463,6 +463,16 @@ sub host_prefix_url ($$) { "$scheme://$host_port". ($env->{SCRIPT_NAME} || '/') . $url; } +sub base_url { # for coderepos, PSGI-only + my ($self, $env) = @_; # env - PSGI env + my $url = host_prefix_url($env, ''); + # for mount in Plack::Builder + $url .= '/' if substr($url, -1, 1) ne '/'; + $url . $self->{nick} . '/'; +} + +sub isrch {} # TODO + sub pub_urls { my ($self, $env) = @_; if (my $urls = $self->{cgit_url}) { @@ -518,11 +528,11 @@ sub description { } sub cloneurl { - my ($self) = @_; + my ($self, $env) = @_; $self->{cloneurl} // do { my @urls = split(/\s+/s, try_cat("$self->{git_dir}/cloneurl")); scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef; - } // []; + } // [ substr(base_url($self, $env), 0, -1) ]; } # for grokmirror, which doesn't read gitweb.description diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm index b32c2fd3..613dbf7e 100644 --- a/lib/PublicInbox/GitAsyncCat.pm +++ b/lib/PublicInbox/GitAsyncCat.pm @@ -45,6 +45,16 @@ sub event_step { } } +sub watch_cat { + my ($git) = @_; + $git->{async_cat} //= do { + my $self = bless { git => $git }, __PACKAGE__; + $git->{in}->blocking(0); + $self->SUPER::new($git->{in}, EPOLLIN|EPOLLET); + \undef; # this is a true ref() + }; +} + sub ibx_async_cat ($$$$) { my ($ibx, $oid, $cb, $arg) = @_; my $git = $ibx->{git} // $ibx->git; @@ -60,12 +70,7 @@ sub ibx_async_cat ($$$$) { \undef; } else { # read-only end of git-cat-file pipe $git->cat_async($oid, $cb, $arg); - $git->{async_cat} //= do { - my $self = bless { git => $git }, __PACKAGE__; - $git->{in}->blocking(0); - $self->SUPER::new($git->{in}, EPOLLIN|EPOLLET); - \undef; # this is a true ref() - }; + watch_cat($git); } } diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 1df5572d..d0e20fb5 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -197,7 +197,9 @@ sub news_cgit_fallback ($) { my $www = $ctx->{www}; my $env = $ctx->{env}; my $res = $www->news_www->call($env); - $res->[0] == 404 ? $www->cgit->call($env) : $res; + $res = $www->cgit->call($env) if $res->[0] == 404; + $res = $www->coderepo->srv($ctx) if $res->[0] == 404; + $res; } # returns undef if valid, array ref response if invalid @@ -494,6 +496,14 @@ sub cgit { } } +sub coderepo { + my ($self) = @_; + $self->{coderepo} //= do { + require PublicInbox::WwwCoderepo; + PublicInbox::WwwCoderepo->new($self->{pi_cfg}); + } +} + # GET $INBOX/manifest.js.gz sub get_inbox_manifest ($$$) { my ($ctx, $inbox, $key) = @_; diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm new file mode 100644 index 00000000..4b1a4f9b --- /dev/null +++ b/lib/PublicInbox/WwwCoderepo.pm @@ -0,0 +1,185 @@ +# Copyright (C) all contributors +# License: AGPL-3.0+ +# +# Standalone code repository viewer for users w/o cgit +package PublicInbox::WwwCoderepo; +use v5.12; +use File::Temp 0.19 (); # newdir +use PublicInbox::ViewVCS; +use PublicInbox::WwwStatic qw(r); +use PublicInbox::GitHTTPBackend; +use PublicInbox::Git; +use PublicInbox::GitAsyncCat; +use PublicInbox::WwwStream; +use PublicInbox::Hval qw(ascii_html); + +my $EACH_REF = "git for-each-ref --sort=-creatordate --format='%(HEAD)%00". + join('%00', map { "%($_)" } + qw(objectname refname:short subject creatordate:short))."'"; + +# shared with PublicInbox::Cgit +sub prepare_coderepos { + my ($self) = @_; + my $pi_cfg = $self->{pi_cfg}; + + # TODO: support gitweb and other repository viewers? + if (defined(my $cgitrc = $pi_cfg->{-cgitrc_unparsed})) { + $pi_cfg->parse_cgitrc($cgitrc, 0); + } + my $code_repos = $pi_cfg->{-code_repos}; + for my $k (grep(/\Acoderepo\.(?:.+)\.dir\z/, keys %$pi_cfg)) { + $k = substr($k, length('coderepo.'), -length('.dir')); + $code_repos->{$k} //= $pi_cfg->fill_code_repo($k); + } + while (my ($nick, $repo) = each %$code_repos) { + $self->{"\0$nick"} = $repo; + } +} + +sub new { + my ($cls, $pi_cfg) = @_; + my $self = bless { pi_cfg => $pi_cfg }, $cls; + prepare_coderepos($self); + $self->{$_} = 10 for qw(summary_branches summary_tags); + $self->{$_} = 10 for qw(summary_log); + $self; +} + +sub summary_finish { + my ($ctx) = @_; + my $wcb = delete($ctx->{env}->{'qspawn.wcb'}) or return; # already done + my @x = split(/\n\n/sm, delete($ctx->{-each_refs})); + PublicInbox::WwwStream::html_init($ctx); + my $zfh = $ctx->zfh; + + # git log + my @r = split(/\n/s, pop(@x) // ''); + my $last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_log}; + print $zfh '
$ '.
+		"git log --pretty=format:'%h %s (%cs)%d'\n";
+	for (@r) {
+		my $d; # decorations
+		s/^ \(([^\)]+)\)// and $d = $1;
+		substr($_, 0, 1, '');
+		my ($H, $h, $cs, $s) = split(/ /, $_, 4);
+		print $zfh "$h ", ascii_html($s),
+			" (", $cs, ")\n";
+		print $zfh "\t(", ascii_html($d), ")\n" if $d;
+	}
+	print $zfh "# no commits, yet\n" if !@r;
+	print $zfh "...\n" if $last;
+
+	# README
+	my ($bref, $oid, $ref_path) = @{delete $ctx->{-readme}};
+	if ($bref) {
+		my $l = PublicInbox::Linkify->new;
+		$$bref =~ s/\s*\z//sm;
+		print $zfh "\n\$ " .
+			"git cat-file blob ",
+			ascii_html($ref_path), "\n",
+			$l->to_html($$bref), '

';
+	}
+
+	# refs/heads
+	print $zfh "# heads (aka `branches'):\n\$ " .
+		"git for-each-ref --sort=-creatordate refs/heads" .
+		" \\\n\t--format='%(HEAD) ". # no space for %(align:) hint
+		"%(refname:short) %(subject) (%(creatordate:short))'\n";
+	@r = split(/^/sm, shift(@x) // '');
+	$last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_branches};
+	for (@r) {
+		my ($pfx, $oid, $ref, $s, $cd) = split(/\0/);
+		utf8::decode($_) for ($ref, $s);
+		chomp $cd;
+		my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+		print $zfh "$pfx ", ascii_html($ref),
+			"$align ", ascii_html($s), " ($cd)\n";
+	}
+	print $zfh "# no heads (branches) yet...\n" if !@r;
+	print $zfh "...\n" if $last;
+	print $zfh "\n# tags:\n\$ " .
+		"git for-each-ref --sort=-creatordate refs/tags" .
+		" \\\n\t--format='". # no space for %(align:) hint
+		"%(refname:short) %(subject) (%(creatordate:short))'\n";
+	@r = split(/^/sm, shift(@x) // '');
+	$last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_tags};
+	for (@r) {
+		my (undef, $oid, $ref, $s, $cd) = split(/\0/);
+		utf8::decode($_) for ($ref, $s);
+		chomp $cd;
+		my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+		print $zfh "", ascii_html($ref),
+			"$align ", ascii_html($s), " ($cd)\n";
+	}
+	print $zfh "# no tags yet...\n" if !@r;
+	print $zfh "...\n" if $last;
+	$wcb->($ctx->html_done('
')); +} + +sub capture_refs ($$) { # psgi_qx callback to capture git-for-each-ref + git-log + my ($bref, $ctx) = @_; + my $qsp_err = delete $ctx->{-qsp_err}; + $ctx->{-each_refs} = $$bref; + summary_finish($ctx) if $ctx->{-readme}; +} + +sub set_readme { # git->cat_async callback + my ($bref, $oid, $type, $size, $ctx) = @_; + my $ref_path = shift @{$ctx->{-nr_readme_tries}}; # e.g. HEAD:README + if ($type eq 'blob' && !$ctx->{-readme}) { + $ctx->{-readme} = [ $bref, $oid, $ref_path ]; + } elsif (scalar @{$ctx->{-nr_readme_tries}} == 0) { + $ctx->{-readme} //= []; # nothing left to try + } # or try another README... + summary_finish($ctx) if $ctx->{-each_refs} && $ctx->{-readme}; +} + +sub summary { + my ($self, $ctx) = @_; + $ctx->{wcr} = $self; + my $nb = $self->{summary_branches} + 1; + my $nt = $self->{summary_tags} + 1; + my $nl = $self->{summary_log} + 1; + my $qsp = PublicInbox::Qspawn->new([qw(/bin/sh -c), + "$EACH_REF --count=$nb refs/heads; echo && " . + "$EACH_REF --count=$nt refs/tags; echo && " . + "git log -$nl --pretty=format:'%d %H %h %cs %s' --" ], + { GIT_DIR => $ctx->{git}->{git_dir} }); + $qsp->{qsp_err} = \($ctx->{-qsp_err} = ''); + my @try = qw(HEAD:README HEAD:README.md); # TODO: configurable + $ctx->{-nr_readme_tries} = [ @try ]; + $ctx->{git}->cat_async($_, \&set_readme, $ctx) for @try; + if ($ctx->{env}->{'pi-httpd.async'}) { + PublicInbox::GitAsyncCat::watch_cat($ctx->{git}); + } else { # synchronous + $ctx->{git}->cat_async_wait; + } + sub { # $_[0] => PublicInbox::HTTP::{Identity,Chunked} + $ctx->{env}->{'qspawn.wcb'} = $_[0]; + $qsp->psgi_qx($ctx->{env}, undef, \&capture_refs, $ctx); + } +} + +sub srv { # endpoint called by PublicInbox::WWW + my ($self, $ctx) = @_; + my $path_info = $ctx->{env}->{PATH_INFO}; + my $git; + # handle clone requests + if ($path_info =~ m!\A/(.+?)/($PublicInbox::GitHTTPBackend::ANY)\z!x) { + $git = $self->{"\0$1"} and return + PublicInbox::GitHTTPBackend::serve($ctx->{env},$git,$2); + } + $path_info =~ m!\A/(.+?)/\z! and + ($ctx->{git} = $self->{"\0$1"}) and return summary($self, $ctx); + if ($path_info =~ m!\A/(.+?)\z! and ($git = $self->{"\0$1"})) { + my $qs = $ctx->{env}->{QUERY_STRING}; + my $url = $git->base_url($ctx->{env}); + $url .= "?$qs" if $qs ne ''; + [ 301, [ Location => $url, 'Content-Type' => 'text/plain' ], + [ "Redirecting to $url\n" ] ]; + } else { + r(404); + } +} + +1; diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm index 16442d51..92d243eb 100644 --- a/lib/PublicInbox/WwwStream.pm +++ b/lib/PublicInbox/WwwStream.pm @@ -18,7 +18,7 @@ https://public-inbox.org/public-inbox.git) ]; sub base_url ($) { my $ctx = shift; - my $base_url = $ctx->{ibx}->base_url($ctx->{env}); + my $base_url = ($ctx->{ibx} // $ctx->{git})->base_url($ctx->{env}); chop $base_url; # no trailing slash for clone $base_url; } @@ -40,7 +40,7 @@ sub async_eml { # for async_blob_cb sub html_top ($) { my ($ctx) = @_; - my $ibx = $ctx->{ibx}; + my $ibx = $ctx->{ibx} // $ctx->{git}; my $desc = ascii_html($ibx->description); my $title = delete($ctx->{-title_html}) // $desc; my $upfx = $ctx->{-upfx} || ''; @@ -84,8 +84,11 @@ sub html_top ($) { ''. $top . (delete($ctx->{-html_tip}) // ''); } +sub inboxes { () } # TODO + sub coderepos ($) { my ($ctx) = @_; + $ctx->{ibx} // return inboxes($ctx); my $cr = $ctx->{ibx}->{coderepo} // return (); my $cfg = $ctx->{www}->{pi_cfg}; my $upfx = ($ctx->{-upfx} // ''). '../'; @@ -114,8 +117,8 @@ sub _html_end { my ($ctx) = @_; my $upfx = $ctx->{-upfx} || ''; my $m = "${upfx}_/text/mirror/"; - my $x; - if ($ctx->{ibx}->can('cloneurl')) { + my $x = ''; + if ($ctx->{ibx} && $ctx->{ibx}->can('cloneurl')) { $x = <mirroring instructions @@ -139,12 +142,15 @@ as well as URLs for IMAP folder(s). EOM } } - } else { + } elsif ($ctx->{ibx}) { # extindex $x = <mirroring instructions on how to clone and mirror all data and code used by this external index. EOF + } elsif ($ctx->{git}) { # coderepo + $x = join('', map { "git clone $_\n" } + @{$ctx->{git}->cloneurl($ctx->{env})}); } chomp $x; '
'.join("\n\n", coderepos($ctx), $x).'
' diff --git a/t/solver_git.t b/t/solver_git.t index e347c711..d6936c47 100644 --- a/t/solver_git.t +++ b/t/solver_git.t @@ -9,7 +9,9 @@ require_git(2.6); use PublicInbox::ContentHash qw(git_sha); use PublicInbox::Spawn qw(popen_rd); require_mods(qw(DBD::SQLite Search::Xapian Plack::Util)); -my $git_dir = xqx([qw(git rev-parse --git-dir)], undef, {2 => \(my $null)}); +my $rdr = { 2 => \(my $null) }; +my $git_dir = xqx([qw(git rev-parse --git-common-dir)], undef, $rdr); +$git_dir = xqx([qw(git rev-parse --git-dir)], undef, $rdr) if $? != 0; $? == 0 or plan skip_all => "$0 must be run from a git working tree"; chomp $git_dir; @@ -300,6 +302,35 @@ EOF is($res->code, 200, 'shows commit w/ utf8.eml'); like($res->content, qr/Eléanor/, 'UTF-8 commit shown properly'); + + # WwwCoderepo + my $olderr; + if (defined $ENV{PLACK_TEST_EXTERNALSERVER_URI}) { + ok(!-s "$tmpdir/stderr.log", + 'nothing in stderr.log, yet'); + } else { + open $olderr, '>&', \*STDERR or xbail "open: $!"; + open STDERR, '+>>', "$tmpdir/stderr.log" or + xbail "open: $!"; + } + $res = $cb->(GET('/binfoo/')); + defined($ENV{PLACK_TEST_EXTERNALSERVER_URI}) or + open STDERR, '>&', $olderr or xbail "open: $!"; + is($res->code, 200, 'coderepo summary (binfoo)'); + if (ok(-s "$tmpdir/stderr.log")) { + open my $fh, '<', "$tmpdir/stderr.log" or xbail $!; + my $s = do { local $/; <$fh> }; + open $fh, '>', "$tmpdir/stderr.log" or xbail $!; + ok($s =~ s/^fatal: your current branch.*?\n//sm, + 'got current branch warning'); + ok($s =~ s/^.*? exit status=[1-9]+ .*?\n//sm, + 'got exit status warning'); + is($s, '', 'no unexpected warnings on empty coderepo'); + } + $res = $cb->(GET('/public-inbox/')); + is($res->code, 200, 'coderepo summary (public-inbox)'); + $res = $cb->(GET('/public-inbox')); + is($res->code, 301, 'redirected'); }; test_psgi(sub { $www->call(@_) }, $client); my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };