From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 10B021FA18 for ; Mon, 23 Nov 2020 07:06:03 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 03/12] git: add manifest_entry method Date: Mon, 23 Nov 2020 07:05:53 +0000 Message-Id: <20201123070602.9698-4-e@80x24.org> In-Reply-To: <20201123070602.9698-1-e@80x24.org> References: <20201123070602.9698-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We'll be using this for MiscIdx and pre-generating the necessary JSON for manifest.js.gz, so make it easier to share code for generating per-repo JSON entries for grokmirror. --- lib/PublicInbox/Git.pm | 53 +++++++++++++++++++++++++++++ lib/PublicInbox/ManifestJsGz.pm | 59 ++------------------------------- t/www_listing.t | 5 ++- 3 files changed, 58 insertions(+), 59 deletions(-) diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm index 86343ac9..917fa4a1 100644 --- a/lib/PublicInbox/Git.pm +++ b/lib/PublicInbox/Git.pm @@ -14,10 +14,12 @@ use POSIX (); use IO::Handle; # ->autoflush use Errno qw(EINTR); use File::Glob qw(bsd_glob GLOB_NOSORT); +use File::Spec (); use Time::HiRes qw(stat); use PublicInbox::Spawn qw(popen_rd); use PublicInbox::Tmpfile; use Carp qw(croak); +use Digest::SHA (); our @EXPORT_OK = qw(git_unquote git_quote); our $PIPE_BUFSIZ = 65536; # Linux default our $in_cleanup; @@ -475,6 +477,57 @@ sub modified ($) { $modified || time; } +# for grokmirror, which doesn't read gitweb.description +# templates/hooks--update.sample and git-multimail in git.git +# only match "Unnamed repository", not the full contents of +# templates/this--description in git.git +sub manifest_entry { + my ($self, $epoch, $default_desc) = @_; + my ($fh, $pid) = $self->popen('show-ref'); + my $dig = Digest::SHA->new(1); + while (read($fh, my $buf, 65536)) { + $dig->add($buf); + } + close $fh; + waitpid($pid, 0); + return if $?; # empty, uninitialized git repo + my $git_dir = $self->{git_dir}; + my $ent = { + fingerprint => $dig->hexdigest, + reference => undef, + modified => modified($self), + }; + chomp(my $owner = $self->qx('config', 'gitweb.owner')); + utf8::decode($owner); + $ent->{owner} = $owner eq '' ? undef : $owner; + my $desc = ''; + if (open($fh, '<', "$git_dir/description")) { + local $/ = "\n"; + chomp($desc = <$fh>); + utf8::decode($desc); + } + $desc = 'Unnamed repository' if $desc eq ''; + if (defined $epoch && $desc =~ /\AUnnamed repository/) { + $desc = "$default_desc [epoch $epoch]"; + } + $ent->{description} = $desc; + if (open($fh, '<', "$git_dir/objects/info/alternates")) { + # n.b.: GitPython doesn't seem to handle comments or C-quoted + # strings like native git does; and we don't for now, either. + local $/ = "\n"; + chomp(my @alt = <$fh>); + + # grokmirror only supports 1 alternate for "reference", + if (scalar(@alt) == 1) { + my $objdir = "$git_dir/objects"; + my $ref = File::Spec->rel2abs($alt[0], $objdir); + $ref =~ s!/[^/]+/?\z!!; # basename + $ent->{reference} = $ref; + } + } + $ent; +} + 1; __END__ =pod diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm index ab1478af..3d8a38ae 100644 --- a/lib/PublicInbox/ManifestJsGz.pm +++ b/lib/PublicInbox/ManifestJsGz.pm @@ -6,15 +6,12 @@ package PublicInbox::ManifestJsGz; use strict; use v5.10.1; use parent qw(PublicInbox::WwwListing); -use Digest::SHA (); -use File::Spec (); use bytes (); # length use PublicInbox::Inbox; use PublicInbox::Config; use PublicInbox::Git; use IO::Compress::Gzip qw(gzip); use HTTP::Date qw(time2str); -*try_cat = \&PublicInbox::Inbox::try_cat; our $json = PublicInbox::Config::json(); @@ -26,21 +23,6 @@ sub url_regexp { $ctx->SUPER::url_regexp('publicInbox.grokManifest', 'match=domain'); } -sub fingerprint ($) { - my ($git) = @_; - # TODO: convert to qspawn for fairness when there's - # thousands of repos - my ($fh, $pid) = $git->popen('show-ref'); - my $dig = Digest::SHA->new(1); - while (read($fh, my $buf, 65536)) { - $dig->add($buf); - } - close $fh; - waitpid($pid, 0); - return if $?; # empty, uninitialized git repo - $dig->hexdigest; -} - sub manifest_add ($$;$$) { my ($ctx, $ibx, $epoch, $default_desc) = @_; my $url_path = "/$ibx->{name}"; @@ -51,48 +33,13 @@ sub manifest_add ($$;$$) { } return unless -d $git_dir; my $git = PublicInbox::Git->new($git_dir); - my $fingerprint = fingerprint($git) or return; # no empty repos - - chomp(my $owner = $git->qx('config', 'gitweb.owner')); - chomp(my $desc = try_cat("$git_dir/description")); - utf8::decode($owner); - utf8::decode($desc); - $owner = undef if $owner eq ''; - $desc = 'Unnamed repository' if $desc eq ''; - - # templates/hooks--update.sample and git-multimail in git.git - # only match "Unnamed repository", not the full contents of - # templates/this--description in git.git - if ($desc =~ /\AUnnamed repository/) { - $desc = "$default_desc [epoch $epoch]" if defined($epoch); - } - - my $reference; - chomp(my $alt = try_cat("$git_dir/objects/info/alternates")); - if ($alt) { - # n.b.: GitPython doesn't seem to handle comments or C-quoted - # strings like native git does; and we don't for now, either. - my @alt = split(/\n+/, $alt); - - # grokmirror only supports 1 alternate for "reference", - if (scalar(@alt) == 1) { - my $objdir = "$git_dir/objects"; - $reference = File::Spec->rel2abs($alt[0], $objdir); - $reference =~ s!/[^/]+/?\z!!; # basename - } - } + my $ent = $git->manifest_entry($epoch, $default_desc) or return; $ctx->{-abs2urlpath}->{$git_dir} = $url_path; - my $modified = $git->modified; + my $modified = $ent->{modified}; if ($modified > ($ctx->{-mtime} // 0)) { $ctx->{-mtime} = $modified; } - $ctx->{manifest}->{$url_path} = { - owner => $owner, - reference => $reference, - description => $desc, - modified => $modified, - fingerprint => $fingerprint, - }; + $ctx->{manifest}->{$url_path} = $ent; } sub ibx_entry { diff --git a/t/www_listing.t b/t/www_listing.t index 4309a5e1..63613371 100644 --- a/t/www_listing.t +++ b/t/www_listing.t @@ -21,8 +21,7 @@ use_ok 'PublicInbox::Git'; my ($tmpdir, $for_destroy) = tmpdir(); my $bare = PublicInbox::Git->new("$tmpdir/bare.git"); PublicInbox::Import::init_bare($bare->{git_dir}); -is(PublicInbox::ManifestJsGz::fingerprint($bare), undef, - 'empty repo has no fingerprint'); +is($bare->manifest_entry, undef, 'empty repo has no manifest entry'); { my $fi_data = './t/git.fast-import-data'; open my $fh, '<', $fi_data or die "open $fi_data: $!"; @@ -31,7 +30,7 @@ is(PublicInbox::ManifestJsGz::fingerprint($bare), undef, 'fast-import'); } -like(PublicInbox::ManifestJsGz::fingerprint($bare), qr/\A[a-f0-9]{40}\z/, +like($bare->manifest_entry->{fingerprint}, qr/\A[a-f0-9]{40}\z/, 'got fingerprint with non-empty repo'); sub tiny_test {