From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 5435C1F8C6 for ; Sun, 12 Sep 2021 09:12:39 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] fetch: fix half-baked v1 manifest.js.gz handling Date: Sun, 12 Sep 2021 09:12:39 +0000 Message-Id: <20210912091239.20745-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: The v1 code path was totally half-baked after the change to use manifest.js.gz :x Fixes: ffb7fbda6869db4b ("fetch: use manifest.js.gz for v1") --- lib/PublicInbox/Fetch.pm | 15 ++++++++------- lib/PublicInbox/LeiMirror.pm | 18 +++++++++--------- t/lei-mirror.t | 8 ++++++++ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm index 9613a582..2c6ae86a 100644 --- a/lib/PublicInbox/Fetch.pm +++ b/lib/PublicInbox/Fetch.pm @@ -77,9 +77,9 @@ sub do_manifest ($$$) { my $t1 = $cur->{modified} // next; delete($mdiff->{$k}) if $f0 eq $f1 && $t0 == $t1; } - my ($path_pfx, $v1_bare, @v2_epochs) = + my (undef, $v1_path, @v2_epochs) = PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path); - [ 200, $path_pfx, $v1_bare, \@v2_epochs, $muri, $ft, $mf ]; + [ 200, $v1_path, \@v2_epochs, $muri, $ft, $mf ]; } sub do_fetch { @@ -117,7 +117,7 @@ EOM } $lei->qerr("# inbox URL: $ibx_uri/"); my $res = do_manifest($lei, $dir, $ibx_uri) or return; - my ($code, $path_pfx, $v1_bare, $v2_epochs, $muri, $ft, $mf) = @$res; + my ($code, $v1_path, $v2_epochs, $muri, $ft, $mf) = @$res; return if $code == 304; if ($code == 404) { # any pre-manifest.js.gz instances running? Just fetch all @@ -130,14 +130,13 @@ EOM $code == 200 or die "BUG unexpected code $code\n"; } if ($ibx_ver == 2) { - defined($v1_bare) and warn <, WTF? + defined($v1_path) and warn <, WTF? EOM @git_dir = map { "$dir/git/$_.git" } sort { $a <=> $b } map { my ($nr) = (m!/([0-9]+)\.git\z!g) } @$v2_epochs; } else { - $v1_bare eq $dir or warn "$v1_bare != $dir"; - $git_dir[0] = $v1_bare // $dir; + $git_dir[0] = $dir; } # n.b. this expects all epochs are from the same host my $torsocks = $lei->{curl}->torsocks($lei, $muri); @@ -150,6 +149,8 @@ EOM } else { my $e_uri = $ibx_uri->clone; my ($epath) = ($d =~ m!/(git/[0-9]+\.git)\z!); + defined($epath) or + die "BUG: $d is not an epoch to clone\n"; $e_uri->path($ibx_uri->path.$epath); $cmd = [ @$torsocks, PublicInbox::LeiMirror::clone_cmd($lei, $opt), diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 23813dcf..254848c9 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -238,15 +238,15 @@ failed to extract epoch number from $src # PSGI mount prefixes and manifest.js.gz prefixes don't always align... sub deduce_epochs ($$) { my ($m, $path) = @_; - my ($v1_bare, @v2_epochs); + my ($v1_ent, @v2_epochs); my $path_pfx = ''; $path =~ s!/+\z!!; do { - $v1_bare = $m->{$path}; + $v1_ent = $m->{$path}; @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m); - } while (!defined($v1_bare) && !@v2_epochs && + } while (!defined($v1_ent) && !@v2_epochs && $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1); - ($path_pfx, $v1_bare, @v2_epochs); + ($path_pfx, $v1_ent ? $path : undef, @v2_epochs); } sub decode_manifest ($$$) { @@ -282,20 +282,20 @@ sub try_manifest { return $lei->child_error($cerr, "@$cmd failed"); } my $m = decode_manifest($ft, $fn, $uri); - my ($path_pfx, $v1_bare, @v2_epochs) = deduce_epochs($m, $path); + my ($path_pfx, $v1_path, @v2_epochs) = deduce_epochs($m, $path); if (@v2_epochs) { # It may be possible to have v1 + v2 in parallel someday: - $lei->err(<err(<path($path_pfx.$_); $uri->clone } @v2_epochs; clone_v2($self, \@v2_epochs); - } elsif (defined $v1_bare) { + } elsif (defined $v1_path) { clone_v1($self); } else { die "E: confused by <$uri>, possible matches:\n\t", diff --git a/t/lei-mirror.t b/t/lei-mirror.t index 35b77cf7..7db49e15 100644 --- a/t/lei-mirror.t +++ b/t/lei-mirror.t @@ -110,6 +110,14 @@ SKIP: { ok(!-e "$d/t1/mirror.done", 'no leftover file'); ok(run_script([qw(-fetch -q -C), "$d/t1"], undef, $opt), 'fetching v1 works'); + unlink("$d/t1/manifest.js.gz") or xbail "unlink $!"; + my $before = [ glob("$d/t1/*") ]; + ok(run_script([qw(-fetch -q -C), "$d/t1"], undef, $opt), + 'fetching v1 works w/o manifest.js.gz'); + unlink("$d/t1/FETCH_HEAD"); # git internal + ok(unlink("$d/t1/manifest.js.gz"), 'manifest created'); + my $after = [ glob("$d/t1/*") ]; + is_deeply($before, $after, 'no new files created'); } ok($td->kill, 'killed -httpd');