From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 9DA8C203BE for ; Mon, 28 Nov 2022 05:32:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613569; bh=4aaN/riJnjdNXz8tQ1JpOORq7v1Mz7te+zuA59TDQfE=; h=From:To:Subject:Date:In-Reply-To:References:From; b=aCwz5kXulSNIRaYWgckkNMRDTRthcrHb/xrLSqkcEF5yDkEM48LgmmLaA0uSV1ao2 BKeCRGs/zcm6Qj6inl9p7ItWTiu/K0n5XRMHV2UKlxPPl0yLzFu0vyyoY4omxhW4bj H1/vp8sG+unPS4outjxrHjslSOtMlLRIMxCzjQV0= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 79/95] lei_mirror: update fingerprints when writing local manifest.js.gz Date: Mon, 28 Nov 2022 05:32:16 +0000 Message-Id: <20221128053232.291618-80-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We need our local manifest to match the actual data we store, not what we're mirroring. --- lib/PublicInbox/LeiMirror.pm | 70 +++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 2e94a2fa..b982f919 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -692,17 +692,46 @@ sub reap_cmd { # async, called via SIGCHLD $self->{lei}->child_error($cerr, "@$cmd failed (\$?=$cerr)") if $cerr; } +sub up_fp_done { + my ($self) = @_; + return if !keep_going($self); + my $fh = delete $self->{-show_ref_up} // die 'BUG: no show-ref output'; + seek($fh, SEEK_SET, 0) or die "seek(show_ref): $!"; + $self->{-ent} // die 'BUG: no -ent'; + my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint'; + my $B = sha1_hex(do { local $/; <$fh> } // die("read(show_ref): $!")); + return if $A eq $B; + $self->{-ent}->{fingerprint} = $B; + push @{$self->{chg}->{fp_mismatch}}, $self->{-key}; +} + +sub update_ent { + my ($self) = @_; + my $key = $self->{-key} // die 'BUG: no -key'; + my $new = $self->{-ent}->{fingerprint}; + my $cur = $self->{-local_manifest}->{$key}->{fingerprint} // "\0"; + my $dst = $self->{cur_dst} // $self->{dst}; + if (defined($new) && $new ne $cur) { + my $cmd = ['git', "--git-dir=$dst", 'show-ref']; + my $opt = { 2 => $self->{lei}->{2} }; + open($opt->{1}, '+>', undef) or die "open(tmp): $!"; + $self->{-show_ref_up} = $opt->{1}; + my $done = PublicInbox::OnDestroy->new($$, \&up_fp_done, $self); + start_cmd($self, $cmd, $opt, $done); + } + $new = $self->{-ent}->{owner} // return; + $cur = $self->{-local_manifest}->{$key}->{owner} // "\0"; + return if $cur eq $new; + my $cmd = [ qw(git config -f), "$dst/config", 'gitweb.owner', $new ]; + start_cmd($self, $cmd, { 2 => $self->{lei}->{2} }); +} + sub v1_done { # called via OnDestroy my ($self) = @_; return if $self->{dry_run} || !keep_going($self); _write_inbox_config($self); my $dst = $self->{cur_dst} // $self->{dst}; - if (defined(my $o = $self->{-ent} ? $self->{-ent}->{owner} : undef)) { - my $key = $self->{-key} // die 'BUG: no -key'; - my $cur = $self->{-local_manifest}->{$key}->{owner} // "\0"; - $cur eq $o or run_die([qw(git config -f), - "$dst/config", 'gitweb.owner', $o]); - } + update_ent($self) if $self->{-ent}; my $o = "$dst/objects"; if (open(my $fh, '<', my $fn = "$o/info/alternates")) {; my $base = File::Spec->rel2abs($o); @@ -797,7 +826,7 @@ failed to extract epoch number from $src } } # filter out the epochs we skipped - $self->{-culled_manifest} = 1 if $m && delete(@$m{@skip}); + $self->{chg}->{manifest} = 1 if $m && delete(@$m{@skip}); (!$self->{dry_run} && !-d $dst) and File::Path::mkpath($dst); @@ -854,8 +883,8 @@ sub multi_inbox ($$$) { $self->{lei}->glob2re($_) // qr/\A\Q$_\E/ } @$incl).'\\z)'; my @gone = delete @$v2{grep(!/$re/, keys %$v2)}; - delete @$m{map { @$_ } @gone} and $self->{-culled_manifest} = 1; - delete @$m{grep(!/$re/, @v1)} and $self->{-culled_manifest} = 1; + delete @$m{map { @$_ } @gone} and $self->{chg}->{manifest} = 1; + delete @$m{grep(!/$re/, @v1)} and $self->{chg}->{manifest} = 1; @v1 = grep(/$re/, @v1); } if (defined $excl) { @@ -863,8 +892,8 @@ sub multi_inbox ($$$) { $self->{lei}->glob2re($_) // qr/\A\Q$_\E/ } @$excl).'\\z)'; my @gone = delete @$v2{grep(/$re/, keys %$v2)}; - delete @$m{map { @$_ } @gone} and $self->{-culled_manifest} = 1; - delete @$m{grep(/$re/, @v1)} and $self->{-culled_manifest} = 1; + delete @$m{map { @$_ } @gone} and $self->{chg}->{manifest} = 1; + delete @$m{grep(/$re/, @v1)} and $self->{chg}->{manifest} = 1; @v1 = grep(!/$re/, @v1); } my $ret; # { v1 => [ ... ], v2 => { "/$inbox_name" => [ epochs ] }} @@ -963,6 +992,7 @@ sub try_manifest { } my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', %opt); my $cmd = $curl->for_uri($lei, $uri, qw(-f -R -o), $ft->filename); + my $mf_url = "$uri"; %opt = map { $_ => $lei->{$_} } (0..2); my $cerr = run_reap($lei, $cmd, \%opt); if ($cerr) { @@ -974,6 +1004,7 @@ sub try_manifest { warn $@; return try_scrape($self); } + local $self->{chg} = {}; local $self->{-local_manifest} = load_current_manifest($self); my ($path_pfx, $n, $multi) = multi_inbox($self, \$path, $m); return $lei->child_error(1, $multi) if !ref($multi); @@ -1034,7 +1065,22 @@ EOM return if $self->{dry_run} || !keep_going($self); # set by clone_v2_prep/-I/--exclude - dump_manifest($m => $ft) if delete $self->{-culled_manifest}; + my $mis = delete $self->{chg}->{fp_mismatch}; + if ($mis) { + my $t = (stat($ft))[9]; + require POSIX; + $t = POSIX::strftime('%Y-%m-%d %k:%M:%S %z', localtime($t)); + warn <{lei}->{opt}->{prune}; +W: The above fingerprints may never match without --prune +EOM + } + dump_manifest($m => $ft) if delete($self->{chg}->{manifest}) || $mis; ft_rename($ft, $manifest, 0666); open my $x, '>', "$self->{dst}/mirror.done"; # for _wq_done_wait }