From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.6 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id A87981F626 for ; Wed, 15 Feb 2023 22:20:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1676499623; bh=sT0Qma5uzwx7W8R5dXMwwj4bp42HSEc2or42yQBM+Aw=; h=From:To:Subject:Date:From; b=oH2jsN0iFDm22nqJyOI01VCjLKfo7O5tnCDTpeQLJ/T7y1qjrGVcuqzO8YJN6Nf2U mElcCrLzhKxtXat7VfG1QyhzpydrnaurefcgWWeOP8hs8j/+A3WBVX+UEQQ7GXj3Vu EYktjgVJMIHccb29tL3up0xt95K9aX+DNtomzR0U= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] lei_mirror: use fetch.hideRefs to speed up connectivity check Date: Wed, 15 Feb 2023 22:20:23 +0000 Message-Id: <20230215222023.3096908-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: `git fetch' runs an expensive connectivity check against all refs, which is unnecessarily expensive for incremental fetches on RAM-constrained systems. This depends on the proposal to support `fetch.hideRefs' for `git fetch': https://public-inbox.org/git/20230212090426.M558990@dcvr/ --- I'll commit this if it lands in git.git lib/PublicInbox/LeiMirror.pm | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 4dedac9b..95486f37 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -407,13 +407,13 @@ sub fgrp_fetch_all { while (my ($osdir, $fgrp_old_new) = each %$todo) { my $f = "$osdir/config"; return if !keep_going($self); - my ($fgrpv, $new) = @$fgrp_old_new; - @$fgrpv = sort { $b->{-sort} <=> $a->{-sort} } @$fgrpv; - push @$fgrpv, @$new; # $new is ordered by references - + my ($old, $new) = @$fgrp_old_new; + @$old = sort { $b->{-sort} <=> $a->{-sort} } @$old; + # $new is ordered by {references} my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ]; - # clobber group from previous run atomically - for ("remotes.$grp") { # TODO: hideRefs + + # clobber settings from previous run atomically + for ("remotes.$grp", 'fetch.hideRefs') { my $c = [ @$cmd, '--unset-all', $_ ]; $self->{lei}->qerr("# @$c"); next if $self->{dry_run}; @@ -424,7 +424,7 @@ sub fgrp_fetch_all { # permanent configs: my $cfg = PublicInbox::Config->git_config_dump($f); - for my $fgrp (@$fgrpv) { + for my $fgrp (@$old, @$new) { my $u = $fgrp->{-uri} // die 'BUG: no {-uri}'; my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}'; for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*", @@ -446,14 +446,24 @@ sub fgrp_fetch_all { or die "open($f.lock): $!"; open my $fh, '>>', $f or die "open(>>$f): $!"; $fh->autoflush(1); - my $buf = join('', "[remotes]\n", - map { "\t$grp = $_->{-remote}\n" } @$fgrpv); + my $buf = ''; + if (@$old) { + $buf = "[fetch]\n\thideRefs = refs\n"; + $buf .= join('', map { + "\thideRefs = !refs/remotes/" . + "$_->{-remote}/\n"; + } @$old); + } + $buf .= join('', "[remotes]\n", + (map { "\t$grp = $_->{-remote}\n" } @$old), + (map { "\t$grp = $_->{-remote}\n" } @$new)); print $fh $buf or die "print($f): $!"; close $fh or die "close($f): $!"; unlink("$f.lock") or die "unlink($f.lock): $!"; } - $cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ]; - my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv); + $cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ]; + push @$old, @$new; + my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $old); start_cmd($self, $cmd, $opt, $end); } }