unofficial mirror of meta@public-inbox.org
 help / color / mirror / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH] lei_mirror: use fetch.hideRefs to speed up connectivity check
Date: Wed, 15 Feb 2023 22:20:23 +0000	[thread overview]
Message-ID: <20230215222023.3096908-1-e@80x24.org> (raw)

`git fetch' runs an expensive connectivity check against all
refs, which is unnecessarily expensive for incremental fetches
on RAM-constrained systems.

This depends on the proposal to support `fetch.hideRefs' for `git fetch':
https://public-inbox.org/git/20230212090426.M558990@dcvr/
---
 I'll commit this if it lands in git.git

 lib/PublicInbox/LeiMirror.pm | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 4dedac9b..95486f37 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -407,13 +407,13 @@ sub fgrp_fetch_all {
 	while (my ($osdir, $fgrp_old_new) = each %$todo) {
 		my $f = "$osdir/config";
 		return if !keep_going($self);
-		my ($fgrpv, $new) = @$fgrp_old_new;
-		@$fgrpv = sort { $b->{-sort} <=> $a->{-sort} } @$fgrpv;
-		push @$fgrpv, @$new; # $new is ordered by references
-
+		my ($old, $new) = @$fgrp_old_new;
+		@$old = sort { $b->{-sort} <=> $a->{-sort} } @$old;
+		# $new is ordered by {references}
 		my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ];
-		# clobber group from previous run atomically
-		for ("remotes.$grp") { # TODO: hideRefs
+
+		# clobber settings from previous run atomically
+		for ("remotes.$grp", 'fetch.hideRefs') {
 			my $c = [ @$cmd, '--unset-all', $_ ];
 			$self->{lei}->qerr("# @$c");
 			next if $self->{dry_run};
@@ -424,7 +424,7 @@ sub fgrp_fetch_all {
 
 		# permanent configs:
 		my $cfg = PublicInbox::Config->git_config_dump($f);
-		for my $fgrp (@$fgrpv) {
+		for my $fgrp (@$old, @$new) {
 			my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
 			my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
 			for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*",
@@ -446,14 +446,24 @@ sub fgrp_fetch_all {
 				or die "open($f.lock): $!";
 			open my $fh, '>>', $f or die "open(>>$f): $!";
 			$fh->autoflush(1);
-			my $buf = join('', "[remotes]\n",
-				map { "\t$grp = $_->{-remote}\n" } @$fgrpv);
+			my $buf = '';
+			if (@$old) {
+				$buf = "[fetch]\n\thideRefs = refs\n";
+				$buf .= join('', map {
+					"\thideRefs = !refs/remotes/" .
+						"$_->{-remote}/\n";
+				} @$old);
+			}
+			$buf .= join('', "[remotes]\n",
+				(map { "\t$grp = $_->{-remote}\n" } @$old),
+				(map { "\t$grp = $_->{-remote}\n" } @$new));
 			print $fh $buf or die "print($f): $!";
 			close $fh or die "close($f): $!";
 			unlink("$f.lock") or die "unlink($f.lock): $!";
 		}
-		$cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ];
-		my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv);
+		$cmd  = [ @git, "--git-dir=$osdir", @fetch, $grp ];
+		push @$old, @$new;
+		my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $old);
 		start_cmd($self, $cmd, $opt, $end);
 	}
 }

             reply	other threads:[~2023-02-15 22:20 UTC|newest]

Thread overview: 2+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-02-15 22:20 Eric Wong [this message]
2023-03-23  8:33 ` [PATCH] lei_mirror: use fetch.hideRefs to speed up connectivity check Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20230215222023.3096908-1-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).