From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.6 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 6DA1B1F624 for ; Sun, 12 Feb 2023 23:19:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1676243985; bh=ZS8t87xTOLUCchyUSLUXDsGXw6ZGG3hY3+gupeMdztE=; h=From:To:Subject:Date:In-Reply-To:References:From; b=0GHkcpkxnx07Bp4p18dx8NMdNUP49+4pRkjJdNHlEUlZEZHx8PUVHlZ0VQo3GvIgk qsnTZi7x3SdGP172ZmiKyxAjUFfYwGb8dub4r5uq6Kli2NazUgrb6Aoda1Mm+8Jgsu nG+bbgg/rOkOsyNTH8zUCQlKKyAl3NuPtz7ZAX9Q= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 1/2] lei_mirror: further reduce `git config' calls Date: Sun, 12 Feb 2023 23:18:27 +0000 Message-Id: <20230212231828.33336-2-e@80x24.org> In-Reply-To: <20230212231828.33336-1-e@80x24.org> References: <20230212231828.33336-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We can parse the config at once and avoid clobbering variables which do not need changing. We'll also do some prep work for fetch.hideRefs proposal being discussed at --- lib/PublicInbox/LeiMirror.pm | 62 +++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index d959b6b6..dd6356bb 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -401,23 +401,42 @@ sub fgrp_fetch_all { my $opt = {}; my @fetch = do { local $self->{lei}->{opt}->{jobs} = 1; - (fetch_args($self->{lei}, $opt), - qw(--no-tags --multiple)); + (fetch_args($self->{lei}, $opt), qw(--no-tags --multiple)); }; push(@fetch, "-j$j") if $j; while (my ($osdir, $fgrpv) = each %$todo) { my $f = "$osdir/config"; return if !keep_going($self); + my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ]; # clobber group from previous run atomically - my $cmd = ['git', "--git-dir=$osdir", qw(config -f), - $f, '--unset-all', "remotes.$grp"]; - $self->{lei}->qerr("# @$cmd"); - if (!$self->{dry_run}) { - my $pid = spawn($cmd, undef, { 2 => $self->{lei}->{2} }); + for ("remotes.$grp") { # TODO: hideRefs + my $c = [ @$cmd, '--unset-all', $_ ]; + $self->{lei}->qerr("# @$c"); + next if $self->{dry_run}; + my $pid = spawn($c, undef, $opt); waitpid($pid, 0) // die "waitpid: $!"; - die "E: @$cmd: \$?=$?" if ($? && ($? >> 8) != 5); + die "E: @$c \$?=$?" if ($? && ($? >> 8) != 5); + } + # permanent configs: + my $cfg = PublicInbox::Config->git_config_dump($f); + for my $fgrp (@$fgrpv) { + my $u = $fgrp->{-uri} // die 'BUG: no {-uri}'; + my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}'; + for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*", + 'tagopt=--no-tags') { + my ($k, $v) = split(/=/, $_, 2); + $k = "remote.$rn.$k"; + next if ($cfg->{$k} // '') eq $v; + my $c = [@$cmd, $k, $v]; + $fgrp->{lei}->qerr("# @$c"); + next if $fgrp->{dry_run}; + run_die($c, undef, $opt); + } + } + + if (!$self->{dry_run}) { # update the config atomically via O_APPEND while # respecting git-config locking sysopen(my $lk, "$f.lock", O_CREAT|O_EXCL|O_WRONLY) @@ -430,7 +449,6 @@ sub fgrp_fetch_all { close $fh or die "close($f): $!"; unlink("$f.lock") or die "unlink($f.lock): $!"; } - $cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ]; my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv); start_cmd($self, $cmd, $opt, $end); @@ -446,12 +464,15 @@ sub forkgroup_prep { my $dir = "$os/$fg.git"; if (!-d $dir && !$self->{dry_run}) { PublicInbox::Import::init_bare($dir); - my @cmd = ('git', "--git-dir=$dir", 'config'); - my $opt = { 2 => $self->{lei}->{2} }; - for ('repack.useDeltaIslands=true', - 'pack.island=refs/remotes/([^/]+)/') { - run_die([@cmd, split(/=/, $_, 2)], undef, $opt); - } + my $f = "$dir/config"; + open my $fh, '+>>', $f or die "open:($f): $!"; + print $fh <{-key} // die 'BUG: no -key'; my $rn = substr(sha256_hex($key), 0, 16); @@ -546,17 +567,6 @@ sub resume_fetch { sub fgrp_enqueue { my ($fgrp, $end) = @_; # $end calls fgrp_fetch_all return if !keep_going($fgrp); - my $opt = { 2 => $fgrp->{lei}->{2} }; - # --no-tags is required to avoid conflicts - my $u = $fgrp->{-uri} // die 'BUG: no {-uri}'; - my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}'; - my @cmd = ('git', "--git-dir=$fgrp->{-osdir}", 'config'); - for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*", 'tagopt=--no-tags') { - my @kv = split(/=/, $_, 2); - $kv[0] = "remote.$rn.$kv[0]"; - $fgrp->{dry_run} ? $fgrp->{lei}->qerr("# @cmd @kv") : - run_die([@cmd, @kv], undef, $opt); - } ++$fgrp->{chg}->{nr_chg}; push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp; }