From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 941A92017A for ; Mon, 28 Nov 2022 05:32:37 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613557; bh=tujzeMTh0FQb/2Bi6B5Mh+62bRUroeF5I+NFeOd5G14=; h=From:To:Subject:Date:In-Reply-To:References:From; b=FZcAdVStjcW5vsXDlG+KrIfImfyTOUJaIRPuL4nrpus4lLlK6ahFz/61+/Mjuyplo z6R0WdGAvrI3Iys/GNmVgLLCYrjuB5H3KySSLCyul+484ISyqhHjzpmjOwPfEpbWLP shFPsPWdifNTMdPhoQrZSMTuPnG/6NgAGsquL+4s= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 24/95] clone: support --inbox-config option Date: Mon, 28 Nov 2022 05:31:21 +0000 Message-Id: <20221128053232.291618-25-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This allows avoiding 404s when trying _/text/config/raw on code repositories. --- Documentation/public-inbox-clone.pod | 11 +++++++++++ lib/PublicInbox/LeiMirror.pm | 13 ++++++++++--- script/public-inbox-clone | 1 + 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod index 178d952a..52c89cfd 100644 --- a/Documentation/public-inbox-clone.pod +++ b/Documentation/public-inbox-clone.pod @@ -65,6 +65,17 @@ When cloning a top-level with multiple inboxes, ignore inboxes and repositories matching the given wildcard pattern. Supports the same wildcards as L +=item --inbox-config=always|v2|v1|never + +Whether or not to retrieve the C<$INBOX/_/text/config/raw> HTTP(S) +endpoint when cloning. + +Since we can't deduce v1 inboxes from code repositories, setting this +to C or C can allow faster clones of code repositories if +no v1 inboxes are present. + +Default: C + =item -n =item --dry-run diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index f81f6094..44d7a524 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -259,7 +259,8 @@ sub clone_v1 { push @$cmd, '--reference', "$self->{dst}$ref"; start_clone($self, $cmd, $opt, $fini); - _get_txt_start($self, '_/text/config/raw', $fini); + $lei->{opt}->{'inbox-config'} =~ /\A(?:always|v1)\z/s and + _get_txt_start($self, '_/text/config/raw', $fini); my $d = $self->{-ent} ? $self->{-ent}->{description} : undef; defined($d) ? ($self->{'txt.description'} = $d) : _get_txt_start($self, 'description', $fini); @@ -445,7 +446,9 @@ failed to extract epoch number from $src my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock'; my $fini = PublicInbox::OnDestroy->new($$, \&v2_done, $task); - _get_txt_start($task, '_/text/config/raw', $fini); + $lei->{opt}->{'inbox-config'} =~ /\A(?:always|v2)\z/s and + _get_txt_start($task, '_/text/config/raw', $fini); + _get_txt_start($self, 'description', $fini); $task->{-locked} = $lk->lock_for_scope($$) if !$self->{dry_run}; @@ -649,11 +652,15 @@ sub start_clone_url { die "TODO: non-HTTP/HTTPS clone of $self->{src} not supported, yet"; } -sub do_mirror { # via wq_io_do +sub do_mirror { # via wq_io_do or public-inbox-clone my ($self) = @_; my $lei = $self->{lei}; umask($lei->{client_umask}) if defined $lei->{client_umask}; eval { + my $ic = $lei->{opt}->{'inbox-config'} //= 'always'; + $ic =~ /\A(?:v1|v2|always|never)\z/s or die <<""; +--inbox-config must be one of `always', `v2', `v1', or `never' + my $iv = $lei->{opt}->{'inbox-version'}; if (defined $iv) { local $LIVE; diff --git a/script/public-inbox-clone b/script/public-inbox-clone index 22ffc0fc..3d980c97 100755 --- a/script/public-inbox-clone +++ b/script/public-inbox-clone @@ -23,6 +23,7 @@ options: -C DIR chdir to specified directory EOF GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@ include|I=s@ exclude=s@ + inbox-config=s dry-run|n jobs|j=i no-torsocks torsocks=s epoch=s)) or die $help; if ($opt->{help}) { print $help; exit }; require PublicInbox::Admin; # loads Config