* [PATCH 1/5] fetch: support --exit-code switch
2021-09-15 21:35 [PATCH 0/5] make(1) inspired things Eric Wong
@ 2021-09-15 21:35 ` Eric Wong
2021-09-15 21:35 ` [PATCH 2/5] TODO: update some items Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2021-09-15 21:35 UTC (permalink / raw)
To: meta
As noted in the new manpage entry, this is useful for avoiding
public-inbox-index invocations when there's nothing to update.
We use 127 to match "grok-pull", and also because it doesn't
conflict with any of the current curl(1) exit codes.
---
Documentation/public-inbox-fetch.pod | 30 ++++++++++++++++++++++++++--
lib/PublicInbox/Fetch.pm | 21 ++++++++++++++++++-
script/public-inbox-fetch | 4 +++-
t/lei-mirror.t | 12 +++++++----
t/v2mirror.t | 3 ++-
5 files changed, 61 insertions(+), 9 deletions(-)
diff --git a/Documentation/public-inbox-fetch.pod b/Documentation/public-inbox-fetch.pod
index 7944fdcd..28d5638d 100644
--- a/Documentation/public-inbox-fetch.pod
+++ b/Documentation/public-inbox-fetch.pod
@@ -4,7 +4,7 @@ public-inbox-fetch - "git fetch" wrapper for v2 inbox mirrors
=head1 SYNOPSIS
-public-inbox-fetch -C INBOX_DIR
+public-inbox-fetch [--exit-code] -C INBOX_DIR
=head1 DESCRIPTION
@@ -31,6 +31,15 @@ file to speed up future invocations.
Quiets down progress messages, also passed to L<git-fetch(1)>.
+=item --exit-code
+
+Exit with C<127> if no updates are done. This can be used in
+shell scripts to avoid invoking L<public-inbox-index(1)> when
+there are no updates:
+
+ public-inbox-fetch -q --exit-code && public-inbox-index
+ test $? -eq 0 || exit $?
+
=item -v
=item --verbose
@@ -45,6 +54,23 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
Default: C<auto>
+=back
+
+=head1 EXIT CODES
+
+=over
+
+=item 127
+
+no updates when L</--exit-code> is used above
+
+=back
+
+public-inbox-fetch will also exit with curl L<curl(1)/EXIT CODES>
+as documented in the L<curl(1)> manpage (e.g. C<7> when curl cannot
+reach a host). Likewise, L<git-fetch(1)> failures are also
+propagated to the user.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -60,4 +86,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<public-inbox-index(1)>
+L<public-inbox-index(1)>, L<curl(1)>
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 9ea55e9d..0539fe50 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -75,12 +75,22 @@ sub do_manifest ($$$) {
my $t1 = $cur->{modified} // next;
delete($mdiff->{$k}) if $f0 eq $f1 && $t0 == $t1;
}
- return unless keys %$mdiff;
+ unless (keys %$mdiff) {
+ $lei->child_error(127 << 8) if $lei->{opt}->{'exit-code'};
+ return;
+ }
my (undef, $v1_path, @v2_epochs) =
PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
[ 200, $v1_path, \@v2_epochs, $muri, $ft, $mf ];
}
+sub get_fingerprint2 {
+ my ($git_dir) = @_;
+ require Digest::SHA;
+ my $rd = popen_rd([qw(git show-ref)], undef, { -C => $git_dir });
+ Digest::SHA::sha256(do { local $/; <$rd> });
+}
+
sub do_fetch {
my ($cls, $lei, $cd) = @_;
my $ibx_ver;
@@ -136,11 +146,14 @@ EOM
}
# n.b. this expects all epochs are from the same host
my $torsocks = $lei->{curl}->torsocks($lei, $muri);
+ my $fp2 = $lei->{opt}->{'exit-code'} ? [] : undef;
+ my $xit = 127;
for my $d (@git_dir) {
my $cmd;
my $opt = {}; # for spawn
if (-d $d) {
$opt->{-C} = $d;
+ $fp2->[0] = get_fingerprint2($d) if $fp2;
$cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
} else {
my $e_uri = $ibx_uri->clone;
@@ -152,6 +165,7 @@ EOM
PublicInbox::LeiMirror::clone_cmd($lei, $opt),
$$e_uri, $d];
push @new_epoch, substr($epath, 5, -4) + 0;
+ $xit = 0;
}
my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
# do not bail on clone failure if we didn't have a manifest
@@ -159,6 +173,10 @@ EOM
$lei->child_error($cerr, "@$cmd failed");
return;
}
+ if ($fp2 && $xit) {
+ $fp2->[1] = get_fingerprint2($d);
+ $xit = 0 if $fp2->[0] ne $fp2->[1];
+ }
}
for my $i (@new_epoch) { $mg->epoch_cfg_set($i) }
if ($ft) {
@@ -166,6 +184,7 @@ EOM
rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";
$ft->unlink_on_destroy(0);
}
+ $lei->child_error($xit << 8) if $fp2 && $xit;
}
1;
diff --git a/script/public-inbox-fetch b/script/public-inbox-fetch
index 5d303574..d7d4ba47 100755
--- a/script/public-inbox-fetch
+++ b/script/public-inbox-fetch
@@ -16,12 +16,13 @@ options:
--torsocks VAL whether or not to wrap git and curl commands with
torsocks (default: `auto')
Must be one of: `auto', `no' or `yes'
+ --exit-code exit with 127 if no updates
--verbose | -v increase verbosity (may be repeated)
--quiet | -q increase verbosity (may be repeated)
-C DIR chdir to specified directory
EOF
GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@
- no-torsocks torsocks=s)) or die $help;
+ no-torsocks torsocks=s exit-code)) or die $help;
if ($opt->{help}) { print $help; exit };
require PublicInbox::Fetch; # loads Admin
PublicInbox::Admin::do_chdir(delete $opt->{C});
@@ -33,3 +34,4 @@ my $lei = bless {
0 => *STDIN{GLOB}, 1 => *STDOUT{GLOB}, 2 => *STDERR{GLOB},
}, 'PublicInbox::LEI';
PublicInbox::Fetch->do_fetch($lei, '.');
+exit(($lei->{child_error} // 0) >> 8);
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 5238b67c..9fdda5aa 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -111,12 +111,14 @@ SKIP: {
'all.git alternates created');
ok(-f "$d/t2/manifest.js.gz", 'manifest saved');
ok(!-e "$d/t2/mirror.done", 'no leftover mirror.done');
- ok(run_script([qw(-fetch -C), "$d/t2"], undef, $opt),
+ ok(!run_script([qw(-fetch --exit-code -C), "$d/t2"], undef, $opt),
'-fetch succeeds w/ manifest.js.gz');
+ is($? >> 8, 127, '--exit-code gave 127');
unlike($err, qr/git fetch/, 'no fetch done w/ manifest');
unlink("$d/t2/manifest.js.gz") or xbail "unlink $!";
- ok(run_script([qw(-fetch -C), "$d/t2"], undef, $opt),
+ ok(!run_script([qw(-fetch --exit-code -C), "$d/t2"], undef, $opt),
'-fetch succeeds w/o manifest.js.gz');
+ is($? >> 8, 127, '--exit-code gave 127');
like($err, qr/git fetch/, 'fetch forced w/o manifest');
ok(run_script([qw(-clone -q -C), $d, "$http/t1"], undef, $opt),
@@ -124,13 +126,15 @@ SKIP: {
ok(-d "$d/t1", 'v1 cloned');
ok(!-e "$d/t1/mirror.done", 'no leftover file');
ok(-f "$d/t1/manifest.js.gz", 'manifest saved');
- ok(run_script([qw(-fetch -C), "$d/t1"], undef, $opt),
+ ok(!run_script([qw(-fetch --exit-code -C), "$d/t1"], undef, $opt),
'fetching v1 works');
+ is($? >> 8, 127, '--exit-code gave 127');
unlike($err, qr/git fetch/, 'no fetch done w/ manifest');
unlink("$d/t1/manifest.js.gz") or xbail "unlink $!";
my $before = [ glob("$d/t1/*") ];
- ok(run_script([qw(-fetch -C), "$d/t1"], undef, $opt),
+ ok(!run_script([qw(-fetch --exit-code -C), "$d/t1"], undef, $opt),
'fetching v1 works w/o manifest.js.gz');
+ is($? >> 8, 127, '--exit-code gave 127');
unlink("$d/t1/FETCH_HEAD"); # git internal
like($err, qr/git fetch/, 'no fetch done w/ manifest');
ok(unlink("$d/t1/manifest.js.gz"), 'manifest created');
diff --git a/t/v2mirror.t b/t/v2mirror.t
index 54ad6945..3df5d053 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -98,8 +98,9 @@ $ibx->cleanup;
my @new_epochs;
my $fetch_each_epoch = sub {
my %before = map { $_ => 1 } glob("$tmpdir/m/git/*");
- run_script([qw(-fetch -q)], undef, {-C => "$tmpdir/m"}) or
+ run_script([qw(-fetch --exit-code -q)], undef, {-C => "$tmpdir/m"}) or
xbail '-fetch fail';
+ is($?, 0, '--exit-code 0 after fetch updated');
my @after = grep { !$before{$_} } glob("$tmpdir/m/git/*");
push @new_epochs, @after;
};
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 2/5] TODO: update some items
2021-09-15 21:35 [PATCH 0/5] make(1) inspired things Eric Wong
2021-09-15 21:35 ` [PATCH 1/5] fetch: support --exit-code switch Eric Wong
@ 2021-09-15 21:35 ` Eric Wong
2021-09-15 21:35 ` [PATCH 3/5] clone|fetch|--mirror: add convenience $INBOX_DIR/Makefile Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2021-09-15 21:35 UTC (permalink / raw)
To: meta
Some things (extindex) done, many more to do; and -fetch should
set us up for v1 => v2 migrations at some point...
---
TODO | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
diff --git a/TODO b/TODO
index a3ab2daf..f14e6c97 100644
--- a/TODO
+++ b/TODO
@@ -32,7 +32,7 @@ all need to be considered for everything we introduce)
archive locations to avoid SPOF.
* optional Cache::FastMmap support so production deployments won't
- need Varnish (Varnish doesn't protect NNTP or IMAP, either)
+ need Varnish (Varnish doesn't protect NNTP nor IMAP, either)
* dogfood and take advantage of new kernel APIs (while maintaining
portability to older Linux, free BSDs and maybe Hurd).
@@ -111,7 +111,8 @@ all need to be considered for everything we introduce)
* code repository integration (cgit: done, TODO: gitweb, etc...)
-* migration path to v2 without breaking v1 "git fetch" cronjobs
+* migration path to v2 (making it transparent for "git fetch"
+ may not be possible, but "public-inbox-fetch" will handle it)
* imperfect scraper importers for obfuscated list archives
(e.g. obfuscated Mailman stuff, Google Groups, etc...)
@@ -124,22 +125,14 @@ all need to be considered for everything we introduce)
as extensions. If JMAP, it should have HTTP(S) analogues to
various IMAP extensions.
-* search across multiple inboxes, or admin-definable groups of inboxes
-
- This will require a new detached Xapian index that can be used in
- parallel with existing per-inbox indices. Using ->add_database
- with hundreds of shards is unusable in current Xapian as of
- August 2020 (acknowledged by Xapian upstream).
-
* scalability to tens/hundreds of thousands of inboxes
- - pagination for WwwListing
-
- inotify-based manifest.js.gz updates
...
* lei - see %CMD in lib/PublicInbox/LEI.pm
+ (there's a truckload here..)
* make "git cat-file --batch" detect unlinked packfiles so we don't
have to restart processes (very long-term)
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 3/5] clone|fetch|--mirror: add convenience $INBOX_DIR/Makefile
2021-09-15 21:35 [PATCH 0/5] make(1) inspired things Eric Wong
2021-09-15 21:35 ` [PATCH 1/5] fetch: support --exit-code switch Eric Wong
2021-09-15 21:35 ` [PATCH 2/5] TODO: update some items Eric Wong
@ 2021-09-15 21:35 ` Eric Wong
2021-09-15 22:45 ` Kyle Meyer
2021-09-15 21:35 ` [PATCH 4/5] fetch|clone|--mirror: shorten paths for progress output Eric Wong
2021-09-15 21:35 ` [PATCH 5/5] support -C (chdir) for most non-daemon commands Eric Wong
4 siblings, 1 reply; 8+ messages in thread
From: Eric Wong @ 2021-09-15 21:35 UTC (permalink / raw)
To: meta
Since the beginning of time, I've been dropping Makefiles
in $INBOX_DIR (and above hiearchies) to organize groups
of commands.
make(1) is widely available in various flavors and a familiar
tool for our target audience. It is easy to run in the right
directory, typically has built-in shell completion, and doesn't
silently ignore errors by default like Bourne shell.
---
lib/PublicInbox/Fetch.pm | 3 +-
lib/PublicInbox/LeiMirror.pm | 61 +++++++++++++++++++++++++++++++++++-
t/lei-mirror.t | 2 ++
3 files changed, 64 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 0539fe50..184b4f26 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -91,7 +91,7 @@ sub get_fingerprint2 {
Digest::SHA::sha256(do { local $/; <$rd> });
}
-sub do_fetch {
+sub do_fetch { # main entry point
my ($cls, $lei, $cd) = @_;
my $ibx_ver;
$lei->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
@@ -124,6 +124,7 @@ Unable to infer inbox URL from <$git_url>
EOM
$ibx_uri = URI->new($inbox_url);
}
+ PublicInbox::LeiMirror::write_makefile($dir, $ibx_ver);
$lei->qerr("# inbox URL: $ibx_uri/");
my $res = do_manifest($lei, $dir, $ibx_uri) or return;
my ($code, $v1_path, $v2_epochs, $muri, $ft, $mf) = @$res;
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index c113c9de..749c6b7d 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -9,7 +9,7 @@ use parent qw(PublicInbox::IPC);
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
use PublicInbox::Spawn qw(popen_rd spawn run_die);
use File::Temp ();
-use Fcntl qw(SEEK_SET);
+use Fcntl qw(SEEK_SET O_CREAT O_EXCL O_WRONLY);
sub do_finish_mirror { # dwaitpid callback
my ($arg, $pid) = @_;
@@ -201,6 +201,7 @@ sub clone_v1 {
my $cerr = run_reap($lei, $cmd, $opt);
return $lei->child_error($cerr, "@$cmd failed") if $cerr;
_try_config($self);
+ write_makefile($self->{dst}, 1);
index_cloned_inbox($self, 1);
}
@@ -234,6 +235,7 @@ failed to extract epoch number from $src
my $mg = PublicInbox::MultiGit->new($dst, 'all.git', 'git');
$mg->fill_alternates;
for my $i ($mg->git_epochs) { $mg->epoch_cfg_set($i) }
+ write_makefile($self->{dst}, 2);
undef $on_destroy; # unlock
index_cloned_inbox($self, 2);
}
@@ -356,4 +358,61 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
+sub write_makefile {
+ my ($dir, $ibx_ver) = @_;
+ my $f = "$dir/Makefile";
+ if (sysopen my $fh, $f, O_CREAT|O_EXCL|O_WRONLY) {
+ print $fh <<EOM or die "print($f) $!";
+# This is a v$ibx_ver public-inbox, see the public-inbox-v$ibx_ver-format(5)
+# manpage for more information on the format. This Makefile is
+# intended as a familiar wrapper for users unfamiliar with
+# public-inbox-* commands.
+#
+# See the respective manpages for public-inbox-fetch(1),
+# public-inbox-index(1), etc for more information on
+# some of the commands used by this Makefile.
+#
+# This Makefile will not modified nor read by public-inbox,
+# so you may edit it freely with your own convenience targets
+# and notes. public-inbox-fetch will recreate it if removed.
+EOM
+ print $fh <<'EOM' or die "print($f): $!";
+# the default target:
+help :
+ @echo Common targets:
+ @echo ' make fetch - fetch from remote git repostorie(s)'
+ @echo ' make update - fetch and update index '
+ @echo
+ @echo Rarely needed targets:
+ @echo ' make reindex - may be needed for new features/bugfixes'
+ @echo ' make compact - rewrite Xapian storage to save space'
+
+fetch :
+ public-inbox-fetch
+update :
+ @if ! public-inbox-fetch --exit-code; \
+ then \
+ c=$$?; \
+ test $$c -eq 127 && exit 0; \
+ exit $$c; \
+ elif test -f msgmap.sqlite3 || test -f public-inbox/msgmap.sqlite3; \
+ then \
+ public-inbox-index; \
+ else \
+ echo 'public-inbox index not initialized'; \
+ echo 'see public-inbox-index(1) man page'; \
+ fi
+reindex :
+ public-inbox-index --reindex
+compact :
+ public-inbox-compact
+
+.PHONY : help fetch update reindex compact
+EOM
+ close $fh or die "close($f): $!";
+ } else {
+ die "open($f): $!" unless $!{EEXIST};
+ }
+}
+
1;
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 9fdda5aa..3eb7f36a 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -19,6 +19,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(-f "$t1/public-inbox/msgmap.sqlite3", 't1-mirror indexed');
is(PublicInbox::Inbox::try_cat("$t1/description"),
"mirror of $http/t1/\n", 'description set');
+ ok(-f "$t1/Makefile", 'convenience Makefile added (v1)');
lei_ok('ls-external');
like($lei_out, qr!\Q$t1\E!, 't1 added to ls-externals');
@@ -27,6 +28,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('add-external', $t2, '--mirror', "$http/t2/", \'--mirror v2');
ok(-f "$t2/msgmap.sqlite3", 't2-mirror indexed');
ok(-f "$t2/description", 't2 description');
+ ok(-f "$t2/Makefile", 'convenience Makefile added (v2)');
is(PublicInbox::Inbox::try_cat("$t2/description"),
"mirror of $http/t2/\n", 'description set');
^ permalink raw reply related [flat|nested] 8+ messages in thread* Re: [PATCH 3/5] clone|fetch|--mirror: add convenience $INBOX_DIR/Makefile
2021-09-15 21:35 ` [PATCH 3/5] clone|fetch|--mirror: add convenience $INBOX_DIR/Makefile Eric Wong
@ 2021-09-15 22:45 ` Kyle Meyer
2021-09-15 23:15 ` Eric Wong
0 siblings, 1 reply; 8+ messages in thread
From: Kyle Meyer @ 2021-09-15 22:45 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> Since the beginning of time, I've been dropping Makefiles
> in $INBOX_DIR (and above hiearchies) to organize groups
> of commands.
>
> make(1) is widely available in various flavors and a familiar
> tool for our target audience. It is easy to run in the right
> directory, typically has built-in shell completion, and doesn't
> silently ignore errors by default like Bourne shell.
Sounds like a nice idea.
[...]
> +sub write_makefile {
> + my ($dir, $ibx_ver) = @_;
> + my $f = "$dir/Makefile";
> + if (sysopen my $fh, $f, O_CREAT|O_EXCL|O_WRONLY) {
> + print $fh <<EOM or die "print($f) $!";
> +# This is a v$ibx_ver public-inbox, see the public-inbox-v$ibx_ver-format(5)
> +# manpage for more information on the format. This Makefile is
> +# intended as a familiar wrapper for users unfamiliar with
> +# public-inbox-* commands.
> +#
> +# See the respective manpages for public-inbox-fetch(1),
> +# public-inbox-index(1), etc for more information on
> +# some of the commands used by this Makefile.
> +#
> +# This Makefile will not modified nor read by public-inbox,
missing "be" before "modified"
^ permalink raw reply [flat|nested] 8+ messages in thread* Re: [PATCH 3/5] clone|fetch|--mirror: add convenience $INBOX_DIR/Makefile
2021-09-15 22:45 ` Kyle Meyer
@ 2021-09-15 23:15 ` Eric Wong
0 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2021-09-15 23:15 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> Eric Wong writes:
> > +# This Makefile will not modified nor read by public-inbox,
>
> missing "be" before "modified"
Thanks, will squash the below in. Also, welcome back!
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 749c6b7d..23a2156c 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -372,7 +372,7 @@ sub write_makefile {
# public-inbox-index(1), etc for more information on
# some of the commands used by this Makefile.
#
-# This Makefile will not modified nor read by public-inbox,
+# This Makefile will not be modified nor read by public-inbox,
# so you may edit it freely with your own convenience targets
# and notes. public-inbox-fetch will recreate it if removed.
EOM
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 4/5] fetch|clone|--mirror: shorten paths for progress output
2021-09-15 21:35 [PATCH 0/5] make(1) inspired things Eric Wong
` (2 preceding siblings ...)
2021-09-15 21:35 ` [PATCH 3/5] clone|fetch|--mirror: add convenience $INBOX_DIR/Makefile Eric Wong
@ 2021-09-15 21:35 ` Eric Wong
2021-09-15 21:35 ` [PATCH 5/5] support -C (chdir) for most non-daemon commands Eric Wong
4 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2021-09-15 21:35 UTC (permalink / raw)
To: meta
The full pathname for "curl -o ..." was too noisy and confusing.
Reduce confusion by adding the ".tmp" suffix and relying on
"-C". We'll also avoid displaying "-C" in run_reap() and
rely on "--git-dir=" with "git fetch" to display progress for
users.
---
lib/PublicInbox/Fetch.pm | 18 +++++++++---------
lib/PublicInbox/LeiMirror.pm | 12 +++++++-----
t/lei-mirror.t | 8 ++++----
3 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 184b4f26..993e5b19 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -15,9 +15,9 @@ use File::Temp ();
sub new { bless {}, __PACKAGE__ }
-sub fetch_cmd ($$) {
+sub fetch_args ($$) {
my ($lei, $opt) = @_;
- my @cmd = qw(git);
+ my @cmd; # (git --git-dir=...) to be added by caller
$opt->{$_} = $lei->{$_} for (0..2);
# we support "-c $key=$val" for arbitrary git config options
# e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
@@ -41,10 +41,9 @@ sub remote_url ($$) {
sub do_manifest ($$$) {
my ($lei, $dir, $ibx_uri) = @_;
my $muri = URI->new("$ibx_uri/manifest.js.gz");
- my $ft = File::Temp->new(TEMPLATE => 'manifest-XXXX',
- UNLINK => 1, DIR => $dir);
+ my $ft = File::Temp->new(TEMPLATE => 'm-XXXX',
+ UNLINK => 1, DIR => $dir, SUFFIX => '.tmp');
my $fn = $ft->filename;
- my @opt = (qw(-R -o), $fn);
my $mf = "$dir/manifest.js.gz";
my $m0; # current manifest.js.gz contents
if (open my $fh, '<', $mf) {
@@ -53,8 +52,9 @@ sub do_manifest ($$$) {
};
$lei->err($@) if $@;
}
- my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, @opt);
- my $opt = {};
+ my ($bn) = ($fn =~ m!/([^/]+)\z!);
+ my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, qw(-R -o), $bn);
+ my $opt = { -C => $dir };
$opt->{$_} = $lei->{$_} for (0..2);
my $cerr = PublicInbox::LeiMirror::run_reap($lei, $curl_cmd, $opt);
if ($cerr) {
@@ -153,9 +153,9 @@ EOM
my $cmd;
my $opt = {}; # for spawn
if (-d $d) {
- $opt->{-C} = $d;
$fp2->[0] = get_fingerprint2($d) if $fp2;
- $cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
+ $cmd = [ @$torsocks, 'git', "--git-dir=$d",
+ fetch_args($lei, $opt) ];
} else {
my $e_uri = $ibx_uri->clone;
my ($epath) = ($d =~ m!(/git/[0-9]+\.git)\z!);
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 749c6b7d..0e048070 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -181,7 +181,7 @@ sub index_cloned_inbox {
sub run_reap {
my ($lei, $cmd, $opt) = @_;
- $lei->qerr("# @$cmd" . ($opt->{-C} ? " (in $opt->{-C})" : ''));
+ $lei->qerr("# @$cmd");
$opt->{pgid} = 0 if $lei->{sock};
my $pid = spawn($cmd, undef, $opt);
my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
@@ -276,11 +276,13 @@ sub try_manifest {
$uri->path($path . '/manifest.js.gz');
my $pdir = $lei->rel2abs($self->{dst});
$pdir =~ s!/[^/]+/?\z!!;
- my $ft = File::Temp->new(TEMPLATE => 'manifest-XXXX',
- UNLINK => 1, DIR => $pdir);
+ my $ft = File::Temp->new(TEMPLATE => 'm-XXXX',
+ UNLINK => 1, DIR => $pdir, SUFFIX => '.tmp');
my $fn = $ft->filename;
- my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $fn);
- my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} };
+ my ($bn) = ($fn =~ m!/([^/]+)\z!);
+ my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $bn);
+ my $opt = { -C => $pdir };
+ $opt->{$_} = $lei->{$_} for (0..2);
my $cerr = run_reap($lei, $cmd, $opt);
if ($cerr) {
return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 3eb7f36a..7dd03b26 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -116,12 +116,12 @@ SKIP: {
ok(!run_script([qw(-fetch --exit-code -C), "$d/t2"], undef, $opt),
'-fetch succeeds w/ manifest.js.gz');
is($? >> 8, 127, '--exit-code gave 127');
- unlike($err, qr/git fetch/, 'no fetch done w/ manifest');
+ unlike($err, qr/git --git-dir=\S+ fetch/, 'no fetch done w/ manifest');
unlink("$d/t2/manifest.js.gz") or xbail "unlink $!";
ok(!run_script([qw(-fetch --exit-code -C), "$d/t2"], undef, $opt),
'-fetch succeeds w/o manifest.js.gz');
is($? >> 8, 127, '--exit-code gave 127');
- like($err, qr/git fetch/, 'fetch forced w/o manifest');
+ like($err, qr/git --git-dir=\S+ fetch/, 'fetch forced w/o manifest');
ok(run_script([qw(-clone -q -C), $d, "$http/t1"], undef, $opt),
'cloning v1 works');
@@ -131,14 +131,14 @@ SKIP: {
ok(!run_script([qw(-fetch --exit-code -C), "$d/t1"], undef, $opt),
'fetching v1 works');
is($? >> 8, 127, '--exit-code gave 127');
- unlike($err, qr/git fetch/, 'no fetch done w/ manifest');
+ unlike($err, qr/git --git-dir=\S+ fetch/, 'no fetch done w/ manifest');
unlink("$d/t1/manifest.js.gz") or xbail "unlink $!";
my $before = [ glob("$d/t1/*") ];
ok(!run_script([qw(-fetch --exit-code -C), "$d/t1"], undef, $opt),
'fetching v1 works w/o manifest.js.gz');
is($? >> 8, 127, '--exit-code gave 127');
unlink("$d/t1/FETCH_HEAD"); # git internal
- like($err, qr/git fetch/, 'no fetch done w/ manifest');
+ like($err, qr/git --git-dir=\S+ fetch/, 'no fetch done w/ manifest');
ok(unlink("$d/t1/manifest.js.gz"), 'manifest created');
my $after = [ glob("$d/t1/*") ];
is_deeply($before, $after, 'no new files created');
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 5/5] support -C (chdir) for most non-daemon commands
2021-09-15 21:35 [PATCH 0/5] make(1) inspired things Eric Wong
` (3 preceding siblings ...)
2021-09-15 21:35 ` [PATCH 4/5] fetch|clone|--mirror: shorten paths for progress output Eric Wong
@ 2021-09-15 21:35 ` Eric Wong
4 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2021-09-15 21:35 UTC (permalink / raw)
To: meta
Because make(1), git(1), tar(1) all support -C in this form, as
do our newer commands such as lei, public-inbox-{clone,fetch}.
---
script/public-inbox-compact | 3 ++-
script/public-inbox-convert | 6 +++---
script/public-inbox-edit | 3 ++-
script/public-inbox-extindex | 3 ++-
script/public-inbox-index | 3 ++-
script/public-inbox-init | 4 +++-
script/public-inbox-purge | 3 ++-
script/public-inbox-xcpdb | 3 ++-
8 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/script/public-inbox-compact b/script/public-inbox-compact
index 6e34aaeb..80d0224b 100755
--- a/script/public-inbox-compact
+++ b/script/public-inbox-compact
@@ -17,7 +17,7 @@ options:
See public-inbox-compact(1) man page for full documentation.
EOF
-GetOptions($opt, qw(all help|h),
+GetOptions($opt, qw(all C=s@ help|h),
# compact options:
qw(jobs|j=i quiet|q blocksize|b=s no-full|n fuller|F),
) or die $help;
@@ -25,6 +25,7 @@ if ($opt->{help}) { print $help; exit 0 };
require PublicInbox::Admin;
PublicInbox::Admin::require_or_die('-index');
+PublicInbox::Admin::do_chdir(delete $opt->{C});
PublicInbox::Admin::progress_prepare($opt);
require PublicInbox::InboxWritable;
diff --git a/script/public-inbox-convert b/script/public-inbox-convert
index 01af846a..42955a48 100755
--- a/script/public-inbox-convert
+++ b/script/public-inbox-convert
@@ -33,7 +33,7 @@ my $opt = {
quiet => -1, compact => 0, maxsize => undef, fsync => 1,
reindex => 1, # we always reindex
};
-GetOptions($opt, qw(jobs|j=i index! help|h),
+GetOptions($opt, qw(jobs|j=i index! help|h C=s@),
# index options
qw(verbose|v+ rethread compact|c+ fsync|sync!
indexlevel|index-level|L=s max_size|max-size=s
@@ -41,13 +41,14 @@ GetOptions($opt, qw(jobs|j=i index! help|h),
sequential-shard|seq-shard
)) or die $help;
if ($opt->{help}) { print $help; exit 0 };
+require PublicInbox::Admin;
+PublicInbox::Admin::do_chdir(delete $opt->{C});
my $old_dir = shift(@ARGV) // '';
my $new_dir = shift(@ARGV) // '';
die $help if (scalar(@ARGV) || $new_dir eq '' || $old_dir eq '');
die "$new_dir exists\n" if -d $new_dir;
die "$old_dir not a directory\n" unless -d $old_dir;
-require PublicInbox::Admin;
require PublicInbox::Config;
require PublicInbox::InboxWritable;
@@ -62,7 +63,6 @@ if (delete $old->{-unconfigured}) {
}
die "Only conversion from v1 inboxes is supported\n" if $old->version >= 2;
-require PublicInbox::Admin;
my $detected = PublicInbox::Admin::detect_indexlevel($old);
$old->{indexlevel} //= $detected;
my $env;
diff --git a/script/public-inbox-edit b/script/public-inbox-edit
index 9498038b..1fbaf5a7 100755
--- a/script/public-inbox-edit
+++ b/script/public-inbox-edit
@@ -33,9 +33,10 @@ See public-inbox-edit(1) man page for full documentation.
EOF
my $opt = { verbose => 1, all => 0, -min_inbox_version => 2, raw => 0 };
-my @opt = qw(mid|m=s file|F=s raw);
+my @opt = qw(mid|m=s file|F=s raw C=s@);
GetOptions($opt, @PublicInbox::AdminEdit::OPT, @opt) or die $help;
if ($opt->{help}) { print $help; exit 0 };
+PublicInbox::Admin::do_chdir(delete $opt->{C});
my $cfg = PublicInbox::Config->new;
my $editor = $ENV{MAIL_EDITOR}; # e.g. "mutt -f"
diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
index 327980d2..1572a1d2 100755
--- a/script/public-inbox-extindex
+++ b/script/public-inbox-extindex
@@ -30,7 +30,7 @@ GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i
indexlevel|index-level|L=s max_size|max-size=s
batch_size|batch-size=s
dedupe:s@ gc commit-interval=i watch scan! dry-run|n
- all help|h))
+ all C=s@ help|h))
or die $help;
if ($opt->{help}) { print $help; exit 0 };
die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0;
@@ -40,6 +40,7 @@ STDERR->autoflush(1);
local $SIG{USR1} = 'IGNORE'; # to be overridden in eidx_sync
# require lazily to speed up --help
require PublicInbox::Admin;
+PublicInbox::Admin::do_chdir(delete $opt->{C});
my $cfg = PublicInbox::Config->new;
my $eidx_dir = shift(@ARGV);
unless (defined $eidx_dir) {
diff --git a/script/public-inbox-index b/script/public-inbox-index
index 0034d44c..ca190a2e 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -42,7 +42,7 @@ GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
batch_size|batch-size=s
sequential-shard|seq-shard
no-update-extindex update-extindex|E=s@
- fast-noop|F skip-docdata all help|h))
+ fast-noop|F skip-docdata all C=s@ help|h))
or die $help;
if ($opt->{help}) { print $help; exit 0 };
die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0;
@@ -56,6 +56,7 @@ if ($opt->{reindex} && delete($opt->{'fast-noop'})) {
# require lazily to speed up --help
require PublicInbox::Admin;
PublicInbox::Admin::require_or_die('-index');
+PublicInbox::Admin::do_chdir(delete $opt->{C});
my $cfg = PublicInbox::Config->new; # Config is loaded by Admin
$opt->{-use_cwd} = 1;
diff --git a/script/public-inbox-init b/script/public-inbox-init
index 78a4d3bd..1223d47e 100755
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -36,7 +36,7 @@ PublicInbox::Admin::require_or_die('-base');
my ($version, $indexlevel, $skip_epoch, $skip_artnum, $jobs, $show_help);
my $skip_docdata;
my $ng = '';
-my @c_extra;
+my (@c_extra, @chdir);
my %opts = (
'V|version=i' => \$version,
'L|index-level|indexlevel=s' => \$indexlevel,
@@ -47,6 +47,7 @@ my %opts = (
'skip-docdata' => \$skip_docdata,
'help|h' => \$show_help,
'c=s@' => \@c_extra,
+ 'C=s@' => \@chdir,
);
my $usage_cb = sub {
print STDERR $help;
@@ -59,6 +60,7 @@ my $inboxdir = shift @ARGV or $usage_cb->();
my $http_url = shift @ARGV or $usage_cb->();
my (@address) = @ARGV;
@address or $usage_cb->();
++PublicInbox::Admin::do_chdir(\@chdir);
@c_extra = map {
my ($k, $v) = split(/=/, $_, 2);
diff --git a/script/public-inbox-purge b/script/public-inbox-purge
index 59c03150..121027cc 100755
--- a/script/public-inbox-purge
+++ b/script/public-inbox-purge
@@ -26,9 +26,10 @@ See public-inbox-purge(1) man page for full documentation.
EOF
my $opt = { verbose => 1, all => 0, -min_inbox_version => 2 };
-GetOptions($opt, @PublicInbox::AdminEdit::OPT) or die $help;
+GetOptions($opt, @PublicInbox::AdminEdit::OPT, 'C=s@') or die $help;
if ($opt->{help}) { print $help; exit 0 };
+PublicInbox::Admin::do_chdir(delete $opt->{C});
my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt);
PublicInbox::AdminEdit::check_editable(\@ibxs);
diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
index 81d1a85b..24fc5a25 100755
--- a/script/public-inbox-xcpdb
+++ b/script/public-inbox-xcpdb
@@ -33,11 +33,12 @@ GetOptions($opt, qw(
sequential-shard|seq-shard
jobs|j=i quiet|q verbose|v
blocksize|b=s no-full|n fuller|F
- all help|h)) or die $help;
+ all C=s@ help|h)) or die $help;
if ($opt->{help}) { print $help; exit 0 };
use PublicInbox::Admin;
PublicInbox::Admin::require_or_die('-search');
+PublicInbox::Admin::do_chdir(delete $opt->{C});
require PublicInbox::Config;
my $cfg = PublicInbox::Config->new;
^ permalink raw reply related [flat|nested] 8+ messages in thread