* [PATCH 2/6] another step towards git SHA-256 support
2022-10-20 8:43 [PATCH 0/6] trivial clone|fetch-related stuff Eric Wong
2022-10-20 8:43 ` [PATCH 1/6] clone|fetch: preserve mtime of modified manifest.js.gz Eric Wong
@ 2022-10-20 8:43 ` Eric Wong
2022-10-20 8:43 ` [PATCH 3/6] githttpbackend: remove unused $BIN variable Eric Wong
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Eric Wong @ 2022-10-20 8:43 UTC (permalink / raw)
To: meta
While SHA-256 isn't supported for inboxes, yet
xt/git-http-backend.t now runs properly against a SHA-256 code
repository
---
lib/PublicInbox/GitHTTPBackend.pm | 12 +++++-------
t/git.t | 4 ++--
t/import.t | 4 ++--
t/mda.t | 4 ++--
xt/git-http-backend.t | 4 ++--
5 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index 3aae5454..72b8e6c3 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -23,10 +23,8 @@ my @text = qw[HEAD info/refs info/attributes
objects/info/(?:http-alternates|alternates|packs)
cloneurl description];
-my @binary = qw!
- objects/[a-f0-9]{2}/[a-f0-9]{38}
- objects/pack/pack-[a-f0-9]{40}\.(?:pack|idx)
- !;
+my @binary = ('objects/[a-f0-9]{2}/[a-f0-9]{38,62}',
+ 'objects/pack/pack-[a-f0-9]{40,64}\.(?:pack|idx)');
our $ANY = join('|', @binary, @text, 'git-upload-pack');
my $BIN = join('|', @binary);
@@ -62,13 +60,13 @@ sub serve_dumb {
my $h = [];
my $type;
- if ($path =~ m!\Aobjects/[a-f0-9]{2}/[a-f0-9]{38}\z!) {
+ if ($path =~ m!\Aobjects/[a-f0-9]{2}/[a-f0-9]{38,62}\z!) {
$type = 'application/x-git-loose-object';
cache_one_year($h);
- } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40}\.pack\z!) {
+ } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40,64}\.pack\z!) {
$type = 'application/x-git-packed-objects';
cache_one_year($h);
- } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40}\.idx\z!) {
+ } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40,64}\.idx\z!) {
$type = 'application/x-git-packed-objects-toc';
cache_one_year($h);
} elsif ($path =~ /\A(?:$TEXT)\z/o) {
diff --git a/t/git.t b/t/git.t
index 56fc8d95..d8957e42 100644
--- a/t/git.t
+++ b/t/git.t
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use Test::More;
@@ -44,7 +44,7 @@ use PublicInbox::Git;
my $f = 'HEAD:foo.txt';
my @x = $gcf->check($f);
is(scalar @x, 3, 'returned 3 element array for existing file');
- like($x[0], qr/\A[a-f0-9]{40}\z/, 'returns obj ID in 1st element');
+ like($x[0], qr/\A[a-f0-9]{40,64}\z/, 'returns obj ID in 1st element');
is('blob', $x[1], 'returns obj type in 2nd element');
like($x[2], qr/\A\d+\z/, 'returns obj size in 3rd element');
diff --git a/t/import.t b/t/import.t
index ae76858b..4ba74022 100644
--- a/t/import.t
+++ b/t/import.t
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use warnings;
@@ -30,7 +30,7 @@ my $smsg = bless {}, 'PublicInbox::Smsg' if $v2;
like($im->add($mime, undef, $smsg), qr/\A:[0-9]+\z/, 'added one message');
if ($v2) {
- like($smsg->{blob}, qr/\A[a-f0-9]{40}\z/, 'got last object_id');
+ like($smsg->{blob}, qr/\A[a-f0-9]{40,64}\z/, 'got last object_id');
my @cmd = ('git', "--git-dir=$git->{git_dir}", qw(hash-object --stdin));
open my $in, '+<', undef or BAIL_OUT "open(+<): $!";
print $in $mime->as_string or die "write failed: $!";
diff --git a/t/mda.t b/t/mda.t
index d20cdb92..e3c5cdff 100644
--- a/t/mda.t
+++ b/t/mda.t
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use warnings;
@@ -97,7 +97,7 @@ EOF
local $ENV{PATH} = $main_path;
ok(run_script(['-mda'], undef, { 0 => \$in }));
my $rev = $git->qx(qw(rev-list HEAD));
- like($rev, qr/\A[a-f0-9]{40}/, "good revision committed");
+ like($rev, qr/\A[a-f0-9]{40,64}/, "good revision committed");
chomp $rev;
my $cmt = $git->cat_file($rev);
like($$cmt, qr/^author Me <me\@example\.com> 0 \+0000\n/m,
diff --git a/xt/git-http-backend.t b/xt/git-http-backend.t
index adadebb0..1f3ba063 100644
--- a/xt/git-http-backend.t
+++ b/xt/git-http-backend.t
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Ensure buffering behavior in -httpd doesn't cause runaway memory use
@@ -53,7 +53,7 @@ SKIP: {
}
}
skip "no packs found in $git_dir" unless defined $pack;
- if ($pack !~ m!(/objects/pack/pack-[a-f0-9]{40}.pack)\z!) {
+ if ($pack !~ m!(/objects/pack/pack-[a-f0-9]{40,64}.pack)\z!) {
skip "bad pack name: $pack";
}
my $url = $1;
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 4/6] treewide: replace /^I: / prefix with /^# /
2022-10-20 8:43 [PATCH 0/6] trivial clone|fetch-related stuff Eric Wong
` (2 preceding siblings ...)
2022-10-20 8:43 ` [PATCH 3/6] githttpbackend: remove unused $BIN variable Eric Wong
@ 2022-10-20 8:43 ` Eric Wong
2022-10-20 8:43 ` [PATCH 5/6] lei_mirror: make _finish_add_external call more obvious Eric Wong
2022-10-20 8:43 ` [PATCH 6/6] lei_mirror: delimit names by `\n' to improve die message Eric Wong
5 siblings, 0 replies; 7+ messages in thread
From: Eric Wong @ 2022-10-20 8:43 UTC (permalink / raw)
To: meta
This is like more familiar to readers of TAP (Test Anywhere
Protocol) output, as well as shell and Perl scripters which also
use `#' for comments.
AFAIK, nobody is parsing our stderr, and I'm not sure how
standardized the `I:' prefix is (nor `W:' and `E:' are). It's
already the prevailing style in Lei* code, too, so things have
been moving in that direction for a bit.
---
examples/grok-pull.post_update_hook.sh | 2 +-
lib/PublicInbox/ExtSearchIdx.pm | 26 ++++++++++++--------------
lib/PublicInbox/Gcf2.pm | 4 ++--
lib/PublicInbox/InboxIdle.pm | 8 +++-----
lib/PublicInbox/MiscIdx.pm | 2 +-
lib/PublicInbox/OverIdx.pm | 6 +++---
lib/PublicInbox/Watch.pm | 4 ++--
script/public-inbox-watch | 6 +++---
t/extsearch.t | 6 +++---
t/imapd.t | 6 +++---
10 files changed, 33 insertions(+), 37 deletions(-)
diff --git a/examples/grok-pull.post_update_hook.sh b/examples/grok-pull.post_update_hook.sh
index 77489472..4d303c03 100755
--- a/examples/grok-pull.post_update_hook.sh
+++ b/examples/grok-pull.post_update_hook.sh
@@ -111,7 +111,7 @@ case $cfg_dir in
"publicinbox.$inbox_name.infourl" "$url"
done
curl -sSfv "$remote_inbox_url"/description >"$inbox_dir"/description
- echo "I: $inbox_name at $inbox_dir ($addresses) $local_url"
+ echo "# $inbox_name at $inbox_dir ($addresses) $local_url"
;;
esac
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 7c44a1a4..401b18d0 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -406,14 +406,14 @@ EOM
while (my ($ibx_id, $eidx_key) = $ibx_ck->fetchrow_array) {
next if $self->{ibx_map}->{$eidx_key};
$self->{midx}->remove_eidx_key($eidx_key);
- warn "I: deleting messages for $eidx_key...\n";
+ warn "# deleting messages for $eidx_key...\n";
$x3_doc->execute($ibx_id);
my $ibx = { -ibx_id => $ibx_id, -gc_eidx_key => $eidx_key };
while (my ($docid, $xnum, $oid) = $x3_doc->fetchrow_array) {
my $r = _unref_doc($sync, $docid, $ibx, $xnum, $oid);
$oid = unpack('H*', $oid);
$r = $r ? 'unref' : 'remove';
- warn "I: $r #$docid $eidx_key $oid\n";
+ warn "# $r #$docid $eidx_key $oid\n";
if (checkpoint_due($sync)) {
$x3_doc = $ibx_ck = undef;
reindex_checkpoint($self, $sync);
@@ -433,12 +433,12 @@ SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
$lc_i->execute("lc-%:$pat//%", '\\');
while (my ($key) = $lc_i->fetchrow_array) {
next if $key !~ m!\Alc-v[1-9]+:\Q$eidx_key\E//!;
- warn "I: removing $key\n";
+ warn "# removing $key\n";
$self->{oidx}->dbh->do(<<'', undef, $key);
DELETE FROM eidx_meta WHERE key = ?
}
- warn "I: $eidx_key removed\n";
+ warn "# $eidx_key removed\n";
}
}
@@ -447,20 +447,20 @@ sub eidx_gc_scan_shards ($$) { # TODO: use for lei/store
my $nr = $self->{oidx}->dbh->do(<<'');
DELETE FROM xref3 WHERE docid NOT IN (SELECT num FROM over)
- warn "I: eliminated $nr stale xref3 entries\n" if $nr != 0;
+ warn "# eliminated $nr stale xref3 entries\n" if $nr != 0;
reindex_checkpoint($self, $sync) if checkpoint_due($sync);
# fixup from old bugs:
$nr = $self->{oidx}->dbh->do(<<'');
DELETE FROM over WHERE num > 0 AND num NOT IN (SELECT docid FROM xref3)
- warn "I: eliminated $nr stale over entries\n" if $nr != 0;
+ warn "# eliminated $nr stale over entries\n" if $nr != 0;
reindex_checkpoint($self, $sync) if checkpoint_due($sync);
$nr = $self->{oidx}->dbh->do(<<'');
DELETE FROM eidxq WHERE docid NOT IN (SELECT num FROM over)
- warn "I: eliminated $nr stale reindex queue entries\n" if $nr != 0;
+ warn "# eliminated $nr stale reindex queue entries\n" if $nr != 0;
reindex_checkpoint($self, $sync) if checkpoint_due($sync);
my ($cur) = $self->{oidx}->dbh->selectrow_array(<<EOM);
@@ -490,7 +490,7 @@ SELECT num FROM over WHERE num >= ? ORDER BY num ASC LIMIT 10000
reindex_checkpoint($self, $sync);
}
}
- warn "I: eliminated $nr stale Xapian documents\n" if $nr != 0;
+ warn "# eliminated $nr stale Xapian documents\n" if $nr != 0;
}
sub eidx_gc {
@@ -731,13 +731,11 @@ sub eidxq_lock_acquire ($) {
my $t = strftime('%Y-%m-%d %k:%M %z', localtime($time));
local $self->{current_info} = 'eidxq';
if ($euid == $> && $ident eq host_ident) {
- if (kill(0, $pid)) {
- warn <<EOM; return;
-I: PID:$pid (re)indexing since $t, it will continue our work
+ kill(0, $pid) and warn <<EOM and return;
+# PID:$pid (re)indexing since $t, it will continue our work
EOM
- }
if ($!{ESRCH}) {
- warn "I: eidxq_lock is stale ($cur), clobbering\n";
+ warn "# eidxq_lock is stale ($cur), clobbering\n";
return _eidxq_take($self);
}
warn "E: kill(0, $pid) failed: $!\n"; # fall-through:
@@ -837,7 +835,7 @@ sub reindex_unseen ($$$$) {
xnum => $xsmsg->{num},
# {mids} and {chash} will be filled in at _reindex_unseen
};
- warn "I: reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
+ warn "# reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
$self->git->cat_async($xsmsg->{blob}, \&_reindex_unseen, $req);
}
diff --git a/lib/PublicInbox/Gcf2.pm b/lib/PublicInbox/Gcf2.pm
index 5c353095..54b3d6aa 100644
--- a/lib/PublicInbox/Gcf2.pm
+++ b/lib/PublicInbox/Gcf2.pm
@@ -128,14 +128,14 @@ sub loop (;$) {
# retry once if missing. We only get unabbreviated OIDs
# from SQLite or Xapian DBs, here, so malicious clients
# can't trigger excessive retries:
- warn "I: $$ $oid missing, retrying in $git_dir\n";
+ warn "# $$ $oid missing, retrying in $git_dir\n";
$gcf2 = new();
%seen = ($git_dir => add_alt($gcf2, $git_dir));
$check_at = clock_gettime(CLOCK_MONOTONIC) + $exp;
if ($gcf2->cat_oid(1, $oid)) {
- warn "I: $$ $oid found after retry\n";
+ warn "# $$ $oid found after retry\n";
} else {
warn "W: $$ $oid missing after retry\n";
print "$oid missing\n"; # mimic git-cat-file
diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm
index ffbbfea7..005e2636 100644
--- a/lib/PublicInbox/InboxIdle.pm
+++ b/lib/PublicInbox/InboxIdle.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# fields:
@@ -48,11 +48,9 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback
$self->{on_unlock}->{$w->name} = $ibx;
} else {
warn "E: ".ref($inot)."->watch($lock, IN_MODIFY) failed: $!\n";
- if ($!{ENOSPC} && $^O eq 'linux') {
- warn <<"";
-I: consider increasing /proc/sys/fs/inotify/max_user_watches
+ warn <<"" if $!{ENOSPC} && $^O eq 'linux';
+# consider increasing /proc/sys/fs/inotify/max_user_watches
- }
}
# TODO: detect deleted packs (and possibly other files)
diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm
index 76b33b16..19200b92 100644
--- a/lib/PublicInbox/MiscIdx.pm
+++ b/lib/PublicInbox/MiscIdx.pm
@@ -72,7 +72,7 @@ sub remove_eidx_key {
}
for my $docid (@docids) {
$xdb->delete_document($docid);
- warn "I: remove inbox docid #$docid ($eidx_key)\n";
+ warn "# remove inbox docid #$docid ($eidx_key)\n";
}
}
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index a49ca6db..6cc86d5d 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI
@@ -509,12 +509,12 @@ EOF
next;
}
$pr->(<<EOM) if $pr;
-I: ghost $r->{num} <$mid> THREADID=$r->{tid} culled
+# ghost $r->{num} <$mid> THREADID=$r->{tid} culled
EOM
}
delete_by_num($self, $r->{num});
}
- $pr->("I: rethread culled $total ghosts\n") if $pr && $total;
+ $pr->("# rethread culled $total ghosts\n") if $pr && $total;
}
# used for cross-inbox search
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 3f6fe21b..082ecfb9 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -328,7 +328,7 @@ sub imap_idle_once ($$$$) {
my ($self, $mic, $intvl, $uri) = @_;
my $i = $intvl //= (29 * 60);
my $end = now() + $intvl;
- warn "I: $uri idling for ${intvl}s\n";
+ warn "# $uri idling for ${intvl}s\n";
local $0 = "IDLE $0";
return if $self->{quit};
unless ($mic->idle) {
@@ -517,7 +517,7 @@ sub poll_fetch_reap {
if ($?) {
warn "W: PID=$pid died: \$?=$?\n", map { "$_\n" } @$uris;
}
- warn("I: will check $_ in ${intvl}s\n") for @$uris;
+ warn("# will check $_ in ${intvl}s\n") for @$uris;
add_timer($intvl, \&poll_fetch_fork, $self, $intvl, $uris);
}
diff --git a/script/public-inbox-watch b/script/public-inbox-watch
index af02d8f3..4c50461f 100755
--- a/script/public-inbox-watch
+++ b/script/public-inbox-watch
@@ -1,5 +1,5 @@
#!/usr/bin/perl -w
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
my $help = <<EOF;
usage: public-inbox-watch
@@ -27,7 +27,7 @@ my $reload = sub {
$watch->quit;
$watch = PublicInbox::Watch->new(PublicInbox::Config->new);
if ($watch) {
- warn("I: reloaded\n");
+ warn "# reloaded\n";
} else {
warn("E: reloading failed\n");
$watch = $prev;
@@ -37,7 +37,7 @@ my $reload = sub {
if ($watch) {
my $scan = sub {
return if !$watch;
- warn "I: scanning\n";
+ warn "# scanning\n";
$watch->trigger_scan('full');
};
my $quit = sub {
diff --git a/t/extsearch.t b/t/extsearch.t
index 6cbe5262..03bcad95 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -309,7 +309,7 @@ if ('reindex catches missed messages') {
is($oidx->eidx_meta($lc_key), $cmt_b, 'lc-v2 stays unchanged');
my @err = split(/^/, $err);
is(scalar(@err), 1, 'only one warning') or diag "err=$err";
- like($err[0], qr/I: reindex_unseen/, 'got reindex_unseen message');
+ like($err[0], qr/# reindex_unseen/, 'got reindex_unseen message');
my $new = $oidx->get_art($max + 1);
is($new->{subject}, $eml->header('Subject'), 'new message added');
@@ -415,8 +415,8 @@ if ('remove v1test and test gc') {
my $opt = { 2 => \(my $err = '') };
ok(run_script([qw(-extindex --gc), "$home/extindex"], undef, $opt),
'extindex --gc');
- like($err, qr/^I: remove #1 v1\.example /ms, 'removed v1 message');
- is(scalar(grep(!/^I:/, split(/^/m, $err))), 0,
+ like($err, qr/^# remove #1 v1\.example /ms, 'removed v1 message');
+ is(scalar(grep(!/^#/, split(/^/m, $err))), 0,
'no non-informational messages');
$misc->{xdb}->reopen;
@it = $misc->mset('')->items;
diff --git a/t/imapd.t b/t/imapd.t
index 43de8675..3c74aefd 100644
--- a/t/imapd.t
+++ b/t/imapd.t
@@ -470,7 +470,7 @@ SKIP: {
PublicInbox::DS::event_loop();
diag 'inbox unlocked on initial fetch, waiting for IDLE';
- tick until (grep(/I: \S+ idling/, <$err>));
+ tick until (grep(/# \S+ idling/, <$err>));
open my $fh, '<', 't/iso-2202-jp.eml' or BAIL_OUT $!;
$old_env->{ORIGINAL_RECIPIENT} = $addr;
ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }),
@@ -487,7 +487,7 @@ SKIP: {
or BAIL_OUT "git config $?";
$w->kill('HUP');
diag 'waiting for -watch reload + initial fetch';
- tick until (grep(/I: will check/, <$err>));
+ tick until (grep(/# will check/, <$err>));
open $fh, '<', 't/psgi_attach.eml' or BAIL_OUT $!;
ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }),
@@ -516,7 +516,7 @@ SKIP: {
my @t0 = times;
$w = start_script(['-watch'], undef, { 2 => $err_wr });
seek($err, 0, 0);
- tick until (grep(/I: \S+ idling/, <$err>));
+ tick until (grep(/# \S+ idling/, <$err>));
diag 'killing imapd, waiting for CPU spins';
my $delay = 0.11;
$td->kill(9);
^ permalink raw reply related [flat|nested] 7+ messages in thread