* [PATCH 1/3] t/extindex: add --max-size test
2024-12-14 14:58 [PATCH 0/3] indexing odds and ends Eric Wong
@ 2024-12-14 14:58 ` Eric Wong
2024-12-14 14:58 ` [PATCH 2/3] admin: common warn_cb Eric Wong
2024-12-14 14:58 ` [PATCH 3/3] lei: use PublicInbox::Eml::warn_ignore_cb Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2024-12-14 14:58 UTC (permalink / raw)
To: meta
I noticed this missing coverage while working on another
series of changes which aren't quite ready, yet.
---
t/extsearch.t | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/t/extsearch.t b/t/extsearch.t
index 1a212bc8..7b75c208 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -703,4 +703,16 @@ EOF
is $args[0]->size, 0, 'isrch->async_mset altid miss works';
}
+if ('max-size') {
+ my $dir = "$home/extindex-max";
+ my $rdr = { 2 => \(my $err) };
+ ok run_script([qw(-extindex --max-size=500 --all -vvv), $dir],
+ undef, $rdr), 'extindex with max-size';
+ my $es = PublicInbox::ExtSearch->new($dir);
+ my $mset = $es->mset('z:500..');
+ is $mset->size, 0, 'no hits w/ max-size=500';
+ like $err, qr/ skipping [a-f0-9]{40,} .*? > 500\b/,
+ 'noted skipping messages in stderr';
+}
+
done_testing;
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 2/3] admin: common warn_cb
2024-12-14 14:58 [PATCH 0/3] indexing odds and ends Eric Wong
2024-12-14 14:58 ` [PATCH 1/3] t/extindex: add --max-size test Eric Wong
@ 2024-12-14 14:58 ` Eric Wong
2024-12-14 14:58 ` [PATCH 3/3] lei: use PublicInbox::Eml::warn_ignore_cb Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2024-12-14 14:58 UTC (permalink / raw)
To: meta
I noticed verbose output from t/extsearch.t was not correctly
commenting out (`# ') informational messages when the
{current_info} prefix is in use. This also appears to affect
other indexers with the exception of the new -cindex.
So consolidate all the common warning code into
PublicInbox::Admin module so we can handle {current_info}
properly in all contexts.
-cindex gets some unnecessary Eml warning filtering, but that's
probably not a real problem and worth the overall code savings
since PublicInbox::Admin is loaded by script/public-inbox-cindex
anyways.
---
lib/PublicInbox/Admin.pm | 17 +++++++++++++----
lib/PublicInbox/CodeSearchIdx.pm | 9 ++-------
lib/PublicInbox/ExtSearchIdx.pm | 12 ++----------
t/extsearch.t | 2 +-
4 files changed, 18 insertions(+), 22 deletions(-)
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index fb745cf8..f1aec53d 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -266,6 +266,18 @@ sub index_terminate {
$ibx->git->cleanup;
}
+sub warn_cb ($) {
+ my ($self) = @_;
+ my $cb = $SIG{__WARN__} || \&CORE::warn;
+ sub {
+ return if PublicInbox::Eml::warn_ignore(@_);
+ my @m = @_;
+ $self->{current_info} ne '' && @m && $m[0] =~ s/\A# // and
+ unshift @m, '# ', $self->{current_info}, ': ';
+ $cb->(@m);
+ }
+}
+
sub index_inbox {
my ($ibx, $im, $opt) = @_;
require PublicInbox::InboxWritable;
@@ -276,10 +288,7 @@ sub index_inbox {
local @SIG{keys %SIG} = values %SIG;
setup_signals(\&index_terminate, $ibx);
my $idx = { current_info => $ibx->{inboxdir} };
- local $SIG{__WARN__} = sub {
- return if PublicInbox::Eml::warn_ignore(@_);
- warn($idx->{current_info}, ': ', @_);
- };
+ local $SIG{__WARN__} = warn_cb $idx;
if ($ibx->version == 2) {
eval { require PublicInbox::V2Writable };
die "v2 requirements not met: $@\n" if $@;
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 8b5f5ad0..6dc2a7a9 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -51,6 +51,7 @@ package PublicInbox::CodeSearchIdx;
use v5.12;
# parent order matters, we want ->DESTROY from IPC, not SearchIdx
use parent qw(PublicInbox::CodeSearch PublicInbox::IPC PublicInbox::SearchIdx);
+use PublicInbox::Admin;
use PublicInbox::DS qw(awaitpid);
use PublicInbox::PktOp;
use PublicInbox::IPC qw(nproc_shards);
@@ -1334,13 +1335,7 @@ sub cidx_run { # main entry point
};
local @PRUNE_BATCH = @PRUNE_BATCH;
$MY_SIG->{$_} = \&parent_quit for qw(TERM QUIT INT);
- my $cb = $SIG{__WARN__} || \&CORE::warn;
- local $SIG{__WARN__} = sub {
- my $m = shift @_;
- $self->{current_info} eq '' or
- $m =~ s/\A(#?\s*)/$1$self->{current_info}: /;
- $cb->($m, @_);
- };
+ local $SIG{__WARN__} = PublicInbox::Admin::warn_cb $self;
load_existing($self) unless $self->{-cidx_internal};
if ($self->{-opt}->{reindex}) {
require PublicInbox::SharedKV;
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 52d7c3b1..4a479978 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1110,12 +1110,8 @@ EOS
sub eidx_sync { # main entry point
my ($self, $opt) = @_;
- my $warn_cb = $SIG{__WARN__} || \&CORE::warn;
local $self->{current_info} = '';
- local $SIG{__WARN__} = sub {
- return if PublicInbox::Eml::warn_ignore(@_);
- $warn_cb->($self->{current_info}, ': ', @_);
- };
+ local $SIG{__WARN__} = PublicInbox::Admin::warn_cb $self;
$self->idx_init($opt); # acquire lock via V2Writable::_idx_init
$self->{oidx}->rethread_prepare($opt);
local $self->{need_checkpoint} = 0;
@@ -1386,11 +1382,7 @@ sub eidx_watch { # public-inbox-extindex --watch main loop
return if $sync->{quit};
my $oldset = PublicInbox::DS::block_signals();
local $self->{current_info} = '';
- my $cb = $SIG{__WARN__} || \&CORE::warn;
- local $SIG{__WARN__} = sub {
- return if PublicInbox::Eml::warn_ignore(@_);
- $cb->($self->{current_info}, ': ', @_);
- };
+ local $SIG{__WARN__} = PublicInbox::Admin::warn_cb $self;
my $sig = {
HUP => sub { eidx_reload($self, $idler) },
USR1 => sub { eidx_resync_start($self) },
diff --git a/t/extsearch.t b/t/extsearch.t
index 7b75c208..6bcfdd84 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -292,7 +292,7 @@ if ('reindex catches missed messages') {
is($oidx->eidx_meta($lc_key), $cmt_b, 'lc-v2 stays unchanged');
my @err = split(/^/, $err);
is(scalar(@err), 1, 'only one warning') or diag "err=$err";
- like($err[0], qr/# reindex_unseen/, 'got reindex_unseen message');
+ like $err[0], qr/# .*? reindex_unseen/, 'got reindex_unseen message';
my $new = $oidx->get_art($max + 1);
is($new->{subject}, $eml->header('Subject'), 'new message added');
^ permalink raw reply related [flat|nested] 4+ messages in thread