[PATCH 0/3] indexing odds and ends

[PATCH 0/3] indexing odds and ends

[Eric Wong]

Possibly bigger changes comming, but small things first and
a small code reduction in lib/

Eric Wong (3):
  t/extindex: add --max-size test
  admin: common warn_cb
  lei: use PublicInbox::Eml::warn_ignore_cb

 lib/PublicInbox/Admin.pm         | 17 +++++++++++++----
 lib/PublicInbox/CodeSearchIdx.pm |  9 ++-------
 lib/PublicInbox/ExtSearchIdx.pm  | 12 ++----------
 lib/PublicInbox/LEI.pm           |  8 ++------
 t/extsearch.t                    | 14 +++++++++++++-
 5 files changed, 32 insertions(+), 28 deletions(-)

[PATCH 1/3] t/extindex: add --max-size test

[Eric Wong]

I noticed this missing coverage while working on another
series of changes which aren't quite ready, yet.
---
 t/extsearch.t | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/t/extsearch.t b/t/extsearch.t
index 1a212bc8..7b75c208 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -703,4 +703,16 @@ EOF
 	is $args[0]->size, 0, 'isrch->async_mset altid miss works';
 }
 
+if ('max-size') {
+	my $dir = "$home/extindex-max";
+	my $rdr = { 2 => \(my $err) };
+	ok run_script([qw(-extindex --max-size=500 --all -vvv), $dir],
+			undef, $rdr), 'extindex with max-size';
+	my $es = PublicInbox::ExtSearch->new($dir);
+	my $mset = $es->mset('z:500..');
+	is $mset->size, 0, 'no hits w/ max-size=500';
+	like $err, qr/ skipping [a-f0-9]{40,} .*? > 500\b/,
+		'noted skipping messages in stderr';
+}
+
 done_testing;

[PATCH 2/3] admin: common warn_cb

[Eric Wong]

I noticed verbose output from t/extsearch.t was not correctly
commenting out (`# ') informational messages when the
{current_info} prefix is in use.  This also appears to affect
other indexers with the exception of the new -cindex.

So consolidate all the common warning code into
PublicInbox::Admin module so we can handle {current_info}
properly in all contexts.

-cindex gets some unnecessary Eml warning filtering, but that's
probably not a real problem and worth the overall code savings
since PublicInbox::Admin is loaded by script/public-inbox-cindex
anyways.
---
 lib/PublicInbox/Admin.pm         | 17 +++++++++++++----
 lib/PublicInbox/CodeSearchIdx.pm |  9 ++-------
 lib/PublicInbox/ExtSearchIdx.pm  | 12 ++----------
 t/extsearch.t                    |  2 +-
 4 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index fb745cf8..f1aec53d 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -266,6 +266,18 @@ sub index_terminate {
 	$ibx->git->cleanup;
 }
 
+sub warn_cb ($) {
+	my ($self) = @_;
+	my $cb = $SIG{__WARN__} || \&CORE::warn;
+	sub {
+		return if PublicInbox::Eml::warn_ignore(@_);
+		my @m = @_;
+		$self->{current_info} ne '' && @m && $m[0] =~ s/\A# // and
+			unshift @m, '# ', $self->{current_info}, ': ';
+		$cb->(@m);
+	}
+}
+
 sub index_inbox {
 	my ($ibx, $im, $opt) = @_;
 	require PublicInbox::InboxWritable;
@@ -276,10 +288,7 @@ sub index_inbox {
 	local @SIG{keys %SIG} = values %SIG;
 	setup_signals(\&index_terminate, $ibx);
 	my $idx = { current_info => $ibx->{inboxdir} };
-	local $SIG{__WARN__} = sub {
-		return if PublicInbox::Eml::warn_ignore(@_);
-		warn($idx->{current_info}, ': ', @_);
-	};
+	local $SIG{__WARN__} = warn_cb $idx;
 	if ($ibx->version == 2) {
 		eval { require PublicInbox::V2Writable };
 		die "v2 requirements not met: $@\n" if $@;
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 8b5f5ad0..6dc2a7a9 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -51,6 +51,7 @@ package PublicInbox::CodeSearchIdx;
 use v5.12;
 # parent order matters, we want ->DESTROY from IPC, not SearchIdx
 use parent qw(PublicInbox::CodeSearch PublicInbox::IPC PublicInbox::SearchIdx);
+use PublicInbox::Admin;
 use PublicInbox::DS qw(awaitpid);
 use PublicInbox::PktOp;
 use PublicInbox::IPC qw(nproc_shards);
@@ -1334,13 +1335,7 @@ sub cidx_run { # main entry point
 	};
 	local @PRUNE_BATCH = @PRUNE_BATCH;
 	$MY_SIG->{$_} = \&parent_quit for qw(TERM QUIT INT);
-	my $cb = $SIG{__WARN__} || \&CORE::warn;
-	local $SIG{__WARN__} = sub {
-		my $m = shift @_;
-		$self->{current_info} eq '' or
-			$m =~ s/\A(#?\s*)/$1$self->{current_info}: /;
-		$cb->($m, @_);
-	};
+	local $SIG{__WARN__} = PublicInbox::Admin::warn_cb $self;
 	load_existing($self) unless $self->{-cidx_internal};
 	if ($self->{-opt}->{reindex}) {
 		require PublicInbox::SharedKV;
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 52d7c3b1..4a479978 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1110,12 +1110,8 @@ EOS
 sub eidx_sync { # main entry point
 	my ($self, $opt) = @_;
 
-	my $warn_cb = $SIG{__WARN__} || \&CORE::warn;
 	local $self->{current_info} = '';
-	local $SIG{__WARN__} = sub {
-		return if PublicInbox::Eml::warn_ignore(@_);
-		$warn_cb->($self->{current_info}, ': ', @_);
-	};
+	local $SIG{__WARN__} = PublicInbox::Admin::warn_cb $self;
 	$self->idx_init($opt); # acquire lock via V2Writable::_idx_init
 	$self->{oidx}->rethread_prepare($opt);
 	local $self->{need_checkpoint} = 0;
@@ -1386,11 +1382,7 @@ sub eidx_watch { # public-inbox-extindex --watch main loop
 	return if $sync->{quit};
 	my $oldset = PublicInbox::DS::block_signals();
 	local $self->{current_info} = '';
-	my $cb = $SIG{__WARN__} || \&CORE::warn;
-	local $SIG{__WARN__} = sub {
-		return if PublicInbox::Eml::warn_ignore(@_);
-		$cb->($self->{current_info}, ': ', @_);
-	};
+	local $SIG{__WARN__} = PublicInbox::Admin::warn_cb $self;
 	my $sig = {
 		HUP => sub { eidx_reload($self, $idler) },
 		USR1 => sub { eidx_resync_start($self) },
diff --git a/t/extsearch.t b/t/extsearch.t
index 7b75c208..6bcfdd84 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -292,7 +292,7 @@ if ('reindex catches missed messages') {
 	is($oidx->eidx_meta($lc_key), $cmt_b, 'lc-v2 stays unchanged');
 	my @err = split(/^/, $err);
 	is(scalar(@err), 1, 'only one warning') or diag "err=$err";
-	like($err[0], qr/# reindex_unseen/, 'got reindex_unseen message');
+	like $err[0], qr/# .*? reindex_unseen/, 'got reindex_unseen message';
 	my $new = $oidx->get_art($max + 1);
 	is($new->{subject}, $eml->header('Subject'), 'new message added');
 

[PATCH 3/3] lei: use PublicInbox::Eml::warn_ignore_cb

[Eric Wong]

The open-coded callback was identical to the existing callback
generated by our Eml package, so just use the existing code
instead unless a user uses `lei import --noisy'
---
 lib/PublicInbox/LEI.pm | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 34ef95a1..1e8dc17f 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -625,12 +625,8 @@ sub _lei_atfork_child {
 	eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
 	undef $errors_log;
 	$quit = \&CORE::exit;
-	if (!$self->{opt}->{noisy}) { # only "lei import" sets this atm
-		my $cb = $SIG{__WARN__} // \&CORE::warn;
-		$SIG{__WARN__} = sub {
-			$cb->(@_) unless PublicInbox::Eml::warn_ignore(@_)
-		};
-	}
+	$self->{opt}->{noisy} or # only "lei import" sets this atm
+		$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$SIG{TERM} = $term_handler;
 	$current_lei = $persist ? undef : $self; # for SIG{__WARN__}
 }