[PATCH 0/2] unrelated coderepo-related fixes

[PATCH 0/2] unrelated coderepo-related fixes

[Eric Wong]

1/2 affects -httpd/netd/imapd/pop3d, and 2/2 slightly improves
things with cindex + `public-inbox-clone --purge' running in parallel

Eric Wong (2):
  git: cleanup un-associated coderepo processes
  cindex: clarify fatal vs non-fatal messages

 lib/PublicInbox/CodeSearchIdx.pm | 13 +++++++------
 lib/PublicInbox/Git.pm           |  6 ++++--
 lib/PublicInbox/Inbox.pm         |  4 ----
 3 files changed, 11 insertions(+), 12 deletions(-)

[PATCH 1/2] git: cleanup un-associated coderepo processes

[Eric Wong]

It's possible to have many coderepos with no inbox association
that never see git->cleanup.  So instead of tying git->cleanup
to inboxes, ensure it gets armed when ->watch_async is called
(since it's only called in our -netd or -httpd servers).
---
 lib/PublicInbox/Git.pm   | 6 ++++--
 lib/PublicInbox/Inbox.pm | 4 ----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 9c26d8bf..f4a24f2a 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -668,8 +668,10 @@ sub event_step {
 
 # idempotently registers with DS epoll/kqueue/select/poll
 sub watch_async ($) {
-	$_[0]->{epwatch} //= do {
-		$_[0]->SUPER::new($_[0]->{sock}, EPOLLIN);
+	my ($self) = @_;
+	PublicInbox::DS::add_uniq_timer($self+0, 30, \&cleanup, $self, 1);
+	$self->{epwatch} //= do {
+		$self->SUPER::new($self->{sock}, EPOLLIN);
 		\undef;
 	}
 }
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 3dad7004..b31f3fff 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -33,9 +33,6 @@ sub do_cleanup {
 	}
 	my $srch = $ibx->{search} // $ibx;
 	delete @$srch{qw(xdb qp)};
-	for my $git (@{$ibx->{-repo_objs} // []}) {
-		$live = 1 if $git->cleanup(1);
-	}
 	PublicInbox::DS::add_uniq_timer($ibx+0, 5, \&do_cleanup, $ibx) if $live;
 }
 
@@ -116,7 +113,6 @@ sub git {
 		my $g = PublicInbox::Git->new($git_dir);
 		my $lim = $self->{-httpbackend_limiter};
 		$g->{-httpbackend_limiter} = $lim if $lim;
-		_cleanup_later($self);
 		$g;
 	};
 }

[PATCH 2/2] cindex: clarify fatal vs non-fatal messages

[Eric Wong]

cindex must be able to handle coderepos being deleted mid-run
since `public-inbox-clone --purge' may be running at the same
time.  This is a step towards handling parallel invocations
of -cindex and public-inbox-clone as gracefully as possible
by improving error messages.
---
 lib/PublicInbox/CodeSearchIdx.pm | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 33080664..bf410734 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -249,12 +249,12 @@ EOM
 }
 
 sub cidx_reap_log { # awaitpid cb
-	my ($pid, $self, $op_p) = @_;
+	my ($pid, $cmd, $self, $op_p) = @_;
 	if (!$? || ($DO_QUIT && (($? & 127) == $DO_QUIT ||
 				($? & 127) == POSIX::SIGPIPE))) {
 		send($op_p, "shard_done $self->{shard}", 0);
 	} else {
-		warn "E: git @LOG_STDIN: \$?=$?\n";
+		warn "W: @$cmd (\$?=$?)\n";
 		$self->{xdb}->cancel_transaction;
 	}
 }
@@ -265,8 +265,9 @@ sub shard_index { # via wq_io_do in IDX_SHARDS
 	my $in = delete($self->{0}) // die 'BUG: no {0} input';
 	my $op_p = delete($self->{1}) // die 'BUG: no {1} op_p';
 	sysseek($in, 0, SEEK_SET);
-	my $rd = popen_rd($git->cmd(@LOG_STDIN), undef, { 0 => $in },
-				\&cidx_reap_log, $self, $op_p);
+	my $cmd = $git->cmd(@LOG_STDIN);
+	my $rd = popen_rd($cmd, undef, { 0 => $in },
+				\&cidx_reap_log, $cmd, $self, $op_p);
 	close $in;
 	PublicInbox::CidxLogP->new($rd, $self, $git, $roots);
 	# CidxLogP->event_step will call cidx_read_log_p once there's input
@@ -372,7 +373,7 @@ sub docids_by_postlist ($$) { # consider moving to PublicInbox::Search
 sub _cb { # run_await cb
 	my ($pid, $cmd, undef, $opt, $cb, $self, $git, @arg) = @_;
 	return if $DO_QUIT;
-	($git->{-cidx_err} = $?) ? warn("@$cmd error: \$?=$?\n") :
+	($git->{-cidx_err} = $?) ? warn("W: @$cmd (\$?=$?)\n") :
 				$cb->($opt, $self, $git, @arg);
 }
 
@@ -864,7 +865,7 @@ sub prep_alternate_start {
 
 sub cmd_done { # run_await cb for sort, xapian-delve, sed failures
 	my ($pid, $cmd, undef, undef, $run_on_destroy) = @_;
-	$? and die "@$cmd failed: \$?=$?";
+	$? and die "fatal: @$cmd (\$?=$?)\n";
 	# $run_on_destroy calls associate() or run_prune()
 }