[PATCH] cindex: dump cidx shards before inboxes

[PATCH] cindex: dump cidx shards before inboxes

[Eric Wong]

Since cidx shards used for associations are typically bigger
than individual inboxes, we'll dump them first to get better
work scheduling for xap_helper processes.

This gives roughly a 5% performance improvement with doing
a full associate on (git+lore).kernel.org
---
 lib/PublicInbox/CodeSearchIdx.pm | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 404d6826..a9a7d313 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -511,8 +511,10 @@ sub assoc_max_init ($) {
 	$max < 0 ? ((2 ** 31) - 1) : $max;
 }
 
-sub dump_roots_once {
+sub dump_roots_start {
 	my ($self, $associate) = @_;
+	($XHC, $XH_PID) = PublicInbox::XapClient::start_helper("-j$NPROC");
+	awaitpid($XH_PID, \&cmd_done, ['xap_helper', "-j$NPROC"]);
 	$associate // die 'BUG: no $associate';
 	$TODO{associating} = 1; # keep shards_active() happy
 	progress($self, 'dumping IDs from coderepos');
@@ -571,7 +573,6 @@ sub dump_ibx_start {
 	my $fold_pid = spawn(\@UNIQ_FOLD, $CMD_ENV, { 0 => $fold_r, 1 => $fh });
 	awaitpid($sort_pid, \&cmd_done, \@sort, $associate);
 	awaitpid($fold_pid, \&cmd_done, [@UNIQ_FOLD, '(ibx)'], $associate);
-	($XHC, $XH_PID) = PublicInbox::XapClient::start_helper("-j$NPROC");
 }
 
 sub index_next ($) {
@@ -586,11 +587,11 @@ sub index_next ($) {
 		fp_start($self, $git, $prep_repo);
 		ct_start($self, $git, $prep_repo);
 	} elsif ($TMPDIR) {
+		return if delete($TODO{dump_roots_start});
 		delete $TODO{dump_ibx_start}; # runs OnDestroy once
 		return dump_ibx($self, shift @IBXQ) if @IBXQ;
-		progress($self, 'done dumping inboxes') if $DUMP_IBX_WPIPE;
 		undef $DUMP_IBX_WPIPE; # done dumping inboxes, dump roots
-		dump_roots_once($self, delete($TODO{associate}) // return);
+		delete $TODO{associate};
 	}
 	# else: wait for shards_active (post_loop_do) callback
 }
@@ -934,6 +935,8 @@ EOM
 	$TODO{associate} = PublicInbox::OnDestroy->new($$, \&associate, $self);
 	$TODO{dump_ibx_start} = PublicInbox::OnDestroy->new($$,
 				\&dump_ibx_start, $self, $TODO{associate});
+	$TODO{dump_roots_start} = PublicInbox::OnDestroy->new($$,
+				\&dump_roots_start, $self, $TODO{associate});
 	my $id = -1;
 	@IBXQ = map { ++$id } @IBX;
 }

[FAIL?] cindex: maximize SMP when dumping inboxes for --associate

[Eric Wong]

  Note: this appears to result in a slight performance regression,
  I think, but I'm on a busy system...

  In theory I expect it to work since smaller work units means
  better work distribution when we're dealing with a few gigantic
  inboxes (LKML) and many smaller ones; but perhaps the extra
  IPC overhead from the multitudes of small inboxes on my lore
  mirror ends up hurting.

Instead of dumping all Xapian shards of an inbox with a
single xap_helper command, issue one xap_helper command
per-shard (as we already do with the cidx shards) to get
smaller work units and improve work sharing.
---
 lib/PublicInbox/CodeSearchIdx.pm | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index a9a7d313..d6348740 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -551,14 +551,18 @@ sub dump_roots_start {
 sub dump_ibx { # sends to xap_helper.h
 	my ($self, $ibx_id) = @_;
 	my $ibx = $IBX[$ibx_id] // die "BUG: no IBX[$ibx_id]";
-	my @cmd = ('dump_ibx', $ibx->isrch->xh_args,
-			(map { ('-A', $_) } @ASSOC_PFX),
-			$ibx_id, $QRY_STR);
-	pipe(my ($r, $w)) or die "pipe: $!";
-	$XHC->mkreq([$DUMP_IBX_WPIPE, $w], @cmd);
+	my @args = map { ('-A', $_) } @ASSOC_PFX;
 	my $ekey = $ibx->eidx_key;
-	$self->{PENDING}->{$ekey} = $TODO{associate};
-	PublicInbox::CidxXapHelperAux->new($r, $self, $ekey);
+	my $es = $ibx->isrch->{es};
+	push(@args, '-O', $ekey) if defined $es;
+	push(@args, $ibx_id, $QRY_STR);
+	for my $d ($es ? $es->shard_dirs : $ibx->isrch->shard_dirs) {
+		pipe(my ($r, $w)) or die "pipe: $!";
+		$XHC->mkreq([$DUMP_IBX_WPIPE, $w], qw(dump_ibx -d), $d, @args);
+		my $desc = "$ekey -d $d";
+		$self->{PENDING}->{$desc} = $TODO{associate};
+		PublicInbox::CidxXapHelperAux->new($r, $self, $desc);
+	}
 }
 
 sub dump_ibx_start {