From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id F0BB01FA13 for ; Sun, 3 Jan 2021 02:06:17 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/7] searchidxshard: IPC conversion, part 2 Date: Sun, 3 Jan 2021 02:06:13 +0000 Message-Id: <20210103020617.15719-4-e@80x24.org> In-Reply-To: <20210103020617.15719-1-e@80x24.org> References: <20210103020617.15719-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We can remove some now-pointless wrapper functions by using ->ipc_do in even more places. --- lib/PublicInbox/ExtSearchIdx.pm | 23 ++++++++++++----------- lib/PublicInbox/LeiStore.pm | 13 +++++++------ lib/PublicInbox/SearchIdx.pm | 8 +------- lib/PublicInbox/SearchIdxShard.pm | 30 ------------------------------ lib/PublicInbox/V2Writable.pm | 3 +-- 5 files changed, 21 insertions(+), 56 deletions(-) diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm index c3581628..064d9939 100644 --- a/lib/PublicInbox/ExtSearchIdx.pm +++ b/lib/PublicInbox/ExtSearchIdx.pm @@ -102,7 +102,7 @@ sub do_xpost ($$) { if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message my $xnum = $req->{xnum}; $self->{oidx}->add_xref3($docid, $xnum, $oid, $eidx_key); - $idx->shard_add_eidx_info($docid, $eidx_key, $eml); + $idx->ipc_do('add_eidx_info', $docid, $eidx_key, $eml); check_batch_limit($req); } else { # 'd' my $rm_eidx_info; @@ -110,9 +110,10 @@ sub do_xpost ($$) { \$rm_eidx_info); if ($nr == 0) { $self->{oidx}->eidxq_del($docid); - $idx->shard_remove($docid); + $idx->ipc_do('xdb_remove', $docid); } elsif ($rm_eidx_info) { - $idx->shard_remove_eidx_info($docid, $eidx_key, $eml); + $idx->ipc_do('remove_eidx_info', + $docid, $eidx_key, $eml); $self->{oidx}->eidxq_add($docid); # yes, add } } @@ -327,7 +328,7 @@ DELETE FROM xref3 WHERE docid = ? AND ibx_id = ? } } else { warn "I: remove #$docid $eidx_key @oid\n"; - $self->idx_shard($docid)->shard_remove($docid); + $self->idx_shard($docid)->ipc_do('xdb_remove', $docid); } } @@ -440,7 +441,7 @@ sub _reindex_finalize ($$$) { for my $x (reverse @$stable) { $ibx = _ibx_for($self, $sync, $x); my $hdr = delete $x->{hdr} // die 'BUG: no {hdr}'; - $idx->shard_add_eidx_info($docid, $ibx->eidx_key, $hdr); + $idx->ipc_do('add_eidx_info', $docid, $ibx->eidx_key, $hdr); } return if $nr == 1; # likely, all good @@ -483,12 +484,12 @@ sub _reindex_oid { # git->cat_async callback my $remain = $self->{oidx}->remove_xref3($docid, $expect_oid); if ($remain == 0) { warn "W: #$docid gone or corrupted\n"; - $self->idx_shard($docid)->shard_remove($docid); + $self->idx_shard($docid)->ipc_do('xdb_remove', $docid); } elsif (my $next_oid = $req->{xr3r}->[++$req->{ix}]->[2]) { $self->git->cat_async($next_oid, \&_reindex_oid, $req); } else { warn "BUG: #$docid gone (UNEXPECTED)\n"; - $self->idx_shard($docid)->shard_remove($docid); + $self->idx_shard($docid)->ipc_do('xdb_remove', $docid); } return; } @@ -522,7 +523,7 @@ sub _reindex_smsg ($$$) { BUG? #$docid $smsg->{blob} is not referenced by inboxes during reindex $self->{oidx}->delete_by_num($docid); - $self->idx_shard($docid)->shard_remove($docid); + $self->idx_shard($docid)->ipc_do('xdb_remove', $docid); return; } @@ -799,10 +800,10 @@ DELETE FROM xref3 WHERE ibx_id = ? AND xnum = ? AND oidbin = ? if (scalar(@$xr3) == 0) { # all gone $self->{oidx}->delete_by_num($docid); $self->{oidx}->eidxq_del($docid); - $idx->shard_remove($docid); + $idx->ipc_do('xdb_remove', $docid); } else { # enqueue for reindex of remaining messages - $idx->shard_remove_eidx_info($docid, - $ibx->eidx_key); + $idx->ipc_do('remove_eidx_info', + $docid, $ibx->eidx_key); $self->{oidx}->eidxq_add($docid); # yes, add } } diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index 07a3198a..d686e95a 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -146,7 +146,7 @@ sub set_eml_keywords { my $eidx = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_set_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('set_keywords', $docid, @kw); } \@docids; } @@ -156,7 +156,7 @@ sub add_eml_keywords { my $eidx = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_add_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('add_keywords', $docid, @kw); } \@docids; } @@ -166,7 +166,7 @@ sub remove_eml_keywords { my $eidx = eidx_init($self); my @docids = _docids_for($self, $eml); for my $docid (@docids) { - $eidx->idx_shard($docid)->shard_remove_keywords($docid, @kw); + $eidx->idx_shard($docid)->ipc_do('remove_keywords', $docid, @kw) } \@docids; } @@ -205,8 +205,9 @@ sub add_eml { for my $docid (@docids) { my $idx = $eidx->idx_shard($docid); $oidx->add_xref3($docid, -1, $smsg->{blob}, '.'); - $idx->shard_add_eidx_info($docid, '.', $eml); # List-Id - $idx->shard_add_keywords($docid, @kw) if @kw; + # add_eidx_info for List-Id + $idx->ipc_do('add_eidx_info', $docid, '.', $eml); + $idx->ipc_do('add_keywords', $docid, @kw) if @kw; } \@docids; } else { @@ -215,7 +216,7 @@ sub add_eml { $oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.'); my $idx = $eidx->idx_shard($smsg->{num}); $idx->index_raw($msgref, $eml, $smsg); - $idx->shard_add_keywords($smsg->{num}, @kw) if @kw; + $idx->ipc_do('add_keywords', $smsg->{num}, @kw) if @kw; $smsg; } } diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index d83fd4ca..da3ac2e3 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -552,6 +552,7 @@ sub smsg_from_doc ($) { sub xdb_remove { my ($self, @docids) = @_; + $self->begin_txn_lazy; my $xdb = $self->{xdb} or return; for my $docid (@docids) { eval { $xdb->delete_document($docid) }; @@ -559,13 +560,6 @@ sub xdb_remove { } } -sub remove_by_docid { - my ($self, $num) = @_; - die "BUG: remove_by_docid is v2-only\n" if $self->{oidx}; - $self->begin_txn_lazy; - xdb_remove($self, $num) if need_xapian($self); -} - sub index_git_blob_id { my ($doc, $pfx, $objid) = @_; diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm index 68644bc0..43dad959 100644 --- a/lib/PublicInbox/SearchIdxShard.pm +++ b/lib/PublicInbox/SearchIdxShard.pm @@ -54,16 +54,6 @@ sub index_raw { $self->ipc_do('add_message', $eml, $smsg); } -sub shard_add_eidx_info { - my ($self, $docid, $eidx_key, $eml) = @_; - $self->ipc_do('add_eidx_info', $docid, $eidx_key, $eml); -} - -sub shard_remove_eidx_info { - my ($self, $docid, $eidx_key, $eml) = @_; - $self->ipc_do('remove_eidx_info', $docid, $eidx_key, $eml); -} - # needed when there's multiple IPC workers and the parent forking # causes newer siblings to inherit older siblings sockets sub shard_atfork_child { @@ -93,26 +83,6 @@ sub shard_close { $self->ipc_worker_stop; } -sub shard_remove { - my ($self, $num) = @_; - $self->ipc_do('remove_by_docid', $num); -} - -sub shard_set_keywords { - my ($self, $docid, @kw) = @_; - $self->ipc_do('set_keywords', $docid, @kw); -} - -sub shard_remove_keywords { - my ($self, $docid, @kw) = @_; - $self->ipc_do('remove_keywords', $docid, @kw); -} - -sub shard_add_keywords { - my ($self, $docid, @kw) = @_; - $self->ipc_do('add_keywords', $docid, @kw); -} - sub shard_over_check { my ($self, $over) = @_; if ($self->{-ipc_sock} && $over->{dbh}) { diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index cad559c5..885edbe9 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -1133,8 +1133,7 @@ sub unindex_oid_aux ($$$) { my ($self, $oid, $mid) = @_; my @removed = $self->{oidx}->remove_oid($oid, $mid); for my $num (@removed) { - my $idx = idx_shard($self, $num); - $idx->shard_remove($num); + idx_shard($self, $num)->ipc_do('xdb_remove', $num); } }