From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 3/7] searchidxshard: IPC conversion, part 2
Date: Sun, 3 Jan 2021 02:06:13 +0000 [thread overview]
Message-ID: <20210103020617.15719-4-e@80x24.org> (raw)
In-Reply-To: <20210103020617.15719-1-e@80x24.org>
We can remove some now-pointless wrapper functions by using
->ipc_do in even more places.
---
lib/PublicInbox/ExtSearchIdx.pm | 23 ++++++++++++-----------
lib/PublicInbox/LeiStore.pm | 13 +++++++------
lib/PublicInbox/SearchIdx.pm | 8 +-------
lib/PublicInbox/SearchIdxShard.pm | 30 ------------------------------
lib/PublicInbox/V2Writable.pm | 3 +--
5 files changed, 21 insertions(+), 56 deletions(-)
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index c3581628..064d9939 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -102,7 +102,7 @@ sub do_xpost ($$) {
if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message
my $xnum = $req->{xnum};
$self->{oidx}->add_xref3($docid, $xnum, $oid, $eidx_key);
- $idx->shard_add_eidx_info($docid, $eidx_key, $eml);
+ $idx->ipc_do('add_eidx_info', $docid, $eidx_key, $eml);
check_batch_limit($req);
} else { # 'd'
my $rm_eidx_info;
@@ -110,9 +110,10 @@ sub do_xpost ($$) {
\$rm_eidx_info);
if ($nr == 0) {
$self->{oidx}->eidxq_del($docid);
- $idx->shard_remove($docid);
+ $idx->ipc_do('xdb_remove', $docid);
} elsif ($rm_eidx_info) {
- $idx->shard_remove_eidx_info($docid, $eidx_key, $eml);
+ $idx->ipc_do('remove_eidx_info',
+ $docid, $eidx_key, $eml);
$self->{oidx}->eidxq_add($docid); # yes, add
}
}
@@ -327,7 +328,7 @@ DELETE FROM xref3 WHERE docid = ? AND ibx_id = ?
}
} else {
warn "I: remove #$docid $eidx_key @oid\n";
- $self->idx_shard($docid)->shard_remove($docid);
+ $self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
}
}
@@ -440,7 +441,7 @@ sub _reindex_finalize ($$$) {
for my $x (reverse @$stable) {
$ibx = _ibx_for($self, $sync, $x);
my $hdr = delete $x->{hdr} // die 'BUG: no {hdr}';
- $idx->shard_add_eidx_info($docid, $ibx->eidx_key, $hdr);
+ $idx->ipc_do('add_eidx_info', $docid, $ibx->eidx_key, $hdr);
}
return if $nr == 1; # likely, all good
@@ -483,12 +484,12 @@ sub _reindex_oid { # git->cat_async callback
my $remain = $self->{oidx}->remove_xref3($docid, $expect_oid);
if ($remain == 0) {
warn "W: #$docid gone or corrupted\n";
- $self->idx_shard($docid)->shard_remove($docid);
+ $self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
} elsif (my $next_oid = $req->{xr3r}->[++$req->{ix}]->[2]) {
$self->git->cat_async($next_oid, \&_reindex_oid, $req);
} else {
warn "BUG: #$docid gone (UNEXPECTED)\n";
- $self->idx_shard($docid)->shard_remove($docid);
+ $self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
}
return;
}
@@ -522,7 +523,7 @@ sub _reindex_smsg ($$$) {
BUG? #$docid $smsg->{blob} is not referenced by inboxes during reindex
$self->{oidx}->delete_by_num($docid);
- $self->idx_shard($docid)->shard_remove($docid);
+ $self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
return;
}
@@ -799,10 +800,10 @@ DELETE FROM xref3 WHERE ibx_id = ? AND xnum = ? AND oidbin = ?
if (scalar(@$xr3) == 0) { # all gone
$self->{oidx}->delete_by_num($docid);
$self->{oidx}->eidxq_del($docid);
- $idx->shard_remove($docid);
+ $idx->ipc_do('xdb_remove', $docid);
} else { # enqueue for reindex of remaining messages
- $idx->shard_remove_eidx_info($docid,
- $ibx->eidx_key);
+ $idx->ipc_do('remove_eidx_info',
+ $docid, $ibx->eidx_key);
$self->{oidx}->eidxq_add($docid); # yes, add
}
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 07a3198a..d686e95a 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -146,7 +146,7 @@ sub set_eml_keywords {
my $eidx = eidx_init($self);
my @docids = _docids_for($self, $eml);
for my $docid (@docids) {
- $eidx->idx_shard($docid)->shard_set_keywords($docid, @kw);
+ $eidx->idx_shard($docid)->ipc_do('set_keywords', $docid, @kw);
}
\@docids;
}
@@ -156,7 +156,7 @@ sub add_eml_keywords {
my $eidx = eidx_init($self);
my @docids = _docids_for($self, $eml);
for my $docid (@docids) {
- $eidx->idx_shard($docid)->shard_add_keywords($docid, @kw);
+ $eidx->idx_shard($docid)->ipc_do('add_keywords', $docid, @kw);
}
\@docids;
}
@@ -166,7 +166,7 @@ sub remove_eml_keywords {
my $eidx = eidx_init($self);
my @docids = _docids_for($self, $eml);
for my $docid (@docids) {
- $eidx->idx_shard($docid)->shard_remove_keywords($docid, @kw);
+ $eidx->idx_shard($docid)->ipc_do('remove_keywords', $docid, @kw)
}
\@docids;
}
@@ -205,8 +205,9 @@ sub add_eml {
for my $docid (@docids) {
my $idx = $eidx->idx_shard($docid);
$oidx->add_xref3($docid, -1, $smsg->{blob}, '.');
- $idx->shard_add_eidx_info($docid, '.', $eml); # List-Id
- $idx->shard_add_keywords($docid, @kw) if @kw;
+ # add_eidx_info for List-Id
+ $idx->ipc_do('add_eidx_info', $docid, '.', $eml);
+ $idx->ipc_do('add_keywords', $docid, @kw) if @kw;
}
\@docids;
} else {
@@ -215,7 +216,7 @@ sub add_eml {
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
my $idx = $eidx->idx_shard($smsg->{num});
$idx->index_raw($msgref, $eml, $smsg);
- $idx->shard_add_keywords($smsg->{num}, @kw) if @kw;
+ $idx->ipc_do('add_keywords', $smsg->{num}, @kw) if @kw;
$smsg;
}
}
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index d83fd4ca..da3ac2e3 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -552,6 +552,7 @@ sub smsg_from_doc ($) {
sub xdb_remove {
my ($self, @docids) = @_;
+ $self->begin_txn_lazy;
my $xdb = $self->{xdb} or return;
for my $docid (@docids) {
eval { $xdb->delete_document($docid) };
@@ -559,13 +560,6 @@ sub xdb_remove {
}
}
-sub remove_by_docid {
- my ($self, $num) = @_;
- die "BUG: remove_by_docid is v2-only\n" if $self->{oidx};
- $self->begin_txn_lazy;
- xdb_remove($self, $num) if need_xapian($self);
-}
-
sub index_git_blob_id {
my ($doc, $pfx, $objid) = @_;
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 68644bc0..43dad959 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -54,16 +54,6 @@ sub index_raw {
$self->ipc_do('add_message', $eml, $smsg);
}
-sub shard_add_eidx_info {
- my ($self, $docid, $eidx_key, $eml) = @_;
- $self->ipc_do('add_eidx_info', $docid, $eidx_key, $eml);
-}
-
-sub shard_remove_eidx_info {
- my ($self, $docid, $eidx_key, $eml) = @_;
- $self->ipc_do('remove_eidx_info', $docid, $eidx_key, $eml);
-}
-
# needed when there's multiple IPC workers and the parent forking
# causes newer siblings to inherit older siblings sockets
sub shard_atfork_child {
@@ -93,26 +83,6 @@ sub shard_close {
$self->ipc_worker_stop;
}
-sub shard_remove {
- my ($self, $num) = @_;
- $self->ipc_do('remove_by_docid', $num);
-}
-
-sub shard_set_keywords {
- my ($self, $docid, @kw) = @_;
- $self->ipc_do('set_keywords', $docid, @kw);
-}
-
-sub shard_remove_keywords {
- my ($self, $docid, @kw) = @_;
- $self->ipc_do('remove_keywords', $docid, @kw);
-}
-
-sub shard_add_keywords {
- my ($self, $docid, @kw) = @_;
- $self->ipc_do('add_keywords', $docid, @kw);
-}
-
sub shard_over_check {
my ($self, $over) = @_;
if ($self->{-ipc_sock} && $over->{dbh}) {
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index cad559c5..885edbe9 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1133,8 +1133,7 @@ sub unindex_oid_aux ($$$) {
my ($self, $oid, $mid) = @_;
my @removed = $self->{oidx}->remove_oid($oid, $mid);
for my $num (@removed) {
- my $idx = idx_shard($self, $num);
- $idx->shard_remove($num);
+ idx_shard($self, $num)->ipc_do('xdb_remove', $num);
}
}
next prev parent reply other threads:[~2021-01-03 2:06 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-03 2:06 [PATCH 0/7] v2: swap in new IPC package Eric Wong
2021-01-03 2:06 ` [PATCH 1/7] ipc: some documentation comments Eric Wong
2021-01-03 2:06 ` [PATCH 2/7] searchidxshard: use PublicInbox::IPC to kill lots of code Eric Wong
2021-01-03 2:06 ` Eric Wong [this message]
2021-01-03 2:06 ` [PATCH 4/7] searchidxshard: replace index_raw with index_eml Eric Wong
2021-01-03 2:06 ` [PATCH 5/7] use Eml (or MIME) objects for all indexing paths Eric Wong
2021-01-03 2:06 ` [PATCH 6/7] ipc: switch to one-way pipes Eric Wong
2021-01-03 2:06 ` [PATCH 7/7] searchidxshard: use add_xapian directly for v2 Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210103020617.15719-4-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).