From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 097DF20078 for ; Tue, 27 Oct 2020 07:54:56 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 16/52] inboxwritable: eidx_key for external index Date: Tue, 27 Oct 2020 07:54:17 +0000 Message-Id: <20201027075453.19163-17-e@80x24.org> In-Reply-To: <20201027075453.19163-1-e@80x24.org> References: <20201027075453.19163-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This is preferable to open-coding "newsgroup // inboxdir" everywhere. --- lib/PublicInbox/InboxWritable.pm | 2 ++ lib/PublicInbox/SearchIdx.pm | 12 ++++++------ lib/PublicInbox/SearchIdxShard.pm | 32 ++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm index e97c7e2d..d3c255c7 100644 --- a/lib/PublicInbox/InboxWritable.pm +++ b/lib/PublicInbox/InboxWritable.pm @@ -319,4 +319,6 @@ sub git_dir_latest { $latest; } +sub eidx_key { $_[0]->{newsgroup} // $_[0]->{inboxdir} } + 1; diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 283bdd6c..061a8153 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -368,9 +368,9 @@ sub add_xapian ($$$$) { $tg->set_document($doc); index_headers($self, $smsg); - if (my $ng_or_dir = $self->{ng_or_dir}) { # external index + if (defined(my $eidx_key = $smsg->{eidx_key})) { $doc->add_boolean_term('P'. - "$ng_or_dir:$smsg->{num}:$smsg->{blob}"); + "$eidx_key:$smsg->{num}:$smsg->{blob}"); } msg_iter($eml, \&index_xapian, [ $self, $doc ]); index_ids($self, $doc, $eml, $mids); @@ -456,21 +456,21 @@ sub _get_doc ($$$) { } sub add_xref3 { - my ($self, $docid, $xnum, $oid, $ng_or_dir, $eml) = @_; + my ($self, $docid, $xnum, $oid, $eidx_key, $eml) = @_; begin_txn_lazy($self); my $doc = _get_doc($self, $docid, $oid) or return; term_generator($self)->set_document($doc); - $doc->add_boolean_term('P'."$ng_or_dir:$xnum:$oid"); + $doc->add_boolean_term('P'."$eidx_key:$xnum:$oid"); index_list_id($self, $doc, $eml); $self->{xdb}->replace_document($docid, $doc); } sub remove_xref3 { - my ($self, $docid, $oid, $ng_or_dir, $eml) = @_; + my ($self, $docid, $oid, $eidx_key, $eml) = @_; begin_txn_lazy($self); my $doc = _get_doc($self, $docid, $oid) or return; my $xref3 = PublicInbox::Smsg::xref3(undef, $doc); - for (grep(/\A\Q$ng_or_dir\E:[0-9]+:\Q$oid\E\z/, @$xref3)) { + for (grep(/\A\Q$eidx_key\E:[0-9]+:\Q$oid\E\z/, @$xref3)) { $doc->remove_term('P' . $_); } for my $l ($eml->header_raw('List-Id')) { diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm index 8ff9ab8b..fa77a9f9 100644 --- a/lib/PublicInbox/SearchIdxShard.pm +++ b/lib/PublicInbox/SearchIdxShard.pm @@ -74,17 +74,21 @@ sub shard_worker_loop ($$$$$) { } elsif ($line =~ /\AD ([a-f0-9]{40,}) ([0-9]+)\n\z/s) { $self->remove_by_oid($1, $2 + 0); } elsif ($line =~ s/\A\+X //) { - my ($len, $docid, $xnum, $oid, $ng_or_dir) = + my ($len, $docid, $xnum, $oid, $eidx_key) = split(/ /, $line, 5); - $self->add_xref3($docid, $xnum, $oid, $ng_or_dir, + $self->add_xref3($docid, $xnum, $oid, $eidx_key, eml($r, $len)); } elsif ($line =~ s/\A-X //) { - my ($len, $docid, $xnum, $oid, $ng_or_dir) = + my ($len, $docid, $xnum, $oid, $eidx_key) = split(/ /, $line, 5); $self->remove_xref3($docid, $xnum, $oid, - $ng_or_dir, eml($r, $len)); + $eidx_key, eml($r, $len)); } else { chomp $line; + my $eidx_key; + if ($line =~ s/\AX(.+)\0//) { + $eidx_key = $1; + } # n.b. $mid may contain spaces(!) my ($len, $bytes, $num, $oid, $ds, $ts, $tid, $mid) = split(/ /, $line, 8); @@ -98,6 +102,7 @@ sub shard_worker_loop ($$$$$) { ds => $ds, ts => $ts, }, 'PublicInbox::Smsg'; + $smsg->{eidx_key} = $eidx_key if defined($eidx_key); $self->add_message(eml($r, $len), $smsg); } } @@ -105,8 +110,12 @@ sub shard_worker_loop ($$$$$) { } sub index_raw { - my ($self, $msgref, $eml, $smsg) = @_; + my ($self, $msgref, $eml, $smsg, $ibx) = @_; if (my $w = $self->{w}) { + if ($ibx) { + print $w 'X', $ibx->eidx_key, "\0" or die + "failed to write shard: $!\n"; + } $msgref //= \($eml->as_string); $smsg->{raw_bytes} //= length($$msgref); # mid must be last, it can contain spaces (but not LF) @@ -120,33 +129,34 @@ sub index_raw { $eml = PublicInbox::Eml->new($msgref); } $self->begin_txn_lazy; + $smsg->{eidx_key} = $ibx->eidx_key if $ibx; $self->add_message($eml, $smsg); } } sub shard_add_xref3 { my ($self, $docid, $xnum, $oid, $xibx, $eml) = @_; - my $ng_or_dir = $xibx->{newsgroup} // $xibx->{inboxdir}; + my $eidx_key = $xibx->eidx_key; if (my $w = $self->{w}) { my $hdr = $eml->header_obj->as_string; my $len = length($hdr); - print $w "+X $len $docid $xnum $oid $ng_or_dir\n", $hdr or + print $w "+X $len $docid $xnum $oid $eidx_key\n", $hdr or die "failed to write shard: $!"; } else { - $self->add_xref3($docid, $xnum, $oid, $ng_or_dir, $eml); + $self->add_xref3($docid, $xnum, $oid, $eidx_key, $eml); } } sub shard_remove_xref3 { my ($self, $docid, $oid, $xibx, $eml) = @_; - my $ng_or_dir = $xibx->{newsgroup} // $xibx->{inboxdir}; + my $eidx_key = $xibx->eidx_key; if (my $w = $self->{w}) { my $hdr = $eml->header_obj->as_string; my $len = length($hdr); - print $w "-X $len $docid $oid $ng_or_dir\n", $hdr or + print $w "-X $len $docid $oid $eidx_key\n", $hdr or die "failed to write shard: $!"; } else { - $self->remove_xref3($docid, $oid, $ng_or_dir, $eml); + $self->remove_xref3($docid, $oid, $eidx_key, $eml); } }