From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id D62352009E for ; Tue, 27 Oct 2020 07:55:00 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 43/52] searchidx: remove xref3 support for Xapian Date: Tue, 27 Oct 2020 07:54:44 +0000 Message-Id: <20201027075453.19163-44-e@80x24.org> In-Reply-To: <20201027075453.19163-1-e@80x24.org> References: <20201027075453.19163-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: It doesn't seem worth storing xref3 data in Xapian now that the same info is in over.sqlite3. --- lib/PublicInbox/ExtSearchIdx.pm | 10 +++-- lib/PublicInbox/SearchIdx.pm | 64 +++++++++++-------------------- lib/PublicInbox/SearchIdxShard.pm | 28 +++++++------- lib/PublicInbox/Smsg.pm | 13 ------- t/search.t | 26 +------------ 5 files changed, 46 insertions(+), 95 deletions(-) diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm index 790ee921..026e1377 100644 --- a/lib/PublicInbox/ExtSearchIdx.pm +++ b/lib/PublicInbox/ExtSearchIdx.pm @@ -116,9 +116,10 @@ sub do_xpost ($$) { my $eml = $req->{eml}; if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message my $xnum = $req->{xnum}; - $idx->shard_add_xref3($docid, $xnum, $oid, $xibx, $eml); + $self->{oidx}->add_xref3($docid, $xnum, $oid, $xibx->eidx_key); + $idx->shard_add_eidx_info($docid, $oid, $xibx, $eml); } else { # 'd' - $idx->shard_remove_xref3($docid, $oid, $xibx, $eml); + $idx->shard_remove_eidx_info($docid, $oid, $xibx, $eml); } } @@ -135,7 +136,10 @@ sub index_unseen ($) { $new_smsg->{num} = $docid; my $idx = $self->idx_shard($docid); $self->{oidx}->add_overview($eml, $new_smsg); - $idx->index_raw(undef, $eml, $new_smsg, $req->{ibx}); + my $oid = $new_smsg->{blob}; + my $ibx = delete $req->{ibx} or die 'BUG: {ibx} unset'; + $self->{oidx}->add_xref3($docid, $req->{xnum}, $oid, $ibx->eidx_key); + $idx->index_raw(undef, $eml, $new_smsg, $ibx); } sub do_finalize ($) { diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 32fa16f5..569efbb0 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -370,8 +370,6 @@ sub add_xapian ($$$$) { if (defined(my $eidx_key = $smsg->{eidx_key})) { $doc->add_boolean_term('O'.$eidx_key); - $doc->add_boolean_term('P'. - "$eidx_key:$smsg->{num}:$smsg->{blob}"); } msg_iter($eml, \&index_xapian, [ $self, $doc ]); index_ids($self, $doc, $eml, $mids); @@ -456,57 +454,41 @@ sub _get_doc ($$$) { } } -sub add_xref3 { - my ($self, $docid, $xnum, $oid, $eidx_key, $eml) = @_; +sub add_eidx_info { + my ($self, $docid, $oid, $eidx_key, $eml) = @_; begin_txn_lazy($self); my $doc = _get_doc($self, $docid, $oid) or return; term_generator($self)->set_document($doc); $doc->add_boolean_term('O'.$eidx_key); - $doc->add_boolean_term('P'."$eidx_key:$xnum:$oid"); index_list_id($self, $doc, $eml); $self->{xdb}->replace_document($docid, $doc); } -sub remove_xref3 { +sub remove_eidx_info { my ($self, $docid, $oid, $eidx_key, $eml) = @_; begin_txn_lazy($self); my $doc = _get_doc($self, $docid, $oid) or return; - my $xref3 = PublicInbox::Smsg::xref3(undef, $doc); - my %x3 = map { $_ => undef } @$xref3; - for (grep(/\A\Q$eidx_key\E:[0-9]+:\Q$oid\E\z/, @$xref3)) { - delete $x3{$_}; - $doc->remove_term('P' . $_); - } - if (scalar(keys(%x3)) == 0) { - $self->{xdb}->delete_document($docid); - if (my $del_fh = $self->{del_fh}) { # TODO - print $del_fh $docid, "\n" or die "E: print $!"; - } - } else { - if (!grep(/\A\Q$eidx_key\E:/, keys %x3)) { - $doc->remove_term('O'.$eidx_key); - } - for my $l ($eml->header_raw('List-Id')) { - $l =~ /<([^>]+)>/ or next; - my $lid = lc $1; - $doc->remove_term('G' . $lid); - - # nb: we don't remove the XL probabilistic terms - # since terms may overlap if cross-posted. - # - # IOW, a message which has both - # and would have overlapping - # "XLexample" and "XLcom" as terms and which we - # wouldn't know if they're safe to remove if we just - # unindex while preserving - # . - # - # In any case, this entire sub is will likely never - # be needed and users using the "l:" prefix are probably - # rarer. - } - $self->{xdb}->replace_document($docid, $doc); + $doc->remove_term('O'.$eidx_key); + for my $l ($eml->header_raw('List-Id')) { + $l =~ /<([^>]+)>/ or next; + my $lid = lc $1; + $doc->remove_term('G' . $lid); + + # nb: we don't remove the XL probabilistic terms + # since terms may overlap if cross-posted. + # + # IOW, a message which has both + # and would have overlapping + # "XLexample" and "XLcom" as terms and which we + # wouldn't know if they're safe to remove if we just + # unindex while preserving + # . + # + # In any case, this entire sub is will likely never + # be needed and users using the "l:" prefix are probably + # rarer. } + $self->{xdb}->replace_document($docid, $doc); } sub get_val ($$) { diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm index ac01340c..644d8b58 100644 --- a/lib/PublicInbox/SearchIdxShard.pm +++ b/lib/PublicInbox/SearchIdxShard.pm @@ -75,15 +75,15 @@ sub shard_worker_loop ($$$$$) { } elsif ($line =~ /\AD ([a-f0-9]{40,}) ([0-9]+)\n\z/s) { $self->remove_by_oid($1, $2 + 0); } elsif ($line =~ s/\A\+X //) { - my ($len, $docid, $xnum, $oid, $eidx_key) = - split(/ /, $line, 5); - $self->add_xref3($docid, $xnum, $oid, $eidx_key, - eml($r, $len)); + my ($len, $docid, $oid, $eidx_key) = + split(/ /, $line, 4); + $self->add_eidx_info($docid, $oid, $eidx_key, + eml($r, $len)); } elsif ($line =~ s/\A-X //) { - my ($len, $docid, $xnum, $oid, $eidx_key) = - split(/ /, $line, 5); - $self->remove_xref3($docid, $xnum, $oid, - $eidx_key, eml($r, $len)); + my ($len, $docid, $oid, $eidx_key) = + split(/ /, $line, 4); + $self->remove_eidx_info($docid, $oid, $eidx_key, + eml($r, $len)); } else { chomp $line; my $eidx_key; @@ -135,20 +135,20 @@ sub index_raw { } } -sub shard_add_xref3 { - my ($self, $docid, $xnum, $oid, $xibx, $eml) = @_; +sub shard_add_eidx_info { + my ($self, $docid, $oid, $xibx, $eml) = @_; my $eidx_key = $xibx->eidx_key; if (my $w = $self->{w}) { my $hdr = $eml->header_obj->as_string; my $len = length($hdr); - print $w "+X $len $docid $xnum $oid $eidx_key\n", $hdr or + print $w "+X $len $docid $oid $eidx_key\n", $hdr or die "failed to write shard: $!"; } else { - $self->add_xref3($docid, $xnum, $oid, $eidx_key, $eml); + $self->add_eidx_info($docid, $oid, $eidx_key, $eml); } } -sub shard_remove_xref3 { +sub shard_remove_eidx_info { my ($self, $docid, $oid, $xibx, $eml) = @_; my $eidx_key = $xibx->eidx_key; if (my $w = $self->{w}) { @@ -157,7 +157,7 @@ sub shard_remove_xref3 { print $w "-X $len $docid $oid $eidx_key\n", $hdr or die "failed to write shard: $!"; } else { - $self->remove_xref3($docid, $oid, $eidx_key, $eml); + $self->remove_eidx_info($docid, $oid, $eidx_key, $eml); } } diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm index c0fd85fd..14086538 100644 --- a/lib/PublicInbox/Smsg.pm +++ b/lib/PublicInbox/Smsg.pm @@ -137,17 +137,4 @@ sub subject_normalized ($) { $subj; } -sub xref3 { - my ($self, $doc) = @_; - my $end = $doc->termlist_end; - my $it = $doc->termlist_begin; - $it->skip_to('P'); - my @ret; - for (; $it != $end; $it++) { - my $val = $it->get_termname; - $val =~ s/\AP// and push @ret, $val; - } - \@ret; -} - 1; diff --git a/t/search.t b/t/search.t index e789b81e..da9acb07 100644 --- a/t/search.t +++ b/t/search.t @@ -341,14 +341,6 @@ $ibx->with_umask(sub { my $uid = PublicInbox::SearchIdx::get_val($doc, $col); is($uid, $smsg->{num}, 'UID column matches {num}'); is($uid, $m->get_docid, 'UID column matches docid'); - - # check ->xref3 for external index: - is_deeply($smsg->xref3($doc), [], 'xref3 empty by default'); - my $exp = "inbox.com.example:$uid:deadbeef"; - $doc->add_boolean_term('P'.$exp); - is_deeply($smsg->xref3($doc), [ $exp ], 'xref3 can be set'); - $doc->remove_term('P'.$exp); - is_deeply($smsg->xref3($doc), [], 'xref3 can be unset'); } $mset = $ibx->search->mset('tc:list@example.com'); @@ -521,13 +513,8 @@ $ibx->with_umask(sub { $rw_commit->(); my $doc_id = $rw->add_message(eml_load('t/data/message_embed.eml')); ok($doc_id > 0, 'messages within messages'); - - my $eml = PublicInbox::Eml->new(< - -EOF - $rw->add_xref3($doc_id, 1, 'deadbeef', 'newsgroup1.example', $eml); - $rw_commit->(); + $rw->commit_txn_lazy; + $ibx->search->reopen; my $n_test_eml = $query->('n:test.eml'); is(scalar(@$n_test_eml), 1, 'got a result'); my $n_embed2x_eml = $query->('n:embed2x.eml'); @@ -545,15 +532,6 @@ EOF is($query->('s:"mail header experiments"')->[0]->{mid}, '20200418222508.GA13918@dcvr', 'Subject search reaches inside message/rfc822'); - is($query->('l:blahblah.example.com')->[0]->{num}, $doc_id, - 'xref3 List-Id probabilistic works'); - is($query->('lid:blahblah.example.com')->[0]->{num}, $doc_id, - 'xref3 List-Id boolean term works'); - $rw->remove_xref3($doc_id, 'deadbeef', 'newsgroup1.example', $eml); - $rw->commit_txn_lazy; - $ibx->search->reopen; - my $res = $query->('lid:blahblah.example.com'); - is_deeply($res, [], '->remove_xref3 dropped boolean term'); }); done_testing();