From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: AS6315 166.70.0.0/16 X-Spam-Status: No, score=-3.7 required=3.0 tests=AWL,BAYES_00, RCVD_IN_DNSWL_LOW,SPF_PASS shortcircuit=no autolearn=ham autolearn_force=no version=3.4.1 Received: from out01.mta.xmission.com (out01.mta.xmission.com [166.70.13.231]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by dcvr.yhbt.net (Postfix) with ESMTPS id ECE041F597; Wed, 18 Jul 2018 16:54:04 +0000 (UTC) Received: from in01.mta.xmission.com ([166.70.13.51]) by out01.mta.xmission.com with esmtps (TLS1.2:ECDHE_RSA_AES_128_GCM_SHA256:128) (Exim 4.87) (envelope-from ) id 1ffpiG-0005S5-87; Wed, 18 Jul 2018 10:54:04 -0600 Received: from [97.119.167.31] (helo=x220.int.ebiederm.org) by in01.mta.xmission.com with esmtpsa (TLS1.2:ECDHE_RSA_AES_128_GCM_SHA256:128) (Exim 4.87) (envelope-from ) id 1ffpiE-00065S-98; Wed, 18 Jul 2018 10:54:04 -0600 From: "Eric W. Biederman" To: Eric Wong Cc: meta@public-inbox.org, "Eric W. Biederman" Date: Wed, 18 Jul 2018 11:53:25 -0500 Message-Id: <20180718165325.19834-3-ebiederm@xmission.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <87pnzkh4fx.fsf_-_@xmission.com> References: <87pnzkh4fx.fsf_-_@xmission.com> X-XM-SPF: eid=1ffpiE-00065S-98;;;mid=<20180718165325.19834-3-ebiederm@xmission.com>;;;hst=in01.mta.xmission.com;;;ip=97.119.167.31;;;frm=ebiederm@xmission.com;;;spf=neutral X-XM-AID: U2FsdGVkX18FhyKa8GCparQ75dwMYt/su+wLEK6QN84= X-SA-Exim-Connect-IP: 97.119.167.31 X-SA-Exim-Mail-From: ebiederm@xmission.com Subject: [PATCH v2 3/3] SearchIdx: Allow the amount of indexing be configured X-SA-Exim-Version: 4.2.1 (built Thu, 05 May 2016 13:38:54 -0600) X-SA-Exim-Scanned: Yes (on in01.mta.xmission.com) List-Id: This adds a new inbox configuration option 'indexlevel' that can take the values 'full', 'medium', and 'basic'. When set to 'full' everything is indexed including the positions of all terms. When set to 'medium' everything except the positions of terms is indexed. When set to 'basic' terms and positions are not indexed. Just the Overview database for NNTP is created. Which is still quite good and allows searching for messages by Message-ID. But there are no indexes to support searching inside the email messages themselves. Update the reindex tests to exercise the full medium and basic code paths Signed-off-by: "Eric W. Biederman" --- lib/PublicInbox/Config.pm | 2 +- lib/PublicInbox/SearchIdx.pm | 8 +++++++ t/v1reindex.t | 43 +++++++++++++++++++++++++++++++++++- t/v2reindex.t | 40 +++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index 289c36a6ccf1..78586560cf0f 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -158,7 +158,7 @@ sub _fill { foreach my $k (qw(mainrepo filter url newsgroup infourl watch watchheader httpbackendmax - replyto feedmax nntpserver)) { + replyto feedmax nntpserver indexlevel)) { my $v = $self->{"$pfx.$k"}; $rv->{$k} = $v if defined $v; } diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 8978914ab087..04e853069f06 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -47,6 +47,7 @@ sub git_unquote ($) { sub new { my ($class, $ibx, $creat, $part) = @_; + my $levels = qr/\A(?:full|medium|basic)\z/; my $mainrepo = $ibx; # for "public-inbox-index" w/o entry in config my $git_dir = $mainrepo; my ($altid, $git); @@ -62,6 +63,13 @@ sub new { PublicInbox::AltId->new($ibx, $_); } @$altid ]; } + if ($ibx->{indexlevel}) { + if ($ibx->{indexlevel} =~ $levels) { + $indexlevel = $ibx->{indexlevel}; + } else { + die("Invalid indexlevel $ibx->{indexlevel}\n"); + } + } } else { # v1 $ibx = { mainrepo => $git_dir, version => 1 }; } diff --git a/t/v1reindex.t b/t/v1reindex.t index 0df36d3fee89..ff32750fc3c3 100644 --- a/t/v1reindex.t +++ b/t/v1reindex.t @@ -20,6 +20,7 @@ my $ibx_config = { mainrepo => $mainrepo, name => 'test-v1reindex', -primary_address => 'test@example.com', + indexlevel => 'full', }; my $ibx = PublicInbox::Inbox->new($ibx_config); my $mime = PublicInbox::MIME->create( @@ -74,15 +75,32 @@ is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); + $rw = PublicInbox::SearchIdx->new($ibx, 1); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + eval { $rw->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is(scalar(@warn), 0, 'no warnings from reindexing'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); + delete $ibx->{mm}; + is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); +} +ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); + +$rw = PublicInbox::SearchIdx->new($ibx, 1); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; eval { $rw->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); - is(scalar(@warn), 0, 'no warnings from reindexing'); + is_deeply(\@warn, [], 'no warnings'); $im->done; ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; @@ -91,9 +109,31 @@ ok(!-d $xap, 'Xapian directories removed again'); ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); + +$ibx_config->{index_level} = 'medium'; +$ibx = PublicInbox::Inbox->new($ibx_config); $rw = PublicInbox::SearchIdx->new($ibx, 1); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + eval { $rw->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); + delete $ibx->{mm}; + is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); +} + +ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); + +$ibx_config->{index_level} = 'basic'; +$ibx = PublicInbox::Inbox->new($ibx_config); +$rw = PublicInbox::SearchIdx->new($ibx, 1); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; @@ -106,4 +146,5 @@ ok(!-d $xap, 'Xapian directories removed again'); is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); } + done_testing(); diff --git a/t/v2reindex.t b/t/v2reindex.t index 8af30991f858..20903967bf25 100644 --- a/t/v2reindex.t +++ b/t/v2reindex.t @@ -19,6 +19,7 @@ my $ibx_config = { name => 'test-v2writable', version => 2, -primary_address => 'test@example.com', + indexlevel => 'full', }; my $ibx = PublicInbox::Inbox->new($ibx_config); my $mime = PublicInbox::MIME->create( @@ -95,4 +96,43 @@ ok(!-d $xap, 'Xapian directories removed again'); is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); } +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); + +$ibx_config->{index_level} = 'medium'; +$ibx = PublicInbox::Inbox->new($ibx_config); +$im = PublicInbox::V2Writable->new($ibx); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + eval { $im->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); + delete $ibx->{mm}; + is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); +} + + +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); + +$ibx_config->{index_level} = 'basic'; +$ibx = PublicInbox::Inbox->new($ibx_config); +$im = PublicInbox::V2Writable->new($ibx); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + eval { $im->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); + delete $ibx->{mm}; + is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); +} + done_testing(); -- 2.17.1