[PATCH] v1: allow upgrading indexlevel=basic to 'medium' or 'full'

[Eric Wong <e@80x24.org>]

"Eric W. Biederman" <ebiederm@xmission.com> wrote:
> Eric Wong <e@80x24.org> writes:
> > I think being able to transition from an existing "basic" to
> > (full|medium) can be straightforward, too, with the untested
> > patch below.
> >
> > But most other transitions between levels will require an expensive
> > --reindex.   (full|medium) => basic would be easy, too, but
> > it'll be up to the user to remove the Xapian files.
> 
> I would still use "$self->{indexlevel} =~ m/(full|medium)/" instead of
> ne 'basic' just in case we start recognizing another level.
 
Good catch.

> Otherwise your change seems to make sense.  I have not really dug into
> the reindex case where the point is to pick up things that were
> previously missed.  I can definitely see that code should be guarded
> with an indexlevel check.

Unfortunately, this only works with v1 right now; and supporting
it for v2 would not be as easy... (I wouldn't consider it a priority,
either).

--------8<------
Subject: [PATCH] v1: allow upgrading indexlevel=basic to 'medium' or 'full'

For v1 repos, we don't need to write any metadata to Xapian
and changing from 'basic' to 'medium' or 'full' will work.

For v2, the metadata for indexing is stored in msgmap (because
the Xapian databases are partitioned for parallelism), so a
reindex is required.
---
 lib/PublicInbox/SearchIdx.pm |  5 +++--
 t/v1reindex.t                | 21 ++++++++++++++++++++-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index bb60506..1d259a8 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -27,6 +27,8 @@ use constant {
 	DEBUG => !!$ENV{DEBUG},
 };
 
+my $xapianlevels = qr/\A(?:full|medium)\z/;
+
 my %GIT_ESC = (
 	a => "\a",
 	b => "\b",
@@ -365,7 +367,6 @@ sub add_xapian ($$$$$) {
 sub add_message {
 	# mime = Email::MIME object
 	my ($self, $mime, $bytes, $num, $oid, $mid0) = @_;
-	my $xapianlevels = qr/\A(?:full|medium)\z/;
 	my $mids = mids($mime->header_obj);
 	$mid0 = $mids->[0] unless defined $mid0; # v1 compatibility
 	unless (defined $num) { # v1
@@ -714,7 +715,7 @@ sub _index_sync {
 			}
 			$dbh->commit;
 		}
-		if ($mkey && $newest) {
+		if ($mkey && $newest && $self->{indexlevel} =~ $xapianlevels) {
 			my $cur = $xdb->get_metadata($mkey);
 			if (need_update($self, $cur, $newest)) {
 				$xdb->set_metadata($mkey, $newest);
diff --git a/t/v1reindex.t b/t/v1reindex.t
index d97938d..75380f0 100644
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -124,9 +124,10 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
 	ok(-d $xap, 'Xapian directories recreated');
 	delete $ibx->{mm};
 	is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+	my $mset = $ibx->search->query('hello world', {mset=>1});
+	isnt(0, $mset->size, 'got Xapian search results');
 }
 
-
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
@@ -144,7 +145,25 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
 	ok(-d $xap, 'Xapian directories recreated');
 	delete $ibx->{mm};
 	is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+	my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
+	is(0, $mset->size, "no Xapian search results");
 }
 
+# upgrade existing basic to medium
+# note: changing indexlevels is not yet supported in v2,
+# and may not be without more effort
+$ibx_config->{indexlevel} = 'medium';
+$ibx = PublicInbox::Inbox->new($ibx_config);
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
+# no removals
+{
+	my @warn;
+	local $SIG{__WARN__} = sub { push @warn, @_ };
+	eval { $rw->index_sync };
+	is($@, '', 'no error from indexing');
+	is_deeply(\@warn, [], 'no warnings');
+	my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
+	isnt(0, $mset->size, 'search OK after basic -> medium');
+}
 
 done_testing();