From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: AS6315 166.70.0.0/16 X-Spam-Status: No, score=-3.7 required=3.0 tests=AWL,BAYES_00, RCVD_IN_DNSWL_LOW,RCVD_IN_MSPIKE_H3,RCVD_IN_MSPIKE_WL,SPF_PASS shortcircuit=no autolearn=ham autolearn_force=no version=3.4.1 Received: from out01.mta.xmission.com (out01.mta.xmission.com [166.70.13.231]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by dcvr.yhbt.net (Postfix) with ESMTPS id F3F1F1F597; Wed, 1 Aug 2018 16:45:49 +0000 (UTC) Received: from in01.mta.xmission.com ([166.70.13.51]) by out01.mta.xmission.com with esmtps (TLS1.2:ECDHE_RSA_AES_128_GCM_SHA256:128) (Exim 4.87) (envelope-from ) id 1fkuFx-0002Iz-FT; Wed, 01 Aug 2018 10:45:49 -0600 Received: from [97.119.167.31] (helo=x220.int.ebiederm.org) by in01.mta.xmission.com with esmtpsa (TLS1.2:ECDHE_RSA_AES_128_CBC_SHA256:128) (Exim 4.87) (envelope-from ) id 1fkuFw-0001mm-5n; Wed, 01 Aug 2018 10:45:49 -0600 From: "Eric W. Biederman" To: Eric Wong Cc: meta@public-inbox.org, "Eric W. Biederman" Date: Wed, 1 Aug 2018 11:43:37 -0500 Message-Id: <20180801164344.7911-6-ebiederm@xmission.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <878t5qkpis.fsf@xmission.com> References: <878t5qkpis.fsf@xmission.com> X-XM-SPF: eid=1fkuFw-0001mm-5n;;;mid=<20180801164344.7911-6-ebiederm@xmission.com>;;;hst=in01.mta.xmission.com;;;ip=97.119.167.31;;;frm=ebiederm@xmission.com;;;spf=neutral X-XM-AID: U2FsdGVkX1+3tFutbxLOIVJnv69cxLovyzJN9bUZ7q4= X-SA-Exim-Connect-IP: 97.119.167.31 X-SA-Exim-Mail-From: ebiederm@xmission.com Subject: [PATCH 06/13] t/v[12]reindex.t: Test incremental indexing works X-SA-Exim-Version: 4.2.1 (built Thu, 05 May 2016 13:38:54 -0600) X-SA-Exim-Scanned: Yes (on in01.mta.xmission.com) List-Id: Capture interesting commits of the test repository in mark variables. Use those marks to build interesting scenarios where index_sync proceeds as if those marks are the heads of the repositor. Use this capability to test what happens when adds and deletes are mixed within a repository. Be sad because things don't yet work as they should. Signed-off-by: "Eric W. Biederman" --- t/v1reindex.t | 194 +++++++++++++++++++++++++++++++++++++++++++++++++ t/v2reindex.t | 195 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+) diff --git a/t/v1reindex.t b/t/v1reindex.t index 877e70390910..8e78aa761333 100644 --- a/t/v1reindex.t +++ b/t/v1reindex.t @@ -33,6 +33,7 @@ my $mime = PublicInbox::MIME->create( ); my $minmax; my $msgmap; +my ($mark1, $mark2, $mark3, $mark4); { my %config = %$ibx_config; my $ibx = PublicInbox::Inbox->new(\%config); @@ -41,13 +42,17 @@ my $msgmap; $mime->header_set('Message-Id', "<$i\@example.com>"); ok($im->add($mime), "message $i added"); if ($i == 4) { + $mark1 = $im->get_mark($im->{tip}); $im->remove($mime); + $mark2 = $im->get_mark($im->{tip}); } } if ('test remove later') { + $mark3 = $im->get_mark($im->{tip}); $mime->header_set('Message-Id', "<5\@example.com>"); $im->remove($mime); + $mark4 = $im->get_mark($im->{tip}); } $im->done; @@ -222,4 +227,193 @@ ok(!-d $xap, 'Xapian directories removed again'); is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } +# An incremental indexing test +ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark1 4 simple additions in the same index_sync + eval { $rw->index_sync({ref => $mark1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 4, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [4, '4@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark2 A delete separated form and add in the same index_sync + eval { $rw->index_sync({ref => $mark2}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark3 adds following the delete at mark2 + eval { $rw->index_sync({ref => $mark3}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark4 A delete of an older message + eval { $rw->index_sync({ref => $mark4}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + + +# Another incremental indexing test +ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark2 an add and it's delete in the same index_sync + eval { $rw->index_sync({ref => $mark2}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark3 adds following the delete at mark2 + eval { $rw->index_sync({ref => $mark3}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark4 A delete of an older message + eval { $rw->index_sync({ref => $mark4}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + + done_testing(); diff --git a/t/v2reindex.t b/t/v2reindex.t index 31e61d5b2578..ce13a07c39c0 100644 --- a/t/v2reindex.t +++ b/t/v2reindex.t @@ -33,21 +33,27 @@ my $mime = PublicInbox::MIME->create( local $ENV{NPROC} = 2; my $minmax; my $msgmap; +my ($mark1, $mark2, $mark3, $mark4); { my %config = %$ibx_config; my $ibx = PublicInbox::Inbox->new(\%config); my $im = PublicInbox::V2Writable->new($ibx, 1); + my $im0 = $im->importer(); foreach my $i (1..10) { $mime->header_set('Message-Id', "<$i\@example.com>"); ok($im->add($mime), "message $i added"); if ($i == 4) { + $mark1 = $im0->get_mark($im0->{tip}); $im->remove($mime); + $mark2 = $im0->get_mark($im0->{tip}); } } if ('test remove later') { + $mark3 = $im0->get_mark($im0->{tip}); $mime->header_set('Message-Id', "<5\@example.com>"); $im->remove($mime); + $mark4 = $im0->get_mark($im0->{tip}); } $im->done; @@ -211,4 +217,193 @@ ok(!-d $xap, 'Xapian directories removed again'); is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } + +# An incremental indexing test +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark1 4 simple additions in the same index_sync + $ibx->{ref_head} = $mark1; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 4, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [4, '4@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark2 A delete separated from an add in the same index_sync + $ibx->{ref_head} = $mark2; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark3 adds following the delete at mark2 + $ibx->{ref_head} = $mark3; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark4 A delete of an older message + $ibx->{ref_head} = $mark4; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + + +# Another incremental indexing test +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark2 an add and it's delete in the same index_sync + $ibx->{ref_head} = $mark2; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark3 adds following the delete at mark2 + $ibx->{ref_head} = $mark3; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark4 A delete of an older message + $ibx->{ref_head} = $mark4; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + done_testing(); -- 2.17.1