From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 6DF841F9E0 for ; Fri, 7 Aug 2020 01:14:06 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 1/7] xapcmd: quietly no-op on indexlevel=basic Date: Fri, 7 Aug 2020 01:14:00 +0000 Message-Id: <20200807011406.12285-2-e@yhbt.net> In-Reply-To: <20200807011406.12285-1-e@yhbt.net> References: <20200807011406.12285-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: I find myself mindlessly adding "-c" to public-inbox-index, and other users may do the same. Instead of erroring out, we'll just silently ignore it, for now and allow public-inbox-compact to work on SQLite-only inboxes. We'll only check for xapian-compact if search exists, since it won't be needed in case we support SQLite VACUUM. --- lib/PublicInbox/Xapcmd.pm | 14 ++++++++------ t/indexlevels-mirror.t | 24 +++++++++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm index f1c80831..8d09ed27 100644 --- a/lib/PublicInbox/Xapcmd.pm +++ b/lib/PublicInbox/Xapcmd.pm @@ -167,9 +167,11 @@ sub prepare_run { my ($ibx, $opt) = @_; my $tmp = {}; # old shard dir => File::Temp->newdir object or undef my @queue; # ([old//src,newdir]) - list of args for cpdb() or compact() - - my $old = $ibx->search->xdir(1); - -d $old or die "$old does not exist\n"; + my $old; + if (my $srch = $ibx->search) { + $old = $srch->xdir(1); + -d $old or die "$old does not exist\n"; + } my $reshard = $opt->{reshard}; if (defined $reshard && $reshard <= 0) { die "--reshard must be a positive number\n"; @@ -177,7 +179,7 @@ sub prepare_run { # we want temporary directories to be as deep as possible, # so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS. - if ($ibx->version == 1) { + if ($old && $ibx->version == 1) { if (defined $reshard) { warn "--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n"; @@ -189,7 +191,7 @@ sub prepare_run { $tmp->{$old} = $wip; nodatacow_dir($wip->dirname); push @queue, [ $old, $wip ]; - } else { + } elsif ($old) { opendir my $dh, $old or die "Failed to opendir $old: $!\n"; my @old_shards; while (defined(my $dn = readdir($dh))) { @@ -255,7 +257,7 @@ sub run { PublicInbox::Admin::progress_prepare($opt ||= {}); defined(my $dir = $ibx->{inboxdir}) or die "no inboxdir defined\n"; -d $dir or die "inboxdir=$dir does not exist\n"; - check_compact() if $opt->{compact}; + check_compact() if $opt->{compact} && $ibx->search; my $reindex; # v1:{ from => $x40 }, v2:{ from => [ $x40, $x40, .. ] } } if (!$opt->{-coarse_lock}) { diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t index 44313e40..859c2c17 100644 --- a/t/indexlevels-mirror.t +++ b/t/indexlevels-mirror.t @@ -6,11 +6,13 @@ use Test::More; use PublicInbox::Eml; use PublicInbox::Inbox; use PublicInbox::InboxWritable; +use PublicInbox::Spawn qw(which); require PublicInbox::Admin; use PublicInbox::TestCommon; my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2; require_git('2.6') if $PI_TEST_VERSION == 2; require_mods(qw(DBD::SQLite)); +my $have_xapian_compact = which($ENV{XAPIAN_COMPACT} || 'xapian-compact'); my $mime = PublicInbox::Eml->new(<<'EOF'); From: a@example.com @@ -21,8 +23,9 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000 hello world EOF -sub import_index_incremental { +my $import_index_incremental = sub { my ($v, $level, $mime) = @_; + my $err = ''; my $this = "pi-$v-$level-indexlevels"; my ($tmpdir, $for_destroy) = tmpdir(); local $ENV{PI_CONFIG} = "$tmpdir/config"; @@ -39,8 +42,9 @@ sub import_index_incremental { $im->done; # index master (required for v1) - ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]), - 'index master OK'); + my @cmd = (qw(-index -j0), $ibx->{inboxdir}, "-L$level"); + push @cmd, '-c' if $have_xapian_compact; + ok(run_script(\@cmd, undef, { 2 => \$err }), 'index master'); my $ro_master = PublicInbox::Inbox->new({ inboxdir => $ibx->{inboxdir}, indexlevel => $level @@ -50,7 +54,7 @@ sub import_index_incremental { is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed'); # clone - my @cmd = (qw(git clone --mirror -q)); + @cmd = (qw(git clone --mirror -q)); my $mirror = "$tmpdir/mirror-$v"; if ($v == 1) { push @cmd, $ibx->{inboxdir}, $mirror; @@ -157,17 +161,23 @@ sub import_index_incremental { is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level, 'indexlevel detectable by Admin '.$v.$level); -} + + SKIP: { + skip 'xapian-compact missing', 1 if !$have_xapian_compact; + my $cmd = [ qw(-compact), $mirror ]; + ok(run_script($cmd, undef, { 2 => \$err}), "compact $level"); + } +}; # we can probably cull some other tests -import_index_incremental($PI_TEST_VERSION, 'basic', $mime); +$import_index_incremental->($PI_TEST_VERSION, 'basic', $mime); SKIP: { require PublicInbox::Search; PublicInbox::Search::load_xapian() or skip('Xapian perl binding missing', 2); foreach my $l (qw(medium full)) { - import_index_incremental($PI_TEST_VERSION, $l, $mime); + $import_index_incremental->($PI_TEST_VERSION, $l, $mime); } }