From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 9FCDD1F462 for ; Thu, 23 May 2019 09:37:07 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 08/26] xcpdb: new tool which wraps Xapian's copydatabase(1) Date: Thu, 23 May 2019 09:36:46 +0000 Message-Id: <20190523093704.18367-9-e@80x24.org> In-Reply-To: <20190523093704.18367-1-e@80x24.org> References: <20190523093704.18367-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: copydatabase(1) is an existing Xapian tool which is the recommended way to upgrade existing DBs to the latest Xapian database format (currently "glass" for stable/released versions). Our use of Xapian relies on preserving document IDs, so we'll wrap it like we do xapian-compact(1) and use the "--no-renumber" switch. I could not name the tool "public-inbox-copydatabase" since it would be ambiguous as to which DB it's actually copying. So, I abbreviated the suffix to "xcpdb" (Xapian CoPy DataBase), which I hope is acceptable and unambiguous. --- Documentation/include.mk | 6 ++-- Documentation/public-inbox-xcpdb.pod | 51 ++++++++++++++++++++++++++++ MANIFEST | 2 ++ script/public-inbox-xcpdb | 18 ++++++++++ t/indexlevels-mirror.t | 22 ++++++++++++ 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 Documentation/public-inbox-xcpdb.pod create mode 100755 script/public-inbox-xcpdb diff --git a/Documentation/include.mk b/Documentation/include.mk index 6415338..27d6ea6 100644 --- a/Documentation/include.mk +++ b/Documentation/include.mk @@ -26,11 +26,13 @@ podtext = $(PODTEXT) $(PODTEXT_OPTS) # MakeMaker only seems to support manpage sections 1 and 3... m1 = -m1 += public-inbox-mda +m1 += public-inbox-compact m1 += public-inbox-httpd +m1 += public-inbox-index +m1 += public-inbox-mda m1 += public-inbox-nntpd m1 += public-inbox-watch -m1 += public-inbox-index +m1 += public-inbox-xcpdb m5 = m5 += public-inbox-config m5 += public-inbox-v1-format diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod new file mode 100644 index 0000000..4ff5186 --- /dev/null +++ b/Documentation/public-inbox-xcpdb.pod @@ -0,0 +1,51 @@ +=head1 NAME + +public-inbox-xcpdb - copy Xapian DBs (for format upgrades) + +=head1 SYNOPSIS + + public-inbox-xcpdb INBOX_DIR + +=head1 DESCRIPTION + +public-inbox-xcpdb is a wrapper for L for +upgrading to the latest database format supported by Xapian +(e.g. "glass" or "honey"). + +It locks the inbox and prevents other processes such as +L and L from +writing while it operates. + +This is intended for upgrading the database format used by +Xapian. It DOES NOT upgrade the schema used by the +public-inbox search interface (see L). + +=head1 ENVIRONMENT + +=over 8 + +=item PI_CONFIG + +The default config file, normally "~/.public-inbox/config". +See L + +=back + +=head1 UPGRADING + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2019 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L diff --git a/MANIFEST b/MANIFEST index dfc1f66..efd5658 100644 --- a/MANIFEST +++ b/MANIFEST @@ -20,6 +20,7 @@ Documentation/public-inbox-overview.pod Documentation/public-inbox-v1-format.pod Documentation/public-inbox-v2-format.pod Documentation/public-inbox-watch.pod +Documentation/public-inbox-xcpdb.pod Documentation/standards.perl Documentation/txt2pre HACKING @@ -154,6 +155,7 @@ script/public-inbox-mda script/public-inbox-nntpd script/public-inbox-purge script/public-inbox-watch +script/public-inbox-xcpdb script/public-inbox.cgi scripts/dc-dlvr scripts/dc-dlvr.pre diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb new file mode 100755 index 0000000..cbf9f55 --- /dev/null +++ b/script/public-inbox-xcpdb @@ -0,0 +1,18 @@ +#!/usr/bin/perl -w +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ +# xcpdb: Xapian copy database, a wrapper around Xapian's copydatabase(1) +use PublicInbox::InboxWritable; +use PublicInbox::Xapcmd; +use PublicInbox::Admin; +PublicInbox::Admin::require_or_die('-search'); +my $usage = "Usage: public-inbox-xcpdb INBOX_DIR\n"; +my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage; +my $cmd = [qw(copydatabase --no-renumber)]; +open my $null, '>', '/dev/null' or die "failed to open /dev/null: $!\n"; +my $rdr = { 1 => fileno($null) }; +foreach (@ibxs) { + my $ibx = PublicInbox::InboxWritable->new($_); + # we rely on --no-renumber to keep docids synched to NNTP + PublicInbox::Xapcmd::run($ibx, $cmd, undef, $rdr); +} diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t index d124c75..61053b6 100644 --- a/t/indexlevels-mirror.t +++ b/t/indexlevels-mirror.t @@ -18,6 +18,7 @@ foreach my $mod (qw(DBD::SQLite)) { my $path = 'blib/script'; my $index = "$path/public-inbox-index"; +my $xcpdb = "$path/public-inbox-xcpdb"; my $mime = PublicInbox::MIME->create( header => [ @@ -108,6 +109,13 @@ sub import_index_incremental { ok($im->remove($mime), '2nd message removed'); $im->done; + if ($level ne 'basic') { + is(system($xcpdb, $mirror), 0, "v$v xcpdb OK"); + delete $ro_mirror->{$_} for (qw(over search)); + ($nr, $msgs) = $ro_mirror->search->query('m:m@2'); + is($nr, 1, "v$v found m\@2 via Xapian on $level"); + } + # sync the mirror is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK'); is(system($index, $mirror), 0, "v$v index mirror again OK"); @@ -120,6 +128,10 @@ sub import_index_incremental { is_deeply([glob("$ibx->{mainrepo}/xap*/?/")], [], 'no Xapian partition directories for v2 basic'); } + if ($level ne 'basic') { + ($nr, $msgs) = $ro_mirror->search->reopen->query('m:m@2'); + is($nr, 0, "v$v m\@2 gone from Xapian in mirror on $level"); + } } # we can probably cull some other tests and put full/medium tests, here @@ -131,4 +143,14 @@ for my $level (qw(basic)) { } } +SKIP: { + require PublicInbox::Search; + PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2; + for my $v (1..2) { + subtest("v$v indexlevel=medium" => sub { + import_index_incremental($v, 'medium'); + }) + } +} + done_testing(); -- EW