From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 867D91FB05 for ; Tue, 6 Mar 2018 08:42:44 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Subject: [PATCH 34/34] v2writable: detect and use previous partition count Date: Tue, 6 Mar 2018 08:42:42 +0000 Message-Id: <20180306084242.19988-35-e@80x24.org> In-Reply-To: <20180306084242.19988-1-e@80x24.org> References: <20180306084242.19988-1-e@80x24.org> List-Id: We need to detect the number of partitions the repository was created with to ensure Xapian DBs can work across different machines (or even CPU affinity changes) without leaving messages unaffected by search. --- lib/PublicInbox/V2Writable.pm | 22 ++++++++++++++++++++-- t/v2writable.t | 11 ++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 3bcea37..7728b91 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -19,7 +19,9 @@ use PublicInbox::Inbox; my $PACKING_FACTOR = 0.4; # assume 2 cores if GNU nproc(1) is not available -my $NPROC = int($ENV{NPROC} || `nproc 2>/dev/null` || 2); +sub nproc () { + int($ENV{NPROC} || `nproc 2>/dev/null` || 2); +} sub new { my ($class, $v2ibx, $creat) = @_; @@ -32,12 +34,28 @@ sub new { die "$dir does not exist\n"; } } + + my $nparts = 0; + my $xpfx = "$dir/xap" . PublicInbox::Search::SCHEMA_VERSION; + + # always load existing partitions in case core count changes: + if (-d $xpfx) { + foreach my $part (<$xpfx/*>) { + -d $part && $part =~ m!/\d+\z! or next; + eval { + Search::Xapian::Database->new($part)->close; + $nparts++; + }; + } + } + $nparts = nproc() if ($nparts == 0); + my $self = { -inbox => $v2ibx, im => undef, # PublicInbox::Import xap_rw => undef, # PublicInbox::V2SearchIdx xap_ro => undef, - partitions => $NPROC, + partitions => $nparts, transact_bytes => 0, # limit each repo to 1GB or so rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR), diff --git a/t/v2writable.t b/t/v2writable.t index 2d35aca..404c865 100644 --- a/t/v2writable.t +++ b/t/v2writable.t @@ -30,7 +30,11 @@ my $mime = PublicInbox::MIME->create( body => "hello world\n", ); -my $im = PublicInbox::V2Writable->new($ibx, 1); +my $im = eval { + local $ENV{NPROC} = '1'; + PublicInbox::V2Writable->new($ibx, 1); +}; +is($im->{partitions}, 1, 'one partition when forced'); ok($im->add($mime), 'ordinary message added'); if ('ensure git configs are correct') { @@ -182,5 +186,10 @@ EOF } is_deeply([sort keys %nn], [sort keys %uniq]); }; +{ + local $ENV{NPROC} = 2; + $im = PublicInbox::V2Writable->new($ibx, 1); + is($im->{partitions}, 1, 'detected single partition from previous'); +} done_testing(); -- EW