From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id C49D41FB05 for ; Wed, 2 Sep 2020 11:04:23 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 11/11] v2writable: reuse read-only shard counting code Date: Wed, 2 Sep 2020 11:04:21 +0000 Message-Id: <20200902110421.30905-12-e@80x24.org> In-Reply-To: <20200902110421.30905-1-e@80x24.org> References: <20200902110421.30905-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We'll also fix the read-only code to ensure we notice missing Xapian shards, since gaps would throw off our expectation that Xapian document IDs and NNTP article numbers are interchangeable. --- lib/PublicInbox/Search.pm | 5 ++++- lib/PublicInbox/V2Writable.pm | 23 +++-------------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index b07f4ea6..fb35b747 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -7,6 +7,7 @@ package PublicInbox::Search; use strict; use parent qw(Exporter); our @EXPORT_OK = qw(mdocid); +use List::Util qw(max); # values for searching, changing the numeric value breaks # compatibility with old indices (so don't change them it) @@ -203,7 +204,9 @@ sub _xdb ($) { # We need numeric sorting so shard[0] is first for reading # Xapian metadata, if needed - for (sort { $a <=> $b } grep(/\A[0-9]+\z/, readdir($dh))) { + my $last = max(grep(/\A[0-9]+\z/, readdir($dh))); + return if !defined($last); + for (0..$last) { my $shard_dir = "$dir/$_"; if (-d $shard_dir && -r _) { push @xdb, $X{Database}->new($shard_dir); diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index c8334645..a1f6048f 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -65,28 +65,11 @@ sub nproc_shards ($) { sub count_shards ($) { my ($self) = @_; - my $n = 0; - my $xpfx = $self->{xpfx}; - # always load existing shards in case core count changes: # Also, shard count may change while -watch is running - # due to "xcpdb --reshard" - if (-d $xpfx) { - my $XapianDatabase; - foreach my $shard (<$xpfx/*>) { - -d $shard && $shard =~ m!/[0-9]+\z! or next; - $XapianDatabase //= do { - require PublicInbox::Search; - PublicInbox::Search::load_xapian(); - $PublicInbox::Search::X{Database}; - }; - eval { - $XapianDatabase->new($shard)->close; - $n++; - }; - } - } - $n; + my $srch = $self->{ibx}->search or return 0; + delete $self->{ibx}->{search}; + $srch->{nshard} // 0 } sub new {