From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B45831FF9E for ; Tue, 27 Oct 2020 07:54:53 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 02/52] search: hoist out _xdb_sharded for v2 inboxes Date: Tue, 27 Oct 2020 07:54:03 +0000 Message-Id: <20201027075453.19163-3-e@80x24.org> In-Reply-To: <20201027075453.19163-1-e@80x24.org> References: <20201027075453.19163-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We'll be using this in detached (ext) Xapian indexes in cross inbox search. --- lib/PublicInbox/Search.pm | 58 +++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index 0321ca93..6346d788 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -190,38 +190,42 @@ sub xdir ($;$) { } } -sub _xdb ($) { +sub _xdb_sharded { + my ($self, $xpfx) = @_; + opendir(my $dh, $xpfx) or return; # not initialized yet + + # We need numeric sorting so shard[0] is first for reading + # Xapian metadata, if needed + my $last = max(grep(/\A[0-9]+\z/, readdir($dh))); + return if !defined($last); + my (@xdb, $slow_phrase); + for (0..$last) { + my $shard_dir = "$xpfx/$_"; + if (-d $shard_dir && -r _) { + push @xdb, $X{Database}->new($shard_dir); + $slow_phrase ||= -f "$shard_dir/iamchert"; + } else { # gaps from missing epochs throw off mdocid() + warn "E: $shard_dir missing or unreadable\n"; + return; + } + } + $self->{qp_flags} |= FLAG_PHRASE() if !$slow_phrase; + $self->{nshard} = scalar(@xdb); + my $xdb = shift @xdb; + $xdb->add_database($_) for @xdb; + $xdb; +} + +sub _xdb { my ($self) = @_; my $dir = xdir($self, 1); - my ($xdb, $slow_phrase); - my $qpf = \($self->{qp_flags} ||= $QP_FLAGS); + $self->{qp_flags} //= $QP_FLAGS; if ($self->{ibx_ver} >= 2) { - my @xdb; - opendir(my $dh, $dir) or return; # not initialized yet - - # We need numeric sorting so shard[0] is first for reading - # Xapian metadata, if needed - my $last = max(grep(/\A[0-9]+\z/, readdir($dh))); - return if !defined($last); - for (0..$last) { - my $shard_dir = "$dir/$_"; - if (-d $shard_dir && -r _) { - push @xdb, $X{Database}->new($shard_dir); - $slow_phrase ||= -f "$shard_dir/iamchert"; - } else { # gaps from missing epochs throw off mdocid() - warn "E: $shard_dir missing or unreadable\n"; - return; - } - } - $self->{nshard} = scalar(@xdb); - $xdb = shift @xdb; - $xdb->add_database($_) for @xdb; + _xdb_sharded($self, $dir); } else { - $slow_phrase = -f "$dir/iamchert"; - $xdb = $X{Database}->new($dir); + $self->{qp_flags} |= FLAG_PHRASE() if !-f "$dir/iamchert"; + $X{Database}->new($dir); } - $$qpf |= FLAG_PHRASE() unless $slow_phrase; - $xdb; } # v2 Xapian docids don't conflict, so they're identical to