From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 7F0521FC0B for ; Thu, 26 Aug 2021 12:33:39 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 8/8] move ->ids_after from mm to over Date: Thu, 26 Aug 2021 12:33:38 +0000 Message-Id: <20210826123338.694-9-e@80x24.org> In-Reply-To: <20210826123338.694-1-e@80x24.org> References: <20210826123338.694-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Since we favor ->over in WWW and IMAP, move this method to ->over to reduce open files in common cases. This fixes the /$EXTINDEX_NAME/all.mbox.gz endpoint for extindex entries (which may get expensive...). --- lib/PublicInbox/Mbox.pm | 10 ++++------ lib/PublicInbox/Msgmap.pm | 11 ----------- lib/PublicInbox/NNTP.pm | 2 +- lib/PublicInbox/Over.pm | 11 +++++++++++ t/extindex-psgi.t | 3 +++ 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 844099aa..f72af26b 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -161,19 +161,17 @@ sub all_ids_cb { my $smsg = $ctx->{over}->get_art($num) or next; return $smsg; } - $ctx->{ids} = $ids = $ctx->{mm}->ids_after(\($ctx->{prev})); + $ctx->{ids} = $ids = $ctx->{over}->ids_after(\($ctx->{prev})); } while (@$ids); } sub mbox_all_ids { my ($ctx) = @_; - my $ibx = $ctx->{ibx}; my $prev = 0; - my $mm = $ctx->{mm} = $ibx->mm; - my $ids = $mm->ids_after(\$prev) or return - [404, [qw(Content-Type text/plain)], ["No results found\n"]]; - $ctx->{over} = $ibx->over or + $ctx->{over} = $ctx->{ibx}->over or return PublicInbox::WWW::need($ctx, 'Overview'); + my $ids = $ctx->{over}->ids_after(\$prev) or return + [404, [qw(Content-Type text/plain)], ["No results found\n"]]; $ctx->{ids} = $ids; $ctx->{prev} = $prev; require PublicInbox::MboxGz; diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index 16a9a476..3887a9e6 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -189,17 +189,6 @@ CREATE TABLE IF NOT EXISTS meta ( } -# used by NNTP.pm -sub ids_after { - my ($self, $num) = @_; - my $ids = $self->{dbh}->selectcol_arrayref(<<'', undef, $$num); -SELECT num FROM msgmap WHERE num > ? -ORDER BY num ASC LIMIT 1000 - - $$num = $ids->[-1] if @$ids; - $ids; -} - sub msg_range { my ($self, $beg, $end, $cols) = @_; $cols //= 'num,mid'; diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index aea04c05..ea9ce183 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -210,7 +210,7 @@ sub listgroup_range_i { sub listgroup_all_i { my ($self, $num) = @_; - my $ary = $self->{ibx}->mm(1)->ids_after($num); + my $ary = $self->{ibx}->over(1)->ids_after($num); scalar(@$ary) or return; more($self, join("\r\n", @$ary)); 1; diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm index 58fdea0e..19da056a 100644 --- a/lib/PublicInbox/Over.pm +++ b/lib/PublicInbox/Over.pm @@ -371,4 +371,15 @@ SELECT COUNT(*) FROM xref3 WHERE oidbin = ? sub blob_exists { oidbin_exists($_[0], pack('H*', $_[1])) } +# used by NNTP.pm +sub ids_after { + my ($self, $num) = @_; + my $ids = dbh($self)->selectcol_arrayref(<<'', undef, $$num); +SELECT num FROM over WHERE num > ? +ORDER BY num ASC LIMIT 1000 + + $$num = $ids->[-1] if @$ids; + $ids; +} + 1; diff --git a/t/extindex-psgi.t b/t/extindex-psgi.t index b9acc979..d4761641 100644 --- a/t/extindex-psgi.t +++ b/t/extindex-psgi.t @@ -52,6 +52,9 @@ my $client = sub { my $cfg = PublicInbox::Config->git_config_dump($f); is($?, 0, 'no errors from git-config parsing'); ok($cfg->{'extindex.all.topdir'}, 'extindex.topdir defined'); + + $res = $cb->(GET('/all/all.mbox.gz')); + is($res->code, 200, 'all.mbox.gz'); }; test_psgi(sub { $www->call(@_) }, $client); %$env = (%$env, TMPDIR => $tmpdir, PI_CONFIG => $pi_config);