From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 25A2C1FB0B for ; Thu, 20 Aug 2020 20:25:01 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 17/23] searchview: convert nested and Atom display to over.sqlite3 Date: Thu, 20 Aug 2020 20:24:51 +0000 Message-Id: <20200820202457.21042-18-e@yhbt.net> In-Reply-To: <20200820202457.21042-1-e@yhbt.net> References: <20200820202457.21042-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: git blob retrieval dominates on these, "&x=t" (nested) is roughly the same due to increased overhead for ->get_percent storage balancing out the mass-loading from SQLite. Atom "&x=A" is sped up slightly and uses less memory in the long-lived response. --- lib/PublicInbox/SearchView.pm | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm index a3527057..ef1b9767 100644 --- a/lib/PublicInbox/SearchView.pm +++ b/lib/PublicInbox/SearchView.pm @@ -255,20 +255,13 @@ sub get_pct ($) { $n > 99 ? 99 : $n; } -sub load_msgs { - my ($mset) = @_; - [ map { - my $mi = $_; - my $smsg = PublicInbox::Smsg::from_mitem($mi); - $smsg->{pct} = get_pct($mi); - $smsg; - } ($mset->items) ] -} - sub mset_thread { my ($ctx, $mset, $q) = @_; my $ibx = $ctx->{-inbox}; - my $msgs = $ibx->search->retry_reopen(\&load_msgs, $mset); + my $nshard = $ibx->search->{nshard} // 1; + my %pct = map { mdocid($nshard, $_) => get_pct($_) } $mset->items; + my $msgs = $ibx->over->get_all(keys %pct); + $_->{pct} = $pct{$_->{num}} for @$msgs; my $r = $q->{r}; my $rootset = PublicInbox::SearchThread::thread($msgs, $r ? \&sort_relevance : \&PublicInbox::View::sort_ds, @@ -323,7 +316,8 @@ sub ctx_prepare { sub adump { my ($cb, $mset, $q, $ctx) = @_; - $ctx->{items} = [ $mset->items ]; + my $nshard = $ctx->{-inbox}->search->{nshard} // 1; + $ctx->{ids} = [ map { mdocid($nshard, $_) } $mset->items ]; $ctx->{search_query} = $q; # used by WwwAtomStream::atom_header PublicInbox::WwwAtomStream->response($ctx, 200, \&adump_i); } @@ -331,11 +325,8 @@ sub adump { # callback for PublicInbox::WwwAtomStream::getline sub adump_i { my ($ctx) = @_; - while (my $mi = shift @{$ctx->{items}}) { - my $srch = $ctx->{-inbox}->search(undef, $ctx) or return; - my $smsg = eval { - PublicInbox::Smsg::from_mitem($mi, $srch); - } or next; + while (my $num = shift @{$ctx->{ids}}) { + my $smsg = eval { $ctx->{-inbox}->over->get_art($num) } or next; return $smsg; } }