From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B1E651FB0D for ; Sat, 28 Nov 2020 05:09:16 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 1/5] nntp: NEWGROUPS uses long_response Date: Sat, 28 Nov 2020 05:09:12 +0000 Message-Id: <20201128050916.5586-2-e@80x24.org> In-Reply-To: <20201128050916.5586-1-e@80x24.org> References: <20201128050916.5586-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We can amortize the cost of NEWGROUPS time filtering using the long_response API. This lets us handle hundreds/thousands of inboxes without monopolizing the event loop for this command. Further speedup is possible using MiscSearch, but that requires not-yet-done indexing changes to MiscIdx. --- lib/PublicInbox/NNTP.pm | 21 +++++++++++++++------ lib/PublicInbox/NNTPD.pm | 5 +---- t/nntp.t | 1 - 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 8eec6b91..cc6534b9 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -263,6 +263,19 @@ sub group_line ($$) { more($self, "$ng->{newsgroup} $max $min n"); } +sub newgroups_i { + my ($self, $ts, $i, $groupnames) = @_; + my $end = $$i + 100; + my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup}; + while ($$i < $end) { + my $ngname = $groupnames->[$$i++] // return; + my $ibx = $groups->{$ngname} or next; # expired on reload + next unless (eval { $ibx->uidvalidity } // 0) > $ts; + group_line($self, $ibx); + } + 1; +} + sub cmd_newgroups ($$$;$$) { my ($self, $date, $time, $gmt, $dists) = @_; my $ts = eval { parse_time($date, $time, $gmt) }; @@ -270,12 +283,8 @@ sub cmd_newgroups ($$$;$$) { # TODO dists more($self, '231 list of new newsgroups follows'); - foreach my $ng (@{$self->{nntpd}->{grouplist}}) { - my $c = eval { $ng->uidvalidity } // 0; - next unless $c > $ts; - group_line($self, $ng); - } - '.' + long_response($self, \&newgroups_i, $ts, \(my $i = 0), + $self->{nntpd}->{groupnames}); } sub wildmat2re (;$) { diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm index 4de1944b..33bc5fda 100644 --- a/lib/PublicInbox/NNTPD.pm +++ b/lib/PublicInbox/NNTPD.pm @@ -24,7 +24,6 @@ sub new { groups => {}, err => \*STDERR, out => \*STDOUT, - grouplist => [], pi_config => $pi_config, servername => $name, greet => \"201 $name ready - post via email\r\n", @@ -60,9 +59,7 @@ sub refresh_groups { delete $groups->{$ngname}; } }); - my @names = sort(keys %$groups); - $self->{grouplist} = [ map { $groups->{$_} } @names ]; - $self->{groupnames} = \@names; + $self->{groupnames} = [ sort(keys %$groups) ]; $self->{pi_config} = $pi_config; # this will destroy old groups that got deleted $self->{groups} = $groups; diff --git a/t/nntp.t b/t/nntp.t index 91a2aff7..ea2ef876 100644 --- a/t/nntp.t +++ b/t/nntp.t @@ -112,7 +112,6 @@ use PublicInbox::Config; my $hdr = $mime->header_obj; my $mock_self = { nntpd => { - grouplist => [], servername => 'example.com', pi_config => bless {}, 'PublicInbox::Config', },