unofficial mirror of meta@public-inbox.org
 help / color / mirror / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 03/12] nntp: use long response API for LISTGROUP
Date: Sat, 19 Sep 2015 02:03:31 +0000	[thread overview]
Message-ID: <20150919020340.6484-4-e@80x24.org> (raw)
In-Reply-To: <20150919020340.6484-1-e@80x24.org>

LISTGROUP can be expensive for giant groups, too.  Use the
long response API to improve fairness and prevent excessive
buffering.
---
 lib/PublicInbox/Msgmap.pm | 24 ++++++++----------------
 lib/PublicInbox/NNTP.pm   | 23 +++++++++++++----------
 2 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index c0fc636..2f64d90 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -155,25 +155,17 @@ sub create_tables {
 			'val VARCHAR(255) NOT NULL)');
 }
 
-sub each_id_batch {
-	my ($self, $cb) = @_;
+sub id_batch {
+	my ($self, $num, $cb) = @_;
 	my $dbh = $self->{dbh};
-	my $n = 0;
-	my $total = 0;
-	my $nr;
 	my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '.
 				'ORDER BY num ASC LIMIT 1000');
-	while (1) {
-		$sth->execute($n);
-		my $ary = $sth->fetchall_arrayref;
-		@$ary = map { $_->[0] } @$ary;
-		$nr = scalar @$ary;
-		last if $nr == 0;
-		$total += $nr;
-		$n = $ary->[-1];
-		$cb->($ary);
-	}
-	$total;
+	$sth->execute($num);
+	my $ary = $sth->fetchall_arrayref;
+	@$ary = map { $_->[0] } @$ary;
+	my $nr = scalar @$ary;
+	$cb->($ary) if $nr;
+	$nr;
 }
 
 1;
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index f86c633..5d770bd 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -14,6 +14,7 @@ use POSIX qw(strftime);
 use Time::HiRes qw(gettimeofday tv_interval ualarm);
 use constant {
 	r501 => '501 command syntax error',
+	long_response_limit => 0xffffffff,
 };
 
 my @OVERVIEW = qw(Subject From Date Message-ID References Bytes Lines);
@@ -142,13 +143,17 @@ sub cmd_listgroup {
 		more($self, $res);
 	}
 
-	my $ng = $self->{ng} or return '412 no newsgroup selected';
-	# Ugh this can be silly expensive for big groups
-	$ng->mm->each_id_batch(sub {
-		my ($ary) = @_;
-		more($self, join("\r\n", @$ary));
+	$self->{ng} or return '412 no newsgroup selected';
+	$self->long_response(0, long_response_limit, sub {
+		my ($i) = @_;
+		my $nr = $self->{ng}->mm->id_batch($$i, sub {
+			my ($ary) = @_;
+			more($self, join("\r\n", @$ary));
+		});
+
+		# -1 to adjust for implicit increment in long_response
+		$$i = $nr ? $$i + $nr - 1 : long_response_limit;
 	});
-	'.'
 }
 
 sub parse_time {
@@ -245,9 +250,7 @@ sub cmd_newnews {
 
 	$ts .= '..';
 	my $opts = { asc => 1, limit => 1000, offset => 0 };
-
-	my $end = 0xffffffff; # would like to read 4 billion messages?
-	$self->long_response(0, $end, sub {
+	$self->long_response(0, long_response_limit, sub {
 		my ($i) = @_;
 		my $srch = $srch[0];
 		my $res = $srch->query($ts, $opts);
@@ -262,7 +265,7 @@ sub cmd_newnews {
 			if (@srch) { # continue onto next newsgroup
 				$opts->{offset} = 0;
 			} else { # break out of the long response.
-				$$i = $end;
+				$$i = long_response_limit;
 			}
 		}
 	});
-- 
EW


  parent reply	other threads:[~2015-09-19  2:03 UTC|newest]

Thread overview: 13+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2015-09-19  2:03 [PATCH 0/12] nntp: misc updates Eric Wong
2015-09-19  2:03 ` [PATCH 01/12] nntp: use write_buf_size instead write_buf Eric Wong
2015-09-19  2:03 ` [PATCH 02/12] nntp: introduce long response API for streaming Eric Wong
2015-09-19  2:03 ` Eric Wong [this message]
2015-09-19  2:03 ` [PATCH 04/12] nntp: implement command argument checking Eric Wong
2015-09-19  2:03 ` [PATCH 05/12] nntp: XOVER does not require range Eric Wong
2015-09-19  2:03 ` [PATCH 06/12] nntp: speed up XHDR for the Message-ID case Eric Wong
2015-09-19  2:03 ` [PATCH 07/12] nntp: implement XROVER, speed up XHDR for some cases Eric Wong
2015-09-19  2:03 ` [PATCH 08/12] nntp: implement XPATH Eric Wong
2015-09-19  2:03 ` [PATCH 09/12] nntp: fix logging of long responses Eric Wong
2015-09-19  2:03 ` [PATCH 10/12] nntp: fix ARTICLE/HEAD/BODY/STAT Eric Wong
2015-09-19  2:03 ` [PATCH 11/12] nntp: log to FDs given by the Nntpd module Eric Wong
2015-09-19  2:03 ` [PATCH 12/12] nntp: article lookups by Message-ID may cross newsgroups Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20150919020340.6484-4-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).