[PATCH 03/12] nntp: use long response API for LISTGROUP

[Eric Wong <e@80x24.org>]

LISTGROUP can be expensive for giant groups, too.  Use the
long response API to improve fairness and prevent excessive
buffering.
---
 lib/PublicInbox/Msgmap.pm | 24 ++++++++----------------
 lib/PublicInbox/NNTP.pm   | 23 +++++++++++++----------
 2 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index c0fc636..2f64d90 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -155,25 +155,17 @@ sub create_tables {
 			'val VARCHAR(255) NOT NULL)');
 }
 
-sub each_id_batch {
-	my ($self, $cb) = @_;
+sub id_batch {
+	my ($self, $num, $cb) = @_;
 	my $dbh = $self->{dbh};
-	my $n = 0;
-	my $total = 0;
-	my $nr;
 	my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '.
 				'ORDER BY num ASC LIMIT 1000');
-	while (1) {
-		$sth->execute($n);
-		my $ary = $sth->fetchall_arrayref;
-		@$ary = map { $_->[0] } @$ary;
-		$nr = scalar @$ary;
-		last if $nr == 0;
-		$total += $nr;
-		$n = $ary->[-1];
-		$cb->($ary);
-	}
-	$total;
+	$sth->execute($num);
+	my $ary = $sth->fetchall_arrayref;
+	@$ary = map { $_->[0] } @$ary;
+	my $nr = scalar @$ary;
+	$cb->($ary) if $nr;
+	$nr;
 }
 
 1;
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index f86c633..5d770bd 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -14,6 +14,7 @@ use POSIX qw(strftime);
 use Time::HiRes qw(gettimeofday tv_interval ualarm);
 use constant {
 	r501 => '501 command syntax error',
+	long_response_limit => 0xffffffff,
 };
 
 my @OVERVIEW = qw(Subject From Date Message-ID References Bytes Lines);
@@ -142,13 +143,17 @@ sub cmd_listgroup {
 		more($self, $res);
 	}
 
-	my $ng = $self->{ng} or return '412 no newsgroup selected';
-	# Ugh this can be silly expensive for big groups
-	$ng->mm->each_id_batch(sub {
-		my ($ary) = @_;
-		more($self, join("\r\n", @$ary));
+	$self->{ng} or return '412 no newsgroup selected';
+	$self->long_response(0, long_response_limit, sub {
+		my ($i) = @_;
+		my $nr = $self->{ng}->mm->id_batch($$i, sub {
+			my ($ary) = @_;
+			more($self, join("\r\n", @$ary));
+		});
+
+		# -1 to adjust for implicit increment in long_response
+		$$i = $nr ? $$i + $nr - 1 : long_response_limit;
 	});
-	'.'
 }
 
 sub parse_time {
@@ -245,9 +250,7 @@ sub cmd_newnews {
 
 	$ts .= '..';
 	my $opts = { asc => 1, limit => 1000, offset => 0 };
-
-	my $end = 0xffffffff; # would like to read 4 billion messages?
-	$self->long_response(0, $end, sub {
+	$self->long_response(0, long_response_limit, sub {
 		my ($i) = @_;
 		my $srch = $srch[0];
 		my $res = $srch->query($ts, $opts);
@@ -262,7 +265,7 @@ sub cmd_newnews {
 			if (@srch) { # continue onto next newsgroup
 				$opts->{offset} = 0;
 			} else { # break out of the long response.
-				$$i = $end;
+				$$i = long_response_limit;
 			}
 		}
 	});
-- 
EW