[PATCH 0/8] nntp: more fixes and tiny speedups

[PATCH 0/8] nntp: more fixes and tiny speedups

[Eric Wong]

Still trying my best to avoid introducing more metadata or Xapian
index overhead to support OVER/XOVER, but it seems futile at this
point.

Eric Wong (8):
      nntp: HDR allows metadata prefixed with ':'
      nntp: consistently use 501 for unsupported LIST
      searchidx: remove unused sub: next_doc_id
      nntp: do not repeat result on blocked write
      nntp: prefix FD on every log line
      git: signal-safety for pipe writes
      git: use fields for GitCatFile
      nntp: avoid signals for long responses

 lib/PublicInbox/GitCatFile.pm | 17 +++++++----------
 lib/PublicInbox/NNTP.pm       | 30 ++++++++++++++----------------
 lib/PublicInbox/SearchIdx.pm  |  2 --
 3 files changed, 21 insertions(+), 28 deletions(-)

[PATCH 1/8] nntp: HDR allows metadata prefixed with ':'

[Eric Wong]

RFC 3977, section 8.5.2 states metadata lookups can be done
with HDR.
---
 lib/PublicInbox/NNTP.pm | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 6c661a1..c2f9717 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -480,9 +480,8 @@ sub get_range ($$) {
 
 sub hdr_val ($$) {
 	my ($r, $header) = @_;
-	$header = lc $header;
-	return $r->[3] if ($header eq 'bytes');
-	return $r->[4] if ($header eq 'lines');
+	return $r->[3] if $header =~ /\A:?bytes\z/i;
+	return $r->[4] if $header =~ /\A:?lines\z/i;
 	$r = $r->[2]->header_obj->header($header);
 	defined $r or return;
 	$r =~ s/[\r\n\t]+/ /sg;
-- 
EW

[PATCH 2/8] nntp: consistently use 501 for unsupported LIST

[Eric Wong]

This is required by RFC 3977, section 3.2.1
---
 lib/PublicInbox/NNTP.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index c2f9717..dd033e8 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -123,14 +123,14 @@ sub list_newsgroups ($;$) {
 	}
 }
 
-# LIST SUBSCRIPTIONS not supported
+# LIST SUBSCRIPTIONS, DISTRIB.PATS are not supported
 sub cmd_list ($;$$) {
 	my ($self, @args) = @_;
 	if (scalar @args) {
 		my $arg = shift @args;
 		$arg =~ tr/A-Z./a-z_/;
 		$arg = "list_$arg";
-		return '503 function not performed' if $DISABLED{$arg};
+		return r501 if $DISABLED{$arg};
 
 		$arg = eval {
 			no strict 'refs';
-- 
EW

[PATCH 3/8] searchidx: remove unused sub: next_doc_id

[Eric Wong]

It seems like it was never used
---
 lib/PublicInbox/SearchIdx.pm | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 351450c..8724326 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -175,8 +175,6 @@ sub term_generator { # write-only
 	$self->{term_generator} = $tg;
 }
 
-sub next_doc_id { $_[0]->{xdb}->get_lastdocid + 1 }
-
 # increments last_thread_id counter
 # returns a 64-bit integer represented as a hex string
 sub next_thread_id {
-- 
EW

[PATCH 4/8] nntp: do not repeat result on blocked write

[Eric Wong]

Oops, we must increment our range even if we yield or
get blocked on output buffering.
---
 lib/PublicInbox/NNTP.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index dd033e8..7e2c2ab 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -508,8 +508,8 @@ sub long_response ($$$$) {
 		my $err;
 		do {
 			eval { $cb->(\$beg) };
-		} until (($err = $@) || $self->{closed} || $yield ||
-			 $self->{write_buf_size} || ++$beg > $end);
+		} until (($err = $@) || $self->{closed} ||
+			 ++$beg > $end || $yield || $self->{write_buf_size});
 		ualarm(0);
 
 		if ($err || $self->{closed}) {
-- 
EW

[PATCH 5/8] nntp: prefix FD on every log line

[Eric Wong]

This can help us track down what request patterns clients
will perform when we have multiple clients.
---
 lib/PublicInbox/NNTP.pm | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 7e2c2ab..3490a09 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -492,11 +492,12 @@ sub long_response ($$$$) {
 	my ($self, $beg, $end, $cb) = @_;
 	die "BUG: nested long response" if $self->{long_res};
 
+	my $fd = $self->{fd};
+	defined $fd or return;
 	# make sure we disable reading during a long response,
 	# clients should not be sending us stuff and making us do more
 	# work while we are stream a response to them
 	$self->watch_read(0);
-	my $fd = fileno $self->{sock};
 	my $t0 = now();
 	$self->{long_res} = sub {
 		# limit our own running time for fairness with other
@@ -885,10 +886,11 @@ sub event_read {
 	while ($r > 0 && $self->{rbuf} =~ s/\A\s*([^\r\n]+)\r?\n//) {
 		my $line = $1;
 		my $t0 = now();
+		my $fd = $self->{fd};
 		$r = eval { $self->process_line($line) };
 		my $d = $self->{long_res} ?
-			' deferred['.fileno($self->{sock}).']' : '';
-		out($self, "$line - %0.6f$d", now() - $t0);
+			" deferred[$fd]" : '';
+		out($self, "[$fd] $line - %0.6f$d", now() - $t0);
 	}
 
 	return $self->close if $r < 0;
-- 
EW

[PATCH 6/8] git: signal-safety for pipe writes

[Eric Wong]

I've yet to hit it, but syswrite has chance of returning EINTR
on a blocking pipe.
---
 lib/PublicInbox/GitCatFile.pm | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/lib/PublicInbox/GitCatFile.pm b/lib/PublicInbox/GitCatFile.pm
index 48ae673..3fced28 100644
--- a/lib/PublicInbox/GitCatFile.pm
+++ b/lib/PublicInbox/GitCatFile.pm
@@ -7,6 +7,7 @@ package PublicInbox::GitCatFile;
 use strict;
 use warnings;
 use POSIX qw(dup2);
+require IO::Handle;
 
 sub new {
 	my ($class, $git_dir) = @_;
@@ -31,6 +32,7 @@ sub _cat_file_begin {
 	}
 	close $out_r or die "close failed: $!\n";
 	close $in_w or die "close failed: $!\n";
+	$out_w->autoflush(1);
 
 	$self->{in} = $in_r;
 	$self->{out} = $out_w;
@@ -40,16 +42,8 @@ sub _cat_file_begin {
 sub cat_file {
 	my ($self, $object, $sizeref) = @_;
 
-	$object .= "\n";
-	my $len = length($object);
-
 	$self->_cat_file_begin;
-	my $written = syswrite($self->{out}, $object);
-	if (!defined $written) {
-		die "pipe write error: $!\n";
-	} elsif ($written != $len) {
-		die "wrote too little to pipe ($written < $len)\n";
-	}
+	print { $self->{out} } $object, "\n" or die "pipe write error: $!\n";
 
 	my $in = $self->{in};
 	my $head = <$in>;
-- 
EW

[PATCH 7/8] git: use fields for GitCatFile

[Eric Wong]

Micro-optimization, but it make using Danga::Socket for watching
pipe readability easier at some point.
---
 lib/PublicInbox/GitCatFile.pm | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/GitCatFile.pm b/lib/PublicInbox/GitCatFile.pm
index 3fced28..5403696 100644
--- a/lib/PublicInbox/GitCatFile.pm
+++ b/lib/PublicInbox/GitCatFile.pm
@@ -8,10 +8,13 @@ use strict;
 use warnings;
 use POSIX qw(dup2);
 require IO::Handle;
+use fields qw(git_dir pid in out);
 
 sub new {
 	my ($class, $git_dir) = @_;
-	bless { git_dir => $git_dir }, $class;
+	my $self = fields::new($class);
+	$self->{git_dir} = $git_dir;
+	$self;
 }
 
 sub _cat_file_begin {
-- 
EW

[PATCH 8/8] nntp: avoid signals for long responses

[Eric Wong]

Using a signal-based timer can hurt throughput on a machine that's
overloaded.  Ensure there's always forward progress and reduce the
number of syscalls we make, too.
---
 lib/PublicInbox/NNTP.pm | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 3490a09..95aa4af 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -11,7 +11,7 @@ use PublicInbox::MID qw(mid2path);
 use Email::MIME;
 use Data::Dumper qw(Dumper);
 use POSIX qw(strftime);
-use Time::HiRes qw(clock_gettime ualarm CLOCK_MONOTONIC);
+use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use constant {
 	r501 => '501 command syntax error',
 	r221 => '221 Header follows',
@@ -502,16 +502,13 @@ sub long_response ($$$$) {
 	$self->{long_res} = sub {
 		# limit our own running time for fairness with other
 		# clients and to avoid buffering too much:
-		my $yield;
-		local $SIG{ALRM} = sub { $yield = 1 };
-		ualarm(100000);
+		my $lim = 100;
 
 		my $err;
 		do {
 			eval { $cb->(\$beg) };
 		} until (($err = $@) || $self->{closed} ||
-			 ++$beg > $end || $yield || $self->{write_buf_size});
-		ualarm(0);
+			 ++$beg > $end || !--$lim || $self->{write_buf_size});
 
 		if ($err || $self->{closed}) {
 			$self->{long_res} = undef;
@@ -527,7 +524,7 @@ sub long_response ($$$$) {
 			} else {
 				$self->watch_read(1);
 			}
-		} elsif ($yield || $self->{write_buf_size}) {
+		} elsif (!$lim || $self->{write_buf_size}) {
 			# no recursion, schedule another call ASAP
 			# but only after all pending writes are done
 			Danga::Socket->AddTimer(0, sub {
-- 
EW