From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 048A21F4BA for ; Mon, 24 Jun 2019 02:56:51 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 34/57] allow use of PerlIO layers for filesystem writes Date: Mon, 24 Jun 2019 02:52:35 +0000 Message-Id: <20190624025258.25592-35-e@80x24.org> In-Reply-To: <20190624025258.25592-1-e@80x24.org> References: <20190624025258.25592-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: It may make sense to use PerlIO::mmap or PerlIO::scalar for DS write buffering with IO::Socket::SSL or similar (since we can't use MSG_MORE), so that means we need to go through buffering in userspace for the common case; while still being easily compatible with slow clients. And it also simplifies GitHTTPBackend slightly. Maybe it can make sense for HTTP input buffering, too... --- lib/PublicInbox/DS.pm | 32 ++++++++++++------------------- lib/PublicInbox/GitHTTPBackend.pm | 18 ++++++++--------- lib/PublicInbox/HTTP.pm | 24 ++++++++++++++++++----- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm index 8735e888..486af40e 100644 --- a/lib/PublicInbox/DS.pm +++ b/lib/PublicInbox/DS.pm @@ -21,7 +21,7 @@ use IO::Handle qw(); use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD SEEK_SET); use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC); use parent qw(Exporter); -our @EXPORT_OK = qw(now msg_more write_in_full); +our @EXPORT_OK = qw(now msg_more); use warnings; use 5.010_001; @@ -422,8 +422,8 @@ sub close { sub psendfile ($$$) { my ($sock, $fh, $off) = @_; - sysseek($fh, $$off, SEEK_SET) or return; - defined(my $to_write = sysread($fh, my $buf, 16384)) or return; + seek($fh, $$off, SEEK_SET) or return; + defined(my $to_write = read($fh, my $buf, 16384)) or return; my $written = 0; while ($to_write > 0) { if (defined(my $w = syswrite($sock, $buf, $to_write, $written))) { @@ -482,29 +482,18 @@ sub do_read ($$$$) { $! == EAGAIN ? $self->watch_in1 : $self->close; } -sub write_in_full ($$$$) { - my ($fh, $bref, $len, $off) = @_; - my $rv = 0; - while ($len > 0) { - my $w = syswrite($fh, $$bref, $len, $off); - return ($rv ? $rv : $w) unless $w; # undef or 0 - $rv += $w; - $len -= $w; - $off += $w; - } - $rv -} - +# n.b.: use ->write/->read for this buffer to allow compatibility with +# PerlIO::mmap or PerlIO::scalar if needed sub tmpbuf ($$) { my ($bref, $off) = @_; # open(my $fh, '+>>', undef) doesn't set O_APPEND my ($fh, $path) = tempfile('wbuf-XXXXXXX', TMPDIR => 1); open $fh, '+>>', $path or die "open: $!"; + $fh->autoflush(1); unlink $path; my $to_write = bytes::length($$bref) - $off; - my $w = write_in_full($fh, $bref, $to_write, $off); - die "write_in_full ($to_write): $!" unless defined $w; - $w == $to_write ? $fh : die("short write $w < $to_write"); + $fh->write($$bref, $to_write, $off) or die "write ($to_write): $!"; + $fh; } =head2 C<< $obj->write( $data ) >> @@ -534,7 +523,10 @@ sub write { } else { my $last = $wbuf->[-1]; if (ref($last) eq 'GLOB') { # append to tmp file buffer - write_in_full($last, $bref, bytes::length($$bref), 0); + unless ($last->print($$bref)) { + warn "error buffering: $!"; + return $self->close; + } } else { push @$wbuf, tmpbuf($bref, 0); } diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm index a2a81f8e..303d5073 100644 --- a/lib/PublicInbox/GitHTTPBackend.pm +++ b/lib/PublicInbox/GitHTTPBackend.pm @@ -231,18 +231,16 @@ sub input_prepare { return; } last if $r == 0; - my $off = 0; - while ($r > 0) { - my $w = syswrite($in, $buf, $r, $off); - if (defined $w) { - $r -= $w; - $off += $w; - } else { - err($env, "error writing temporary file: $!"); - return; - } + unless (print $in $buf) { + err($env, "error writing temporary file: $!"); + return; } } + # ensure it's visible to git-http-backend(1): + unless ($in->flush) { + err($env, "error writing temporary file: $!"); + return; + } unless (defined(sysseek($in, 0, SEEK_SET))) { err($env, "error seeking temporary file: $!"); return; diff --git a/lib/PublicInbox/HTTP.pm b/lib/PublicInbox/HTTP.pm index 7697ac5c..a1cb4aca 100644 --- a/lib/PublicInbox/HTTP.pm +++ b/lib/PublicInbox/HTTP.pm @@ -19,7 +19,7 @@ use HTTP::Status qw(status_message); use HTTP::Date qw(time2str); use IO::Handle; require PublicInbox::EvCleanup; -PublicInbox::DS->import(qw(msg_more write_in_full)); +PublicInbox::DS->import(qw(msg_more)); use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT); use constant { CHUNK_START => -1, # [a-f0-9]+\r\n @@ -102,6 +102,15 @@ sub rbuf_process { $len ? read_input($self) : app_dispatch($self); } +# IO::Handle::write returns boolean, this returns bytes written: +sub xwrite ($$$) { + my ($fh, $rbuf, $max) = @_; + my $w = bytes::length($$rbuf); + $w = $max if $w > $max; + $fh->write($$rbuf, $w) or return; + $w; +} + sub read_input ($) { my ($self) = @_; my $env = $self->{env}; @@ -116,7 +125,7 @@ sub read_input ($) { while ($len > 0) { if ($$rbuf ne '') { - my $w = write_in_full($input, $rbuf, $len, 0); + my $w = xwrite($input, $rbuf, $len); return write_err($self, $len) unless $w; $len -= $w; die "BUG: $len < 0 (w=$w)" if $len < 0; @@ -306,6 +315,11 @@ sub response_write { } } +sub input_tmpfile ($) { + open($_[0], '+>', undef); + $_[0]->autoflush(1); +} + sub input_prepare { my ($self, $env) = @_; my $input; @@ -315,10 +329,10 @@ sub input_prepare { quit($self, 413); return; } - open($input, '+>', undef); + input_tmpfile($input); } elsif (env_chunked($env)) { $len = CHUNK_START; - open($input, '+>', undef); + input_tmpfile($input); } else { $input = $null_io; } @@ -399,7 +413,7 @@ sub read_input_chunked { # unlikely... # drain the current chunk until ($len <= 0) { if ($$rbuf ne '') { - my $w = write_in_full($input, $rbuf, $len, 0); + my $w = xwrite($input, $rbuf, $len); return write_err($self, "$len chunk") if !$w; $len -= $w; if ($len == 0) { -- EW