From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 1EE8720713 for ; Sat, 9 Jul 2016 03:18:43 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 5/6] qspawn: allow configurable limiters Date: Sat, 9 Jul 2016 03:18:34 +0000 Message-Id: <20160709031835.21005-6-e@80x24.org> In-Reply-To: <20160709031835.21005-1-e@80x24.org> References: <20160709031835.21005-1-e@80x24.org> List-Id: And bump the default limit to 32 so we match git-daemon behavior. This shall allow us to configure different levels of concurrency for different repositories and prevent clones of giant repos from stalling service to small repos. --- lib/PublicInbox/GitHTTPBackend.pm | 6 +++++- lib/PublicInbox/Qspawn.pm | 38 ++++++++++++++++++++++++++++---------- t/qspawn.t | 10 ++++++---- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm index ebb0850..ed8fdf0 100644 --- a/lib/PublicInbox/GitHTTPBackend.pm +++ b/lib/PublicInbox/GitHTTPBackend.pm @@ -13,6 +13,9 @@ use HTTP::Status qw(status_message); use Plack::Util; use PublicInbox::Qspawn; +# 32 is same as the git-daemon connection limit +my $default_limiter = PublicInbox::Qspawn::Limiter->new(32); + # n.b. serving "description" and "cloneurl" should be innocuous enough to # not cause problems. serving "config" might... my @text = qw[HEAD info/refs @@ -176,6 +179,7 @@ sub prepare_range { # returns undef if 403 so it falls back to dumb HTTP sub serve_smart { my ($env, $git, $path) = @_; + my $limiter = $default_limiter; my $in = $env->{'psgi.input'}; my $fd = eval { fileno($in) }; unless (defined $fd && $fd >= 0) { @@ -248,7 +252,7 @@ sub serve_smart { # holding the input here is a waste of FDs and memory $env->{'psgi.input'} = undef; - $x->start(sub { # may run later, much later... + $x->start($limiter, sub { # may run later, much later... ($rpipe) = @_; $in = undef; if ($async) { diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm index 9299096..cc9c340 100644 --- a/lib/PublicInbox/Qspawn.pm +++ b/lib/PublicInbox/Qspawn.pm @@ -1,12 +1,14 @@ # Copyright (C) 2016 all contributors # License: AGPL-3.0+ + +# Limits the number of processes spawned +# This does not depend on Danga::Socket or any other external +# scheduling mechanism, you just need to call start and finish +# appropriately package PublicInbox::Qspawn; use strict; use warnings; use PublicInbox::Spawn qw(popen_rd); -our $LIMIT = 1; -my $running = 0; -my @run_queue; sub new ($$$;) { my ($class, $cmd, $env, $opt) = @_; @@ -16,9 +18,10 @@ sub new ($$$;) { sub _do_spawn { my ($self, $cb) = @_; my $err; + ($self->{rpipe}, $self->{pid}) = popen_rd(@{$self->{args}}); if (defined $self->{pid}) { - $running++; + $self->{limiter}->{running}++; } else { $self->{err} = $!; } @@ -27,26 +30,41 @@ sub _do_spawn { sub finish ($) { my ($self) = @_; + my $limiter = $self->{limiter}; if (delete $self->{rpipe}) { my $pid = delete $self->{pid}; $self->{err} = $pid == waitpid($pid, 0) ? $? : "PID:$pid still running?"; - $running--; + $limiter->{running}--; } - if (my $next = shift @run_queue) { + if (my $next = shift @{$limiter->{run_queue}}) { _do_spawn(@$next); } $self->{err}; } -sub start ($$) { - my ($self, $cb) = @_; +sub start { + my ($self, $limiter, $cb) = @_; + $self->{limiter} = $limiter; - if ($running < $LIMIT) { + if ($limiter->{running} < $limiter->{limit}) { _do_spawn($self, $cb); } else { - push @run_queue, [ $self, $cb ]; + push @{$limiter->{run_queue}}, [ $self, $cb ]; } } +package PublicInbox::Qspawn::Limiter; +use strict; +use warnings; + +sub new { + my ($class, $limit) = @_; + bless { + limit => $limit || 1, + running => 0, + run_queue => [], + }, $class; +} + 1; diff --git a/t/qspawn.t b/t/qspawn.t index 05072e2..9c42e10 100644 --- a/t/qspawn.t +++ b/t/qspawn.t @@ -2,10 +2,12 @@ # License: AGPL-3.0+ use Test::More; use_ok 'PublicInbox::Qspawn'; + +my $limiter = PublicInbox::Qspawn::Limiter->new(1); { my $x = PublicInbox::Qspawn->new([qw(true)]); my $run = 0; - $x->start(sub { + $x->start($limiter, sub { my ($rpipe) = @_; is(0, sysread($rpipe, my $buf, 1), 'read zero bytes'); ok(!$x->finish, 'no error on finish'); @@ -17,7 +19,7 @@ use_ok 'PublicInbox::Qspawn'; { my $x = PublicInbox::Qspawn->new([qw(false)]); my $run = 0; - $x->start(sub { + $x->start($limiter, sub { my ($rpipe) = @_; is(0, sysread($rpipe, my $buf, 1), 'read zero bytes from false'); my $err = $x->finish; @@ -30,7 +32,7 @@ use_ok 'PublicInbox::Qspawn'; foreach my $cmd ([qw(sleep 1)], [qw(sh -c), 'sleep 1; false']) { my $s = PublicInbox::Qspawn->new($cmd); my @run; - $s->start(sub { + $s->start($limiter, sub { my ($rpipe) = @_; push @run, 'sleep'; is(0, sysread($rpipe, my $buf, 1), 'read zero bytes'); @@ -39,7 +41,7 @@ foreach my $cmd ([qw(sleep 1)], [qw(sh -c), 'sleep 1; false']) { my @t = map { my $i = $n++; my $x = PublicInbox::Qspawn->new([qw(true)]); - $x->start(sub { + $x->start($limiter, sub { my ($rpipe) = @_; push @run, $i; }); -- EW