From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 6/6] www: add configurable limiters
Date: Sat, 9 Jul 2016 03:18:35 +0000 [thread overview]
Message-ID: <20160709031835.21005-7-e@80x24.org> (raw)
In-Reply-To: <20160709031835.21005-1-e@80x24.org>
Currently only for git-http-backend use, this allows limiting
the number of spawned processes per-inbox or by group, if there
are multiple large inboxes amidst a sea of small ones.
For example, a "big" repo limiter could be used for big inboxes:
which would be shared between multiple repos:
[limiter "big"]
max = 4
[publicinbox "git"]
address = git@vger.kernel.org
mainrepo = /path/to/git.git
; shared limiter with giant:
httpbackendmax = big
[publicinbox "giant"]
address = giant@project.org
mainrepo = /path/to/giant.git
; shared limiter with git:
httpbackendmax = big
; This is a tiny inbox, use the default limiter with 32 slots:
[publicinbox "meta"]
address = meta@public-inbox.org
mainrepo = /path/to/meta.git
---
MANIFEST | 1 +
lib/PublicInbox/Config.pm | 13 +++++++++-
lib/PublicInbox/GitHTTPBackend.pm | 10 ++++++--
lib/PublicInbox/Inbox.pm | 25 +++++++++++++++++++-
lib/PublicInbox/Qspawn.pm | 7 +++---
t/config.t | 2 ++
t/config_limiter.t | 50 +++++++++++++++++++++++++++++++++++++++
7 files changed, 101 insertions(+), 7 deletions(-)
create mode 100644 t/config_limiter.t
diff --git a/MANIFEST b/MANIFEST
index ceb1a9d..75bb43e 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -106,6 +106,7 @@ t/cgi.t
t/check-www-inbox.perl
t/common.perl
t/config.t
+t/config_limiter.t
t/emergency.t
t/fail-bin/spamc
t/feed.t
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index d34d11a..d7eaa3e 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -20,6 +20,7 @@ sub new {
$self->{-by_addr} ||= {};
$self->{-by_name} ||= {};
$self->{-by_newsgroup} ||= {};
+ $self->{-limiters} ||= {};
$self;
}
@@ -85,6 +86,15 @@ sub lookup_newsgroup {
undef;
}
+sub limiter {
+ my ($self, $name) = @_;
+ $self->{-limiters}->{$name} ||= do {
+ require PublicInbox::Qspawn;
+ my $key = "limiter.$name.max";
+ PublicInbox::Qspawn::Limiter->new($self->{$key});
+ };
+}
+
sub get {
my ($self, $inbox, $key) = @_;
@@ -131,7 +141,7 @@ sub _fill {
my $rv = {};
foreach my $k (qw(mainrepo address filter url newsgroup
- watch watchheader)) {
+ watch watchheader httpbackendmax)) {
my $v = $self->{"$pfx.$k"};
$rv->{$k} = $v if defined $v;
}
@@ -139,6 +149,7 @@ sub _fill {
my $name = $pfx;
$name =~ s/\Apublicinbox\.//;
$rv->{name} = $name;
+ $rv->{-pi_config} = $self;
$rv = PublicInbox::Inbox->new($rv);
my $v = $rv->{address};
if (ref($v) eq 'ARRAY') {
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index ed8fdf0..d491479 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -179,7 +179,6 @@ sub prepare_range {
# returns undef if 403 so it falls back to dumb HTTP
sub serve_smart {
my ($env, $git, $path) = @_;
- my $limiter = $default_limiter;
my $in = $env->{'psgi.input'};
my $fd = eval { fileno($in) };
unless (defined $fd && $fd >= 0) {
@@ -197,7 +196,14 @@ sub serve_smart {
my $val = $env->{$name};
$env{$name} = $val if defined $val;
}
- my $git_dir = ref $git ? $git->{git_dir} : $git;
+ my ($git_dir, $limiter);
+ if (ref $git) {
+ $limiter = $git->{-httpbackend_limiter} || $default_limiter;
+ $git_dir = $git->{git_dir};
+ } else {
+ $limiter = $default_limiter;
+ $git_dir = $git;
+ }
$env{GIT_HTTP_EXPORT_ALL} = '1';
$env{PATH_TRANSLATED} = "$git_dir/$path";
my %rdr = ( 0 => fileno($in) );
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index dc9980b..23b7721 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -34,6 +34,7 @@ sub new {
my $v = $opts->{address} ||= 'public-inbox@example.com';
my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v;
$opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost';
+ weaken($opts->{-pi_config});
bless $opts, $class;
}
@@ -44,11 +45,33 @@ sub _weaken_fields {
}
}
+sub _set_limiter ($$$) {
+ my ($self, $git, $pfx) = @_;
+ my $lkey = "-${pfx}_limiter";
+ $git->{$lkey} = $self->{$lkey} ||= eval {
+ my $mkey = $pfx.'max';
+ my $val = $self->{$mkey} or return;
+ my $lim;
+ if ($val =~ /\A\d+\z/) {
+ require PublicInbox::Qspawn;
+ $lim = PublicInbox::Qspawn::Limiter->new($val);
+ } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) {
+ $lim = $self->{-pi_config}->limiter($val);
+ warn "$mkey limiter=$val not found\n" if !$lim;
+ } else {
+ warn "$mkey limiter=$val not understood\n";
+ }
+ $lim;
+ }
+}
+
sub git {
my ($self) = @_;
$self->{git} ||= eval {
_weaken_later($self);
- PublicInbox::Git->new($self->{mainrepo});
+ my $g = PublicInbox::Git->new($self->{mainrepo});
+ _set_limiter($self, $g, 'httpbackend');
+ $g;
};
}
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index cc9c340..697c55a 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -47,7 +47,7 @@ sub start {
my ($self, $limiter, $cb) = @_;
$self->{limiter} = $limiter;
- if ($limiter->{running} < $limiter->{limit}) {
+ if ($limiter->{running} < $limiter->{max}) {
_do_spawn($self, $cb);
} else {
push @{$limiter->{run_queue}}, [ $self, $cb ];
@@ -59,9 +59,10 @@ use strict;
use warnings;
sub new {
- my ($class, $limit) = @_;
+ my ($class, $max) = @_;
bless {
- limit => $limit || 1,
+ # 32 is same as the git-daemon connection limit
+ max => $max || 32,
running => 0,
run_queue => [],
}, $class;
diff --git a/t/config.t b/t/config.t
index dc448cd..77e8f4a 100644
--- a/t/config.t
+++ b/t/config.t
@@ -30,6 +30,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
'url' => 'http://example.com/meta',
-primary_address => 'meta@public-inbox.org',
'name' => 'meta',
+ -pi_config => $cfg,
}, "lookup matches expected output");
is($cfg->lookup('blah@example.com'), undef,
@@ -45,6 +46,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
'domain' => 'public-inbox.org',
'name' => 'test',
'url' => 'http://example.com/test',
+ -pi_config => $cfg,
}, "lookup matches expected output for test");
}
diff --git a/t/config_limiter.t b/t/config_limiter.t
new file mode 100644
index 0000000..bfea151
--- /dev/null
+++ b/t/config_limiter.t
@@ -0,0 +1,50 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use PublicInbox::Config;
+my $cfgpfx = "publicinbox.test";
+{
+ my $config = PublicInbox::Config->new({
+ "$cfgpfx.address" => 'test@example.com',
+ "$cfgpfx.mainrepo" => '/path/to/non/existent',
+ "$cfgpfx.httpbackendmax" => 12,
+ });
+ my $ibx = $config->lookup_name('test');
+ my $git = $ibx->git;
+ my $old = "$git";
+ my $lim = $git->{-httpbackend_limiter};
+ ok($lim, 'Limiter exists');
+ is($lim->{max}, 12, 'limiter has expected slots');
+ $git = undef;
+ $ibx->{git} = undef;
+ $git = $ibx->git;
+ isnt($old, "$git", 'got new Git object');
+ is("$git->{-httpbackend_limiter}", "$lim", 'same limiter');
+}
+
+{
+ my $config = PublicInbox::Config->new({
+ 'limiter.named.max' => 3,
+ "$cfgpfx.address" => 'test@example.com',
+ "$cfgpfx.mainrepo" => '/path/to/non/existent',
+ "$cfgpfx.httpbackendmax" => 'named',
+ });
+ my $ibx = $config->lookup_name('test');
+ my $git = $ibx->git;
+ ok($git, 'got git object');
+ my $old = "$git";
+ my $lim = $git->{-httpbackend_limiter};
+ ok($lim, 'Limiter exists');
+ is($lim->{max}, 3, 'limiter has expected slots');
+ $git = undef;
+ $ibx->{git} = undef;
+ PublicInbox::Inbox::weaken_task;
+ $git = $ibx->git;
+ isnt($old, "$git", 'got new Git object');
+ is("$git->{-httpbackend_limiter}", "$lim", 'same limiter');
+ is($lim->{max}, 3, 'limiter has expected slots');
+}
+
+done_testing;
--
EW
prev parent reply other threads:[~2016-07-09 3:18 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-07-09 3:18 [PATCH 0/6] bunch of cleanups and a new feature! Eric Wong
2016-07-09 3:18 ` [PATCH 1/6] www: drop unused constants Eric Wong
2016-07-09 3:18 ` [PATCH 2/6] www: cleanup parameter passing Eric Wong
2016-07-09 3:18 ` [PATCH 3/6] feed: remove dead code and unneeded use Eric Wong
2016-07-09 3:18 ` [PATCH 4/6] cleanup some unnecessary use/requires Eric Wong
2016-07-09 3:18 ` [PATCH 5/6] qspawn: allow configurable limiters Eric Wong
2016-07-09 3:18 ` Eric Wong [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20160709031835.21005-7-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).