From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 09913205BD for ; Tue, 12 Mar 2019 04:00:47 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 11/13] qspawn: wire up RLIMIT_* handling to limiters Date: Tue, 12 Mar 2019 04:00:44 +0000 Message-Id: <20190312040046.4619-12-e@80x24.org> In-Reply-To: <20190312040046.4619-1-e@80x24.org> References: <20190312040046.4619-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This allows users to configure RLIMIT_{CORE,CPU,DATA} using our "limiter" config directive when spawning external processes. --- Documentation/public-inbox-config.pod | 17 +++++++++++ lib/PublicInbox/Config.pm | 4 ++- lib/PublicInbox/Qspawn.pm | 41 +++++++++++++++++++++++++-- lib/PublicInbox/Spawn.pm | 3 +- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod index 9647e4a..dae6998 100644 --- a/Documentation/public-inbox-config.pod +++ b/Documentation/public-inbox-config.pod @@ -228,12 +228,29 @@ large inboxes, it makes sense to put large inboxes on a named limiter with a low max value; while smaller inboxes can use the default limiter. +C keys may be set to enforce resource limits for +a particular limiter. + =over 8 =item publicinboxlimiter..max The maximum number of parallel processes for the given limiter. +=item publicinboxlimiter..rlimitCore + +=item publicinboxlimiter..rlimitCPU + +=item publicinboxlimiter..rlimitData + +The maximum core size, CPU time, or data size processes run with the +given limiter will use. This may be comma-separated to distinguish +soft and hard limits. The word "INFINITY" is accepted as the +RLIM_INFINITY constant (if supported by your OS). + +See L for more info on the behavior of RLIMIT_CORE, +RLIMIT_CPU, and RLIMIT_DATA for you operating system. + =back =head3 EXAMPLE WITH NAMED LIMITERS diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index 15664c6..0f7485f 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -133,7 +133,9 @@ sub limiter { $self->{-limiters}->{$name} ||= do { require PublicInbox::Qspawn; my $max = $self->{"publicinboxlimiter.$name.max"}; - PublicInbox::Qspawn::Limiter->new($max); + my $limiter = PublicInbox::Qspawn::Limiter->new($max); + $limiter->setup_rlimit($name, $self); + $limiter; }; } diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm index 509a441..79cdae7 100644 --- a/lib/PublicInbox/Qspawn.pm +++ b/lib/PublicInbox/Qspawn.pm @@ -43,10 +43,18 @@ sub new ($$$;) { sub _do_spawn { my ($self, $cb) = @_; my $err; + my ($cmd, $env, $opts) = @{$self->{args}}; + my %opts = %{$opts || {}}; + my $limiter = $self->{limiter}; + foreach my $k (PublicInbox::Spawn::RLIMITS()) { + if (defined(my $rlimit = $limiter->{$k})) { + $opts{$k} = $rlimit; + } + } - ($self->{rpipe}, $self->{pid}) = popen_rd(@{$self->{args}}); + ($self->{rpipe}, $self->{pid}) = popen_rd($cmd, $env, \%opts); if (defined $self->{pid}) { - $self->{limiter}->{running}++; + $limiter->{running}++; } else { $self->{err} = $!; } @@ -251,9 +259,38 @@ sub new { max => $max || 32, running => 0, run_queue => [], + # RLIMIT_CPU => undef, + # RLIMIT_DATA => undef, + # RLIMIT_CORE => undef, }, $class; } +sub setup_rlimit { + my ($self, $name, $config) = @_; + foreach my $rlim (PublicInbox::Spawn::RLIMITS()) { + my $k = lc($rlim); + $k =~ tr/_//d; + $k = "publicinboxlimiter.$name.$k"; + defined(my $v = $config->{$k}) or next; + my @rlimit = split(/\s*,\s*/, $v); + if (scalar(@rlimit) == 1) { + push @rlimit, $rlimit[0]; + } elsif (scalar(@rlimit) != 2) { + warn "could not parse $k: $v\n"; + } + eval { require BSD::Resource }; + if ($@) { + warn "BSD::Resource missing for $rlim"; + next; + } + foreach my $i (0..$#rlimit) { + next if $rlimit[$i] ne 'INFINITY'; + $rlimit[$i] = BSD::Resource::RLIM_INFINITY(); + } + $self->{$rlim} = \@rlimit; + } +} + # captures everything into a buffer and executes a callback when done package PublicInbox::Qspawn::Qx; use strict; diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm index 202cfca..fd98160 100644 --- a/lib/PublicInbox/Spawn.pm +++ b/lib/PublicInbox/Spawn.pm @@ -18,6 +18,7 @@ use Symbol qw(gensym); use IO::Handle; use PublicInbox::ProcessPipe; our @EXPORT_OK = qw/which spawn popen_rd/; +sub RLIMITS () { qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA) } my $vfork_spawn = <<'VFORK_SPAWN'; #include @@ -202,7 +203,7 @@ sub spawn ($;$$) { my $err = $opts->{2} || 2; my $rlim = []; - foreach my $l (qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA)) { + foreach my $l (RLIMITS()) { defined(my $v = $opts->{$l}) or next; my ($soft, $hard); if (ref($v)) { -- EW