From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 32A661F4B7; Tue, 14 May 2019 02:04:44 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Cc: "Eric W. Biederman" Subject: [PATCH 2/3] v2writable: allow setting nproc via creat options Date: Tue, 14 May 2019 02:04:42 +0000 Message-Id: <20190514020443.7200-3-e@80x24.org> In-Reply-To: <20190514020443.7200-1-e@80x24.org> References: <20190514020443.7200-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Avoiding reliance on environment variables is a bit cleaner for writing tests --- lib/PublicInbox/V2Writable.pm | 13 +++++++++++-- script/public-inbox-index | 3 +-- t/purge.t | 3 +-- t/v2reindex.t | 3 +-- t/v2writable.t | 8 ++------ 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index b92d8d2..afcac4d 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -24,7 +24,14 @@ use IO::Handle; my $PACKING_FACTOR = 0.4; # assume 2 cores if GNU nproc(1) is not available -sub nproc_parts () { +sub nproc_parts ($) { + my ($creat_opt) = @_; + if (ref($creat_opt) eq 'HASH') { + if (defined(my $n = $creat_opt->{nproc})) { + return $n + } + } + my $n = int($ENV{NPROC} || `nproc 2>/dev/null` || 2); # subtract for the main process and git-fast-import $n -= 1; @@ -52,6 +59,8 @@ sub count_partitions ($) { } sub new { + # $creat may be any true value, or 0/undef. A hashref is true, + # and $creat->{nproc} may be set to an integer my ($class, $v2ibx, $creat) = @_; my $dir = $v2ibx->{mainrepo} or die "no mainrepo in inbox\n"; unless (-d $dir) { @@ -80,7 +89,7 @@ sub new { rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR), last_commit => [], # git repo -> commit }; - $self->{partitions} = count_partitions($self) || nproc_parts(); + $self->{partitions} = count_partitions($self) || nproc_parts($creat); bless $self, $class; } diff --git a/script/public-inbox-index b/script/public-inbox-index index 2f810a5..b353093 100755 --- a/script/public-inbox-index +++ b/script/public-inbox-index @@ -71,8 +71,7 @@ sub index_dir { eval { require PublicInbox::V2Writable }; die "v2 requirements not met: $@\n" if $@; my $v2w = eval { - $jobs and local $ENV{NPROC} = $jobs; - PublicInbox::V2Writable->new($repo); + PublicInbox::V2Writable->new($repo, {nproc=>$jobs}); }; if (defined $jobs) { if ($jobs == 0) { diff --git a/t/purge.t b/t/purge.t index b518c26..574935e 100644 --- a/t/purge.t +++ b/t/purge.t @@ -35,12 +35,11 @@ Hello World EOF -local $ENV{NPROC} = '1'; my $cfgfile = "$tmpdir/config"; local $ENV{PI_CONFIG} = $cfgfile; open my $cfg_fh, '>', $cfgfile or die "open: $!"; -my $v2w = PublicInbox::V2Writable->new($ibx, 1); +my $v2w = PublicInbox::V2Writable->new($ibx, {nproc => 1}); my $mime = PublicInbox::MIME->new($raw); ok($v2w->add($mime), 'add message to be purged'); $v2w->done; diff --git a/t/v2reindex.t b/t/v2reindex.t index 8a3071b..c416629 100644 --- a/t/v2reindex.t +++ b/t/v2reindex.t @@ -39,14 +39,13 @@ my $mime = PublicInbox::MIME->create( ], body => $agpl, ); -local $ENV{NPROC} = 2; my $minmax; my $msgmap; my ($mark1, $mark2, $mark3, $mark4); { my %config = %$ibx_config; my $ibx = PublicInbox::Inbox->new(\%config); - my $im = PublicInbox::V2Writable->new($ibx, 1); + my $im = PublicInbox::V2Writable->new($ibx, {nproc => 1}); my $im0 = $im->importer(); foreach my $i (1..10) { $mime->header_set('Message-Id', "<$i\@example.com>"); diff --git a/t/v2writable.t b/t/v2writable.t index f8ef415..7511015 100644 --- a/t/v2writable.t +++ b/t/v2writable.t @@ -33,10 +33,7 @@ my $mime = PublicInbox::MIME->create( body => "hello world\n", ); -my $im = eval { - local $ENV{NPROC} = '1'; - PublicInbox::V2Writable->new($ibx, 1); -}; +my $im = PublicInbox::V2Writable->new($ibx, {nproc => 1}); is($im->{partitions}, 1, 'one partition when forced'); ok($im->add($mime), 'ordinary message added'); foreach my $f ("$mainrepo/msgmap.sqlite3", @@ -201,11 +198,10 @@ EOF is_deeply([sort keys %lg], [sort keys %$rover], 'XROVER range OK'); }; { - local $ENV{NPROC} = 2; my @log = qw(log --no-decorate --no-abbrev --no-notes --no-color); my @before = $git0->qx(@log, qw(--pretty=oneline)); my $before = $git0->qx(@log, qw(--pretty=raw --raw -r)); - $im = PublicInbox::V2Writable->new($ibx, 1); + $im = PublicInbox::V2Writable->new($ibx, {nproc => 2}); is($im->{partitions}, 1, 'detected single partition from previous'); my $smsg = $im->remove($mime, 'test removal'); $im->done; -- EW