From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 370C51FAE5 for ; Wed, 28 Feb 2018 23:42:08 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Subject: [PATCH 04/21] v2writable: cleanup unused pipes in partitions Date: Wed, 28 Feb 2018 23:41:45 +0000 Message-Id: <20180228234202.8839-5-e@80x24.org> In-Reply-To: <20180228234202.8839-1-e@80x24.org> References: <20180228234202.8839-1-e@80x24.org> List-Id: Leaking these pipes to child processes wasn't harmful, but made determining relationships and dataflow between processes more confusing. --- lib/PublicInbox/Import.pm | 7 +++++++ lib/PublicInbox/SearchIdxPart.pm | 9 +++++---- lib/PublicInbox/SearchIdxThread.pm | 6 ++++-- lib/PublicInbox/V2Writable.pm | 18 +++++++++++++++--- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm index b650e4e..ac46c0c 100644 --- a/lib/PublicInbox/Import.pm +++ b/lib/PublicInbox/Import.pm @@ -372,6 +372,13 @@ sub done { close $lockfh or die "close lock failed: $!"; } +sub atfork_child { + my ($self) = @_; + foreach my $f (qw(in out)) { + close $self->{$f} or die "failed to close import[$f]: $!\n"; + } +} + 1; __END__ =pod diff --git a/lib/PublicInbox/SearchIdxPart.pm b/lib/PublicInbox/SearchIdxPart.pm index 5582d67..64e5263 100644 --- a/lib/PublicInbox/SearchIdxPart.pm +++ b/lib/PublicInbox/SearchIdxPart.pm @@ -14,10 +14,7 @@ sub new { my $pid = fork; defined $pid or die "fork failed: $!\n"; if ($pid == 0) { - foreach my $other (@{$v2writable->{idx_parts}}) { - my $other_w = $other->{w} or next; - close $other_w or die "close other failed: $!\n"; - } + $v2writable->atfork_child; $v2writable = undef; close $w; @@ -74,4 +71,8 @@ sub index_raw { $w->flush or die "failed to flush: $!\n"; } +sub atfork_child { + close $_[0]->{w} or die "failed to close write pipe: $!\n"; +} + 1; diff --git a/lib/PublicInbox/SearchIdxThread.pm b/lib/PublicInbox/SearchIdxThread.pm index 6471309..7df07a6 100644 --- a/lib/PublicInbox/SearchIdxThread.pm +++ b/lib/PublicInbox/SearchIdxThread.pm @@ -7,8 +7,8 @@ use base qw(PublicInbox::SearchIdx); use Storable qw(freeze thaw); sub new { - my ($class, $v2ibx) = @_; - my $self = $class->SUPER::new($v2ibx, 1, 'all'); + my ($class, $v2writable) = @_; + my $self = $class->SUPER::new($v2writable->{-inbox}, 1, 'all'); # create the DB: $self->_xdb_acquire; $self->_xdb_release; @@ -20,6 +20,8 @@ sub new { my $pid = fork; defined $pid or die "fork failed: $!\n"; if ($pid == 0) { + $v2writable->atfork_child; + $v2writable = undef; close $w; eval { thread_worker_loop($self, $r) }; die "thread worker died: $@\n" if $@; diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 29ed23c..3451261 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -84,9 +84,9 @@ sub idx_part { sub idx_init { my ($self) = @_; return if $self->{idx_parts}; - # first time initialization: - my $all = $self->{all} = - PublicInbox::SearchIdxThread->new($self->{-inbox}); + + # first time initialization, first we create the threader pipe: + my $all = $self->{all} = PublicInbox::SearchIdxThread->new($self); # need to create all parts before initializing msgmap FD my $max = $self->{partitions} - 1; @@ -94,6 +94,8 @@ sub idx_init { for my $i (0..$max) { push @$idx, PublicInbox::SearchIdxPart->new($self, $i, $all); } + + # Now that all subprocesses are up, we can open the FD for SQLite: $all->_msgmap_init->{dbh}->begin_work; } @@ -242,4 +244,14 @@ sub lookup_content { undef # TODO } +sub atfork_child { + my ($self) = @_; + if (my $parts = $self->{idx_parts}) { + $_->atfork_child foreach @$parts; + } + if (my $im = $self->{im}) { + $im->atfork_child; + } +} + 1; -- EW