From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.6 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 7046C1F51B for ; Tue, 16 Apr 2024 20:56:30 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1713300990; bh=MOmEtZHvd58jX6jtr6yKks8b6EbccforMwwzpSJuNB0=; h=From:To:Subject:Date:In-Reply-To:References:From; b=SIvs3wn8Z5SCdxOHMgnbO2lYPsWSB5RWgfNhHJYz6ssIjrsBoDMMjedDGrfKncD27 a1eBQG/pVlRxYzy/5XbeQCtxtdqqgsYE4cCNhftyeWZXAHXjgRrVazFHwhmAnaAW3j xSxFCv5xtuiAdyp51OhZ47Xb+cc9vaWcCOXJChSY= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/4] lei/store: stop shard workers + cat-file on idle Date: Tue, 16 Apr 2024 20:56:28 +0000 Message-ID: <20240416205629.3648894-4-e@80x24.org> In-Reply-To: <20240416205629.3648894-1-e@80x24.org> References: <20240416205629.3648894-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Schedule a timer to stop shard workers and the git-cat-file process after a `barrier' command. This allows us to save some memory again when the lei-daemon is idle but preserves the fork overhead reduction when issuing many commands in parallel or in quick succession. --- lib/PublicInbox/LeiStore.pm | 46 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index 162c915f..a054f649 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -571,21 +571,11 @@ sub set_xvmd { sto_export_kw($self, $smsg->{num}, $vmd); } -sub barrier { +sub check_done { my ($self) = @_; - my ($errfh, $lei_sock) = @$self{0, 1}; # via sto_barrier_request - my @err; - if ($self->{im}) { - eval { $self->{im}->barrier }; - push(@err, "E: import barrier: $@\n") if $@; - } - delete $self->{lms}; - eval { $self->{priv_eidx}->barrier }; - push(@err, "E: priv_eidx barrier: $@\n") if $@; - print { $errfh // \*STDERR } @err; - send($lei_sock, 'child_error 256', 0) if @err && $lei_sock; - xchg_stderr($self); - die @err if @err; + $self->git->_active ? + add_uniq_timer("$self-check_done", \&check_done, $self) : + done($self); } sub xchg_stderr { @@ -602,23 +592,33 @@ sub xchg_stderr { undef; } -sub done { - my ($self) = @_; - my ($errfh, $lei_sock) = @$self{0, 1}; +sub _commit ($$) { + my ($self, $cmd) = @_; # cmd is 'done' or 'barrier' + my ($errfh, $lei_sock) = @$self{0, 1}; # via sto_barrier_request my @err; - if (my $im = delete($self->{im})) { - eval { $im->done }; - push(@err, "E: import done: $@\n") if $@; + if ($self->{im}) { + eval { $self->{im}->$cmd }; + push(@err, "E: import $cmd: $@\n") if $@; } delete $self->{lms}; - eval { $self->{priv_eidx}->done }; # V2Writable::done - push(@err, "E: priv_eidx done: $@\n") if $@; - print { $errfh // *STDERR{GLOB} } @err; + eval { $self->{priv_eidx}->$cmd }; + push(@err, "E: priv_eidx $cmd: $@\n") if $@; + print { $errfh // \*STDERR } @err; send($lei_sock, 'child_error 256', 0) if @err && $lei_sock; xchg_stderr($self); die @err if @err; + # $lei_sock goes out-of-scope and script/lei can terminate +} + +sub barrier { + my ($self) = @_; + _commit $self, 'barrier'; + add_uniq_timer("$self-check_done", 5, \&check_done, $self); + undef; } +sub done { _commit $_[0], 'done' } + sub ipc_atfork_child { my ($self) = @_; my $lei = $self->{lei};