* [PATCH] lei_input: reduce IPC traffic with multiple inputs
@ 2021-03-31 7:45 Eric Wong
0 siblings, 0 replies; only message in thread
From: Eric Wong @ 2021-03-31 7:45 UTC (permalink / raw)
To: meta
No point in sending a command for every input when a
single one will do. We'll also trigger LeiStore->done
sooner in the worker rather than later.
---
lib/PublicInbox/LeiConvert.pm | 11 +++++------
lib/PublicInbox/LeiImport.pm | 6 +-----
lib/PublicInbox/LeiInput.pm | 9 +++++++++
lib/PublicInbox/LeiTag.pm | 6 +-----
4 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index fb7a2f3b..0ce49ea9 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -7,6 +7,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiOverview;
+use PublicInbox::DS;
# /^input_/ subs are used by PublicInbox::LeiInput
@@ -32,12 +33,10 @@ sub input_maildir_cb {
$self->{wcb}->(undef, { kw => $kw }, $eml);
}
-sub do_convert { # via wq_do
+sub process_inputs { # via wq_do
my ($self) = @_;
- $PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid
- for my $input (@{$self->{inputs}}) {
- $self->input_path_url($input);
- }
+ local $PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid
+ $self->SUPER::process_inputs;
delete $self->{lei}->{1};
delete $self->{wcb}; # commit
}
@@ -55,7 +54,7 @@ sub lei_convert { # the main "lei convert" method
$self->prepare_inputs($lei, \@inputs) or return;
my ($op_c, $ops) = $lei->workers_start($self, 'lei_convert', 1);
$lei->{cnv} = $self;
- $self->wq_io_do('do_convert', []);
+ $self->wq_io_do('process_inputs', []);
$self->wq_close(1);
$op_c->op_wait_event($ops);
}
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index dbf655b6..d33143ef 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -39,8 +39,6 @@ sub import_done_wait { # dwaitpid callback
my ($arg, $pid) = @_;
my ($imp, $lei) = @$arg;
$lei->child_error($?, 'non-fatal errors during import') if $?;
- my $sto = delete $lei->{sto} // return $lei->fail('BUG: {sto} gone');
- my $wait = $sto->ipc_do('done'); # PublicInbox::LeiStore::done
$lei->dclose;
}
@@ -52,9 +50,7 @@ sub import_done { # EOF callback for main daemon
sub net_merge_complete { # callback used by LeiAuth
my ($self) = @_;
- for my $input (@{$self->{inputs}}) {
- $self->wq_io_do('input_path_url', [], $input);
- }
+ $self->wq_io_do('process_inputs');
$self->wq_close(1);
}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 505b73ff..40d71f9e 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -5,6 +5,7 @@
package PublicInbox::LeiInput;
use strict;
use v5.10.1;
+use PublicInbox::DS;
sub check_input_format ($;$) {
my ($lei, $files) = @_;
@@ -165,6 +166,14 @@ $input is `eml', not --in-format=$in_fmt
$self->{inputs} = $inputs;
}
+sub process_inputs {
+ my ($self) = @_;
+ for my $input (@{$self->{inputs}}) {
+ $self->input_path_url($input);
+ }
+ my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto};
+}
+
sub input_only_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 8b012b16..d572a84a 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -73,8 +73,6 @@ sub tag_done_wait { # dwaitpid callback
my ($arg, $pid) = @_;
my ($tag, $lei) = @$arg;
$lei->child_error($?, 'non-fatal errors during tag') if $?;
- my $sto = delete $lei->{sto};
- my $wait = $sto->ipc_do('done') if $sto; # PublicInbox::LeiStore::done
$lei->dclose;
}
@@ -86,9 +84,7 @@ sub tag_done { # EOF callback for main daemon
sub net_merge_complete { # callback used by LeiAuth
my ($self) = @_;
- for my $input (@{$self->{inputs}}) {
- $self->wq_io_do('input_path_url', [], $input);
- }
+ $self->wq_io_do('process_inputs');
$self->wq_close(1);
}
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2021-03-31 7:45 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-03-31 7:45 [PATCH] lei_input: reduce IPC traffic with multiple inputs Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).