From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id D3A311F9F4 for ; Wed, 9 Jun 2021 07:47:51 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/5] lei tag: parallelize Maildir access Date: Wed, 9 Jun 2021 07:47:49 +0000 Message-Id: <20210609074751.29217-4-e@80x24.org> In-Reply-To: <20210609074751.29217-1-e@80x24.org> References: <20210609074751.29217-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Since Maildir isn't guaranteed to have any sort of order, we can parallelize inputs, here. On a 4-core system, this reduced one of my tag invocations from 5.5 to 1.4s. --- lib/PublicInbox/LeiImport.pm | 8 ++++---- lib/PublicInbox/LeiPmdir.pm | 10 ++-------- lib/PublicInbox/LeiTag.pm | 8 +++++--- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index cddd5619..e3cb69ca 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -35,13 +35,13 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn die "BUG: $f was not from a Maildir?\n"; my $kw = PublicInbox::MdirReader::flags2kw($fl); substr($folder, 0, 0) = 'maildir:'; # add prefix - my $lms = $self->{-lms_ro}; + my $lse = $self->{lse} //= $self->{lei}->{sto}->search; + my $lms = $self->{-lms_ro} //= $lse->lms; # may be 0 or undef my $oidbin = $lms ? $lms->name_oidbin($folder, $bn) : undef; - my @docids = defined($oidbin) ? - $self->{over}->oidbin_exists($oidbin) : (); + my @docids = defined($oidbin) ? $lse->over->oidbin_exists($oidbin) : (); my $vmd = $self->{-import_kw} ? { kw => $kw } : undef; if (scalar @docids) { - $self->{lse}->kw_changed(undef, $kw, \@docids) or return; + $lse->kw_changed(undef, $kw, \@docids) or return; } if (my $eml = eml_from_path($f)) { $vmd->{sync_info} = [ $folder, \$bn ] if $self->{-mail_sync}; diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm index aa9ce713..760f276c 100644 --- a/lib/PublicInbox/LeiPmdir.pm +++ b/lib/PublicInbox/LeiPmdir.pm @@ -30,15 +30,9 @@ sub new { sub ipc_atfork_child { my ($self) = @_; - my $lei = $self->{lei}; - $lei->_lei_atfork_child; my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}'; - $ipt->{lei} = $lei; - $ipt->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}'; - $ipt->{lse} = $ipt->{sto}->search; - $ipt->{over} = $ipt->{lse}->over; - $ipt->{-lms_ro} //= $ipt->{lse}->lms; # may be undef or '0' - $self->SUPER::ipc_atfork_child; + $ipt->{lei} = $self->{lei}; + $ipt->ipc_atfork_child; } sub each_mdir_fn { # maildir_each_file callback diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm index 4b3ce7d8..e0532653 100644 --- a/lib/PublicInbox/LeiTag.pm +++ b/lib/PublicInbox/LeiTag.pm @@ -6,6 +6,7 @@ package PublicInbox::LeiTag; use strict; use v5.10.1; use parent qw(PublicInbox::IPC PublicInbox::LeiInput); +use PublicInbox::InboxWritable qw(eml_from_path); sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh my ($self, $eml) = @_; @@ -24,8 +25,9 @@ sub input_mbox_cb { input_eml_cb($self, $eml); } -sub input_maildir_cb { # maildir_each_eml cb - my ($f, $kw, $eml, $self) = @_; +sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn + my ($self, $f) = @_; + my $eml = eml_from_path($f) or return; input_eml_cb($self, $eml); } @@ -42,12 +44,12 @@ sub lei_tag { # the "lei tag" method $lei->ale; # refresh and prepare my $vmd_mod = $self->vmd_mod_extract(\@argv); return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err}; + $self->{vmd_mod} = $vmd_mod; # before LeiPmdir->new in prepare_inputs $self->prepare_inputs($lei, \@argv) or return; grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or return $lei->fail('no keywords or labels specified'); my $ops = {}; $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; - $self->{vmd_mod} = $vmd_mod; my $j = $self->{-wq_nr_workers} = 1; # locked for now (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); $lei->{wq1} = $self;