From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 26C741F953; Wed, 27 Oct 2021 21:09:19 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Cc: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Subject: [PATCH] lei q: fix remote import accounting Date: Wed, 27 Oct 2021 21:09:19 +0000 Message-Id: <20211027210919.7408-1-e@80x24.org> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit List-Id: We need to update the {-nr_remote_eml} counter regardless of progress display being enabled since it's needed for saved searches. We'll also split out the {-imported} flag separately and only call LeiStore->done if a new message was imported. Note: this change is NOT expected to fix errors reported by Thomas in Cc: Thomas Weißschuh --- lib/PublicInbox/LeiXSearch.pm | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index acc36897d4e3..2a037f2bd79b 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -283,20 +283,22 @@ sub each_remote_eml { # callback for MboxReader->mboxrd my $smsg = bless {}, 'PublicInbox::Smsg'; if ($self->{import_sto} && !$xoids) { my ($res, $kw) = $self->{import_sto}->wq_do('add_eml', $eml); - $smsg = $res if ref($res) eq ref($smsg); # totally new message + if (ref($res) eq ref($smsg)) { # totally new message + $smsg = $res; + $self->{-imported} = 1; + } $smsg->{kw} = $kw; # short-circuit xsmsg_vmd } $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_oid($eml)->hexdigest; _smsg_fill($smsg, $eml); wait_startq($lei); + my $nr = ++$lei->{-nr_remote_eml}; # needed for lss->cfg_set if ($lei->{-progress}) { - ++$lei->{-nr_remote_eml}; my $now = now(); my $next = $lei->{-next_progress} //= ($now + 1); if ($now > $next) { $lei->{-next_progress} = $now + 1; - my $nr = $lei->{-nr_remote_eml}; mset_progress($lei, $lei->{-current_url}, $nr, '?'); } } @@ -374,13 +376,14 @@ sub query_remote_mboxrd { $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1); PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml, $self, $lei, $each_smsg); - my $nr = $lei->{-nr_remote_eml}; - my $wait = $lei->{sto}->wq_do('done') if $nr && $lei->{sto}; + if ($self->{import_sto} && delete($self->{-imported})) { + my $wait = $self->{import_sto}->wq_do('done'); + } $reap_curl->join; if ($? == 0) { # don't update if no results, maybe MTA is down - $key && $nr and - $lei->{lss}->cfg_set($key, $start); + my $nr = $lei->{-nr_remote_eml}; + $lei->{lss}->cfg_set($key, $start) if $key && $nr; mset_progress($lei, $lei->{-current_url}, $nr, $nr); next; }