From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <e@yhbt.net>
X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net
X-Spam-Level: 
X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00
	shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2
Received: from localhost (dcvr.yhbt.net [127.0.0.1])
	by dcvr.yhbt.net (Postfix) with ESMTP id 2FE341F920
	for <meta@public-inbox.org>; Sat, 27 Jun 2020 10:04:04 +0000 (UTC)
From: Eric Wong <e@yhbt.net>
To: meta@public-inbox.org
Subject: [PATCH 21/34] watch: avoid long transaction to IMAPTracker
Date: Sat, 27 Jun 2020 10:03:47 +0000
Message-Id: <20200627100400.9871-22-e@yhbt.net>
In-Reply-To: <20200627100400.9871-1-e@yhbt.net>
References: <20200627100400.9871-1-e@yhbt.net>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
List-Id: <meta.public-inbox.org>

With different polling intervals, multiple processes may
simultaneously write to IMAPtracker.  This ought to reduce
SQLite busy waiting and contention issues when importing
many inboxes in parallel.
---
 lib/PublicInbox/WatchMaildir.pm | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index e0caaa563b2..d492e5d65b7 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -335,12 +335,12 @@ sub mic_for ($$$) { # mic = Mail::IMAPClient
 	$mic;
 }
 
-sub imap_import_msg ($$$$$) {
-	my ($self, $itrk, $r_uidval, $uid, $raw) = @_;
+sub imap_import_msg ($$$$) {
+	my ($self, $url, $uid, $raw) = @_;
 	# our target audience expects LF-only, save storage
 	$$raw =~ s/\r\n/\n/sg;
 
-	my $inboxes = $self->{imap}->{$itrk->{url}};
+	my $inboxes = $self->{imap}->{$url};
 	if (ref($inboxes)) {
 		for my $ibx (@$inboxes) {
 			my $eml = PublicInbox::Eml->new($$raw);
@@ -348,12 +348,11 @@ sub imap_import_msg ($$$$$) {
 		}
 	} elsif ($inboxes eq 'watchspam') {
 		my $eml = PublicInbox::Eml->new($raw);
-		my $arg = [ $self, $eml, "$itrk->{url} UID:$uid" ];
+		my $arg = [ $self, $eml, "$url UID:$uid" ];
 		$self->{config}->each_inbox(\&remove_eml_i, $arg);
 	} else {
 		die "BUG: destination unknown $inboxes";
 	}
-	$itrk->update_last($r_uidval, $uid);
 }
 
 sub imap_fetch_all ($$$) {
@@ -415,8 +414,8 @@ sub imap_fetch_all ($$$) {
 		return if $uids->[0] < $l_uid;
 
 		$l_uid = $uids->[-1] + 1; # for next search
+		my $last_uid;
 
-		$itrk->{dbh}->begin_work;
 		while (defined(($uid = shift(@$uids)))) {
 			local $0 = "UID:$uid $mbx $sec";
 			my $r = $mic->fetch_hash($uid, $req);
@@ -426,11 +425,12 @@ sub imap_fetch_all ($$$) {
 			}
 			# messages get deleted, so holes appear
 			defined(my $raw = delete $r->{$uid}->{$key}) or next;
-			imap_import_msg($self, $itrk, $r_uidval, $uid, \$raw);
+			imap_import_msg($self, $url, $uid, \$raw);
+			$last_uid = $uid;
 			last if $self->{quit};
 		}
 		_done_for_now($self);
-		$itrk->{dbh}->commit;
+		$itrk->update_last($r_uidval, $last_uid) if defined $last_uid;
 	} until ($err || $self->{quit});
 	$err;
 }