[PATCH 3/5] lei import|convert: support mbox locking on reads

[Eric Wong <e@80x24.org>]

In case somebody is writing non-atomically, ensure we
take read locks when opening mbox files for reading.
---
 lib/PublicInbox/LEI.pm        | 13 +++++++++----
 lib/PublicInbox/LeiConvert.pm |  9 ++++++---
 lib/PublicInbox/LeiImport.pm  | 13 +++++++------
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b5bdda21..e133b357 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -172,12 +172,12 @@ our %CMD = ( # sorted in order of importance/use:
 'import' => [ 'LOCATION...|--stdin',
 	'one-time import/update from URL or filesystem',
 	qw(stdin| offset=i recursive|r exclude=s include|I=s
-	in-format|F=s kw|keywords|flags! C=s@),
+	lock=s@ in-format|F=s kw|keywords|flags! C=s@),
 	],
 'convert' => [ 'LOCATION...|--stdin',
 	'one-time conversion from URL or filesystem to another format',
 	qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s quiet|q
-	kw|keywords|flags! C=s@),
+	lock=s@ kw|keywords|flags! C=s@),
 	],
 'config' => [ '[...]', sub {
 		'git-config(1) wrapper for '._config_path($_[0]);
@@ -218,6 +218,9 @@ my %OPTDESC = (
 'help|h' => 'show this built-in help',
 'C=s@' => [ 'DIR', 'chdir to specify to directory' ],
 'quiet|q' => 'be quiet',
+'lock=s@' => [ 'METHOD|dotlock|fcntl|flock|none',
+	'mbox(5) locking method(s) to use (default: fcntl,dotlock)' ],
+
 'globoff|g' => "do not match locations using '*?' wildcards ".
 		"and\xa0'[]'\x{a0}ranges",
 'verbose|v+' => 'be more verbose',
@@ -410,8 +413,10 @@ sub check_input_format ($;$) {
 	return 1 if $fmt eq 'eml';
 	# XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail
 	require PublicInbox::MboxReader;
-	PublicInbox::MboxReader->can($fmt) ||
-				fail($self, "--$opt_key=$fmt unrecognized");
+	PublicInbox::MboxReader->can($fmt) or
+		return fail($self, "--$opt_key=$fmt unrecognized");
+	require PublicInbox::MboxLock if $files;
+	1;
 }
 
 sub out ($;@) {
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 45d42c9c..4c0bbd88 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -62,9 +62,11 @@ sub do_convert { # via wq_do
 			$ifmt = lc $1;
 		}
 		if (-f $input) {
-			open my $fh, '<', $input or
-					return $lei->fail("open $input: $!");
-			convert_fh($self, $ifmt, $fh, $input);
+			my $m = $lei->{opt}->{'lock'} //
+					($ifmt eq 'eml' ? ['none'] :
+					PublicInbox::MboxLock->defaults);
+			my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+			convert_fh($self, $ifmt, $mbl->{fh}, $input);
 		} elsif (-d _) {
 			PublicInbox::MdirReader::maildir_each_eml($input,
 							\&mdir_cb, $self);
@@ -109,6 +111,7 @@ sub call { # the main "lei convert" method
 
 			}
 			if (-f $input_path) {
+				require PublicInbox::MboxLock;
 				require PublicInbox::MboxReader;
 				PublicInbox::MboxReader->can($ifmt) or return
 					$lei->fail("$ifmt not supported");
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7f247b64..c2c98030 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -80,10 +80,11 @@ sub call { # the main "lei import" method
 			my $ifmt = lc $1;
 			if (($fmt // $ifmt) ne $ifmt) {
 				return $lei->fail(<<"");
---format=$fmt and `$ifmt:' conflict
+--in-format=$fmt and `$ifmt:' conflict
 
 			}
 			if (-f $input_path) {
+				require PublicInbox::MboxLock;
 				require PublicInbox::MboxReader;
 				PublicInbox::MboxReader->can($ifmt) or return
 					$lei->fail("$ifmt not supported");
@@ -142,7 +143,7 @@ error reading $input: $!
 			$cb->(undef, $fh, \&_import_eml, $lei->{sto}, $set_kw);
 		}
 	};
-	$lei->child_error(1 << 8, "<stdin>: $@") if $@;
+	$lei->child_error(1 << 8, "$input: $@") if $@;
 }
 
 sub _import_maildir { # maildir_each_file cb
@@ -171,10 +172,10 @@ sub import_path_url {
 		$ifmt = lc $1;
 	}
 	if (-f $input) {
-		open my $fh, '<', $input or return $lei->child_error(1 << 8, <<"");
-unable to open $input: $!
-
-		_import_fh($lei, $fh, $input, $ifmt);
+		my $m = $lei->{opt}->{'lock'} // ($ifmt eq 'eml' ? ['none'] :
+				PublicInbox::MboxLock->defaults);
+		my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+		_import_fh($lei, $mbl->{fh}, $input, $ifmt);
 	} elsif (-d _ && (-d "$input/cur" || -d "$input/new")) {
 		return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
 $input appears to a be a maildir, not $ifmt