From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 9A0221FA00 for ; Fri, 26 Feb 2021 09:41:41 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/5] lei import|convert: support mbox locking on reads Date: Thu, 25 Feb 2021 22:41:39 -1100 Message-Id: <20210226094141.11514-4-e@80x24.org> In-Reply-To: <20210226094141.11514-1-e@80x24.org> References: <20210226094141.11514-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: In case somebody is writing non-atomically, ensure we take read locks when opening mbox files for reading. --- lib/PublicInbox/LEI.pm | 13 +++++++++---- lib/PublicInbox/LeiConvert.pm | 9 ++++++--- lib/PublicInbox/LeiImport.pm | 13 +++++++------ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index b5bdda21..e133b357 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -172,12 +172,12 @@ our %CMD = ( # sorted in order of importance/use: 'import' => [ 'LOCATION...|--stdin', 'one-time import/update from URL or filesystem', qw(stdin| offset=i recursive|r exclude=s include|I=s - in-format|F=s kw|keywords|flags! C=s@), + lock=s@ in-format|F=s kw|keywords|flags! C=s@), ], 'convert' => [ 'LOCATION...|--stdin', 'one-time conversion from URL or filesystem to another format', qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s quiet|q - kw|keywords|flags! C=s@), + lock=s@ kw|keywords|flags! C=s@), ], 'config' => [ '[...]', sub { 'git-config(1) wrapper for '._config_path($_[0]); @@ -218,6 +218,9 @@ my %OPTDESC = ( 'help|h' => 'show this built-in help', 'C=s@' => [ 'DIR', 'chdir to specify to directory' ], 'quiet|q' => 'be quiet', +'lock=s@' => [ 'METHOD|dotlock|fcntl|flock|none', + 'mbox(5) locking method(s) to use (default: fcntl,dotlock)' ], + 'globoff|g' => "do not match locations using '*?' wildcards ". "and\xa0'[]'\x{a0}ranges", 'verbose|v+' => 'be more verbose', @@ -410,8 +413,10 @@ sub check_input_format ($;$) { return 1 if $fmt eq 'eml'; # XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail require PublicInbox::MboxReader; - PublicInbox::MboxReader->can($fmt) || - fail($self, "--$opt_key=$fmt unrecognized"); + PublicInbox::MboxReader->can($fmt) or + return fail($self, "--$opt_key=$fmt unrecognized"); + require PublicInbox::MboxLock if $files; + 1; } sub out ($;@) { diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm index 45d42c9c..4c0bbd88 100644 --- a/lib/PublicInbox/LeiConvert.pm +++ b/lib/PublicInbox/LeiConvert.pm @@ -62,9 +62,11 @@ sub do_convert { # via wq_do $ifmt = lc $1; } if (-f $input) { - open my $fh, '<', $input or - return $lei->fail("open $input: $!"); - convert_fh($self, $ifmt, $fh, $input); + my $m = $lei->{opt}->{'lock'} // + ($ifmt eq 'eml' ? ['none'] : + PublicInbox::MboxLock->defaults); + my $mbl = PublicInbox::MboxLock->acq($input, 0, $m); + convert_fh($self, $ifmt, $mbl->{fh}, $input); } elsif (-d _) { PublicInbox::MdirReader::maildir_each_eml($input, \&mdir_cb, $self); @@ -109,6 +111,7 @@ sub call { # the main "lei convert" method } if (-f $input_path) { + require PublicInbox::MboxLock; require PublicInbox::MboxReader; PublicInbox::MboxReader->can($ifmt) or return $lei->fail("$ifmt not supported"); diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index 7f247b64..c2c98030 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -80,10 +80,11 @@ sub call { # the main "lei import" method my $ifmt = lc $1; if (($fmt // $ifmt) ne $ifmt) { return $lei->fail(<<""); ---format=$fmt and `$ifmt:' conflict +--in-format=$fmt and `$ifmt:' conflict } if (-f $input_path) { + require PublicInbox::MboxLock; require PublicInbox::MboxReader; PublicInbox::MboxReader->can($ifmt) or return $lei->fail("$ifmt not supported"); @@ -142,7 +143,7 @@ error reading $input: $! $cb->(undef, $fh, \&_import_eml, $lei->{sto}, $set_kw); } }; - $lei->child_error(1 << 8, ": $@") if $@; + $lei->child_error(1 << 8, "$input: $@") if $@; } sub _import_maildir { # maildir_each_file cb @@ -171,10 +172,10 @@ sub import_path_url { $ifmt = lc $1; } if (-f $input) { - open my $fh, '<', $input or return $lei->child_error(1 << 8, <<""); -unable to open $input: $! - - _import_fh($lei, $fh, $input, $ifmt); + my $m = $lei->{opt}->{'lock'} // ($ifmt eq 'eml' ? ['none'] : + PublicInbox::MboxLock->defaults); + my $mbl = PublicInbox::MboxLock->acq($input, 0, $m); + _import_fh($lei, $mbl->{fh}, $input, $ifmt); } elsif (-d _ && (-d "$input/cur" || -d "$input/new")) { return $lei->fail(<