From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 7EF881F5AE for ; Wed, 19 May 2021 08:54:13 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] lei: relax rules for "new" in Maildir Date: Wed, 19 May 2021 08:54:13 +0000 Message-Id: <20210519085413.6438-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: mbsync and offlineimap both use ":2," suffixes for filenames in "new/", however my interpretation of the Maildir spec at is that ":2," is only for files in "cur/". My interpretation also matches that of doveecot, but we'll allow what mbsync and offlineimap do given their popularity. --- lib/PublicInbox/LeiMailSync.pm | 13 ++++++++++--- lib/PublicInbox/MdirReader.pm | 5 ++++- t/lei-index.t | 10 +++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm index 803de48c..3bada42d 100644 --- a/lib/PublicInbox/LeiMailSync.pm +++ b/lib/PublicInbox/LeiMailSync.pm @@ -233,9 +233,16 @@ WHERE b.oidbin = ? $b2n->execute(pack('H*', $oidhex)); while (my ($d, $n) = $b2n->fetchrow_array) { substr($d, 0, length('maildir:')) = ''; - my $f = "$d/" . ($n =~ /:2,[a-zA-Z]*\z/ ? "cur/$n" : "new/$n"); - open my $fh, '<', $f or next; - if (-s $fh) { + # n.b. both mbsync and offlineimap use ":2," as a suffix + # in "new/", despite (from what I understand of reading + # ), the ":2," only + # applies to files in "cur/". + my @try = $n =~ /:2,[a-zA-Z]+\z/ ? qw(cur new) : qw(new cur); + for my $x (@try) { + my $f = "$d/$x/$n"; + open my $fh, '<', $f or next; + # some (buggy) Maildir writers are non-atomic: + next unless -s $fh; local $/; my $raw = <$fh>; if ($vrfy && git_sha(1, \$raw)->hexdigest ne $oidhex) { diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm index b49c8ceb..7a0641fb 100644 --- a/lib/PublicInbox/MdirReader.pm +++ b/lib/PublicInbox/MdirReader.pm @@ -61,7 +61,10 @@ sub maildir_each_eml { while (defined(my $bn = readdir($dh))) { next if substr($bn, 0, 1) eq '.'; my @f = split(/:/, $bn, -1); - next if scalar(@f) != 1; + + # mbsync and offlineimap both use "2," in "new/" + next if ($f[1] // '2,') ne '2,' || defined($f[2]); + next if defined($mod) && !shard_ok($bn, $mod, $shard); my $f = $pfx.$bn; my $eml = eml_from_path($f) or next; diff --git a/t/lei-index.t b/t/lei-index.t index 3382d42b..b7dafb71 100644 --- a/t/lei-index.t +++ b/t/lei-index.t @@ -27,6 +27,11 @@ my $expect = do { local $/; <$fh>; }; + +# mbsync and offlineimap both put ":2," in "new/" files: +symlink(File::Spec->rel2abs('t/utf8.eml'), "$tmpdir/md/new/u:2,") or + xbail "symlink $!"; + test_lei({ tmpdir => $tmpdir }, sub { my $store_path = "$ENV{HOME}/.local/share/lei/store/"; @@ -40,13 +45,16 @@ test_lei({ tmpdir => $tmpdir }, sub { lei_ok(qw(q mid:qp@example.com -f text)); like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback'); + lei_ok(qw(q mid:testmessage@example.com -f text)); + lei_ok(qw(-C / blob --mail 9bf1002c49eb075df47247b74d69bcd555e23422)); + my $all_obj = ['git', "--git-dir=$store_path/ALL.git", qw(cat-file --batch-check --batch-all-objects)]; is_deeply([xqx($all_obj)], [], 'no git objects'); lei_ok('import', 't/plack-qp.eml'); ok(grep(/\A$blob blob /, my @objs = xqx($all_obj)), 'imported blob'); - lei_ok(qw(q z:0.. --dedupe=none)); + lei_ok(qw(q m:qp@example.com --dedupe=none)); my $res_b = json_utf8->decode($lei_out); is_deeply($res_b, $res_a, 'no extra DB entries');