[PATCH 0/3] some watch fixes and improvements

[PATCH 0/3] some watch fixes and improvements

[Eric Wong]

Most notably, -watch now supports a Maildir with multiple lists.
(works better with the "watcheader" directive).

It should probably resolve symlinks in the future, too;
since I seem to recall some weirdness in that + Filesys::Notify::Simple

Eric Wong (3):
  watchmaildir: support multiple inboxes in the same Maildir
  watchmaildir: get rid of unused spamdir field
  watchmaildir: normalize Maildir pathnames consistently

 lib/PublicInbox/WatchMaildir.pm | 71 ++++++++++++++++++++-------------
 t/watch_maildir.t               |  9 +++++
 t/watch_maildir_v2.t            | 38 ++++++++++++++++--
 3 files changed, 88 insertions(+), 30 deletions(-)

-- 
EW

[PATCH 1/3] watchmaildir: support multiple inboxes in the same Maildir

[Eric Wong]

Not sure what I was smoking when I originally wrote this code.

cf. https://public-inbox.org/meta/874li887mp.fsf@vuxu.org/
---
 lib/PublicInbox/WatchMaildir.pm | 54 +++++++++++++++++++++------------
 t/watch_maildir_v2.t            | 38 +++++++++++++++++++++--
 2 files changed, 69 insertions(+), 23 deletions(-)

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index b558cda..064cedf 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -20,6 +20,7 @@ package PublicInbox::WatchMaildir;
 sub new {
 	my ($class, $config) = @_;
 	my (%mdmap, @mdir, $spamc, $spamdir);
+	my %uniq;
 
 	# "publicinboxwatch" is the documented namespace
 	# "publicinboxlearn" is legacy but may be supported
@@ -32,7 +33,17 @@ sub new {
 				# skip "new", no MUA has seen it, yet.
 				my $cur = "$dir/cur";
 				$spamdir = $cur;
+				my $old = $mdmap{$cur};
+				if (ref($old)) {
+					foreach my $ibx (@$old) {
+						warn <<"";
+"$cur already watched for `$ibx->{name}'
+
+					}
+					die;
+				}
 				push @mdir, $cur;
+				$uniq{$cur}++;
 				$mdmap{$cur} = 'watchspam';
 			} else {
 				warn "unsupported $k=$dir\n";
@@ -58,10 +69,11 @@ sub new {
 			}
 			my $new = "$watch/new";
 			my $cur = "$watch/cur";
-			push @mdir, $new, $cur;
-			die "$new already in use\n" if $mdmap{$new};
-			die "$cur already in use\n" if $mdmap{$cur};
-			$mdmap{$new} = $mdmap{$cur} = $ibx;
+			push @mdir, $new unless $uniq{$new}++;
+			push @mdir, $cur unless $uniq{$cur}++;
+
+			push @{$mdmap{$new} ||= []}, $ibx;
+			push @{$mdmap{$cur} ||= []}, $ibx;
 		} else {
 			warn "watch unsupported: $k=$watch\n";
 		}
@@ -134,28 +146,30 @@ sub _try_path {
 		warn "unrecognized path: $path\n";
 		return;
 	}
-	my $inbox = $self->{mdmap}->{$1};
-	unless ($inbox) {
+	my $inboxes = $self->{mdmap}->{$1};
+	unless ($inboxes) {
 		warn "unmappable dir: $1\n";
 		return;
 	}
-	if (!ref($inbox) && $inbox eq 'watchspam') {
+	if (!ref($inboxes) && $inboxes eq 'watchspam') {
 		return _remove_spam($self, $path);
 	}
-	my $im = _importer_for($self, $inbox);
-	my $mime = _path_to_mime($path) or return;
-	my $wm = $inbox->{-watchheader};
-	if ($wm) {
-		my $v = $mime->header_obj->header_raw($wm->[0]);
-		return unless ($v && $v =~ $wm->[1]);
-	}
-	if (my $scrub = $inbox->filter) {
-		my $ret = $scrub->scrub($mime) or return;
-		$ret == REJECT() and return;
-		$mime = $ret;
-	}
+	foreach my $ibx (@$inboxes) {
+		my $mime = _path_to_mime($path) or next;
+		my $im = _importer_for($self, $ibx);
 
-	$im->add($mime, $self->{spamcheck});
+		my $wm = $ibx->{-watchheader};
+		if ($wm) {
+			my $v = $mime->header_obj->header_raw($wm->[0]);
+			next unless ($v && $v =~ $wm->[1]);
+		}
+		if (my $scrub = $ibx->filter) {
+			my $ret = $scrub->scrub($mime) or next;
+			$ret == REJECT() and next;
+			$mime = $ret;
+		}
+		$im->add($mime, $self->{spamcheck});
+	}
 }
 
 sub quit { trigger_scan($_[0], 'quit') }
diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index fc002dc..3b5d2b8 100644
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -38,13 +38,14 @@ ok(POSIX::mkfifo("$maildir/cur/fifo", 0777),
 	'create FIFO to ensure we do not get stuck on it :P');
 my $sem = PublicInbox::Emergency->new($spamdir); # create dirs
 
-my $config = PublicInbox::Config->new({
+my %orig = (
 	"$cfgpfx.address" => $addr,
 	"$cfgpfx.mainrepo" => $mainrepo,
 	"$cfgpfx.watch" => "maildir:$maildir",
 	"$cfgpfx.filter" => 'PublicInbox::Filter::Vger',
-	"publicinboxlearn.watchspam" => "maildir:$spamdir",
-});
+	"publicinboxlearn.watchspam" => "maildir:$spamdir"
+);
+my $config = PublicInbox::Config->new({%orig});
 my $ibx = $config->lookup_name('test');
 ok($ibx, 'found inbox by name');
 my $srch = $ibx->search;
@@ -137,4 +138,35 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
 	is($post->{blob}, $msgs->[0]->{blob}, 'same message');
 }
 
+# multiple inboxes in the same maildir
+{
+	my $v1repo = "$tmpdir/v1";
+	my $v1pfx = "publicinbox.v1";
+	my $v1addr = 'v1-public@example.com';
+	is(system(qw(git init -q --bare), $v1repo), 0, 'v1 init OK');
+	my $config = PublicInbox::Config->new({
+		%orig,
+		"$v1pfx.address" => $v1addr,
+		"$v1pfx.mainrepo" => $v1repo,
+		"$v1pfx.watch" => "maildir:$maildir",
+	});
+	my $both = <<EOF;
+From: user\@example.com
+To: $addr, $v1addr
+Subject: both
+Message-Id: <both\@b.com>
+Date: Sat, 18 Jun 2016 00:00:00 +0000
+
+both
+EOF
+	PublicInbox::Emergency->new($maildir)->prepare(\$both);
+	PublicInbox::WatchMaildir->new($config)->scan('full');
+	my ($total, $msgs) = $srch->reopen->query('m:both@b.com');
+	my $v1 = $config->lookup_name('v1');
+	my $msg = $v1->git->cat_file($msgs->[0]->{blob});
+	is($both, $$msg, 'got original message back from v1');
+	$msg = $ibx->git->cat_file($msgs->[0]->{blob});
+	is($both, $$msg, 'got original message back from v2');
+}
+
 done_testing;
-- 
EW

[PATCH 2/3] watchmaildir: get rid of unused spamdir field

[Eric Wong]

Unused since commit 6c2caa791bd5fbf5c4edb1a4a2c1807e527348a7
("watchmaildir: support v2 repositories")
---
 lib/PublicInbox/WatchMaildir.pm | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 064cedf..8d8e817 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -19,7 +19,7 @@ package PublicInbox::WatchMaildir;
 
 sub new {
 	my ($class, $config) = @_;
-	my (%mdmap, @mdir, $spamc, $spamdir);
+	my (%mdmap, @mdir, $spamc);
 	my %uniq;
 
 	# "publicinboxwatch" is the documented namespace
@@ -32,7 +32,6 @@ sub new {
 				$dir =~ s!/+\z!!;
 				# skip "new", no MUA has seen it, yet.
 				my $cur = "$dir/cur";
-				$spamdir = $cur;
 				my $old = $mdmap{$cur};
 				if (ref($old)) {
 					foreach my $ibx (@$old) {
@@ -84,7 +83,6 @@ sub new {
 	$mdre = qr!\A($mdre)/!;
 	bless {
 		spamcheck => $spamcheck,
-		spamdir => $spamdir,
 		mdmap => \%mdmap,
 		mdir => \@mdir,
 		mdre => $mdre,
-- 
EW

[PATCH 3/3] watchmaildir: normalize Maildir pathnames consistently

[Eric Wong]

Remove redundant slashes while we're at it.
---
 lib/PublicInbox/WatchMaildir.pm | 13 +++++++++----
 t/watch_maildir.t               |  9 +++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 8d8e817..84080ba 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -28,8 +28,7 @@ sub new {
 	foreach my $pfx (qw(publicinboxwatch publicinboxlearn)) {
 		my $k = "$pfx.watchspam";
 		if (my $dir = $config->{$k}) {
-			if ($dir =~ s/\Amaildir://) {
-				$dir =~ s!/+\z!!;
+			if (is_maildir($dir)) {
 				# skip "new", no MUA has seen it, yet.
 				my $cur = "$dir/cur";
 				my $old = $mdmap{$cur};
@@ -60,8 +59,7 @@ sub new {
 		my $ibx = $_[0] = PublicInbox::InboxWritable->new($_[0]);
 
 		my $watch = $ibx->{watch} or return;
-		if ($watch =~ s/\Amaildir://) {
-			$watch =~ s!/+\z!!;
+		if (is_maildir($watch)) {
 			if (my $wm = $ibx->{watchheader}) {
 				my ($k, $v) = split(/:/, $wm, 2);
 				$ibx->{-watchheader} = [ $k, qr/\Q$v\E/ ];
@@ -278,4 +276,11 @@ sub _spamcheck_cb {
 	}
 }
 
+sub is_maildir {
+	$_[0] =~ s!\Amaildir:!! or return;
+	$_[0] =~ tr!/!/!s;
+	$_[0] =~ s!/\z!!;
+	$_[0];
+}
+
 1;
diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index 7178f29..b85ddc5 100644
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -124,4 +124,13 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
 	like($$mref, qr/something\n\z/s, 'message scrubbed on import');
 }
 
+sub is_maildir {
+	my ($dir) = @_;
+	PublicInbox::WatchMaildir::is_maildir($dir);
+}
+
+is(is_maildir('maildir:/hello//world'), '/hello/world', 'extra slash gone');
+is(is_maildir('maildir:/hello/world/'), '/hello/world', 'trailing slash gone');
+is(is_maildir('faildir:/hello/world/'), undef, 'non-maildir rejected');
+
 done_testing;
-- 
EW