[PATCH] truncate Message-IDs and References consistently

[PATCH] truncate Message-IDs and References consistently

[Eric Wong (Contractor, The Linux Foundation)]

We need to stop ghost messages from generating longer
Message-IDs than Xapian can handle with terms.
---
 lib/PublicInbox/MID.pm | 18 ++++++++----------
 t/v2writable.t         | 22 ++++++++++++++++++++++
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm
index 117d3c4..c82e840 100644
--- a/lib/PublicInbox/MID.pm
+++ b/lib/PublicInbox/MID.pm
@@ -65,12 +65,6 @@ sub mids ($) {
 			push(@mids, $v);
 		}
 	}
-	foreach my $i (0..$#mids) {
-		next if length($mids[$i]) <= MAX_MID_SIZE;
-		warn "Message-ID: <$mids[$i]> too long, truncating\n";
-		$mids[$i] = substr($mids[$i], 0, MAX_MID_SIZE);
-	}
-
 	uniq_mids(\@mids);
 }
 
@@ -92,10 +86,14 @@ sub uniq_mids ($) {
 	my ($mids) = @_;
 	my @ret;
 	my %seen;
-	foreach (@$mids) {
-		next if $seen{$_};
-		push @ret, $_;
-		$seen{$_} = 1;
+	foreach my $mid (@$mids) {
+		if (length($mid) > MAX_MID_SIZE) {
+			warn "Message-ID: <$mid> too long, truncating\n";
+			$mid = substr($mid, 0, MAX_MID_SIZE);
+		}
+		next if $seen{$mid};
+		push @ret, $mid;
+		$seen{$mid} = 1;
 	}
 	\@ret;
 }
diff --git a/t/v2writable.t b/t/v2writable.t
index 4a7cfb9..7e29ef7 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -235,4 +235,26 @@ EOF
 	$im->done;
 }
 
+{
+	my @warn;
+	my $x = 'x'x250;
+	my $y = 'y'x250;
+	local $SIG{__WARN__} = sub { push @warn, @_ };
+	$mime->header_set('Subject', 'long mid');
+	$mime->header_set('Message-ID', "<$x>");
+	ok($im->add($mime), 'add excessively long Message-ID');
+
+	$mime->header_set('Message-ID', "<$y>");
+	$mime->header_set('References', "<$x>");
+	ok($im->add($mime), 'add excessively long References');
+	$im->barrier;
+
+	my $msgs = $ibx->search->reopen->get_thread('x'x244)->{msgs};
+	is(2, scalar(@$msgs), 'got both messages');
+	is($msgs->[0]->{mid}, 'x'x244, 'stored truncated mid');
+	is($msgs->[1]->{references}, '<'.('x'x244).'>', 'stored truncated ref');
+	is($msgs->[1]->{mid}, 'y'x244, 'stored truncated mid(2)');
+	$im->done;
+}
+
 done_testing();
-- 
EW