[PATCH 0/4] non-cindex-related stuff

[PATCH 0/4] non-cindex-related stuff

[Eric Wong]

Well, I actually found the mail_diff bugs while looking into
micro-optimizing -cindex.

Eric Wong (4):
  lei q: fix --no-import-before completion + docs
  www: mail_diff: fix optional address obfuscation
  www: mail_diff: add final newline before diffing
  www: mail_diff: add missing </pre> tag

 Documentation/lei-q.pod     |  5 +++--
 lib/PublicInbox/LEI.pm      |  1 +
 lib/PublicInbox/MailDiff.pm | 11 ++++++++---
 t/lei-mail-diff.t           |  1 +
 t/lei-q-kw.t                | 19 ++++++++++++++++---
 t/psgi_v2.t                 |  1 +
 6 files changed, 30 insertions(+), 8 deletions(-)

[PATCH 1/4] lei q: fix --no-import-before completion + docs

[Eric Wong]

--no-import-before skips importing entire messages, not just
keywords, so it can cause permanent data loss if -o is pointed
to precious data.
---
 Documentation/lei-q.pod |  5 +++--
 lib/PublicInbox/LEI.pm  |  1 +
 t/lei-q-kw.t            | 19 ++++++++++++++++---
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 4862ce78..95f3f702 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -108,8 +108,9 @@ Augment output destination instead of clobbering it.
 
 =item --no-import-before
 
-Do not import keywords before writing to an existing output
-destination.
+Do not import messages before writing to an existing output destination.
+Be certain you do not need existing data in your output before using
+this, it permanently erases data unless C<--augment> is used.
 
 =item --threads
 
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 86b71fcd..a89bdc51 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -353,6 +353,7 @@ my %OPTDESC = (
 'no-torsocks' => 'alias for --torsocks=no',
 'save!' =>  "do not save a search for `lei up'",
 'import-remote!' => 'do not memoize remote messages into local store',
+'import-before!' => 'do not import before writing to output (DANGEROUS)',
 
 'type=s' => [ 'any|mid|git', 'disambiguate type' ],
 
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 06e1df6c..63e46037 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -9,6 +9,8 @@ use IO::Compress::Gzip qw(gzip);
 use PublicInbox::MboxReader;
 use PublicInbox::LeiToMail;
 use PublicInbox::Spawn qw(popen_rd);
+use File::Path qw(make_path);
+use PublicInbox::IO qw(write_file);
 my $exp = {
 	'<qp@example.com>' => eml_load('t/plack-qp.eml'),
 	'<testmessage@example.com>' => eml_load('t/utf8.eml'),
@@ -42,6 +44,19 @@ lei_ok(qw(q -o), "maildir:$o", qw(m:qp@example.com));
 @fn = glob("$o/cur/*:2,S");
 is(scalar(@fn), 1, "`seen' flag (but not `replied') set on Maildir file");
 
+{
+	$o = "$ENV{HOME}/dst-existing";
+	make_path(map { "$o/$_" } qw(new cur tmp));
+	my $bp = eml_load('t/data/binary.patch');
+	write_file '>', "$o/cur/binary-patch:2,S", $bp->as_string;
+	lei_ok qw(q --no-import-before m:qp@example.com -o), $o;
+	my @g = glob("$o/*/*");
+	is scalar(@g), 1, 'only newly imported message left';
+	is eml_load($g[0])->header_raw('Message-ID'), '<qp@example.com>';
+	lei qw(q m:binary-patch-test@example);
+	is $lei_out, "[null]\n", 'old message not imported';
+}
+
 SKIP: {
 	$o = "$ENV{HOME}/fifo";
 	mkfifo($o, 0600) or skip("mkfifo not supported: $!", 1);
@@ -80,9 +95,7 @@ my $write_file = sub {
 	if ($_[0] =~ /\.gz\z/) {
 		gzip(\($_[1]), $_[0]) or BAIL_OUT 'gzip';
 	} else {
-		open my $fh, '>', $_[0] or BAIL_OUT $!;
-		print $fh $_[1] or BAIL_OUT $!;
-		close $fh or BAIL_OUT;
+		write_file '>', $_[0], $_[1];
 	}
 };
 

[PATCH 2/4] www: mail_diff: fix optional address obfuscation

[Eric Wong]

We need to load the proper package and fully-qualify the sub
call since we shouldn't load Hval in lei.  Some users use this
feature even if its broken, oh well :<
---
 lib/PublicInbox/MailDiff.pm | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/MailDiff.pm b/lib/PublicInbox/MailDiff.pm
index b1c12d6d..98284b23 100644
--- a/lib/PublicInbox/MailDiff.pm
+++ b/lib/PublicInbox/MailDiff.pm
@@ -47,6 +47,8 @@ sub prep_a ($$) {
 	dump_eml($self, "$self->{tmp}/a", $eml);
 }
 
+# WWW-specific stuff below (TODO: split out for non-lei)
+
 sub next_smsg ($) {
 	my ($self) = @_;
 	my $ctx = $self->{ctx};
@@ -62,9 +64,12 @@ sub next_smsg ($) {
 
 sub emit_msg_diff {
 	my ($bref, $self) = @_; # bref is `git diff' output
+	require PublicInbox::Hval;
+
 	# will be escaped to `&#8226;' in HTML
 	$self->{ctx}->{ibx}->{obfuscate} and
-		obfuscate_addrs($self->{ctx}->{ibx}, $$bref, "\x{2022}");
+		PublicInbox::Hval::obfuscate_addrs($self->{ctx}->{ibx},
+						$$bref, "\x{2022}");
 	print { $self->{ctx}->{zfh} } '</pre><hr><pre>' if $self->{nr} > 1;
 	flush_diff($self->{ctx}, $bref);
 	next_smsg($self);

[PATCH 3/4] www: mail_diff: add final newline before diffing

[Eric Wong]

This gets rid of the "\ No newline at end of file"
since it's distracting noise.
---
 lib/PublicInbox/MailDiff.pm | 2 +-
 t/lei-mail-diff.t           | 1 +
 t/psgi_v2.t                 | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/MailDiff.pm b/lib/PublicInbox/MailDiff.pm
index 98284b23..89284e39 100644
--- a/lib/PublicInbox/MailDiff.pm
+++ b/lib/PublicInbox/MailDiff.pm
@@ -24,7 +24,7 @@ sub write_part { # Eml->each_part callback
 	$s //= $part->body;
 	$s =~ s/\r\n/\n/gs; # TODO: consider \r+\n to match View
 	$s =~ s/\s*\z//s;
-	write_file '>:utf8', "$self->{curdir}/$idx.$sfx", $s;
+	write_file '>:utf8', "$self->{curdir}/$idx.$sfx", $s, "\n";
 }
 
 # public
diff --git a/t/lei-mail-diff.t b/t/lei-mail-diff.t
index 9398596a..1a896e51 100644
--- a/t/lei-mail-diff.t
+++ b/t/lei-mail-diff.t
@@ -7,6 +7,7 @@ test_lei(sub {
 	ok(!lei('mail-diff', 't/data/0001.patch', 't/data/binary.patch'),
 		'different messages are different');
 	like($lei_out, qr/^\+/m, 'diff shown');
+	unlike $lei_out, qr/No newline at end of file/;
 	lei_ok('mail-diff', 't/data/0001.patch', 't/data/0001.patch');
 	is($lei_out, '', 'no output if identical');
 });
diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index 5ccaab19..54faae9b 100644
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -287,6 +287,7 @@ my $client1 = sub {
 		'cr_mismatch is only diff context');
 	like($raw, qr!>\-pipe !s, 'pipe diff del line');
 	like($raw, qr!>\+pipe !s, 'pipe diff ins line');
+	unlike $raw, qr/No newline at end of file/;
 };
 
 test_psgi(sub { $www->call(@_) }, $client1);

[PATCH 4/4] www: mail_diff: add missing </pre> tag

[Eric Wong]

Found by tidy(1) while dealing with other stuff.
---
 lib/PublicInbox/MailDiff.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/MailDiff.pm b/lib/PublicInbox/MailDiff.pm
index 89284e39..e4e262ef 100644
--- a/lib/PublicInbox/MailDiff.pm
+++ b/lib/PublicInbox/MailDiff.pm
@@ -56,7 +56,7 @@ sub next_smsg ($) {
 	$self->{smsg} = $over ? $over->next_by_mid(@{$self->{next_arg}})
 			: $ctx->gone('over');
 	if (!$self->{smsg}) {
-		$ctx->write($ctx->_html_end);
+		$ctx->write('</pre>', $ctx->_html_end);
 		return $ctx->close;
 	}
 	PublicInbox::DS::requeue($self) if $ctx->{env}->{'pi-httpd.async'};