[PATCH] lei q: make --save the default

[Eric Wong <e@80x24.org>]

Since "lei up" is more often useful than not and incurs neglible
overhead; enable --save by default and allow --no-save to work.

This also fixes a long-standing when overwriting --output
destinations with saved searches:  dedupe data from previous
searches are reset and no longer influences the new (changed)
search, so results no longer go missing if two sequential
invocations of "lei q --save" point to the same --output.
---
 Documentation/lei-q.pod           |  4 ++--
 lib/PublicInbox/LEI.pm            |  4 ++--
 lib/PublicInbox/LeiSavedSearch.pm | 10 ++++++++++
 lib/PublicInbox/LeiToMail.pm      | 31 +++++++++++++++++++++++++++----
 lib/PublicInbox/LeiUp.pm          |  1 -
 t/lei-q-kw.t                      |  3 ++-
 t/lei-q-save.t                    | 26 ++++++++++++++------------
 t/lei.t                           |  2 +-
 t/lei_to_mail.t                   |  2 +-
 9 files changed, 59 insertions(+), 24 deletions(-)

diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index fbe61920..69a6cdf2 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -26,9 +26,9 @@ TODO: mention curl options?
 
 Read search terms from stdin.
 
-=item --save
+=item --no-save
 
-Save a search for L<lei-up(1)>.
+Do not save the search for L<lei-up(1)>.
 
 =item --output=MFOLDER
 
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e5232f1b..79dc9bf9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -173,7 +173,7 @@ our %CMD = ( # sorted in order of importance/use:
 'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms',
 	'stdin|', # /|\z/ must be first for lone dash
 	@lxs_opt,
-	qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+	qw(save! output|mfolder|o=s format|f=s dedupe|d=s threads|t+
 	sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
 	import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
 	shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
@@ -337,7 +337,7 @@ my %OPTDESC = (
 'torsocks=s' => ['VAL|auto|no|yes',
 		'whether or not to wrap git and curl commands with torsocks'],
 'no-torsocks' => 'alias for --torsocks=no',
-'save' =>  "save a search for `lei up'",
+'save!' =>  "do not save a search for `lei up'",
 'import-remote!' => 'do not memoize remote messages into local store',
 
 'type=s' => [ 'any|mid|git', 'disambiguate type' ],
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 2a0e9321..fd51fe38 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -243,6 +243,16 @@ sub pause_dedupe {
 	$oidx->commit_lazy;
 }
 
+sub reset_dedupe {
+	my ($self) = @_;
+	prepare_dedupe($self);
+	my $lk = $self->lock_for_scope_fast;
+	for my $t (qw(xref3 over id2num)) {
+		$self->{oidx}->{dbh}->do("DELETE FROM $t");
+	}
+	pause_dedupe($self);
+}
+
 sub mm { undef }
 
 sub altid_map { {} }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d782d4c7..be6e178f 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -390,10 +390,16 @@ sub new {
 		-e $dst && !-d _ and die
 				"$dst exists and is not a directory\n";
 		$lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
+		$lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
 	} elsif (substr($fmt, 0, 4) eq 'mbox') {
 		require PublicInbox::MboxReader;
 		$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
 		$self->{base_type} = 'mbox';
+		if ($lei->{cmd} eq 'q' &&
+				(($lei->path_to_fd($dst) // -1) < 0) &&
+				(-f $dst || !-e _)) {
+			$lei->{opt}->{save} //= \1;
+		}
 	} elsif ($fmt =~ /\Aimaps?\z/) {
 		require PublicInbox::NetWriter;
 		require PublicInbox::URIimap;
@@ -408,6 +414,7 @@ sub new {
 		$dst = $lei->{ovv}->{dst} = $$uri; # canonicalized
 		$lei->{net} = $net;
 		$self->{base_type} = 'imap';
+		$lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
 	} elsif ($fmt eq 'text') {
 		require PublicInbox::LeiViewText;
 		$lei->{lvt} = PublicInbox::LeiViewText->new($lei);
@@ -454,9 +461,17 @@ sub _pre_augment_maildir {
 	open $self->{poke_dh}, '<', "${dst}cur" or die "open ${dst}cur: $!";
 }
 
+sub clobber_dst_prepare ($;$) {
+	my ($lei, $f) = @_;
+	my $wait = (defined($f) && $lei->{sto}) ?
+			$lei->{sto}->ipc_do('lms_forget_folders', $f) : undef;
+	my $dedupe = $lei->{dedupe} or return;
+	$dedupe->reset_dedupe if $dedupe->can('reset_dedupe');
+}
+
 sub _do_augment_maildir {
 	my ($self, $lei) = @_;
-	return if ($lei->{opt}->{save} // 0) < 0;
+	return if $lei->{cmd} eq 'up';
 	my $dst = $lei->{ovv}->{dst};
 	my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
 	my $mdr = PublicInbox::MdirReader->new;
@@ -468,9 +483,11 @@ sub _do_augment_maildir {
 			$dedupe->pause_dedupe;
 		}
 	} elsif ($lse) {
+		clobber_dst_prepare($lei, "maildir:$dst");
 		$mdr->{shard_info} = $self->{shard_info};
 		$mdr->maildir_each_eml($dst, \&_md_update, $lei, $lse, 1);
 	} else {# clobber existing Maildir
+		clobber_dst_prepare($lei, "maildir:$dst");
 		$mdr->maildir_each_file($dst, \&_unlink);
 	}
 }
@@ -487,7 +504,7 @@ sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
 
 sub _do_augment_imap {
 	my ($self, $lei) = @_;
-	return if ($lei->{opt}->{save} // 0) < 0;
+	return if $lei->{cmd} eq 'up';
 	my $net = $lei->{net};
 	my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
 	if ($lei->{opt}->{augment}) {
@@ -499,11 +516,13 @@ sub _do_augment_imap {
 		}
 	} elsif ($lse) {
 		my $delete_mic;
+		clobber_dst_prepare($lei, "$self->{uri}");
 		$net->imap_each($self->{uri}, \&_imap_augment_or_delete,
 					$lei, $lse, \$delete_mic);
 		$delete_mic->expunge if $delete_mic;
 	} elsif (!$self->{-wq_worker_nr}) { # undef or 0
 		# clobber existing IMAP folder
+		clobber_dst_prepare($lei, "$self->{uri}");
 		$net->imap_delete_all($self->{uri});
 	}
 }
@@ -563,6 +582,8 @@ sub _pre_augment_mbox {
 			if $imp_before && !ref($imp_before);
 		die "--augment specified but $dst is not seekable\n" if
 			$lei->{opt}->{augment};
+		die "cannot --save with unseekable $dst\n" if
+			$lei->{dedupe} && $lei->{dedupe}->can('reset_dedupe');
 	}
 	if ($self->{zsfx} = PublicInbox::MboxReader::zsfx($dst)) {
 		pipe(my ($r, $w)) or die "pipe: $!";
@@ -581,10 +602,10 @@ sub _do_augment_mbox {
 	my ($self, $lei) = @_;
 	return unless $self->{seekable};
 	my $opt = $lei->{opt};
-	return if ($opt->{save} // 0) < 0;
+	return if $lei->{cmd} eq 'up';
 	my $out = $lei->{1};
 	my ($fmt, $dst) = @{$lei->{ovv}}{qw(fmt dst)};
-	return unless -s $out;
+	return clobber_dst_prepare($lei) unless -s $out;
 	unless ($opt->{augment} || $opt->{'import-before'}) {
 		truncate($out, 0) or die "truncate($dst): $!";
 		return;
@@ -599,6 +620,8 @@ sub _do_augment_mbox {
 	if ($opt->{augment}) {
 		$dedupe = $lei->{dedupe};
 		$dedupe->prepare_dedupe if $dedupe;
+	} else {
+		clobber_dst_prepare($lei);
 	}
 	if ($opt->{'import-before'}) { # the default
 		my $lse = $lei->{lse};
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index c5a01527..ba11761a 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -64,7 +64,6 @@ sub lei_up {
 	my ($lei, @outs) = @_;
 	$lei->{lse} = $lei->_lei_store(1)->search;
 	my $opt = $lei->{opt};
-	$opt->{save} = -1;
 	my @local;
 	if (defined $opt->{all}) {
 		return $lei->fail("--all and @outs incompatible") if @outs;
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 2e6be1f0..4edee72a 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -28,7 +28,8 @@ ok(!glob("$o/cur/*"), 'last result cleared after augment-import');
 
 lei_ok(qw(q -o), "maildir:$o", qw(m:qp@example.com));
 @fn = glob("$o/cur/*:2,S");
-is(scalar(@fn), 1, "`seen' flag set on Maildir file");
+is(scalar(@fn), 1, "`seen' flag set on Maildir file") or
+	diag "$o contents: ", explain([glob("$o/*/*")]);
 
 # ensure --no-import-before works
 my $n = $fn[0];
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index eada2dd4..7aa9b84e 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -25,7 +25,7 @@ test_lei(sub {
 	my $home = $ENV{HOME};
 	my $in = $doc1->as_string;
 	lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
-	lei_ok qw(q -q --save z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
+	lei_ok qw(q -q z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
 	my %before = map { $_ => 1 } glob("$home/md/cur/*");
 	my $f = (keys %before)[0] or xbail({before => \%before});
 	is_deeply(eml_load($f), $doc1, 'doc1 matches');
@@ -52,8 +52,8 @@ test_lei(sub {
 	is_deeply(eml_load($f), $doc2, 'doc2 matches');
 
 	# check stdin
-	lei_ok [qw(q --save - -o), "mboxcl2:mbcl2" ],
-		undef, { -C => $home, %$lei_opt, 0 => \'d:last.week..'};
+	lei_ok [qw(q - -o), "mboxcl2:mbcl2" ], undef,
+		{ -C => $home, %$lei_opt, 0 => \'d:last.week..'};
 	@s = glob("$home/.local/share/lei/saved-searches/mbcl2-*");
 	$cfg = PublicInbox::Config->new("$s[0]/lei.saved-search");
 	is_deeply $cfg->{'lei.q'}, 'd:last.week..',
@@ -68,7 +68,11 @@ test_lei(sub {
 	lei_ok([qw(up mbcl2)], undef, { -C => $home, %$lei_opt });
 	ok(-s "$home/mbcl2" > $size, 'size increased after up');
 
-	ok(!lei(qw(up -q), $home), 'up fails w/o --save');
+	ok(!lei(qw(up -q), $home), 'up fails on unknown dir');
+	like($lei_err, qr/--save was not used/, 'error noted --save');
+
+	lei_ok(qw(q --no-save d:last.week.. -q -o), "$home/no-save");
+	ok(!lei(qw(up -q), "$home/no-save"), 'up fails on --no-save');
 	like($lei_err, qr/--save was not used/, 'error noted --save');
 
 	lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
@@ -94,7 +98,7 @@ test_lei(sub {
 	open my $mb, '>', "$home/mbrd" or xbail "open $!";
 	print $mb $pre_existing;
 	close $mb or xbail "close: $!";
-	lei_ok(qw(q --save -o mboxrd:mbrd m:qp@example.com -C), $home);
+	lei_ok(qw(q -o mboxrd:mbrd m:qp@example.com -C), $home);
 	open $mb, '<', "$home/mbrd" or xbail "open $!";
 	is_deeply([grep(/pre-existing/, <$mb>)], [],
 		'pre-existing messsage gone w/o augment');
@@ -107,7 +111,7 @@ test_lei(sub {
 	open $mb, '>', "$home/mbrd-aug" or xbail "open $!";
 	print $mb $pre_existing;
 	close $mb or xbail "close: $!";
-	lei_ok(qw(q -a --save -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
+	lei_ok(qw(q -a -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
 	open $mb, '<', "$home/mbrd-aug" or xbail "open $!";
 	$mb = do { local $/; <$mb> };
 	like($mb, qr/pre-existing/, 'pre-existing message preserved w/ -a');
@@ -133,7 +137,7 @@ test_lei(sub {
 	my $o = "$home/dd-mid";
 	$in = $doc2->as_string . "\n-------\nappended list sig\n";
 	lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
-	lei_ok(qw(q --dedupe=mid --save m:testmessage@example.com -o), $o);
+	lei_ok(qw(q --dedupe=mid m:testmessage@example.com -o), $o);
 	my @m = glob("$o/cur/*");
 	is(scalar(@m), 1, '--dedupe=mid w/ --save');
 	$in = $doc2->as_string . "\n-------\nanother list sig\n";
@@ -143,8 +147,7 @@ test_lei(sub {
 
 	for my $dd (qw(content)) {
 		$o = "$home/dd-$dd";
-		lei_ok(qw(q --save m:testmessage@example.com -o), $o,
-				"--dedupe=$dd");
+		lei_ok(qw(q m:testmessage@example.com -o), $o, "--dedupe=$dd");
 		@m = glob("$o/cur/*");
 		is(scalar(@m), 3, 'all 3 matches with dedupe='.$dd);
 	}
@@ -153,7 +156,7 @@ test_lei(sub {
 	$o = "$home/dd-oid";
 	my $ibx = create_inbox 'ibx', indexlevel => 'medium',
 			tmpdir => "$home/v1", sub {};
-	lei_ok(qw(q --save --dedupe=oid m:qp@example.com -o), $o,
+	lei_ok(qw(q --dedupe=oid m:qp@example.com -o), $o,
 		'-I', $ibx->{inboxdir});
 	@m = glob("$o/cur/*");
 	is(scalar(@m), 1, 'got first result');
@@ -203,8 +206,7 @@ test_lei(sub {
 	lei_ok([qw(edit-search), $v2s], { VISUAL => 'cat', EDITOR => 'cat' });
 	like($lei_out, qr/^\[lei/sm, 'edit-search can cat');
 
-	lei_ok('-C', "$home/v2s",
-		qw(q -q --save -o ../s m:testmessage@example.com));
+	lei_ok('-C', "$home/v2s", qw(q -q -o ../s m:testmessage@example.com));
 	lei_ok qw(ls-search);
 	unlike $lei_out, qr{/\.\./s$}sm, 'relative path not in ls-search';
 	like $lei_out, qr{^\Q$home\E/s$}sm,
diff --git a/t/lei.t b/t/lei.t
index 8211c01d..dfbcb1f3 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -114,7 +114,7 @@ my $test_completion = sub {
 	%out = map { $_ => 1 } split(/\s+/s, $lei_out);
 	for my $sw (qw(-f --format -o --output --mfolder --augment -a
 			--mua --no-local --local --verbose -v
-			--save --no-remote --remote --torsocks
+			--save --no-save --no-remote --remote --torsocks
 			--reverse -r )) {
 		ok($out{$sw}, "$sw offered as `lei q' completion");
 	}
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 35904706..e8958c64 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -75,7 +75,7 @@ for my $mbox (@MBOX) {
 my ($tmpdir, $for_destroy) = tmpdir();
 local $ENV{TMPDIR} = $tmpdir;
 open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!;
-my $lei = bless { 2 => $err }, 'PublicInbox::LEI';
+my $lei = bless { 2 => $err, cmd => 'test' }, 'PublicInbox::LEI';
 my $commit = sub {
 	$_[0] = undef; # wcb
 	delete $lei->{1};