unofficial mirror of meta@public-inbox.org
 help / color / mirror / Atom feed
* [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives
@ 2021-09-03  8:54 Eric Wong
  2021-09-03  8:54 ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
                   ` (7 more replies)
  0 siblings, 8 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

Back to using syslog more for errors, since it's still less
instrusive.  IMAP R/W is fixed (ugh, lack of automatic tests
since we only have a read-only IMAP server for testing).
Some error/warning fixes, and false-positives on L: and kw:
searches on local (but not remote) externals are avoided.

"lei inspect --stdin" was somewhat useful for figuring out why
L: was false-positiving an external result on me.

Eric Wong (8):
  lei: dump errors to syslog, and not to CLI
  lei/store: quiet down link(2) warnings
  lei: ->child_error less error-prone
  lei: use lei->lms in place of lse->lms in a few places
  lei up --all: avoid double-close on shared STDOUT
  lei inspect: support reading eml from --stdin
  lei_xsearch: avoid false-positives on externals w/ L: and kw:
  lei: fix read/write IMAP access

 MANIFEST                            |  1 +
 lib/PublicInbox/LEI.pm              | 27 ++++++++++--------
 lib/PublicInbox/LeiBlob.pm          |  2 +-
 lib/PublicInbox/LeiIndex.pm         |  2 +-
 lib/PublicInbox/LeiInput.pm         |  4 +--
 lib/PublicInbox/LeiInspect.pm       | 43 ++++++++++++++++++++++++-----
 lib/PublicInbox/LeiLcat.pm          |  2 +-
 lib/PublicInbox/LeiPruneMailSync.pm | 26 ++++++++---------
 lib/PublicInbox/LeiRediff.pm        |  2 +-
 lib/PublicInbox/LeiStore.pm         | 10 ++-----
 lib/PublicInbox/LeiStoreErr.pm      |  4 +--
 lib/PublicInbox/LeiToMail.pm        |  2 +-
 lib/PublicInbox/LeiUp.pm            |  6 +++-
 lib/PublicInbox/LeiXSearch.pm       |  1 +
 lib/PublicInbox/NetReader.pm        |  5 +++-
 lib/PublicInbox/NetWriter.pm        |  2 ++
 lib/PublicInbox/Watch.pm            |  2 ++
 t/lei-daemon.t                      |  5 ----
 t/lei-up.t                          | 39 ++++++++++++++++++++++++++
 19 files changed, 130 insertions(+), 55 deletions(-)
 create mode 100644 t/lei-up.t


^ permalink raw reply	[flat|nested] 9+ messages in thread

* [PATCH 1/8] lei: dump errors to syslog, and not to CLI
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  2021-09-03  8:54 ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
                   ` (6 subsequent siblings)
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

Dumping errors from the previous run can often get lost, so just
spew to syslog since it's a standard place to put errors that
don't make it to a client.  Note: we don't rely on $SIG{__WARN__}
since some of the Net:: stuff will write directly to STDERR
(as will external processes).
---
 lib/PublicInbox/LEI.pm         | 16 +++++++++-------
 lib/PublicInbox/LeiStoreErr.pm |  4 ++--
 t/lei-daemon.t                 |  5 -----
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 41e811ca..9e9aa165 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -27,6 +27,7 @@ use PublicInbox::Eml;
 use Time::HiRes qw(stat); # ctime comparisons for config cache
 use File::Path qw(mkpath);
 use File::Spec;
+use Sys::Syslog qw(openlog syslog closelog);
 our $quit = \&CORE::exit;
 our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
 	$recv_cmd, $send_cmd);
@@ -464,7 +465,6 @@ sub x_it ($$) {
 	my ($self, $code) = @_;
 	# make sure client sees stdout before exit
 	$self->{1}->autoflush(1) if $self->{1};
-	dump_and_clear_log();
 	stop_pager($self);
 	if ($self->{pkt_op_p}) { # to top lei-daemon
 		$self->{pkt_op_p}->pkt_do('x_it', $code);
@@ -765,7 +765,6 @@ sub dispatch {
 	my ($self, $cmd, @argv) = @_;
 	local $current_lei = $self; # for __WARN__
 	$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
-	dump_and_clear_log("from previous run\n");
 	return _help($self, 'no command given') unless defined($cmd);
 	# do not support Getopt bundling for this
 	while ($cmd eq '-C' || $cmd eq '-c') {
@@ -1147,10 +1146,12 @@ sub oldset { $oldset }
 
 sub dump_and_clear_log {
 	if (defined($errors_log) && -s STDIN && seek(STDIN, 0, SEEK_SET)) {
-		my @pfx = @_;
-		unshift(@pfx, "$errors_log ") if @pfx;
-		warn @pfx, do { local $/; <STDIN> };
-		truncate(STDIN, 0) or warn "ftruncate ($errors_log): $!";
+		openlog('lei-daemon', 'pid,nowait,nofatal,ndelay', 'user');
+		chomp(my @lines = <STDIN>);
+		truncate(STDIN, 0) or
+			syslog('warning', "ftruncate (%s): %m", $errors_log);
+		for my $l (@lines) { syslog('warning', '%s', $l) }
+		closelog(); # don't share across fork
 	}
 }
 
@@ -1243,7 +1244,7 @@ sub lazy_start {
 	(-p STDOUT) or die "E: stdout must be a pipe\n";
 	open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
 	STDIN->autoflush(1);
-	dump_and_clear_log("from previous daemon process:\n");
+	dump_and_clear_log();
 	POSIX::setsid() > 0 or die "setsid: $!";
 	my $pid = fork // die "fork: $!";
 	return if $pid;
@@ -1345,6 +1346,7 @@ sub DESTROY {
 	}
 	$self->{1}->autoflush(1) if $self->{1};
 	stop_pager($self);
+	dump_and_clear_log();
 	# preserve $? for ->fail or ->x_it code
 }
 
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index 5f9ba24d..cc085fdc 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -2,7 +2,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # forwards stderr from lei/store process to any lei clients using
-# the same store
+# the same store, falls back to syslog if no matching clients exist.
 package PublicInbox::LeiStoreErr;
 use strict;
 use v5.10.1;
@@ -31,7 +31,7 @@ sub event_step {
 		print $err $$rbuf and $printed = 1;
 	}
 	if (!$printed) {
-		openlog('lei-store', 'pid,nowait,nofatal,ndelay', 'user');
+		openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
 		for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
 		closelog(); # don't share across fork
 	}
diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index a7c4b799..b0c94a87 100644
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -21,14 +21,9 @@ test_lei({ daemon_only => 1 }, sub {
 	ok(kill(0, $pid), 'pid is valid');
 	ok(-S $sock, 'sock created');
 	is(-s $err_log, 0, 'nothing in errors.log');
-	open my $efh, '>>', $err_log or BAIL_OUT $!;
-	print $efh "phail\n" or BAIL_OUT $!;
-	close $efh or BAIL_OUT $!;
-
 	lei_ok('daemon-pid');
 	chomp(my $pid_again = $lei_out);
 	is($pid, $pid_again, 'daemon-pid idempotent');
-	like($lei_err, qr/phail/, 'got mock "phail" error previous run');
 
 	SKIP: {
 		skip 'only testing open files on Linux', 1 if $^O ne 'linux';

^ permalink raw reply related	[flat|nested] 9+ messages in thread

* [PATCH 2/8] lei/store: quiet down link(2) warnings
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
  2021-09-03  8:54 ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  2021-09-03  8:54 ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
                   ` (5 subsequent siblings)
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

ENOENT can be too common due to timing and concurrent access
from MUAs and "lei export-kw", and other mail synchronization
tools (e.g. mbsync and offlineimap).
---
 lib/PublicInbox/LeiStore.pm | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index a91b30f7..f81a8dae 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -215,14 +215,10 @@ sub export1_kw_md ($$$$$) {
 			return;
 		} elsif ($! == EEXIST) { # lost race with "lei export-kw"?
 			return;
-		} elsif ($! == ENOENT) {
-			syslog('warning', "link($src -> $dst): $!")
-		} # else loop @try
+		} elsif ($! != ENOENT) {
+			syslog('warning', "link($src -> $dst): $!");
+		}
 	}
-	my $e = $!;
-	my $src = "$mdir/{".join(',', @try)."}/$orig";
-	my $oidhex = unpack('H*', $oidbin);
-	syslog('warning', "link($src -> $dst) ($oidhex): $e");
 	for (@try) { return if -e "$mdir/$_/$orig" };
 	lms_clear_src($self, "maildir:$mdir", \$orig);
 }

^ permalink raw reply related	[flat|nested] 9+ messages in thread

* [PATCH 3/8] lei: ->child_error less error-prone
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
  2021-09-03  8:54 ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
  2021-09-03  8:54 ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  2021-09-03  8:54 ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
                   ` (4 subsequent siblings)
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

I was calling "child_error(1, ...)" in a few places where I meant
to be calling "child_error(1 << 8, ...)" and inadvertantly
triggering SIGHUP in script/lei.  Since giving a zero exit code
to child_error makes no sense, just allow falsy values to
default to 1 << 8.
---
 lib/PublicInbox/LEI.pm       | 7 ++++---
 lib/PublicInbox/LeiBlob.pm   | 2 +-
 lib/PublicInbox/LeiIndex.pm  | 2 +-
 lib/PublicInbox/LeiInput.pm  | 4 ++--
 lib/PublicInbox/LeiLcat.pm   | 2 +-
 lib/PublicInbox/LeiRediff.pm | 2 +-
 lib/PublicInbox/LeiUp.pm     | 2 +-
 7 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9e9aa165..8b6c1d36 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -532,6 +532,7 @@ sub puts ($;@) { out(shift, map { "$_\n" } @_) }
 
 sub child_error { # passes non-fatal curl exit codes to user
 	my ($self, $child_error, $msg) = @_; # child_error is $?
+	$child_error ||= 1 << 8;
 	$self->err($msg) if $msg;
 	if ($self->{pkt_op_p}) { # to top lei-daemon
 		$self->{pkt_op_p}->pkt_do('child_error', $child_error);
@@ -1341,7 +1342,7 @@ sub DESTROY {
 	if (my $counters = delete $self->{counters}) {
 		for my $k (sort keys %$counters) {
 			my $nr = $counters->{$k};
-			$self->child_error(1 << 8, "$nr $k messages");
+			$self->child_error(0, "$nr $k messages");
 		}
 	}
 	$self->{1}->autoflush(1) if $self->{1};
@@ -1417,7 +1418,7 @@ sub refresh_watches {
 				add_maildir_watch($d, $cfg_f);
 			}
 		} else { # TODO: imap/nntp/jmap
-			$lei->child_error(1, "E: watch $url not supported, yet")
+			$lei->child_error(0, "E: watch $url not supported, yet")
 		}
 	}
 
@@ -1452,7 +1453,7 @@ sub refresh_watches {
 				my $d = canonpath_harder($1);
 				cancel_maildir_watch($d, $cfg_f);
 			} else { # TODO: imap/nntp/jmap
-				$lei->child_error(1, "E: watch $url TODO");
+				$lei->child_error(0, "E: watch $url TODO");
 			}
 		}
 	}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 21003894..b94f67a0 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -32,7 +32,7 @@ sub solver_user_cb { # called by solver when done
 	my $lei = $self->{lei};
 	my $log_buf = delete $lei->{'log_buf'};
 	$$log_buf =~ s/^/# /sgm;
-	ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+	ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
 	$lei->qerr($$log_buf);
 	my ($git, $oid, $type, $size, $di) = @$res;
 	my $gd = $git->{git_dir};
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 5b545998..ef3e4d0b 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -21,7 +21,7 @@ sub input_eml_cb { # used by input_maildir_cb and input_net_cb
 
 sub input_fh { # overrides PublicInbox::LeiInput::input_fh
 	my ($self, $ifmt, $fh, $input, @args) = @_;
-	$self->{lei}->child_error(1<<8, <<EOM);
+	$self->{lei}->child_error(0, <<EOM);
 $input ($ifmt) not yet supported, try `lei import'
 EOM
 }
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 1b28f36f..cb71e97c 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -63,7 +63,7 @@ sub input_fh {
 	my ($self, $ifmt, $fh, $name, @args) = @_;
 	if ($ifmt eq 'eml') {
 		my $buf = do { local $/; <$fh> } //
-			return $self->{lei}->child_error(1 << 8, <<"");
+			return $self->{lei}->child_error(0, <<"");
 error reading $name: $!
 
 		# mutt pipes single RFC822 messages with a "From " line,
@@ -104,7 +104,7 @@ sub handle_http_input ($$@) {
 	my $err = $@;
 	waitpid($pid, 0);
 	$? || $err and
-		$lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+		$lei->child_error($?, "@$cmd failed".$err ? " $err" : '');
 }
 
 sub input_path_url {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 9d95e899..1e54c3bf 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -18,7 +18,7 @@ sub lcat_folder ($$$) {
 	my $err = $lms->arg2folder($lei, $folders);
 	$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
 	if ($err && $err->{fail}) {
-		$lei->child_error(1 << 8, "# unknown folder: $folder");
+		$lei->child_error(0, "# unknown folder: $folder");
 	} else {
 		for my $f (@$folders) {
 			my $fid = $lms->fid_for($f);
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 0ba5897c..60286b06 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -23,7 +23,7 @@ sub rediff_user_cb { # called by solver when done
 	my $lei = $self->{lei};
 	my $log_buf = delete $lei->{log_buf};
 	$$log_buf =~ s/^/# /sgm;
-	ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+	ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
 	$lei->qerr($$log_buf);
 	my ($git, $oid, $type, $size, $di) = @$res;
 	my $oid_want = delete $self->{cur_oid_want};
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 85efd9f5..e1da64aa 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -59,7 +59,7 @@ sub up1_redispatch {
 		up1($l, $out);
 		$l->qerr("# $out done");
 	};
-	$l->child_error(1 << 8, $@) if $@;
+	$l->child_error(0, $@) if $@;
 }
 
 sub lei_up {

^ permalink raw reply related	[flat|nested] 9+ messages in thread

* [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
                   ` (2 preceding siblings ...)
  2021-09-03  8:54 ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  2021-09-03  8:54 ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
                   ` (3 subsequent siblings)
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

We can golf out some code and refcounts this way.
---
 lib/PublicInbox/LeiInspect.pm       |  3 +--
 lib/PublicInbox/LeiPruneMailSync.pm | 26 +++++++++++++-------------
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2d2ff1a0..2ade17af 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -206,8 +206,7 @@ sub lei_inspect {
 
 sub _complete_inspect {
 	my ($lei, @argv) = @_;
-	my $sto = $lei->_lei_store or return;
-	my $lms = $sto->search->lms or return;
+	my $lms = $lei->lms or return;
 	my $match_cb = $lei->complete_url_prepare(\@argv);
 	map { $match_cb->($_) } $lms->folders;
 }
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
index 79f3325d..98239a13 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -40,7 +40,7 @@ sub prune_imap { # lms->each_src callback
 
 sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
 	my ($self, $input, @args) = @_;
-	my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
+	my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
 	if ($input =~ /\Amaildir:(.+)/i) {
 		my $mdir = $1;
 		$lms->each_src($input, \&prune_mdir, $self, $mdir);
@@ -59,24 +59,24 @@ sub lei_prune_mail_sync {
 	my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
 lei/store uninitialized, see lei-import(1)
 EOM
-	my $lse = $sto->search;
-	my $lms = $lse->lms or return $lei->fail(<<EOM);
-lei mail_sync uninitialized, see lei-import(1)
-EOM
-	if (defined(my $all = $lei->{opt}->{all})) {
-		$lms->group2folders($lei, $all, \@folders) or return;
+	if (my $lms = $lei->lms) {
+		if (defined(my $all = $lei->{opt}->{all})) {
+			$lms->group2folders($lei, $all, \@folders) or return;
+		} else {
+			my $err = $lms->arg2folder($lei, \@folders);
+			$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+			return $lei->fail($err->{fail}) if $err->{fail};
+		}
 	} else {
-		my $err = $lms->arg2folder($lei, \@folders);
-		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
-		return $lei->fail($err->{fail}) if $err->{fail};
+		return $lei->fail(<<EOM);
+lei mail_sync.sqlite3 uninitialized, see lei-import(1)
+EOM
 	}
-	delete $lms->{dbh};
 	$sto->write_prepare($lei);
-	my $self = bless { lse => $lse }, __PACKAGE__;
+	my $self = bless {}, __PACKAGE__;
 	$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
 	$self->prepare_inputs($lei, \@folders) or return;
 	my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
-	undef $lms; # for fork
 	my $ops = {};
 	$sto->write_prepare($lei);
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};

^ permalink raw reply related	[flat|nested] 9+ messages in thread

* [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
                   ` (3 preceding siblings ...)
  2021-09-03  8:54 ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  2021-09-03  8:54 ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
                   ` (2 subsequent siblings)
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

This is merely to avoid perl setting errors internally which
were not user visible.  The double-close wasn't a problem in
practice since we open a new file hanlde for the mbox or
mbox.gz anyways, so the new t/lei-up.t test case shows no
regressions nor fixes.
---
 MANIFEST                 |  1 +
 lib/PublicInbox/LeiUp.pm |  4 ++++
 t/lei-up.t               | 39 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+)
 create mode 100644 t/lei-up.t

diff --git a/MANIFEST b/MANIFEST
index be6ec927..fad29622 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -443,6 +443,7 @@ t/lei-q-save.t
 t/lei-q-thread.t
 t/lei-sigpipe.t
 t/lei-tag.t
+t/lei-up.t
 t/lei-watch.t
 t/lei.t
 t/lei_dedupe.t
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index e1da64aa..a39d6047 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -54,6 +54,10 @@ sub up1_redispatch {
 	$l->{opt} = { %{$l->{opt}} };
 	delete $l->{sock};
 	$l->{''} = $op_p; # daemon only
+
+	# make close($l->{1}) happy in lei->dclose
+	open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+	$l->{1} = $fh;
 	eval {
 		$l->qerr("# updating $out");
 		up1($l, $out);
diff --git a/t/lei-up.t b/t/lei-up.t
new file mode 100644
index 00000000..c6f31c74
--- /dev/null
+++ b/t/lei-up.t
@@ -0,0 +1,39 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
+test_lei(sub {
+	my $s = eml_load('t/plack-qp.eml')->as_string;
+	lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+	lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a.mbox.gz";
+	lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b.mbox.gz";
+	lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a";
+	lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b";
+	lei_ok qw(ls-search);
+	$s = eml_load('t/utf8.eml')->as_string;
+	lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+	lei_ok qw(up --all=local);
+	open my $fh, "$ENV{HOME}/a.mbox.gz" or xbail "open: $!";
+	my $gz = do { local $/; <$fh> };
+	my $uc;
+	gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+	open $fh, "$ENV{HOME}/a" or xbail "open: $!";
+
+	my $exp = do { local $/; <$fh> };
+	is($uc, $exp, 'compressed and uncompressed match (a.gz)');
+	like($exp, qr/testmessage\@example.com/, '2nd message added');
+	open $fh, "$ENV{HOME}/b.mbox.gz" or xbail "open: $!";
+
+	$gz = do { local $/; <$fh> };
+	undef $uc;
+	gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+	is($uc, $exp, 'compressed and uncompressed match (b.gz)');
+
+	open $fh, "$ENV{HOME}/b" or xbail "open: $!";
+	$uc = do { local $/; <$fh> };
+	is($uc, $exp, 'uncompressed both match');
+});
+
+done_testing;

^ permalink raw reply related	[flat|nested] 9+ messages in thread

* [PATCH 6/8] lei inspect: support reading eml from --stdin
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
                   ` (4 preceding siblings ...)
  2021-09-03  8:54 ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  2021-09-03  8:54 ` [PATCH 7/8] lei_xsearch: avoid false-positives on externals w/ L: and kw: Eric Wong
  2021-09-03  8:54 ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

This can be useful inside mutt since I was diagnosing why
a label ("L:$FOO") search was giving me a false-positive
search result...
---
 lib/PublicInbox/LEI.pm        |  4 ++--
 lib/PublicInbox/LeiInspect.pm | 40 ++++++++++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8b6c1d36..098a45ba 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -279,8 +279,8 @@ our %CMD = ( # sorted in order of importance/use:
 		'git-config(1) wrapper for '._config_path($_[0]);
 	}, qw(config-file|system|global|file|f=s), # for conflict detection
 	 qw(c=s@ C=s@), pass_through('git config') ],
-'inspect' => [ 'ITEMS...', 'inspect lei/store and/or local external',
-	qw(pretty ascii dir=s), @c_opt ],
+'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
+	qw(stdin| pretty ascii dir=s), @c_opt ],
 
 'init' => [ '[DIRNAME]', sub {
 	"initialize storage, default: ".store_path($_[0]);
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2ade17af..25bd47e7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -9,6 +9,7 @@ package PublicInbox::LeiInspect;
 use strict;
 use v5.10.1;
 use PublicInbox::Config;
+use PublicInbox::MID qw(mids);
 
 sub inspect_blob ($$) {
 	my ($lei, $oidhex) = @_;
@@ -184,6 +185,32 @@ sub inspect1 ($$$) {
 	1;
 }
 
+sub _inspect_argv ($$) {
+	my ($lei, $argv) = @_;
+	my $multi = scalar(@$argv) > 1;
+	$lei->out('[') if $multi;
+	while (defined(my $x = shift @$argv)) {
+		inspect1($lei, $x, scalar(@$argv)) or return;
+	}
+	$lei->out(']') if $multi;
+}
+
+sub ins_add { # InputPipe->consume callback
+	my ($lei) = @_; # $_[1] = $rbuf
+	if (defined $_[1]) {
+		$_[1] eq '' and return eval {
+			my $str = delete $lei->{istr};
+			$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+			my $eml = PublicInbox::Eml->new(\$str);
+			_inspect_argv($lei, [ 'blob:'.$lei->git_blob_id($eml),
+				map { "mid:$_" } @{mids($eml)} ]);
+		};
+		$lei->{istr} .= $_[1];
+	} else {
+		$lei->fail("error reading stdin: $!");
+	}
+}
+
 sub lei_inspect {
 	my ($lei, @argv) = @_;
 	$lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
@@ -196,12 +223,15 @@ sub lei_inspect {
 	}
 	$lei->start_pager if -t $lei->{1};
 	$lei->{1}->autoflush(0);
-	my $multi = scalar(@argv) > 1;
-	$lei->out('[') if $multi;
-	while (defined(my $x = shift @argv)) {
-		inspect1($lei, $x, scalar(@argv)) or return;
+	if ($lei->{opt}->{stdin}) {
+		return $lei->fail(<<'') if @argv;
+no args allowed on command-line with --stdin
+
+		require PublicInbox::InputPipe;
+		PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
+		return;
 	}
-	$lei->out(']') if $multi;
+	_inspect_argv($lei, \@argv);
 }
 
 sub _complete_inspect {

^ permalink raw reply related	[flat|nested] 9+ messages in thread

* [PATCH 7/8] lei_xsearch: avoid false-positives on externals w/ L: and kw:
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
                   ` (5 preceding siblings ...)
  2021-09-03  8:54 ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  2021-09-03  8:54 ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

We need to use LeiSearch->qparse_new to handle (and filter out)
"L:" and "kw:" search prefixes to avoid hitting false positives
when externals are involved.  Unfortunately, this doesn't work
for remote HTTP(S) externals, but those aren't enabled by
default.
---
 lib/PublicInbox/LeiXSearch.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index b9f0d692..b6d7bf2b 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -161,6 +161,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
 	my ($srch, $over) = ($ibxish->search, $ibxish->over);
 	my $dir = $ibxish->{inboxdir} // $ibxish->{topdir};
 	return warn("$dir not indexed by Xapian\n") unless ($srch && $over);
+	bless $srch, 'PublicInbox::LeiSearch'; # for ->qparse_new
 	my $mo = { %{$lei->{mset_opt}} }; # copy
 	my $mset;
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);

^ permalink raw reply related	[flat|nested] 9+ messages in thread

* [PATCH 8/8] lei: fix read/write IMAP access
  2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
                   ` (6 preceding siblings ...)
  2021-09-03  8:54 ` [PATCH 7/8] lei_xsearch: avoid false-positives on externals w/ L: and kw: Eric Wong
@ 2021-09-03  8:54 ` Eric Wong
  7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03  8:54 UTC (permalink / raw)
  To: meta

xt/net_writer-imap.t was completely broken in recent months and
I completely forgot this test.  net->add_url still only accepts
bare scalars (and not scalar refs), so we must set that up
properly.  Furthermore, our changes to do FLAGS-only
synchronization in lei of old messages was causing us to not
handle FLAGS properly for the test.
---
 lib/PublicInbox/LeiToMail.pm | 2 +-
 lib/PublicInbox/NetReader.pm | 5 ++++-
 lib/PublicInbox/NetWriter.pm | 2 ++
 lib/PublicInbox/Watch.pm     | 2 ++
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index be6e178f..6e102a1d 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -406,7 +406,7 @@ sub new {
 		my $net = PublicInbox::NetWriter->new;
 		$net->{quiet} = $lei->{opt}->{quiet};
 		my $uri = PublicInbox::URIimap->new($dst)->canonical;
-		$net->add_url($uri);
+		$net->add_url($$uri);
 		my $err = $net->errors($lei);
 		return $lei->fail($err) if $err;
 		$uri->mailbox or return $lei->fail("No mailbox: $dst");
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 23445e7a..c050c60f 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -493,6 +493,9 @@ sub perm_fl_ok ($) {
 	undef;
 }
 
+# may be overridden in NetWriter or Watch
+sub folder_select { $_[0]->{each_old} ? 'select' : 'examine' }
+
 sub _imap_fetch_all ($$$) {
 	my ($self, $mic, $orig_uri) = @_;
 	my $sec = uri_section($orig_uri);
@@ -501,7 +504,7 @@ sub _imap_fetch_all ($$$) {
 
 	# we need to check for mailbox writability to see if we care about
 	# FLAGS from already-imported messages.
-	my $cmd = $self->{each_old} ? 'select' : 'examine';
+	my $cmd = $self->folder_select;
 	$mic->$cmd($mbx) or return "E: \U$cmd\E $mbx ($sec) failed: $!";
 
 	my ($r_uidval, $r_uidnext, $perm_fl);
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 82288e6b..629a752a 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,6 +26,8 @@ sub imap_append {
 		die "APPEND $folder: $@";
 }
 
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
 sub imap_delete_all {
 	my ($self, $uri) = @_;
 	my $mic = $self->mic_for_folder($uri) or return;
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 86dae91f..482d35c4 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -682,4 +682,6 @@ EOF
 	undef;
 }
 
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
 1;

^ permalink raw reply related	[flat|nested] 9+ messages in thread

end of thread, other threads:[~2021-09-03  8:54 UTC | newest]

Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-03  8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03  8:54 ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
2021-09-03  8:54 ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
2021-09-03  8:54 ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
2021-09-03  8:54 ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
2021-09-03  8:54 ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
2021-09-03  8:54 ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
2021-09-03  8:54 ` [PATCH 7/8] lei_xsearch: avoid false-positives on externals w/ L: and kw: Eric Wong
2021-09-03  8:54 ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).