* [PATCH 00/11] lei: misc fixes, more lcat color support
@ 2021-04-28 7:51 Eric Wong
2021-04-28 7:51 ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
` (10 more replies)
0 siblings, 11 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
I'm seeing some odd test failures in "make check-run"
(but not "make check") that I haven't diagnosed, yet.
So there's some diagnostic changes and some golfing
to make internals more consistent.
Eric Wong (11):
t/lei-p2q: add diagnostics
tests: restore CWD with "lei -C" and run_script
view_diff: minor coding style fixes
lei_p2q: add _lei_wq_eof callback
lei-daemon: note FD count mismatch to client
t/run.perl: add (GNU) tail and strace support
lei: quiet down Eml-related warnings consistently
lei: simple WQ workers use {wq1} field
lei_view_text: improve attachment display
lei_view_text: translate background colors from git
lei (lcat|q): support --no-color and --color
lib/PublicInbox/LEI.pm | 30 ++++++++++++++++++++---------
lib/PublicInbox/LeiBlob.pm | 9 +--------
lib/PublicInbox/LeiConvert.pm | 3 +--
lib/PublicInbox/LeiImport.pm | 10 +++-------
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
lib/PublicInbox/LeiP2q.pm | 6 ++----
lib/PublicInbox/LeiStore.pm | 1 -
lib/PublicInbox/LeiTag.pm | 9 ++-------
lib/PublicInbox/LeiViewText.pm | 35 +++++++++++++++++++++++-----------
lib/PublicInbox/LeiXSearch.pm | 1 -
lib/PublicInbox/TestCommon.pm | 12 ++++++++----
lib/PublicInbox/ViewDiff.pm | 9 ++++-----
script/lei | 4 ++--
t/lei-externals.t | 5 -----
t/lei-p2q.t | 4 ++--
t/lei-q-save.t | 4 ----
t/run.perl | 7 +++++++
t/solver_git.t | 5 -----
19 files changed, 80 insertions(+), 80 deletions(-)
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH 01/11] t/lei-p2q: add diagnostics
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
@ 2021-04-28 7:51 ` Eric Wong
2021-04-28 7:51 ` [PATCH 02/11] tests: restore CWD with "lei -C" and run_script Eric Wong
` (9 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
Maybe this helps fix occasional problems in daemon mode,
but I haven't seen anything on failure, yet...
---
t/lei-p2q.t | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index be2d437c..f8b073cf 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -9,10 +9,10 @@ test_lei(sub {
ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
'p2q fails on bogus arg');
lei_ok(qw(p2q -w dfpost t/data/0001.patch));
- is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname');
+ is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname') or diag $lei_err;
open my $fh, '+<', 't/data/0001.patch' or xbail "open: $!";
lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
- is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin');
+ is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin') or diag $lei_err;
lei_ok(qw(p2q --uri t/data/0001.patch -w), 'dfpost,dfn');
is($lei_out, "dfpost%3A6e006fd73b1d+".
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 02/11] tests: restore CWD with "lei -C" and run_script
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
2021-04-28 7:51 ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
@ 2021-04-28 7:51 ` Eric Wong
2021-04-28 7:51 ` [PATCH 03/11] view_diff: minor coding style fixes Eric Wong
` (8 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
This simplifies test_lei users in t/*.t
---
lib/PublicInbox/TestCommon.pm | 12 ++++++++----
t/lei-externals.t | 5 -----
t/lei-q-save.t | 4 ----
t/solver_git.t | 5 -----
4 files changed, 8 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 09256852..460c9da0 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -12,6 +12,7 @@ use IO::Socket::INET;
use File::Spec;
our @EXPORT;
my $lei_loud = $ENV{TEST_LEI_ERR_LOUD};
+our ($lei_opt, $lei_out, $lei_err, $lei_cwdfh);
BEGIN {
@EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
run_script start_script key2sub xsys xsys_e xqx eml_load tick
@@ -306,14 +307,16 @@ sub run_script ($;$$) {
local %SIG = %SIG;
local $0 = join(' ', @$cmd);
my $orig_io = _prepare_redirects($fhref);
- my $cwdfh;
+ my $cwdfh = $lei_cwdfh;
if (my $d = $opt->{'-C'}) {
- opendir $cwdfh, '.' or die "opendir .: $!";
+ unless ($cwdfh) {
+ opendir $cwdfh, '.' or die "opendir .: $!";
+ }
chdir $d or die "chdir $d: $!";
}
_run_sub($sub, $key, \@argv);
eval { PublicInbox::Inbox::cleanup_task() };
- die "chdir(restore): $!" if $cwdfh && !chdir($cwdfh);
+ die "fchdir(restore): $!" if $cwdfh && !chdir($cwdfh);
_undo_redirects($orig_io);
select STDOUT;
}
@@ -469,7 +472,6 @@ sub have_xapian_compact () {
PublicInbox::Spawn::which($ENV{XAPIAN_COMPACT} || 'xapian-compact');
}
-our ($lei_opt, $lei_out, $lei_err);
# favor lei() or lei_ok() over $lei for new code
sub lei (@) {
my ($cmd, $env, $xopt) = @_;
@@ -515,6 +517,8 @@ sub test_lei {
SKIP: {
my ($cb) = pop @_;
my $test_opt = shift // {};
+ local $lei_cwdfh;
+ opendir $lei_cwdfh, '.' or xbail "opendir .: $!";
require_git(2.6, 1) or skip('git 2.6+ required for lei test', 2);
require_mods(qw(json DBD::SQLite Search::Xapian), 2);
require PublicInbox::Config;
diff --git a/t/lei-externals.t b/t/lei-externals.t
index 2291dd99..16241e02 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -4,7 +4,6 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use Fcntl qw(SEEK_SET);
use PublicInbox::Spawn qw(which);
-use PublicInbox::OnDestroy;
require_git 2.6;
require_mods(qw(json DBD::SQLite Search::Xapian));
use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
@@ -266,10 +265,6 @@ test_lei(sub {
{
skip 'TEST_LEI_DAEMON_PERSIST_DIR in use', 1 if
$ENV{TEST_LEI_DAEMON_PERSIST_DIR};
- opendir my $dh, '.' or BAIL_OUT "opendir(.) $!";
- my $od = PublicInbox::OnDestroy->new($$, sub {
- chdir $dh or BAIL_OUT "chdir: $!"
- });
my @q = qw(q -o mboxcl2:rel.mboxcl2 bye);
lei_ok('-C', $home, @q);
is(unlink("$home/rel.mboxcl2"), 1, '-C works before q');
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 170f7ce5..9f65e4a2 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -36,11 +36,9 @@ test_lei(sub {
# ensure "lei up" works, since it compliments "lei q --save"
$in = $doc2->as_string;
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
- opendir my $dh, '.' or xbail "opendir .: $!";
lei_ok qw(up -q md -C), $home;
lei_ok qw(up -q . -C), "$home/md";
lei_ok qw(up -q), "/$home/md";
- chdir($dh) or xbail "fchdir . $!";
my %after = map { $_ => 1 } glob("$home/md/cur/*");
is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
is(scalar(keys %after), 1, 'one new message added');
@@ -89,7 +87,6 @@ test_lei(sub {
print $mb $pre_existing;
close $mb or xbail "close: $!";
lei_ok(qw(q --save -o mboxrd:mbrd m:qp@example.com -C), $home);
- chdir($dh) or xbail "fchdir . $!";
open $mb, '<', "$home/mbrd" or xbail "open $!";
is_deeply([grep(/pre-existing/, <$mb>)], [],
'pre-existing messsage gone w/o augment');
@@ -103,7 +100,6 @@ test_lei(sub {
print $mb $pre_existing;
close $mb or xbail "close: $!";
lei_ok(qw(q -a --save -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
- chdir($dh) or xbail "fchdir . $!";
open $mb, '<', "$home/mbrd-aug" or xbail "open $!";
$mb = do { local $/; <$mb> };
like($mb, qr/pre-existing/, 'pre-existing message preserved w/ -a');
diff --git a/t/solver_git.t b/t/solver_git.t
index 8acf907f..6875e26b 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -50,11 +50,6 @@ test_lei({tmpdir => $tmpdir}, sub {
'non-existent blob fails');
SKIP: {
skip '/.git exists', 1 if -e '/.git';
- require PublicInbox::OnDestroy;
- opendir my $dh, '.' or xbail "opendir: $!";
- my $end = PublicInbox::OnDestroy->new($$, sub {
- chdir $dh or xbail "chdir: $!";
- });
lei_ok(qw(-C / blob 69df7d5 -I), $ibx->{inboxdir},
"--git-dir=$git_dir");
is($lei_out, $prev, '--git-dir works');
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 03/11] view_diff: minor coding style fixes
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
2021-04-28 7:51 ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
2021-04-28 7:51 ` [PATCH 02/11] tests: restore CWD with "lei -C" and run_script Eric Wong
@ 2021-04-28 7:51 ` Eric Wong
2021-04-28 7:51 ` [PATCH 04/11] lei_p2q: add _lei_wq_eof callback Eric Wong
` (7 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
Prefer "use v5.10", s/base/parent/, rely on "perl -w" for warnings.
We also pass a regexp to the split perlop rather than literal
SV, since split() will compile a new RE every time.
---
lib/PublicInbox/ViewDiff.pm | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index e9a7bf69..d2c5fabe 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -7,10 +7,9 @@
# (or reconstruct) blobs.
package PublicInbox::ViewDiff;
-use 5.010_001;
use strict;
-use warnings;
-use base qw(Exporter);
+use v5.10.1;
+use parent qw(Exporter);
our @EXPORT_OK = qw(flush_diff);
use URI::Escape qw(uri_escape_utf8);
use PublicInbox::Hval qw(ascii_html to_attr);
@@ -122,8 +121,8 @@ sub diff_header ($$$) {
my $dctx = { spfx => $spfx };
# get rid of leading "a/" or "b/" (or whatever --{src,dst}-prefix are)
- $pa = (split('/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
- $pb = (split('/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
+ $pa = (split(m'/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
+ $pb = (split(m'/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
if ($pa eq $pb && $pb ne '/dev/null') {
$dctx->{Q} = "?b=".uri_escape_utf8($pb, UNSAFE);
} else {
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 04/11] lei_p2q: add _lei_wq_eof callback
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (2 preceding siblings ...)
2021-04-28 7:51 ` [PATCH 03/11] view_diff: minor coding style fixes Eric Wong
@ 2021-04-28 7:51 ` Eric Wong
2021-04-28 7:51 ` [PATCH 05/11] lei-daemon: note FD count mismatch to client Eric Wong
` (6 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
This may help with some error diagnostics.
---
lib/PublicInbox/LeiP2q.pm | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index 3248afd7..07357e32 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -203,4 +203,10 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
+sub _lei_wq_eof { # EOF callback for main daemon
+ my ($lei) = @_;
+ my $p2q = delete $lei->{p2q} // return $lei->dclose;
+ $p2q->wq_wait_old($lei->can('wq_done_wait'), $lei);
+}
+
1;
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 05/11] lei-daemon: note FD count mismatch to client
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (3 preceding siblings ...)
2021-04-28 7:51 ` [PATCH 04/11] lei_p2q: add _lei_wq_eof callback Eric Wong
@ 2021-04-28 7:51 ` Eric Wong
2021-04-28 7:52 ` [PATCH 06/11] t/run.perl: add (GNU) tail and strace support Eric Wong
` (5 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
This should help in some error diagnostics
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ef72758c..a949ae3e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -998,7 +998,7 @@ sub accept_dispatch { # Listener {post_accept} callback
open($self->{$i++}, '+<&=', $fd) and next;
send($sock, "open(+<&=$fd) (FD=$i): $!", MSG_EOR);
}
- return if scalar(@fds) != 4;
+ $i == 4 or return send($sock, 'not enough FDs='.($i-1), MSG_EOR)
}
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
# $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 06/11] t/run.perl: add (GNU) tail and strace support
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (4 preceding siblings ...)
2021-04-28 7:51 ` [PATCH 05/11] lei-daemon: note FD count mismatch to client Eric Wong
@ 2021-04-28 7:52 ` Eric Wong
2021-04-28 7:52 ` [PATCH 07/11] lei: quiet down Eml-related warnings consistently Eric Wong
` (4 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
This may help track down some "make check-run" failures
that don't seem to manifest under "make check".
---
t/run.perl | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/t/run.perl b/t/run.perl
index f4e235b7..2fbe4033 100755
--- a/t/run.perl
+++ b/t/run.perl
@@ -52,6 +52,13 @@ if (!$ENV{TEST_LEI_DAEMON_PERSIST_DIR} &&
chomp $lei_daemon_pid;
$lei_daemon_pid =~ /\A[0-9]+\z/ or die "no daemon pid: $lei_daemon_pid";
kill(0, $lei_daemon_pid) or die "kill $lei_daemon_pid: $!";
+ if (my $t = $ENV{GNU_TAIL}) {
+ system("$t --pid=$lei_daemon_pid -F " .
+ "$lei_env->{XDG_RUNTIME_DIR}/lei/errors.log >&2 &");
+ }
+ if (my $strace_cmd = $ENV{STRACE_CMD}) {
+ system("$strace_cmd -p $lei_daemon_pid &");
+ }
$owner_pid = $$;
}
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 07/11] lei: quiet down Eml-related warnings consistently
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (5 preceding siblings ...)
2021-04-28 7:52 ` [PATCH 06/11] t/run.perl: add (GNU) tail and strace support Eric Wong
@ 2021-04-28 7:52 ` Eric Wong
2021-04-28 7:52 ` [PATCH 08/11] lei: simple WQ workers use {wq1} field Eric Wong
` (3 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
"lei import" is probably the only place where it users
might care about warnings.
---
lib/PublicInbox/LEI.pm | 3 +++
lib/PublicInbox/LeiBlob.pm | 1 -
lib/PublicInbox/LeiConvert.pm | 1 -
lib/PublicInbox/LeiImport.pm | 1 +
lib/PublicInbox/LeiP2q.pm | 4 +---
lib/PublicInbox/LeiStore.pm | 1 -
lib/PublicInbox/LeiXSearch.pm | 1 -
7 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a949ae3e..cfbf12f0 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -23,6 +23,7 @@ use PublicInbox::Sigfd;
use PublicInbox::DS qw(now dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
+use PublicInbox::Eml;
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
@@ -509,6 +510,8 @@ sub _lei_atfork_child {
%PATH2CFG = ();
undef $errors_log;
$quit = \&CORE::exit;
+ $self->{-eml_noisy} or # only "lei import" sets this atm
+ $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$current_lei = $persist ? undef : $self; # for SIG{__WARN__}
}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 0b96bd04..ff079e65 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -167,7 +167,6 @@ EOM
sub ipc_atfork_child {
my ($self) = @_;
$self->{lei}->_lei_atfork_child;
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 0c324169..14bed901 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -68,7 +68,6 @@ sub ipc_atfork_child {
$net->{mics_cached} = $net->imap_common_init($lei);
$net->{nn_cached} = $net->nntp_common_init($lei);
}
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$l2m->pre_augment($lei);
$l2m->do_augment($lei);
$l2m->post_augment($lei);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index e0d899cc..f2a0c95a 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -93,6 +93,7 @@ sub lei_import { # the main "lei import" method
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
+ $lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
$lei->{imp} = $self;
net_merge_complete($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index 07357e32..deb31974 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -197,9 +197,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
sub ipc_atfork_child {
my ($self) = @_;
- my $lei = $self->{lei};
- $lei->_lei_atfork_child;
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
+ $self->{lei}->_lei_atfork_child;
$self->SUPER::ipc_atfork_child;
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 1cf7ffc1..fcc9224d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -411,7 +411,6 @@ sub ipc_atfork_child {
close $err->[0];
$self->{-err_wr} = $err->[1];
}
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 018b60f9..b3fd79d0 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -443,7 +443,6 @@ sub incr_start_query { # called whenever an l2m shard starts do_post_auth
sub ipc_atfork_child {
my ($self) = @_;
$self->{lei}->_lei_atfork_child;
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 08/11] lei: simple WQ workers use {wq1} field
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (6 preceding siblings ...)
2021-04-28 7:52 ` [PATCH 07/11] lei: quiet down Eml-related warnings consistently Eric Wong
@ 2021-04-28 7:52 ` Eric Wong
2021-04-28 7:52 ` [PATCH 09/11] lei_view_text: improve attachment display Eric Wong
` (2 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
This lets us share more code and reduces cognitive overhead when
it comes to picking names (because {lsss} was ridiculous).
We'll need to ensure the first error set in lei is the actual
error we exit with, otherwise things can get confusing and
errors may get lost.
---
lib/PublicInbox/LEI.pm | 16 ++++++++++++----
lib/PublicInbox/LeiBlob.pm | 8 +-------
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 9 ++-------
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
lib/PublicInbox/LeiP2q.pm | 8 +-------
lib/PublicInbox/LeiTag.pm | 9 ++-------
script/lei | 4 ++--
9 files changed, 24 insertions(+), 38 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index cfbf12f0..403f9ed8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -382,7 +382,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol lsss); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
sub _drop_wq {
my ($self) = @_;
@@ -542,7 +542,7 @@ sub workers_start {
'child_error' => [ \&child_error, $lei ],
($ops ? %$ops : ()),
};
- $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&dclose, $lei ];
+ $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
delete $lei->{pkt_op_p};
@@ -1237,9 +1237,17 @@ sub DESTROY {
sub wq_done_wait { # dwaitpid callback
my ($arg, $pid) = @_;
- my ($wq, $lei, $e) = @$arg;
- $? and $lei->child_error($?, $e ? "$e errors during $lei->{cmd}" : ());
+ my ($wq, $lei) = @$arg;
+ my $err_type = $lei->{-err_type};
+ $? and $lei->child_error($?,
+ $err_type ? "$err_type errors during $lei->{cmd}" : ());
$lei->dclose;
}
+sub wq_eof { # EOF callback for main daemon
+ my ($lei) = @_;
+ my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
+ $wq1->wq_wait_old(\&wq_done_wait, $lei);
+}
+
1;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index ff079e65..0a957358 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -10,12 +10,6 @@ use parent qw(PublicInbox::IPC);
use PublicInbox::Spawn qw(spawn popen_rd which);
use PublicInbox::DS;
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $sol = delete $lei->{sol} // return $lei->dclose; # already failed
- $sol->wq_wait_old($lei->can('wq_done_wait'), $lei);
-}
-
sub get_git_dir ($$) {
my ($lei, $d) = @_;
return $d if -d "$d/objects" && -d "$d/refs" && -e "$d/HEAD";
@@ -158,7 +152,7 @@ EOM
require PublicInbox::SolverGit;
my $self = bless { lxs => $lxs, oid_b => $blob }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 'lei-blob', 1);
- $lei->{sol} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_solve_blob', []);
$self->wq_close(1);
$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 14bed901..cefcaf65 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -53,7 +53,7 @@ sub lei_convert { # the main "lei convert" method
$lei->{opt}->{augment} = 1 if $devfd < 0;
$self->prepare_inputs($lei, \@inputs) or return;
my ($op_c, $ops) = $lei->workers_start($self, 'lei-convert', 1);
- $lei->{cnv} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('process_inputs', []);
$self->wq_close(1);
$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index f2a0c95a..26127ece 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -53,12 +53,6 @@ sub input_nntp_cb { # nntp_each
input_eml_cb($self, $eml, $self->{-import_kw} ? { kw => $kw } : undef);
}
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $imp = delete $lei->{imp} // return $lei->fail('BUG: {imp} gone');
- $imp->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
-}
-
sub net_merge_complete { # callback used by LeiAuth
my ($self) = @_;
$self->wq_io_do('process_inputs');
@@ -95,7 +89,8 @@ sub lei_import { # the main "lei import" method
$self->{-wq_nr_workers} = $j // 1; # locked
$lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
- $lei->{imp} = $self;
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
net_merge_complete($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 9ac4870f..a00e78fc 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -73,7 +73,7 @@ sub bg_worker ($$$) {
my ($lei, $pfx, $json) = @_;
my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1);
- $lei->{lsss} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
$self->wq_close(1);
$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 50ab4c85..db97b98c 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -26,7 +26,7 @@ sub do_finish_mirror { # dwaitpid callback
sub _lei_wq_eof { # EOF callback for main daemon
my ($lei) = @_;
- my $mrr = delete $lei->{mrr} or return;
+ my $mrr = delete $lei->{wq1} or return $lei->fail;
$mrr->wq_wait_old(\&do_finish_mirror, $lei);
}
@@ -283,7 +283,7 @@ sub start {
require PublicInbox::Admin;
require PublicInbox::InboxWritable;
my ($op, $ops) = $lei->workers_start($self, 'lei_mirror', 1);
- $lei->{mrr} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_mirror', []);
$self->wq_close(1);
$op->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index deb31974..b4893489 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -189,7 +189,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
$self->{input} = $input;
}
my ($op, $ops) = $lei->workers_start($self, 'lei-p2q', 1);
- $lei->{p2q} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_p2q', []);
$self->wq_close(1);
$op->op_wait_event($ops);
@@ -201,10 +201,4 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $p2q = delete $lei->{p2q} // return $lei->dclose;
- $p2q->wq_wait_old($lei->can('wq_done_wait'), $lei);
-}
-
1;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 3cda2eca..989a6954 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -19,12 +19,6 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $tag = delete $lei->{tag} // return $lei->dclose;
- $tag->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
-}
-
sub net_merge_complete { # callback used by LeiAuth
my ($self) = @_;
$self->wq_io_do('process_inputs');
@@ -57,7 +51,8 @@ sub lei_tag { # the "lei tag" method
$self->{vmd_mod} = $vmd_mod;
my $j = $self->{-wq_nr_workers} = 1; # locked for now
(my $op_c, $ops) = $lei->workers_start($self, 'lei-tag', $j, $ops);
- $lei->{tag} = $self;
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
net_merge_complete($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/script/lei b/script/lei
index db302422..90a93839 100755
--- a/script/lei
+++ b/script/lei
@@ -116,10 +116,10 @@ Falling back to (slow) one-shot mode
} elsif ($buf eq '-WINCH') {
kill($buf, @parent); # for MUA
} elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
- $x_it_code = $1 + 0;
+ $x_it_code ||= $1 + 0;
last;
} elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
- $x_it_code = $1 + 0;
+ $x_it_code ||= $1 + 0;
} else {
$sigchld->();
die $buf;
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 09/11] lei_view_text: improve attachment display
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (7 preceding siblings ...)
2021-04-28 7:52 ` [PATCH 08/11] lei: simple WQ workers use {wq1} field Eric Wong
@ 2021-04-28 7:52 ` Eric Wong
2021-04-28 7:52 ` [PATCH 10/11] lei_view_text: translate background colors from git Eric Wong
2021-04-28 7:52 ` [PATCH 11/11] lei (lcat|q): support --no-color and --color Eric Wong
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
Support setting a color to distinguish from user-supplied text.
We'll also put the $BLOB:$IDX identifier on a separate line and
just put the entire corresponding lei command in the form of:
"[-- lei blob $BLOB:$IDX --]" to teach users how to access it.
---
lib/PublicInbox/LeiViewText.pm | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 6f5fca49..5d649840 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -25,6 +25,7 @@ my %DEFAULT_COLOR = (
quoted => 'blue',
hdrdefault => 'cyan',
status => 'bright_cyan', # smsg stuff
+ attachment => 'bright_red',
# git names and defaults, falls back to ~/.gitconfig
new => 'green',
@@ -113,20 +114,21 @@ sub hdr_buf ($$) {
sub attach_note ($$$$;$) {
my ($self, $ct, $p, $fn, $err) = @_;
my ($part, $depth, $idx) = @$p;
- my $obuf = $self->{obuf};
my $nl = $idx eq '1' ? '' : "\n"; # like join("\n", ...)
- $$obuf .= <<EOF if $err;
+ my $abuf = $err ? <<EOF : '';
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
EOF
- my $blob = $self->{-smsg}->{blob} // '';
- $blob .= ':' if $blob ne '';
- $$obuf .= "[-- Attachment $blob$idx ";
+ $abuf .= "[-- Attachment #$idx: ";
_xs($ct);
my $size = length($part->body);
my $ts = "Type: $ct, Size: $size bytes";
my $d = $part->header('Content-Description') // $fn // '';
_xs($d);
- $$obuf .= $d eq '' ? "$ts --]\n" : "$d --]\n[-- $ts --]\n";
+ $abuf .= $d eq '' ? "$ts --]\n" : "$d --]\n[-- $ts --]\n";
+ if (my $blob = $self->{-smsg}->{blob}) {
+ $abuf .= "[-- lei blob $blob:$idx --]\n";
+ }
+ $self->{-colored}->($self, 'attachment', $abuf);
hdr_buf($self, $part) if $part->{is_submsg};
}
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 10/11] lei_view_text: translate background colors from git
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (8 preceding siblings ...)
2021-04-28 7:52 ` [PATCH 09/11] lei_view_text: improve attachment display Eric Wong
@ 2021-04-28 7:52 ` Eric Wong
2021-04-28 7:52 ` [PATCH 11/11] lei (lcat|q): support --no-color and --color Eric Wong
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
This seems to work with or without attributes. We'll deal with
256-color terminal colors when/if somebody cares for it, but the
usual 16 ought to be more than enough.
---
lib/PublicInbox/LeiViewText.pm | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 5d649840..e0d62c0d 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -36,8 +36,11 @@ my %DEFAULT_COLOR = (
context => undef,
);
+my $COLOR = qr/(?:bright)?
+ (?:normal|black|red|green|yellow|blue|magenta|cyan|white)/x;
+
sub my_colored {
- my ($self, $slot) = @_; # $_[2] = buffer
+ my ($self, $slot, $buf) = @_;
my $val = $self->{"color.$slot"} //=
$self->{-leicfg}->{"color.$slot"} //
$self->{-gitcfg}->{"color.diff.$slot"} //
@@ -45,11 +48,19 @@ sub my_colored {
$DEFAULT_COLOR{$slot};
$val = $val->[-1] if ref($val) eq 'ARRAY';
if (defined $val) {
+ $val = lc $val;
# git doesn't use "_", Term::ANSIColor does
- $val =~ s/\Abright([^_])/bright_$1/i;
- ${$self->{obuf}} .= Term::ANSIColor::colored($_[2], lc $val);
+ $val =~ s/\Abright([^_])/bright_$1/ig;
+
+ # git: "green black" => T::A: "green on_black"
+ $val =~ s/($COLOR)(.+?)($COLOR)/$1$2on_$3/;
+
+ # FIXME: convert git #XXXXXX to T::A-compatible colors
+ # for 256-color terminals
+
+ ${$self->{obuf}} .= colored($buf, $val);
} else {
- ${$self->{obuf}} .= $_[2];
+ ${$self->{obuf}} .= $buf;
}
}
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 11/11] lei (lcat|q): support --no-color and --color
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (9 preceding siblings ...)
2021-04-28 7:52 ` [PATCH 10/11] lei_view_text: translate background colors from git Eric Wong
@ 2021-04-28 7:52 ` Eric Wong
10 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
This should be familiar to git users who wish to force color
when writing to pipes or disable color.
---
lib/PublicInbox/LEI.pm | 9 +++++----
lib/PublicInbox/LeiViewText.pm | 2 +-
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 403f9ed8..7ffcf163 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -144,8 +144,8 @@ our %CMD = ( # sorted in order of importance/use:
@lxs_opt,
qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
- import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
- opt_dash('limit|n=i', '[0-9]+') ],
+ import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+ color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
@@ -155,8 +155,8 @@ our %CMD = ( # sorted in order of importance/use:
# some of these options are ridiculous for lcat
@lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i jobs|j=s globoff|g augment|a
- import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
- opt_dash('limit|n=i', '[0-9]+') ],
+ import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+ color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'blob' => [ 'OID', 'show a git blob, reconstructing from mail if necessary',
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@@ -268,6 +268,7 @@ my %OPTDESC = (
'incremental! import' => 'import already seen IMAP and NNTP articles',
'globoff|g' => "do not match locations using '*?' wildcards ".
"and\xa0'[]'\x{a0}ranges",
+'color!' => 'disable color (for --format=text)',
'verbose|v+' => 'be more verbose',
'external!' => 'do not use externals',
'mail!' => 'do not look in mail storage for OID',
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index e0d62c0d..d0f8b7f4 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -69,7 +69,7 @@ sub uncolored { ${$_[0]->{obuf}} .= $_[2] }
sub new {
my ($cls, $lei) = @_;
my $self = bless { %{$lei->{opt}}, -colored => \&uncolored }, $cls;
- return $self unless $self->{color} || -t $lei->{1};
+ return $self unless $self->{color} //= -t $lei->{1};
my $cmd = [ qw(git config -z --includes -l) ];
my ($r, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
my $cfg = PublicInbox::Config::config_fh_parse($r, "\0", "\n");
^ permalink raw reply related [flat|nested] 12+ messages in thread
end of thread, other threads:[~2021-04-28 7:52 UTC | newest]
Thread overview: 12+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-04-28 7:51 [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
2021-04-28 7:51 ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
2021-04-28 7:51 ` [PATCH 02/11] tests: restore CWD with "lei -C" and run_script Eric Wong
2021-04-28 7:51 ` [PATCH 03/11] view_diff: minor coding style fixes Eric Wong
2021-04-28 7:51 ` [PATCH 04/11] lei_p2q: add _lei_wq_eof callback Eric Wong
2021-04-28 7:51 ` [PATCH 05/11] lei-daemon: note FD count mismatch to client Eric Wong
2021-04-28 7:52 ` [PATCH 06/11] t/run.perl: add (GNU) tail and strace support Eric Wong
2021-04-28 7:52 ` [PATCH 07/11] lei: quiet down Eml-related warnings consistently Eric Wong
2021-04-28 7:52 ` [PATCH 08/11] lei: simple WQ workers use {wq1} field Eric Wong
2021-04-28 7:52 ` [PATCH 09/11] lei_view_text: improve attachment display Eric Wong
2021-04-28 7:52 ` [PATCH 10/11] lei_view_text: translate background colors from git Eric Wong
2021-04-28 7:52 ` [PATCH 11/11] lei (lcat|q): support --no-color and --color Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).