* [PATCH 1/3] lei: fix handling of broken lei.saved-search config files
@ 2021-09-11 0:19 53% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
lei shouldn't become unusable if a config file is invalid.
Instead, show the "git config" stderr and attempt to continue
gracefully.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210910141157.6u5adehpx7wftkor@meerkat.local/
---
lib/PublicInbox/Config.pm | 6 +--
lib/PublicInbox/LEI.pm | 21 ++++++--
lib/PublicInbox/LeiEditSearch.pm | 79 +++++++++++++++++++++++++++++--
lib/PublicInbox/LeiSavedSearch.pm | 77 ++++++------------------------
script/lei | 2 +
t/lei-q-save.t | 17 +++++++
6 files changed, 128 insertions(+), 74 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index b3e00ae0..74a1a6f5 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -164,12 +164,12 @@ sub config_fh_parse ($$$) {
}
sub git_config_dump {
- my ($class, $file) = @_;
+ my ($class, $file, $errfh) = @_;
return bless {}, $class unless -e $file;
my $cmd = [ qw(git config -z -l --includes), "--file=$file" ];
- my $fh = popen_rd($cmd);
+ my $fh = popen_rd($cmd, undef, { 2 => $errfh // 2 });
my $rv = config_fh_parse($fh, "\0", "\n");
- close $fh or die "failed to close (@$cmd) pipe: $?";
+ close $fh or die "@$cmd failed: \$?=$?\n";
bless $rv, $class;
}
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bbb6ab7e..0b4c99dc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1029,13 +1029,15 @@ sub path_to_fd {
# caller needs to "-t $self->{1}" to check if tty
sub start_pager {
- my ($self) = @_;
+ my ($self, $new_env) = @_;
my $fh = popen_rd([qw(git var GIT_PAGER)]);
chomp(my $pager = <$fh> // '');
close($fh) or warn "`git var PAGER' error: \$?=$?";
return if $pager eq 'cat' || $pager eq '';
- my $new_env = { LESS => 'FRX', LV => '-c' };
- $new_env->{MORE} = 'FRX' if $^O eq 'freebsd';
+ $new_env //= {};
+ $new_env->{LESS} //= 'FRX';
+ $new_env->{LV} //= '-c';
+ $new_env->{MORE} = $new_env->{LESS} if $^O eq 'freebsd';
pipe(my ($r, $wpager)) or return warn "pipe: $!";
my $rdr = { 0 => $r, 1 => $self->{1}, 2 => $self->{2} };
my $pgr = [ undef, @$rdr{1, 2} ];
@@ -1052,6 +1054,19 @@ sub start_pager {
$self->{pgr} = $pgr;
}
+# display a message for user before spawning full-screen $VISUAL
+sub pgr_err {
+ my ($self, @msg) = @_;
+ return $self->err(@msg) unless $self->{sock} && -t $self->{2};
+ start_pager($self, { LESS => 'RX' }); # no 'F' so we prompt
+ print { $self->{2} } @msg;
+ $self->{2}->autoflush(1);
+ my $pgr = delete($self->{pgr}) or return;
+ $self->{2} = $pgr->[2];
+ $self->{1} = $pgr->[1];
+ send($self->{sock}, 'wait', MSG_EOR); # wait for user to quit pager
+}
+
sub stop_pager {
my ($self) = @_;
my $pgr = delete($self->{pgr}) or return;
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 82dfbf63..47166ce7 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -8,20 +8,89 @@ use v5.10.1;
use PublicInbox::LeiSavedSearch;
use PublicInbox::LeiUp;
-sub lei_edit_search {
- my ($lei, $out) = @_;
- my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
+sub edit_begin {
+ my ($lss, $lei) = @_;
+ if (ref($lss->{-cfg}->{'lei.q.output'})) {
+ delete $lss->{-cfg}->{'lei.q.output'}; # invalid
+ $lei->pgr_err(<<EOM);
+$lss->{-f} has multiple values of lei.q.output
+please remove redundant ones
+EOM
+ }
+ $lei->{-lss_for_edit} = $lss;
+}
+
+sub do_edit ($$;$) {
+ my ($lss, $lei, $reason) = @_;
+ $lei->pgr_err($reason) if defined $reason;
my @cmd = (qw(git config --edit -f), $lss->{'-f'});
$lei->qerr("# spawning @cmd");
- $lss->edit_begin($lei);
+ edit_begin($lss, $lei);
# run in script/lei foreground
require PublicInbox::PktOp;
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
# $op_p will EOF when $EDITOR is done
- $op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
+ $op_c->{ops} = { '' => [\&op_edit_done, $lss, $lei] };
$lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], \@cmd, {});
}
+sub _edit_done {
+ my ($lss, $lei) = @_;
+ my $cfg = $lss->can('cfg_dump')->($lei, $lss->{'-f'}) //
+ return do_edit($lss, $lei, <<EOM);
+$lss->{-f} is unparseable
+EOM
+ my $new_out = $cfg->{'lei.q.output'} // '';
+ return do_edit($lss, $lei, <<EOM) if ref $new_out;
+$lss->{-f} has multiple values of lei.q.output
+EOM
+ return do_edit($lss, $lei, <<EOM) if $new_out eq '';
+$lss->{-f} needs lei.q.output
+EOM
+ my $old_out = $lss->{-cfg}->{'lei.q.output'} // return;
+ return if $old_out eq $new_out;
+ my $old_path = $old_out;
+ my $new_path = $new_out;
+ s!$PublicInbox::LeiSavedSearch::LOCAL_PFX!! for ($old_path, $new_path);
+ my $dir_old = $lss->can('lss_dir_for')->($lei, \$old_path, 1);
+ my $dir_new = $lss->can('lss_dir_for')->($lei, \$new_path);
+ return if $dir_new eq $dir_old;
+
+ ($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
+ return do_edit($lss, $lei, <<EOM);
+conversions from/to v2 inboxes not supported at this time
+EOM
+ return do_edit($lss, $lei, <<EOM) if -e $dir_new;
+lei.q.output changed from `$old_out' to `$new_out'
+However, $dir_new exists
+EOM
+ # start the conversion asynchronously
+ my $old_sq = PublicInbox::Config::squote_maybe($old_out);
+ my $new_sq = PublicInbox::Config::squote_maybe($new_out);
+ $lei->puts("lei.q.output changed from $old_sq to $new_sq");
+ $lei->qerr("# lei convert $old_sq -o $new_sq");
+ my $v = !$lei->{opt}->{quiet};
+ $lei->{opt} = { output => $new_out, verbose => $v };
+ require PublicInbox::LeiConvert;
+ PublicInbox::LeiConvert::lei_convert($lei, $old_out);
+
+ $lei->fail(<<EOM) if -e $dir_old && !rename($dir_old, $dir_new);
+E: rename($dir_old, $dir_new) error: $!
+EOM
+}
+
+sub op_edit_done { # PktOp
+ my ($lss, $lei) = @_;
+ eval { _edit_done($lss, $lei) };
+ $lei->fail($@) if $@;
+}
+
+sub lei_edit_search {
+ my ($lei, $out) = @_;
+ my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
+ do_edit($lss, $lei);
+}
+
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index fd51fe38..8ceaaf01 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -14,7 +14,7 @@ use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::MID qw(mids_for_index);
use Digest::SHA qw(sha256_hex);
-my $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
+our $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
# move this to PublicInbox::Config if other things use it:
my %cquote = ("\n" => '\\n', "\t" => '\\t', "\b" => '\\b');
@@ -29,6 +29,14 @@ sub BOOL_FIELDS () {
qw(external local remote import-remote import-before threads)
}
+sub cfg_dump ($$) {
+ my ($lei, $f) = @_;
+ my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
+ return $ret if !$@;
+ $lei->err($@);
+ undef;
+}
+
sub lss_dir_for ($$;$) {
my ($lei, $dstref, $on_fs) = @_;
my @n;
@@ -56,7 +64,7 @@ sub lss_dir_for ($$;$) {
for my $g ("$n[0]-*", '*') {
my @maybe = glob("$lss_dir$g/lei.saved-search");
for my $f (@maybe) {
- $c = PublicInbox::Config->git_config_dump($f);
+ $c = cfg_dump($lei, $f) // next;
$o = $c->{'lei.q.output'} // next;
$o =~ s!$LOCAL_PFX!! or next;
@st = stat($o) or next;
@@ -80,9 +88,9 @@ sub list {
print $fh "\tpath = ", cquote_val($p), "\n";
}
close $fh or die "close $f: $!";
- my $cfg = PublicInbox::Config->git_config_dump($f);
+ my $cfg = cfg_dump($lei, $f);
unlink($f);
- my $out = $cfg->get_all('lei.q.output') or return ();
+ my $out = $cfg ? $cfg->get_all('lei.q.output') : [];
map {;
s!$LOCAL_PFX!!;
$_;
@@ -105,7 +113,7 @@ sub up { # updating existing saved search via "lei up"
output2lssdir($self, $lei, \$dir, \$f) or
return $lei->fail("--save was not used with $dst cwd=".
$lei->rel2abs('.'));
- $self->{-cfg} = PublicInbox::Config->git_config_dump($f);
+ $self->{-cfg} = cfg_dump($lei, $f) // return $lei->fail;
$self->{-ovf} = "$dir/over.sqlite3";
$self->{'-f'} = $f;
$self->{lock_path} = "$self->{-f}.flock";
@@ -267,7 +275,7 @@ sub output2lssdir {
my $dir = lss_dir_for($lei, \$dst, 1);
my $f = "$dir/lei.saved-search";
if (-f $f && -r _) {
- $self->{-cfg} = PublicInbox::Config->git_config_dump($f);
+ $self->{-cfg} = cfg_dump($lei, $f) // return;
$$dir_ref = $dir;
$$fn_ref = $f;
return 1;
@@ -275,63 +283,6 @@ sub output2lssdir {
undef;
}
-sub edit_begin {
- my ($self, $lei) = @_;
- if (ref($self->{-cfg}->{'lei.q.output'})) {
- delete $self->{-cfg}->{'lei.q.output'}; # invalid
- $lei->err(<<EOM);
-$self->{-f} has multiple values of lei.q.output
-please remove redundant ones
-EOM
- }
- $lei->{-lss_for_edit} = $self;
-}
-
-sub edit_done {
- my ($self, $lei) = @_;
- my $cfg = PublicInbox::Config->git_config_dump($self->{'-f'});
- my $new_out = $cfg->{'lei.q.output'} // '';
- return $lei->fail(<<EOM) if ref $new_out;
-$self->{-f} has multiple values of lei.q.output
-please edit again
-EOM
- return $lei->fail(<<EOM) if $new_out eq '';
-$self->{-f} needs lei.q.output
-please edit again
-EOM
- my $old_out = $self->{-cfg}->{'lei.q.output'} // '';
- return if $old_out eq $new_out;
- my $old_path = $old_out;
- my $new_path = $new_out;
- s!$LOCAL_PFX!! for ($old_path, $new_path);
- my $dir_old = lss_dir_for($lei, \$old_path, 1);
- my $dir_new = lss_dir_for($lei, \$new_path);
- return if $dir_new eq $dir_old; # no change, likely
-
- ($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
- return $lei->fail(<<EOM);
-conversions from/to v2 inboxes not supported at this time
-EOM
-
- return $lei->fail(<<EOM) if -e $dir_new;
-lei.q.output changed from `$old_out' to `$new_out'
-However, $dir_new exists
-EOM
- # start the conversion asynchronously
- my $old_sq = PublicInbox::Config::squote_maybe($old_out);
- my $new_sq = PublicInbox::Config::squote_maybe($new_out);
- $lei->puts("lei.q.output changed from $old_sq to $new_sq");
- $lei->qerr("# lei convert $old_sq -o $new_sq");
- my $v = !$lei->{opt}->{quiet};
- $lei->{opt} = { output => $new_out, verbose => $v };
- require PublicInbox::LeiConvert;
- PublicInbox::LeiConvert::lei_convert($lei, $old_out);
-
- $lei->fail(<<EOM) if -e $dir_old && !rename($dir_old, $dir_new);
-E: rename($dir_old, $dir_new) error: $!
-EOM
-}
-
# cf. LeiDedupe->has_entries
sub has_entries {
my $oidx = $_[0]->{oidx} // die 'BUG: no {oidx}';
diff --git a/script/lei b/script/lei
index 99d94b4e..2d84487a 100755
--- a/script/lei
+++ b/script/lei
@@ -127,6 +127,8 @@ while (1) {
last;
} elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
$x_it_code ||= $1 + 0;
+ } elsif ($buf eq 'wait') {
+ $sigchld->();
} else {
$sigchld->();
die $buf;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 743a7b70..9c17a011 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -215,5 +215,22 @@ test_lei(sub {
'absolute path appears in ls-search';
lei_ok qw(up ../s -C), "$home/v2s", \'relative lei up';
lei_ok qw(up), "$home/s", \'absolute lei up';
+
+ # mess up a config file
+ my @lss = glob("$home/" .
+ '.local/share/lei/saved-searches/*/lei.saved-search');
+ my $out = xqx([qw(git config -f), $lss[0], 'lei.q.output']);
+ xsys($^X, qw(-i -p -e), "s/\\[/\\0/", $lss[0])
+ and xbail "-ipe $lss[0]: $?";
+ lei_ok qw(ls-search);
+ like($lei_err, qr/bad config line.*?\Q$lss[0]\E/,
+ 'git config parse error shown w/ lei ls-search');
+ lei_ok qw(up --all), \'up works with bad config';
+ like($lei_err, qr/bad config line.*?\Q$lss[0]\E/,
+ 'git config parse error shown w/ lei up');
+ xsys($^X, qw(-i -p -e), "s/\\0/\\[/", $lss[0])
+ and xbail "-ipe $lss[0]: $?";
+ lei_ok qw(ls-search);
+ is($lei_err, '', 'no errors w/ fixed config');
});
done_testing;
^ permalink raw reply related [relevance 53%]
* [PATCH 0/1] lei q -f reply ... perhaps my new favorite feature
@ 2021-09-11 8:33 71% Eric Wong
2021-09-11 8:33 40% ` [PATCH 1/1] lei q|lcat: support "-f reply" output format Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-11 8:33 UTC (permalink / raw)
To: meta
I think this deserves a cover-letter even if it's just one patch :>
Eric Wong (1):
lei q|lcat: support "-f reply" output format
Documentation/lei-lcat.pod | 10 +++++
Documentation/lei-q.pod | 11 ++++-
lib/PublicInbox/LeiToMail.pm | 4 +-
lib/PublicInbox/LeiViewText.pm | 82 +++++++++++++++++++++++++++++-----
t/lei-lcat.t | 14 ++++++
5 files changed, 106 insertions(+), 15 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/1] lei q|lcat: support "-f reply" output format
2021-09-11 8:33 71% [PATCH 0/1] lei q -f reply ... perhaps my new favorite feature Eric Wong
@ 2021-09-11 8:33 40% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 8:33 UTC (permalink / raw)
To: meta
When composing replies in "git format-patch" cover letters,
I'd been relying on "lei q -f text ...", but that still requires
several steps to make it suitable for composing a reply:
* s/^/> / to quote the body
* drop existing In-Reply-To+References
* s/^Message-ID:/In-Reply-To:/;
* add an attribute line
...
"lei q -f reply" takes care of most of that and users will
only have to trim "From " lines, unnecessary results and
over-quoted text (and trimming is likely less error-prone
than doing all the steps above manually).
This should also be a good replacement for
"git format-patch --in-reply-to=...", since copying long
Message-IDs can be error-prone (and this lets you include
quoted text in replies).
---
Documentation/lei-lcat.pod | 10 +++++
Documentation/lei-q.pod | 11 ++++-
lib/PublicInbox/LeiToMail.pm | 4 +-
lib/PublicInbox/LeiViewText.pm | 82 +++++++++++++++++++++++++++++-----
t/lei-lcat.t | 14 ++++++
5 files changed, 106 insertions(+), 15 deletions(-)
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
index 656df489..b7887b6c 100644
--- a/Documentation/lei-lcat.pod
+++ b/Documentation/lei-lcat.pod
@@ -20,9 +20,19 @@ Message-ID or link from surrounding text (e.g., a "Link: $URL" line).
=head1 OPTIONS
The following options, described in L<lei-q(1)>, are supported.
+One deviation from L<lei-q(1)> is the default output format is
+C<-f text> when writing to stdout.
=over
+=item --format=FORMAT
+
+=item -f FORMAT
+
+Most commonly C<text> (the default) or C<reply> to
+display the message(s) in a format suitable for trimming
+and sending as a email reply.
+
=item --[no-]remote
=item --no-local
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 69a6cdf2..1d9e66cd 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -60,7 +60,7 @@ Default: C<-> (stdout)
Format of results to stdout. This option exists as a convenient
way to specify the format for the default stdout destination.
-C<text>, C<json>, C<jsonl>, or C<concatjson> are all supported,
+C<reply>, C<text>, C<json>, C<jsonl>, or C<concatjson> are all supported,
as are the various mbox variants described in L</--output>.
When a format isn't specified, it's chosen based on the
@@ -72,7 +72,7 @@ preferred when not writing to stdout.
=item --no-color
-Disable color (for C<--format=text>).
+Disable color (for C<-f reply> and C<-f text>).
=item --pretty
@@ -241,6 +241,13 @@ Default: C<auto>
=back
+=head1 TIPS
+
+C<-f reply> is intended to aid in turning a cover letter
+into a reply (since using C<git format-patch --in-reply-to=...>
+is tedious). Results (including "From " lines) should be edited
+and trimmed in your favorite C<$EDITOR> before sending.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index dbf58df9..15729bda 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -410,9 +410,9 @@ sub new {
$lei->{net} = $net;
$self->{base_type} = 'imap';
$lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
- } elsif ($fmt eq 'text') {
+ } elsif ($fmt eq 'text' || $fmt eq 'reply') {
require PublicInbox::LeiViewText;
- $lei->{lvt} = PublicInbox::LeiViewText->new($lei);
+ $lei->{lvt} = PublicInbox::LeiViewText->new($lei, $fmt);
$self->{base_type} = 'text';
@conflict = qw(mua save);
} elsif ($fmt eq 'v2') {
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 340a6648..34612711 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -13,6 +13,8 @@ use PublicInbox::Hval;
use PublicInbox::ViewDiff;
use PublicInbox::Spawn qw(popen_rd);
use Term::ANSIColor;
+use POSIX ();
+use PublicInbox::Address;
sub _xs {
# xhtml_map works since we don't search for HTML ([&<>'"])
@@ -66,8 +68,9 @@ sub my_colored {
sub uncolored { ${$_[0]->{obuf}} .= $_[2] }
sub new {
- my ($cls, $lei) = @_;
+ my ($cls, $lei, $fmt) = @_;
my $self = bless { %{$lei->{opt}}, -colored => \&uncolored }, $cls;
+ $self->{-quote_reply} = 1 if $fmt eq 'reply';
return $self unless $self->{color} //= -t $lei->{1};
my $cmd = [ qw(git config -z --includes -l) ];
my ($r, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
@@ -83,6 +86,45 @@ sub new {
$self;
}
+sub quote_hdr_buf ($$) {
+ my ($self, $eml) = @_;
+ my $hbuf = '';
+ my $to = $eml->header_raw('Reply-To') //
+ $eml->header_raw('From') //
+ $eml->header_raw('Sender');
+ my $cc = '';
+ for my $f (qw(To Cc)) {
+ for my $v ($eml->header_raw($f)) {
+ next if $v !~ /\S/;
+ $cc .= $v;
+ $to //= $v;
+ }
+ }
+ PublicInbox::View::fold_addresses($to);
+ PublicInbox::View::fold_addresses($cc);
+ _xs($to);
+ _xs($cc);
+ $hbuf .= "To: $to\n" if defined $to && $to =~ /\S/;
+ $hbuf .= "Cc: $cc\n" if $cc =~ /\S/;
+ my $s = $eml->header_str('Subject') // 'your mail';
+ _xs($s);
+ substr($s, 0, 0, 'Re: ') if $s !~ /\bRe:/i;
+ $hbuf .= "Subject: $s\n";
+ if (defined(my $irt = $eml->header_raw('Message-ID'))) {
+ _xs($irt);
+ $hbuf .= "In-Reply-To: $irt\n";
+ }
+ $self->{-colored}->($self, 'hdrdefault', $hbuf);
+ my ($n) = PublicInbox::Address::names($eml->header_str('From') //
+ $eml->header_str('Sender') //
+ $eml->header_str('Reply-To') //
+ 'unknown sender');
+ my $d = $eml->header_raw('Date') // 'some unknown date';
+ _xs($d);
+ _xs($n);
+ ${delete $self->{obuf}} . "\nOn $d, $n wrote:\n";
+}
+
sub hdr_buf ($$) {
my ($self, $eml) = @_;
my $hbuf = '';
@@ -224,25 +266,43 @@ sub add_text_buf { # callback for Eml->each_part
}
}
-# returns an arrayref suitable for $lei->out or print
+# returns a stringref suitable for $lei->out or print
sub eml_to_text {
my ($self, $smsg, $eml) = @_;
local $Term::ANSIColor::EACHLINE = "\n";
$self->{obuf} = \(my $obuf = '');
$self->{-smsg} = $smsg;
$self->{-max_cols} = ($self->{columns} //= 80) - 8; # for header wrap
- my @h = ();
- for my $f (qw(blob pct)) {
- push @h, "$f:$smsg->{$f}" if defined $smsg->{$f};
+ my $h = [];
+ if ($self->{-quote_reply}) {
+ my $blob = $smsg->{blob} // 'unknown-blob';
+ my $pct = $smsg->{pct} // 'unknown';
+ my $t = POSIX::asctime(gmtime($smsg->{ts} // $smsg->{ds} // 0));
+ $h->[0] = "From $blob\@$pct $t";
+ } else {
+ for my $f (qw(blob pct)) {
+ push @$h, "$f:$smsg->{$f}" if defined $smsg->{$f};
+ }
+ @$h = ("# @$h\n") if @$h;
+ for my $f (qw(kw L)) {
+ my $v = $smsg->{$f} or next;
+ push @$h, "# $f:".join(',', @$v)."\n" if @$v;
+ }
}
- @h = ("# @h\n") if @h;
- for my $f (qw(kw L)) {
- my $v = $smsg->{$f} or next;
- push @h, "# $f:".join(',', @$v)."\n" if @$v;
+ $h = join('', @$h);
+ $self->{-colored}->($self, 'status', $h);
+ my $quote_hdr;
+ if ($self->{-quote_reply}) {
+ $quote_hdr = ${delete $self->{obuf}};
+ $quote_hdr .= quote_hdr_buf($self, $eml);
+ } else {
+ hdr_buf($self, $eml);
}
- $self->{-colored}->($self, 'status', join('', @h));
- hdr_buf($self, $eml);
$eml->each_part(\&add_text_buf, $self, 1);
+ if (defined $quote_hdr) {
+ ${$self->{obuf}} =~ s/^/> /sgm;
+ substr(${$self->{obuf}}, 0, 0, $quote_hdr);
+ }
delete $self->{obuf};
}
diff --git a/t/lei-lcat.t b/t/lei-lcat.t
index e5f00706..31a84744 100644
--- a/t/lei-lcat.t
+++ b/t/lei-lcat.t
@@ -11,6 +11,20 @@ test_lei(sub {
lei_ok('import', 't/plack-qp.eml');
lei_ok([qw(lcat --stdin)], undef, { 0 => \$in, %$lei_opt });
like($lei_out, qr/qp\@example\.com/, 'got a result');
+
+ # test Link:, -f reply, and implicit --stdin:
+ my $prev = $lei_out;
+ $in = "\nLink: https://example.com/foo/qp\@example.com/\n";
+ lei_ok([qw(lcat -f reply)], undef, { 0 => \$in, %$lei_opt});
+ my $exp = <<'EOM';
+To: qp@example.com
+Subject: Re: QP
+In-Reply-To: <qp@example.com>
+
+On some unknown date, qp wrote:
+> hi = bye
+EOM
+ like($lei_out, qr/\AFrom [^\n]+\n\Q$exp\E/sm, '-f reply works');
});
done_testing;
^ permalink raw reply related [relevance 40%]
* [PATCH 1/3] t/lei-*.t: guard setup_public_inboxes with test_lei
@ 2021-09-12 8:42 67% ` Eric Wong
2021-09-12 8:42 65% ` [PATCH 2/3] lei sucks: allow it to work without SQLite Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-12 8:42 UTC (permalink / raw)
To: meta
This ensures tests are skipped properly if SQLite or Xapian
are missing and don't bail out.
---
t/lei-auto-watch.t | 3 +--
t/lei-up.t | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index 3b0c1b10..e5e132eb 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -3,14 +3,13 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Basename qw(basename);
-my ($ro_home, $cfg_path) = setup_public_inboxes;
my $have_fast_inotify = eval { require Linux::Inotify2 } ||
eval { require IO::KQueue };
-
$have_fast_inotify or
diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
test_lei(sub {
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
my $x = "$ENV{HOME}/x";
my $y = "$ENV{HOME}/y";
lei_ok qw(add-external), "$ro_home/t1";
diff --git a/t/lei-up.t b/t/lei-up.t
index c6f31c74..6b34774d 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -2,9 +2,9 @@
# Copyright all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
-my ($ro_home, $cfg_path) = setup_public_inboxes;
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
test_lei(sub {
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
my $s = eml_load('t/plack-qp.eml')->as_string;
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a.mbox.gz";
^ permalink raw reply related [relevance 67%]
* [PATCH 2/3] lei sucks: allow it to work without SQLite
2021-09-12 8:42 67% ` [PATCH 1/3] t/lei-*.t: guard setup_public_inboxes with test_lei Eric Wong
@ 2021-09-12 8:42 65% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-12 8:42 UTC (permalink / raw)
To: meta
And try to improve the message about Inline::C while we're at
it, since Socket::Msghdr isn't widely-packaged, yet.
---
lib/PublicInbox/LeiSucks.pm | 4 ++--
lib/PublicInbox/TestCommon.pm | 12 +++++++-----
t/lei.t | 4 ++++
3 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index 3e945d0b..e832f95e 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -11,7 +11,6 @@ use Digest::SHA ();
use Config;
use POSIX ();
use PublicInbox::Config;
-use PublicInbox::Search;
sub lei_sucks {
my ($lei, @argv) = @_;
@@ -41,7 +40,8 @@ sub lei_sucks {
} else {
push @out, "Unable to load DBI / DBD::SQLite: $@\n";
}
- if (PublicInbox::Search::load_xapian()) {
+ if (eval { require PublicInbox::Search } &&
+ PublicInbox::Search::load_xapian()) {
push @out, 'Xapian '.
join('.', map {
$PublicInbox::Search::Xap->can($_)->();
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 14dac03f..d28b32b0 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -521,7 +521,8 @@ SKIP: {
local $lei_cwdfh;
opendir $lei_cwdfh, '.' or xbail "opendir .: $!";
require_git(2.6, 1) or skip('git 2.6+ required for lei test', 2);
- require_mods(qw(json DBD::SQLite Search::Xapian), 2);
+ my $mods = $test_opt->{mods} // [ 'lei' ];
+ require_mods(@$mods, 2);
require PublicInbox::Config;
require File::Path;
local %ENV = %ENV;
@@ -534,10 +535,11 @@ SKIP: {
require PublicInbox::Spawn;
state $lei_daemon = PublicInbox::Spawn->can('send_cmd4') ||
eval { require Socket::MsgHdr; 1 };
- # XXX fix and move this inside daemon-only before 1.7 release
- skip <<'EOM', 1 unless $lei_daemon;
-Socket::MsgHdr missing or Inline::C is unconfigured/missing
-EOM
+ unless ($lei_daemon) {
+ skip('Inline::C unconfigured/missing '.
+'(mkdir -p ~/.cache/public-inbox/inline-c) OR Socket::MsgHdr missing',
+ 1);
+ }
$lei_opt = { 1 => \$lei_out, 2 => \$lei_err };
my ($daemon_pid, $for_destroy, $daemon_xrd);
my $tmpdir = $test_opt->{tmpdir};
diff --git a/t/lei.t b/t/lei.t
index dfbcb1f3..d1f1cbc0 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -176,4 +176,8 @@ test_lei(sub {
$test_fail->();
});
+test_lei({ mods => [] }, sub {
+ lei_ok('sucks', \'no optional modules required');
+});
+
done_testing;
^ permalink raw reply related [relevance 65%]
* [PATCH 4/6] lei up: localize %ENV in redispatch
2021-09-13 20:53 67% ` [PATCH 2/6] lei: stop_pager: restore stdout when done Eric Wong
@ 2021-09-13 20:53 71% ` Eric Wong
2021-09-13 20:53 62% ` [PATCH 6/6] lei up: fix --mua with single output Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-13 20:53 UTC (permalink / raw)
To: meta
We need to restore %ENV of script/lei in case another
lei client has a different %ENV than what daemon has.
---
lib/PublicInbox/LeiUp.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index be476427..a16117c9 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -55,13 +55,14 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
my $l = bless { %$lei }, ref($lei);
- $l->{opt} = { %{$l->{opt}} };
+ $l->{opt} = { %{$l->{opt}} }; # deep copy
delete $l->{sock}; # do not close
$l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
local $PublicInbox::LEI::current_lei = $l;
+ local %ENV = %{$l->{env}};
$l->{1} = $fh;
eval {
$l->qerr("# updating $out");
^ permalink raw reply related [relevance 71%]
* [PATCH 2/6] lei: stop_pager: restore stdout when done
@ 2021-09-13 20:53 67% ` Eric Wong
2021-09-13 20:53 71% ` [PATCH 4/6] lei up: localize %ENV in redispatch Eric Wong
2021-09-13 20:53 62% ` [PATCH 6/6] lei up: fix --mua with single output Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-13 20:53 UTC (permalink / raw)
To: meta
The reason(s) we had for not restoring stdout haven't been valid
since script/lei (and not lei-daemon) spawns the pager, nowadays.
---
lib/PublicInbox/LEI.pm | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6d5d3c03..784e679d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -464,11 +464,11 @@ sub x_it ($$) {
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
- if ($self->{pkt_op_p}) { # to top lei-daemon
+ if ($self->{pkt_op_p}) { # worker => lei-daemon
$self->{pkt_op_p}->pkt_do('x_it', $code);
- } elsif ($self->{sock}) { # to lei(1) client
+ } elsif ($self->{sock}) { # lei->daemon => lei(1) client
send($self->{sock}, "x_it $code", MSG_EOR);
- } elsif ($quit == \&CORE::exit) { # an admin command
+ } elsif ($quit == \&CORE::exit) { # an admin (one-shot) command
exit($code >> 8);
} # else ignore if client disconnected
}
@@ -1065,9 +1065,7 @@ sub pgr_err {
start_pager($self, { LESS => 'RX' }); # no 'F' so we prompt
print { $self->{2} } @msg;
$self->{2}->autoflush(1);
- my $pgr = delete($self->{pgr}) or return;
- $self->{2} = $pgr->[2];
- $self->{1} = $pgr->[1];
+ stop_pager($self);
send($self->{sock}, 'wait', MSG_EOR); # wait for user to quit pager
}
@@ -1075,8 +1073,8 @@ sub stop_pager {
my ($self) = @_;
my $pgr = delete($self->{pgr}) or return;
$self->{2} = $pgr->[2];
- # do not restore original stdout, just close it so we error out
close(delete($self->{1})) if $self->{1};
+ $self->{1} = $pgr->[1];
}
sub accept_dispatch { # Listener {post_accept} callback
^ permalink raw reply related [relevance 67%]
* [PATCH 6/6] lei up: fix --mua with single output
2021-09-13 20:53 67% ` [PATCH 2/6] lei: stop_pager: restore stdout when done Eric Wong
2021-09-13 20:53 71% ` [PATCH 4/6] lei up: localize %ENV in redispatch Eric Wong
@ 2021-09-13 20:53 62% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-13 20:53 UTC (permalink / raw)
To: meta
Oops :x
Fixes: b584a53f053a7629 ("lei up: support --all for IMAP folders")
---
lib/PublicInbox/LeiUp.pm | 25 +++++++++++++++----------
t/lei-up.t | 4 ++++
2 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index a16117c9..53f06dbc 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -54,16 +54,21 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
- my $l = bless { %$lei }, ref($lei);
- $l->{opt} = { %{$l->{opt}} }; # deep copy
- delete $l->{sock}; # do not close
- $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
-
- # make close($l->{1}) happy in lei->dclose
- open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ my $l;
+ if (defined($lei->{opt}->{mua})) { # single output
+ $l = $lei;
+ } else { # multiple outputs
+ $l = bless { %$lei }, ref($lei);
+ $l->{opt} = { %{$l->{opt}} }; # deep copy
+ delete $l->{sock}; # do not close
+ # make close($l->{1}) happy in lei->dclose
+ open my $fh, '>&', $l->{1} or
+ return $l->child_error(0, "dup: $!");
+ $l->{1} = $fh;
+ }
local $PublicInbox::LEI::current_lei = $l;
local %ENV = %{$l->{env}};
- $l->{1} = $fh;
+ $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
eval {
$l->qerr("# updating $out");
up1($l, $out);
@@ -92,7 +97,7 @@ sub lei_up {
$lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
- length($opt->{mua}//'') and return
+ defined($opt->{mua}) and return
$lei->fail('--all and --mua= are incompatible');
@outs = PublicInbox::LeiSavedSearch::list($lei);
if ($all eq 'local') {
@@ -110,7 +115,7 @@ sub lei_up {
$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
- length($opt->{mua} // '')) and return $lei->fail(<<EOM);
+ defined($opt->{mua})) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
if ($self->{remote}) { # setup lei->{auth}
diff --git a/t/lei-up.t b/t/lei-up.t
index 6b34774d..8937cfb1 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -34,6 +34,10 @@ test_lei(sub {
open $fh, "$ENV{HOME}/b" or xbail "open: $!";
$uc = do { local $/; <$fh> };
is($uc, $exp, 'uncompressed both match');
+
+ lei_ok [ 'up', "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
+ undef, { run_mode => 0 };
+ ok(-f "$ENV{HOME}/c", '--mua works with single output');
});
done_testing;
^ permalink raw reply related [relevance 62%]
* [PATCH 1/5] lei: warn on event loop errors
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
@ 2021-09-14 2:39 71% ` Eric Wong
2021-09-14 2:39 71% ` [PATCH 2/5] lei: sto_done_request: add eval guard Eric Wong
2021-09-14 2:39 56% ` [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
This should help us notice (and fix) bugs more easily.
---
lib/PublicInbox/LEI.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 784e679d..f0caac03 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1347,7 +1347,8 @@ sub lazy_start {
open STDERR, '>&STDIN' or die "redirect stderr failed: $!";
open STDOUT, '>&STDIN' or die "redirect stdout failed: $!";
# $daemon pipe to `lei' closed, main loop begins:
- PublicInbox::DS->EventLoop;
+ eval { PublicInbox::DS->EventLoop };
+ warn "event loop error: $@\n" if $@;
dump_and_clear_log();
exit($exit_code // 0);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes
@ 2021-09-14 2:39 71% Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
The 5/5 "lei up" fix has real-world implications if you're
using it in parallel. Otherwise, it's test-only stuff, and
`make check-run TEST_LEI_DAEMON_PERSIST=1 N=$(nproc)`
seems to fail or stall less often than before (but it
still can...).
(persisting the lei-daemon is around 5% faster with check-run)
Eric Wong (5):
lei: warn on event loop errors
lei: sto_done_request: add eval guard
t/run: TEST_LEI_DAEMON_PERSIST: die if pid changes
test_common: remove non-hidden files, first
lei up: fix env/cwd mismatches with multiple folders
lib/PublicInbox/LEI.pm | 33 ++++++++++++++++-----------------
lib/PublicInbox/LeiUp.pm | 15 +++++++--------
lib/PublicInbox/TestCommon.pm | 5 ++++-
t/run.perl | 3 +++
4 files changed, 30 insertions(+), 26 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/5] lei: sto_done_request: add eval guard
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
@ 2021-09-14 2:39 71% ` Eric Wong
2021-09-14 2:39 56% ` [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
Failures here can cause the lei-daemon event loop to break
since PktOp doesn't guard dispatch. Add a guard here (and
not deeper in the stack) so we can use the $lei object to
report errors.
---
lib/PublicInbox/LEI.pm | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f0caac03..e529c86a 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1500,12 +1500,15 @@ sub lms { # read-only LeiMailSync
sub sto_done_request { # only call this from lei-daemon process (not workers)
my ($lei, $sock) = @_;
- if ($sock //= $lei->{sock}) {
- $LIVE_SOCK{"$sock"} = $sock;
- $lei->{sto}->ipc_do('done', "$sock"); # issue, async wait
- } else { # forcibly wait
- my $wait = $lei->{sto}->ipc_do('done');
- }
+ eval {
+ if ($sock //= $lei->{sock}) { # issue, async wait
+ $LIVE_SOCK{"$sock"} = $sock;
+ $lei->{sto}->ipc_do('done', "$sock");
+ } else { # forcibly wait
+ my $wait = $lei->{sto}->ipc_do('done');
+ }
+ };
+ $lei->err($@) if $@;
}
sub sto_done_complete { # called in lei-daemon when LeiStore->done is complete
^ permalink raw reply related [relevance 71%]
* [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
2021-09-14 2:39 71% ` [PATCH 2/5] lei: sto_done_request: add eval guard Eric Wong
@ 2021-09-14 2:39 56% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
By moving %ENV localization and fchdir into ->dispatch,
we can maintain a consistent environment across multiple
dispatches while having different clients.
---
lib/PublicInbox/LEI.pm | 15 +++++----------
lib/PublicInbox/LeiUp.pm | 15 +++++++--------
2 files changed, 12 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e529c86a..0a30bc36 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -769,6 +769,8 @@ sub lazy_cb ($$$) {
sub dispatch {
my ($self, $cmd, @argv) = @_;
+ fchdir($self) or return;
+ local %ENV = %{$self->{env}};
local $current_lei = $self; # for __WARN__
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
return _help($self, 'no command given') unless defined($cmd);
@@ -1104,14 +1106,9 @@ sub accept_dispatch { # Listener {post_accept} callback
my ($argc, @argv) = split(/\0/, $buf, -1);
undef $buf;
my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
- if (chdir($self->{3})) {
- local %ENV = %env;
- $self->{env} = \%env;
- eval { dispatch($self, @argv) };
- send($sock, $@, MSG_EOR) if $@;
- } else {
- send($sock, "fchdir: $!", MSG_EOR); # implicit close
- }
+ $self->{env} = \%env;
+ eval { dispatch($self, @argv) };
+ send($sock, $@, MSG_EOR) if $@;
}
sub dclose {
@@ -1181,7 +1178,6 @@ sub cfg2lei ($) {
open($lei->{1}, '>>&', \*STDOUT) or die "dup 1: $!";
open($lei->{2}, '>>&', \*STDERR) or die "dup 2: $!";
open($lei->{3}, '/') or die "open /: $!";
- chdir($lei->{3}) or die "chdir /': $!";
my ($x, $y);
socketpair($x, $y, AF_UNIX, SOCK_SEQPACKET, 0) or die "socketpair: $!";
$lei->{sock} = $x;
@@ -1199,7 +1195,6 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
my $cfg = $PATH2CFG{$f} // next;
eval {
- local %ENV = %{$cfg->{-env}};
my $lei = cfg2lei($cfg);
$lei->dispatch('note-event',
"maildir:$mdir", $nc, $bn, $fn);
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 53f06dbc..4637cb46 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -11,7 +11,6 @@ use PublicInbox::LeiSavedSearch;
use PublicInbox::DS;
use PublicInbox::PktOp;
use PublicInbox::LeiFinmsg;
-use PublicInbox::LEI;
my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
sub up1 ($$) {
@@ -60,19 +59,16 @@ sub up1_redispatch {
} else { # multiple outputs
$l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} }; # deep copy
+ delete $l->{opt}->{all};
delete $l->{sock}; # do not close
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or
return $l->child_error(0, "dup: $!");
$l->{1} = $fh;
+ $l->qerr("# updating $out");
}
- local $PublicInbox::LEI::current_lei = $l;
- local %ENV = %{$l->{env}};
$l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
- eval {
- $l->qerr("# updating $out");
- up1($l, $out);
- };
+ eval { $l->dispatch('up', $out) };
$lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
}
@@ -94,7 +90,6 @@ sub lei_up {
my ($lei, @outs) = @_;
my $opt = $lei->{opt};
my $self = bless { -mail_sync => 1 }, __PACKAGE__;
- $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
defined($opt->{mua}) and return
@@ -110,10 +105,14 @@ sub lei_up {
} else {
$lei->fail("only --all=$all not understood");
}
+ } elsif ($lei->{lse}) {
+ scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
+ return up1($lei, $outs[0]);
} else {
$self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
+ $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
defined($opt->{mua})) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
^ permalink raw reply related [relevance 56%]
* RFC: lei-daemon and auto-up
@ 2021-09-14 20:42 71% Konstantin Ryabitsev
2021-09-14 22:32 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-14 20:42 UTC (permalink / raw)
To: meta
Hello:
Since lei-daemon is already up and running, would it be possible to tell it to
automatically "lei up" things at certain intervals?
Maybe something like:
[lei]
q = [...]
[lei "q"]
output = [...]
include = https://lore.kernel.org/all/
external = 1
local = 1
remote = 1
refresh = 300
That would allow folks to automatically get updated info without needing to
set up cronjobs or systemd timers.
-K
^ permalink raw reply [relevance 71%]
* Re: RFC: lei-daemon and auto-up
2021-09-14 20:42 71% RFC: lei-daemon and auto-up Konstantin Ryabitsev
@ 2021-09-14 22:32 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 22:32 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Hello:
>
> Since lei-daemon is already up and running, would it be possible to tell it to
> automatically "lei up" things at certain intervals?
Yes, something along those lines... It should at least fall
back to refresh intervals if network connectivity is lost or
HTTP servers w/o support for long-polling.
The ideal thing would be to POST HTTP requests that do
long-polling against -httpd and let curl wait on the response;
similar to IMAP IDLE.
-httpd will probably trickle zero-byte gzipped responses at a
regular intervals to keep connections alive if there is no data,
yet.
There's still a lot of things I want to do w/ inotify and
IDLE(-like) real-time notifications (including the new
public-inbox-fetch command). I don't like polling as a first
choice.
Also. I'm not sure what ways exist to be notified on network
up/down events on Linux or *BSDs w/o polling.
^ permalink raw reply [relevance 71%]
* make menuconfig interface for lei / grok-pull
@ 2021-09-15 21:34 52% Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Luis Chamberlain @ 2021-09-15 21:34 UTC (permalink / raw)
To: meta; +Cc: Luis Chamberlain
Greetings,
I unsubscribed to most mailing lists thanks to public-inbox. Thanks
to public-inbox my inbox went on quite a bit of diet.
Now, the way I work though is... I like to hop onto different systems
every now and then, and not rely on one central place for everything.
This is because when I've done that before I get outages / network
outages / etc.
So I often have to start from scratch.
My use case is I'm subscribed to a few kernel mailign lists and I use
mutt with Maildir. I had configured recently pi-piper and grokmirror
so that I get only the last 1 year of email from a few set of mailing
lists. For this I needed to know the commit IDs for those emails on
the public-inbox git mirror for each mailing list.
To start from scratch I could go into my logs and figure out what that
was again, and start from scratch... but I'm inpsired to just provide
a 'make menuconfig' interface for this so that all I have to do is
select the mailing list I want to git clone and then dump into Maildir
the last 1 year of content for each. Since this time frame of 1 year
can be variable, it would seem a good candidate would be to use kconfig
to allow those interested to associate the content year by year to git
tree and respective commit IDs on each mailing list epoch/git tree.
I was hinted using lei would be better though. But I'm stuck:
lei init
# The next two don't work
lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
I can use nttp but jeesh, really?
Anyway, once the above works, if one didn't want to have the git
entire git tree around I gather that I can use something like
lei index <same-path-as-above>
Then in so far as the date idea I mentioned, I take it the way to go
there might be after this to use something like:
lei q --only=linux-fsdevel <search-term>
but its not clear if a date is supported as a search term.
Also how about doing updates?
Is my intended use case better with grok-pull or is there
hopes I can end up using lei for all what I have intentions for?
My vision is, kernel deveoper would 'make menuconfig' and select
a few mailing lists they want to import, then when one is selected
new options appear so you can pick the starting year from where
you want the import to occur for. Then for each mailing list there
is also a refresh thing, so that we can grok-pull differently
for each mailing list, which will be handled by systemd timers
and a service for each. The reason for differnt times for refresh
is updating linux-kernel is not a priority to me, but linux-fsdevel
and linux-block are, so I want linux-fsdevel and linux-block to
refresh once every 10 minutes, while linux-kernel can update...
I don't know once every 8 hours. Anyway, this should all be
configurable. And my hope is that developers would not have to
know what's going on behind the scenes. All they need to know
is that their ~Mail/linux/ directory will eventually be stuffed
with Maildir directories for the mailing lists they enabled and
they'll be updated as often as specified in their configuration.
Luis
^ permalink raw reply [relevance 52%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
@ 2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
` (2 more replies)
2021-09-15 23:06 57% ` Eric Wong
` (2 subsequent siblings)
3 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-15 21:44 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> # The next two don't work
> lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
Apologies, I need to document this better. This should work:
lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
> Also how about doing updates?
public-inbox-fetch -C SOME_FS_PATH
public-inbox-index SOME_FS_PATH
And I just posted some patches to simplify updates to:
make -C SOME_FS_PATH
In case you reorganize your file system:
lei forget-external OLD_FS_PATH
lei add-external NEW_FS_PATH
I'll read the rest and reply later. "lei index" is still
half-baked, but the above should be fine.
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:44 71% ` Eric Wong
@ 2021-09-15 21:57 71% ` Luis Chamberlain
2021-09-15 23:12 71% ` Eric Wong
2021-09-15 22:36 71% ` Kyle Meyer
2021-09-16 2:59 71% ` Eric Wong
2 siblings, 1 reply; 200+ results
From: Luis Chamberlain @ 2021-09-15 21:57 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Wed, Sep 15, 2021 at 09:44:20PM +0000, Eric Wong wrote:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
> > # The next two don't work
> > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> > lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
>
> Apologies, I need to document this better. This should work:
>
> lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
Groovy thanks! I'll start with that!
Any reason we need to specify SOME_FS_PATH ? Can't we assume a sensible
default somewhere in .local/lei or whatever in the future?
> > Also how about doing updates?
>
> public-inbox-fetch -C SOME_FS_PATH
> public-inbox-index SOME_FS_PATH
Neat!
> And I just posted some patches to simplify updates to:
>
> make -C SOME_FS_PATH
Oh, pretty nice.
> In case you reorganize your file system:
>
> lei forget-external OLD_FS_PATH
> lei add-external NEW_FS_PATH
>
> I'll read the rest and reply later. "lei index" is still
> half-baked, but the above should be fine.
Thanks!
Luis
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
@ 2021-09-15 22:36 71% ` Kyle Meyer
2021-09-15 23:31 71% ` Eric Wong
2021-09-16 2:59 71% ` Eric Wong
2 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-15 22:36 UTC (permalink / raw)
To: Eric Wong; +Cc: Luis Chamberlain, meta
Eric Wong writes:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
>> # The next two don't work
>> lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
>> lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
>
> Apologies, I need to document this better. This should work:
A good chunk of the current documentation was fumbled together by me, so
I'm sure there are many spots for improvement but...
> lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
... hey, the equivalent of the snippet above is one of the few things
that made it into lei-overview :)
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
@ 2021-09-15 23:06 57% ` Eric Wong
2021-09-16 17:43 71% ` Konstantin Ryabitsev
2021-09-19 21:21 71% ` make menuconfig interface for lei / grok-pull Eric Wong
2021-09-16 17:38 71% ` Konstantin Ryabitsev
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
3 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-15 23:06 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> Greetings,
>
> I unsubscribed to most mailing lists thanks to public-inbox. Thanks
> to public-inbox my inbox went on quite a bit of diet.
>
> Now, the way I work though is... I like to hop onto different systems
> every now and then, and not rely on one central place for everything.
> This is because when I've done that before I get outages / network
> outages / etc.
>
> So I often have to start from scratch.
Same here, this goes for project code repos, as well...
At the same time, I don't want too much synchronization since
the systems I hop on to vary hugely in storage capacity.
<snip replied parts>
> lei q --only=linux-fsdevel <search-term>
>
> but its not clear if a date is supported as a search term.
All the search prefixes documented in
https://public-inbox.org/meta/_/text/help/ work for lei.
dt: (date-time), d: (date (mairix-compatible) have been there a while,
rt: (Received-time) is new.
They gained support for git approxidate recently, so "rt:last.week.."
gives you all the messages that were received from last week til now.
Anything w/o spaces which git understands should work.
> Is my intended use case better with grok-pull or is there
> hopes I can end up using lei for all what I have intentions for?
I think introducing support for "lei update-external [--all]"
could be useful combined with my previous note about
add-external.
> My vision is, kernel deveoper would 'make menuconfig' and select
> a few mailing lists they want to import, then when one is selected
> new options appear so you can pick the starting year from where
> you want the import to occur for. Then for each mailing list there
> is also a refresh thing, so that we can grok-pull differently
> for each mailing list, which will be handled by systemd timers
> and a service for each. The reason for differnt times for refresh
> is updating linux-kernel is not a priority to me, but linux-fsdevel
> and linux-block are, so I want linux-fsdevel and linux-block to
> refresh once every 10 minutes, while linux-kernel can update...
> I don't know once every 8 hours. Anyway, this should all be
> configurable. And my hope is that developers would not have to
> know what's going on behind the scenes. All they need to know
> is that their ~Mail/linux/ directory will eventually be stuffed
> with Maildir directories for the mailing lists they enabled and
> they'll be updated as often as specified in their configuration.
Interesting... I suppose it could work. I don't know much
about curses or interactive UIs, but "make menuconfig" isn't
a bad UI :) I've been halfway envisioning something else
that would use curses (more on that some other time).
Does lore.kernel.org run public-inbox-imapd?
Well, at least nntp works for lore:
# populate ~/.cache/lei/uri_folder/.sqlite3
lei ls-mail-source nntp://nntp.lore.kernel.org
# if you're using bash:
. contrib/completion/lei-completion.bash
lei import <TAB><TAB>
lei import will download the entire newsgroup, which may get
huge, unfortunately. It should support ranges (e.g.
nntp://example.com/inbox.foo.example/900-1000) but the URI
package doesn't support it, yet...
public-inbox-imapd IMAP mailbox slices are limited to 50K
messages since typical MUAs can't handle much more, so it's
easier to import just the most recent .[0-9]+ slices.
If you have Tor (on port 9050), libio-socket-socks (IO::Socket::Socks)
# long Tor v3 URL :<
url=imap://rskvuqcfnfizkjg6h5jvovwb3wkikzcwskf54lfpymus6mxrzw67b5ad.onion
# configure IMAP to use Tor on port 9050:
lei config imap.$url.proxy socks5h://0:9050
# git 2.26+ can use a wildcard for all .onions:
# lei config 'imap.imap://*.onion.proxy' socks5h://0:9050
# I also suggest enabling compression for public-inbox-imapd
# (some IMAP servers can't handle it, our -imapd does):
lei config imap.$url.compress 1
# Same as above with NNTP:
lei ls-mail-source $url
lei import <TAB-COMPLETE>
For public-inbox IMAP folders, make sure the \.[0-9]+ is at the
end of the mailbox to get the slice you want (there isn't
recursive import, yet).
Right now, "lei import" can be run periodically. The plan is to
support IMAP IDLE and auto-imports. For read/write IMAP
folders, there'd be auto keyword (FLAG) synchronization
(currently manual: lei export-kw imaps://...)
It mostly works for Maildirs at the moment...
...Which reminds me, I need to add an imapserver config variable
so the WWW interface can advertise IMAP servers like it does
NNTP servers
^ permalink raw reply [relevance 57%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:57 71% ` Luis Chamberlain
@ 2021-09-15 23:12 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-15 23:12 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> On Wed, Sep 15, 2021 at 09:44:20PM +0000, Eric Wong wrote:
> > Luis Chamberlain <mcgrof@kernel.org> wrote:
> > > # The next two don't work
> > > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> > > lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
> >
> > Apologies, I need to document this better. This should work:
> >
> > lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
>
> Groovy thanks! I'll start with that!
>
> Any reason we need to specify SOME_FS_PATH ? Can't we assume a sensible
> default somewhere in .local/lei or whatever in the future?
Probably OK to have a default path for --mirror, the
add-external w/o --mirror works for existing inboxes (which I
also serve via -httpd/-imapd/-nntpd).
I think ~/.local/public-inbox/ would be a better default location
for mirrors, since I assume anything mirrored is public (and
therefor to rehost, as well).
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 22:36 71% ` Kyle Meyer
@ 2021-09-15 23:31 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-15 23:31 UTC (permalink / raw)
To: Kyle Meyer; +Cc: Luis Chamberlain, meta
Kyle Meyer <kyle@kyleam.com> wrote:
> A good chunk of the current documentation was fumbled together by me, so
> I'm sure there are many spots for improvement but...
No worries, my brain is a disorganized mess (always has been,
but pandemic has made it worse) and I've realized a lot of my
design choices around public-inbox + lei is based around that :x
Btw, I'm thinking about replacing prune-mail-sync with
"refresh-mail-sync [--prune]", so probably no point
in documenting it just yet....
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei ls-mail-source: sort IMAP folder names
@ 2021-09-16 2:19 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 2:19 UTC (permalink / raw)
To: meta
Otherwise, public-inbox-imapd will emit mailboxes in random
order (as IMAP servers do not need to guarantee any sort of
ordering). We'll take into account numeric slice numbers
generated by -imapd if they exist, so slice "80" doesn't show up
next to "8".
---
lib/PublicInbox/LeiLsMailSource.pm | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 71e253d9..f012e10e 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -27,6 +27,12 @@ sub input_path_url { # overrides LeiInput version
my $sec = $lei->{net}->can('uri_section')->($uri);
my $mic = $lei->{net}->mic_get($uri);
my $l = $mic->folders_hash($uri->path); # server-side filter
+ @$l = map { $_->[2] } # undo Schwartzian transform below:
+ sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
+ map { # prepare to sort -imapd slices numerically
+ $_->{name} =~ /\A(.+?)\.([0-9]+)\z/ ?
+ [ $1, $2 + 0, $_ ] : [ $_->{name}, -1, $_ ];
+ } @$l;
@f = map { "$sec/$_->{name}" } @$l;
if ($json) {
$_->{url} = "$sec/$_->{name}" for @$l;
^ permalink raw reply related [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
2021-09-15 22:36 71% ` Kyle Meyer
@ 2021-09-16 2:59 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 2:59 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Btw, without having any externals on disk:
lei q [OPTIONS] --only https://lore.kernel.org/linux-fsdevel/ SEARCH_TERMS...
works, and I use it often.
Perl^Wlei: there's more than one way to do it :>
^ permalink raw reply [relevance 71%]
* [PATCH] doc: lei-mail-formats: add "eml" and expand on git things
@ 2021-09-16 7:45 63% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 7:45 UTC (permalink / raw)
To: meta
While "eml" is not an output format, it seems worthy
to document, here, since users are likely to have experience
with *.patch files from "git format-patch".
---
Documentation/lei-mail-formats.pod | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/Documentation/lei-mail-formats.pod b/Documentation/lei-mail-formats.pod
index da723d2b..3c37c880 100644
--- a/Documentation/lei-mail-formats.pod
+++ b/Documentation/lei-mail-formats.pod
@@ -51,7 +51,9 @@ with any number of C<E<gt>> characters and is thus fully
reversible.
This format is emitted by L<PublicInbox::WWW(3pm)> with gzip.
-It is supported by L<git-am(1)> since git 2.10.
+Since git 2.10, C<git am --patch-format=mboxrd> reads this
+format. C<git log> and C<git format-patch --stdout> can also
+generate this format with the C<--pretty=mboxrd> switch.
As with uncompressed L</mboxo>, uncompressed mboxrd are vulnerable
to undetectable truncation.
@@ -96,6 +98,22 @@ Depending on the IMAP server software and configuration, IMAP
servers may use any (or combination) of the aforementioned
formats or a non-standard database backend.
+=head1 eml
+
+A single raw message file. C<eml> is not an output format for lei,
+but accepted by as an C<--input-format> (C<-F>) for read-only
+commands such as L<lei-tag(1)> and L<lei-import(1)>.
+
+Since C<eml> is the suffix for the C<message/rfc822> MIME type
+(according to the C<mime.types> file), lei will infer the type
+based on the C<.eml> suffix if C<--input-format> is unspecified
+
+C<.patch>-suffixed files generated by L<git-format-patch(1)>
+(without C<--stdout>) are C<eml> files with the addition of an
+mbox C<From > header. L<lei(1)> removes C<From > lines to treat
+them as C<eml> when reading these for compatibility with
+C<git-am(1)> and similar tools.
+
=head1 COPYRIGHT
Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
^ permalink raw reply related [relevance 63%]
* [PATCH 0/3] lei refresh-mail-sync
@ 2021-09-16 9:41 71% Eric Wong
2021-09-16 9:41 59% ` [PATCH 1/3] lei: git_oid: replace git_blob_id Eric Wong
2021-09-16 9:41 39% ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-16 9:41 UTC (permalink / raw)
To: meta
The command I need because lei-daemon keeps dying so inotify
ain't useful :P Well, actually, I just restart lei-daemon
frequently because I'm always changing code.
1/3 is a nice cleanup, 2/3 makes the change
AFAIK, refresh-mail-sync (and prune-mail-sync before it)
were the only which could trigger the bug fixed in PATCH 3/3
Eric Wong (3):
lei: git_oid: replace git_blob_id
lei refresh-mail-sync: replace prune-mail-sync
net_reader: ensure IO::Socket::Socks is loaded in all workers
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 13 ++--
lib/PublicInbox/LeiInspect.pm | 3 +-
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 36 +++++++---
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 14 ++--
lib/PublicInbox/LeiXSearch.pm | 3 +-
lib/PublicInbox/NetReader.pm | 5 ++
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 67 +++++++++++++++++++
10 files changed, 121 insertions(+), 25 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (70%)
create mode 100644 t/lei-refresh-mail-sync.t
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei: git_oid: replace git_blob_id
2021-09-16 9:41 71% [PATCH 0/3] lei refresh-mail-sync Eric Wong
@ 2021-09-16 9:41 59% ` Eric Wong
2021-09-16 9:41 39% ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 9:41 UTC (permalink / raw)
To: meta
We'll be using binary SHA-1 and SHA-256 in-memory since that's
what mail_sync.sqlite3 stores.
---
lib/PublicInbox/LEI.pm | 10 +++++++---
lib/PublicInbox/LeiInspect.pm | 3 ++-
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 9 +--------
lib/PublicInbox/LeiXSearch.pm | 3 ++-
5 files changed, 13 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0a30bc36..ec103231 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -24,6 +24,8 @@ use PublicInbox::DS qw(now dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
use PublicInbox::Eml;
+use PublicInbox::Import;
+use PublicInbox::ContentHash qw(git_sha);
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
@@ -1479,9 +1481,11 @@ sub refresh_watches {
}
}
-sub git_blob_id {
- my ($lei, $eml) = @_;
- ($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
+# TODO: support SHA-256
+sub git_oid {
+ my $eml = $_[-1];
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+ git_sha(1, $eml);
}
sub lms { # read-only LeiMailSync
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 25bd47e7..2385f7f8 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -202,7 +202,8 @@ sub ins_add { # InputPipe->consume callback
my $str = delete $lei->{istr};
$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
my $eml = PublicInbox::Eml->new(\$str);
- _inspect_argv($lei, [ 'blob:'.$lei->git_blob_id($eml),
+ _inspect_argv($lei, [
+ 'blob:'.$lei->git_oid($eml)->hexdigest,
map { "mid:$_" } @{mids($eml)} ]);
};
$lei->{istr} .= $_[1];
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 580787c0..8d4ffed0 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -32,7 +32,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
$smsg = $res if ref($res) eq ref($smsg);
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : $lei->git_blob_id($eml);
+ : $lei->git_oid($eml)->hexdigest;
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
push @{$self->{smsg}}, $smsg;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 42f574f2..e8bcb04e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -20,7 +20,7 @@ use PublicInbox::Eml;
use PublicInbox::Import;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash git_sha);
+use PublicInbox::ContentHash qw(content_hash);
use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
@@ -603,13 +603,6 @@ sub write_prepare {
$lei->{sto} = $self;
}
-# TODO: support SHA-256
-sub git_blob_id { # called via LEI->git_blob_id
- my ($self, $eml) = @_;
- $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
- git_sha(1, $eml)->hexdigest;
-}
-
# called by lei-daemon before lei->refresh_watches
sub add_sync_folders {
my ($self, @folders) = @_;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 556ffd58..50cadb5e 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -275,7 +275,8 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
$smsg->{kw} = []; # short-circuit xsmsg_vmd
}
}
- $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
+ : $lei->git_oid($eml)->hexdigest;
_smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
^ permalink raw reply related [relevance 59%]
* [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync
2021-09-16 9:41 71% [PATCH 0/3] lei refresh-mail-sync Eric Wong
2021-09-16 9:41 59% ` [PATCH 1/3] lei: git_oid: replace git_blob_id Eric Wong
@ 2021-09-16 9:41 39% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 9:41 UTC (permalink / raw)
To: meta
Merely pruning mail synchronization information was
insufficient for Maildir: renames are common in Maildir
and we need to detect them after-the-fact when lei-daemon
isn't running.
Running this command could make "lei index" far more
useful...
---
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 3 +-
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 36 +++++++---
lib/PublicInbox/LeiStore.pm | 5 ++
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 67 +++++++++++++++++++
6 files changed, 103 insertions(+), 11 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (70%)
create mode 100644 t/lei-refresh-mail-sync.t
diff --git a/MANIFEST b/MANIFEST
index 640eabd1..221cb992 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -238,6 +238,7 @@ lib/PublicInbox/LeiPmdir.pm
lib/PublicInbox/LeiPruneMailSync.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
+lib/PublicInbox/LeiRefreshMailSync.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiRmWatch.pm
@@ -450,6 +451,7 @@ t/lei-q-kw.t
t/lei-q-remote-import.t
t/lei-q-save.t
t/lei-q-thread.t
+t/lei-refresh-mail-sync.t
t/lei-sigpipe.t
t/lei-tag.t
t/lei-up.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ec103231..9794497b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -263,7 +263,7 @@ our %CMD = ( # sorted in order of importance/use:
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
-'prune-mail-sync' => [ 'LOCATION...|--all',
+'refresh-mail-sync' => [ 'LOCATION...|--all',
'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
@@ -616,6 +616,7 @@ sub pkt_ops {
$ops->{x_it} = [ \&x_it, $lei ];
$ops->{child_error} = [ \&child_error, $lei ];
$ops->{incr} = [ \&incr, $lei ];
+ $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
$ops;
}
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
similarity index 70%
rename from lib/PublicInbox/LeiPruneMailSync.pm
rename to lib/PublicInbox/LeiRefreshMailSync.pm
index 3678bd04..07b0aa52 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -1,16 +1,20 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# "lei prune-mail-sync" drops dangling sync information
-package PublicInbox::LeiPruneMailSync;
+# "lei refresh-mail-sync" drops dangling sync information
+# and attempts to detect moved files
+package PublicInbox::LeiRefreshMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub eml_match ($$) {
my ($eml, $oidbin) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
$oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
}
@@ -20,7 +24,7 @@ sub prune_mdir { # lms->each_src callback
for my $d (@try) {
my $src = "$mdir/$d/$$id";
if ($self->{verify}) {
- my $eml = eml_from_path($src) or next;
+ my $eml = eml_from_path($src) // next;
return if eml_match($eml, $oidbin);
} elsif (-f $src) {
return;
@@ -38,12 +42,27 @@ sub prune_imap { # lms->each_src callback
$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
}
+# detects missed file moves
+sub pmdir_cb { # called via LeiPmdir->each_mdir_fn
+ my ($self, $f, $fl) = @_;
+ my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+ die "BUG: $f was not from a Maildir?";
+ substr($folder, 0, 0) = 'maildir:'; # add prefix
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
+ return if defined($lms->name_oidbin($folder, $bn));
+ my $eml = eml_from_path($f) // return;
+ my $oidbin = $self->{lei}->git_oid($eml)->digest;
+ $self->{lei}->{sto}->ipc_do('lms_set_src', $oidbin, $folder, \$bn);
+}
+
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
- my $mdir = $1;
- $lms->each_src($input, \&prune_mdir, $self, $mdir);
+ $lms->each_src($input, \&prune_mdir, $self, my $mdir = $1);
+ $self->{lse} //= $self->{lei}->{sto}->search;
+ # call pmdir_cb (via maildir_each_file -> each_mdir_fn)
+ PublicInbox::LeiInput::input_path_url($self, $input);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{lei}->{net}->mic_for_folder($uri);
@@ -51,10 +70,10 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
$uids = +{ map { $_ => undef } @$uids };
$lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
} else { die "BUG: $input not supported" }
- my $wait = $self->{lei}->{sto}->ipc_do('done');
+ $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
}
-sub lei_prune_mail_sync {
+sub lei_refresh_mail_sync {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
@@ -78,7 +97,6 @@ EOM
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
- $sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
@@ -89,7 +107,7 @@ EOM
}
no warnings 'once';
-*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e8bcb04e..32f55abd 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -293,6 +293,11 @@ sub set_sync_info {
_lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
}
+sub lms_set_src {
+ my ($self, $oidbin, $folder, $id) = @_;
+ _lms_rw($self)->set_src($oidbin, $folder, $id);
+}
+
sub _remove_if_local { # git->cat_async arg
my ($bref, $oidhex, $type, $size, $self) = @_;
$self->{im}->remove($bref) if $bref;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
index 9531949a..1fe940bb 100644
--- a/t/lei-export-kw.t
+++ b/t/lei-export-kw.t
@@ -6,7 +6,6 @@ use File::Copy qw(cp);
use File::Path qw(make_path);
require_mods(qw(lei -imapd Mail::IMAPClient));
my ($tmpdir, $for_destroy) = tmpdir;
-my ($ro_home, $cfg_path) = setup_public_inboxes;
my $expect = eml_load('t/data/0001.patch');
test_lei({ tmpdir => $tmpdir }, sub {
my $home = $ENV{HOME};
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
new file mode 100644
index 00000000..ff558277
--- /dev/null
+++ b/t/lei-refresh-mail-sync.t
@@ -0,0 +1,67 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+
+my $stop_daemon = sub { # needed since we don't have inotify
+ lei_ok qw(daemon-pid);
+ chomp(my $pid = $lei_out);
+ $pid > 0 or xbail "bad pid: $pid";
+ kill('TERM', $pid) or xbail "kill: $!";
+ for (0..10) {
+ tick;
+ kill(0, $pid) or last;
+ }
+ kill(0, $pid) and xbail "daemon still running (PID:$pid)";
+};
+
+test_lei({ daemon_only => 1 }, sub {
+ my $d = "$ENV{HOME}/d";
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
+ lei_ok qw(daemon-pid);
+ lei_ok qw(add-external), "$ro_home/t2";
+ lei_ok qw(q mid:testmessage@example.com -o), "Maildir:$d";
+ my (@o) = glob("$d/*/*");
+ scalar(@o) == 1 or xbail('multiple results', \@o);
+ my ($bn0) = ($o[0] =~ m!/([^/]+)\z!);
+
+ my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+ lei_ok 'inspect', "blob:$oid";
+ my $before = json_utf8->decode($lei_out);
+ my $exp0 = { 'mail-sync' => { "maildir:$d" => [ $bn0 ] } };
+ is_deeply($before, $exp0, 'inspect shows expected');
+
+ $stop_daemon->();
+ my $dst = $o[0];
+ $dst =~ s/:2,.*\z// and $dst =~ s!/cur/!/new/! and
+ rename($o[0], $dst) or xbail "rename($o[0] => $dst): $!";
+
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out),
+ $before, 'inspect unchanged immediately after restart');
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ my ($bn1) = ($dst =~ m!/([^/]+)\z!);
+ my $exp1 = { 'mail-sync' => { "maildir:$d" => [ $bn1 ] } };
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'refresh-mail-sync updated location');
+
+ $stop_daemon->();
+ rename($dst, "$d/unwatched") or xbail "rename $dst out-of-the-way $!";
+
+ lei_ok 'refresh-mail-sync', $d;
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, '{}', 'no known locations after "removal"');
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+
+ $stop_daemon->();
+ rename("$d/unwatched", $dst) or xbail "rename $dst back";
+
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'replaced file noted again');
+});
+
+done_testing;
^ permalink raw reply related [relevance 39%]
* [PATCH] lei: sqlite: do not forcibly enable WAL
@ 2021-09-16 11:04 66% Eric Wong
2021-09-17 2:16 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-16 11:04 UTC (permalink / raw)
To: meta
I'm not sure why and I can no longer reproduce it; but I've
triggered a SIGBUS in the lei/store inside the WAL commit code
while writing to mail_sync.sqlite3 on my Debian (oldstable)
buster system.
It could be a problem with my ancient SSD, improper use on our
end, a bug fixed in a newer SQLite version, etc.. In any case,
lets minimize the risk and keep using TRUNCATE like we do with
the time-tested public-inbox-* stuff, but respect WAL if a user
wants to set it themselves.
---
lib/PublicInbox/LeiMailSync.pm | 5 ++++-
lib/PublicInbox/LeiSavedSearch.pm | 5 +----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 5a10c127..dc4395f0 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -21,8 +21,11 @@ sub dbh_new {
sqlite_use_immediate_transaction => 1,
});
# no sqlite_unicode, here, all strings are binary
+ # TRUNCATE reduces I/O compared to the default (DELETE).
+ # Allow and preserve user-overridden WAL, but don't force it.
+ my $jm = $dbh->selectrow_array('PRAGMA journal_mode');
+ $dbh->do('PRAGMA journal_mode = TRUNCATE') if $jm ne 'wal';
create_tables($dbh) if $rw;
- $dbh->do('PRAGMA journal_mode = WAL') if $creat;
$dbh->do('PRAGMA case_sensitive_like = ON');
$dbh;
}
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 96ff816e..8fcfa752 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -229,10 +229,7 @@ sub prepare_dedupe {
my $oidx = PublicInbox::OverIdx->new($self->{-ovf});
$oidx->{-no_fsync} = 1;
$oidx->dbh;
- if ($creat) {
- $oidx->{dbh}->do('PRAGMA journal_mode = WAL');
- $oidx->eidx_prep; # for xref3
- }
+ $oidx->eidx_prep if $creat; # for xref3
$oidx
};
}
^ permalink raw reply related [relevance 66%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 23:06 57% ` Eric Wong
@ 2021-09-16 17:38 71% ` Konstantin Ryabitsev
2021-09-16 23:54 71% ` Luis Chamberlain
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
3 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-16 17:38 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
On Wed, Sep 15, 2021 at 02:34:40PM -0700, Luis Chamberlain wrote:
> My use case is I'm subscribed to a few kernel mailign lists and I use
> mutt with Maildir. I had configured recently pi-piper and grokmirror
> so that I get only the last 1 year of email from a few set of mailing
> lists. For this I needed to know the commit IDs for those emails on
> the public-inbox git mirror for each mailing list.
The pi-piper bit was really only useful for this until lei showed up. I wrote
it mostly so we could pipe things to patchwork straight from public-inbox git
archives.
> I was hinted using lei would be better though. But I'm stuck:
FYI, I'm giving a talk about that on Monday.
https://linuxplumbersconf.org/event/11/contributions/983/
Assuming I finish the prep work by then.
Hopefully, you don't live on the US West Coast and don't have to wake up at
7AM to attend.
-K
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 23:06 57% ` Eric Wong
@ 2021-09-16 17:43 71% ` Konstantin Ryabitsev
2021-09-16 20:04 70% ` IMAP vs NNTP [was: make menuconfig interface for lei / grok-pull] Eric Wong
2021-09-19 21:21 71% ` make menuconfig interface for lei / grok-pull Eric Wong
1 sibling, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-16 17:43 UTC (permalink / raw)
To: Eric Wong; +Cc: Luis Chamberlain, meta
On Wed, Sep 15, 2021 at 11:06:05PM +0000, Eric Wong wrote:
> Does lore.kernel.org run public-inbox-imapd?
I'm still not convinced it's useful for huge collections, especially
considering how chatty IMAP is. Is there any benefit to enable it for lei use?
-K
^ permalink raw reply [relevance 71%]
* IMAP vs NNTP [was: make menuconfig interface for lei / grok-pull]
2021-09-16 17:43 71% ` Konstantin Ryabitsev
@ 2021-09-16 20:04 70% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 20:04 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: Luis Chamberlain, meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Wed, Sep 15, 2021 at 11:06:05PM +0000, Eric Wong wrote:
> > Does lore.kernel.org run public-inbox-imapd?
>
> I'm still not convinced it's useful for huge collections, especially
> considering how chatty IMAP is. Is there any benefit to enable it for lei use?
lei tries to treat IMAP and NNTP the same for (import|tag).
One benefit of IMAP over NNTP is it doesn't add Xref or
Newsgroups headers that could throw off OID-based deduplication.
The other is compression (see below).
lei will be able to use IMAP IDLE to detect new messages for
auto-import; AFAIK there's nothing like it in NNTP.
In general, client-side support for read-only IMAP folders is
lacking, but lei isn't too different in that regard.
On the Perl side (which affects lei):
* the Mail::IMAPClient maintainer has been responsive to
bugfixes I've sent, while the Net::NNTP (libnet) maintainer
hasn't been.
* as a result, Net::NNTP doesn't have DEFLATE support, yet
https://rt.cpan.org/Ticket/Display.html?id=129966
https://rt.cpan.org/Ticket/Display.html?id=129967
* Net::NNTP is bundled with Perl, but Mail::IMAPClient is a
separate package (but widely-packaged by distros)
^ permalink raw reply [relevance 70%]
* lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
` (2 preceding siblings ...)
2021-09-16 17:38 71% ` Konstantin Ryabitsev
@ 2021-09-16 21:09 90% ` Eric Wong
2021-09-16 23:53 90% ` Luis Chamberlain
3 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-16 21:09 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> # The next two don't work
> lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
Btw, extra "import" aside; I never intended to support the above
case. However, your post got me thinking we could and probably
should... :>
(epochs are git storage with a loose limit of ~1G packed data,
"slices" are public-inbox-imapd folders capped at 50K to
accomodate limitations of existing clients/filesystems)
^ permalink raw reply [relevance 90%]
* Re: lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
@ 2021-09-16 23:53 90% ` Luis Chamberlain
2021-09-17 0:22 89% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Luis Chamberlain @ 2021-09-16 23:53 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 16, 2021 at 09:09:02PM +0000, Eric Wong wrote:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
> > # The next two don't work
> > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
>
> Btw, extra "import" aside; I never intended to support the above
> case. However, your post got me thinking we could and probably
> should... :>
>
> (epochs are git storage with a loose limit of ~1G packed data,
> "slices" are public-inbox-imapd folders capped at 50K to
> accomodate limitations of existing clients/filesystems)
So... by default all epochs are pulled? If so, yeah geesh.
I don't want to pull all linux-kernel epochs. Only if doing
R&D on that list would I need it.
Maybe a --query-epochs and then a --use-epoch 0 or whatever
as well?
Luis
^ permalink raw reply [relevance 90%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-16 17:38 71% ` Konstantin Ryabitsev
@ 2021-09-16 23:54 71% ` Luis Chamberlain
0 siblings, 0 replies; 200+ results
From: Luis Chamberlain @ 2021-09-16 23:54 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
On Thu, Sep 16, 2021 at 01:38:32PM -0400, Konstantin Ryabitsev wrote:
> On Wed, Sep 15, 2021 at 02:34:40PM -0700, Luis Chamberlain wrote:
> > My use case is I'm subscribed to a few kernel mailign lists and I use
> > mutt with Maildir. I had configured recently pi-piper and grokmirror
> > so that I get only the last 1 year of email from a few set of mailing
> > lists. For this I needed to know the commit IDs for those emails on
> > the public-inbox git mirror for each mailing list.
>
> The pi-piper bit was really only useful for this until lei showed up. I wrote
> it mostly so we could pipe things to patchwork straight from public-inbox git
> archives.
OK got it! In lei we trust.
> > I was hinted using lei would be better though. But I'm stuck:
>
> FYI, I'm giving a talk about that on Monday.
> https://linuxplumbersconf.org/event/11/contributions/983/
>
> Assuming I finish the prep work by then.
>
> Hopefully, you don't live on the US West Coast and don't have to wake up at
> 7AM to attend.
I do.. but hey I sometimes have 5am meetings so I might be able to make it!
Luis
^ permalink raw reply [relevance 71%]
* Re: lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-16 23:53 90% ` Luis Chamberlain
@ 2021-09-17 0:22 89% ` Eric Wong
2021-09-17 0:48 90% ` Luis Chamberlain
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-17 0:22 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> On Thu, Sep 16, 2021 at 09:09:02PM +0000, Eric Wong wrote:
> > Luis Chamberlain <mcgrof@kernel.org> wrote:
> > > # The next two don't work
> > > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> >
> > Btw, extra "import" aside; I never intended to support the above
> > case. However, your post got me thinking we could and probably
> > should... :>
> >
> > (epochs are git storage with a loose limit of ~1G packed data,
> > "slices" are public-inbox-imapd folders capped at 50K to
> > accomodate limitations of existing clients/filesystems)
>
> So... by default all epochs are pulled? If so, yeah geesh.
> I don't want to pull all linux-kernel epochs. Only if doing
> R&D on that list would I need it.
Right now, "lei import" doesn't read nor understand git repos at all.
If you meant add-external, yes, all epochs are pulled when using
"lei add-external --mirror" (and public-inbox-clone). Yes,
there should be a way to do partial mirrors, just haven't
gotten around to it...
"lei import" currently reads IMAP, NNTP, Maildir, mbox*, etc.
It also understands various https?://*/(raw,t.mbox.gz) URLs used by
PublicInbox::WWW, since they're (gzipped) mboxrd.
But none of that is reading from git repos directly...
> Maybe a --query-epochs and then a --use-epoch 0 or whatever
> as well?
Since epochs expose the gittyness of lei, perhaps a limited form
of commitish-like ranges used by git would work:
lei add-external --mirror $URL --epoch=0..9
lei add-external --mirror $URL --epoch=5..
lei add-external --mirror $URL --epoch=~2.. # like HEAD~2..
That only works for contiguous ranges, though; but I'm also
unsure if discontiguous ranges make sense to anyone.
There's other syntaxes (e.g. (sed|Perl)-like), of course,
but I think git-like here would be most natural.
Thoughts?
^ permalink raw reply [relevance 89%]
* Re: lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-17 0:22 89% ` Eric Wong
@ 2021-09-17 0:48 90% ` Luis Chamberlain
0 siblings, 0 replies; 200+ results
From: Luis Chamberlain @ 2021-09-17 0:48 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 16, 2021 at 5:22 PM Eric Wong <e@80x24.org> wrote:
> Since epochs expose the gittyness of lei, perhaps a limited form
> of commitish-like ranges used by git would work:
>
> lei add-external --mirror $URL --epoch=0..9
> lei add-external --mirror $URL --epoch=5..
> lei add-external --mirror $URL --epoch=~2.. # like HEAD~2..
That looks sexy.
I mentioned the query option so that I could end up displaying on
'make menuconfig' the epochs dynamically, it would be a script that
wants the list of epochs, and I'd use this to dynamically populate
kconfig. Dynamic population of kconfig is not a yet well understood
thing, but I've figured out a way to do it :)
> That only works for contiguous ranges, though; but I'm also
> unsure if discontiguous ranges make sense to anyone.
>
> There's other syntaxes (e.g. (sed|Perl)-like), of course,
> but I think git-like here would be most natural.
>
> Thoughts?
I think the above would suffice for sure!
Luis
^ permalink raw reply [relevance 90%]
* [PATCH v2 0/6] lei refresh-mail-sync: another try...
@ 2021-09-17 1:56 71% Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
OK, so a bunch of things were broken/annoying in my initial
patch @ <20210916094116.11457-3-e@80x24.org>
2/6 was only exposed in real-world usage with giant folders;
and a testcase is still pending for that...
Eric Wong (6):
lei refresh-mail-sync: replace prune-mail-sync
lei_mail_sync: don't hold statement handle into callback
lei refresh-mail-sync: remove "gone" notices
lei refresh-mail-sync: drop unused {verify} code path
lei refresh-mail-sync: implicitly remove missing folders
lei refresh-mail-sync: drop old IMAP folder info
MANIFEST | 3 +-
lib/PublicInbox/LEI.pm | 3 +-
lib/PublicInbox/LeiInput.pm | 11 +-
lib/PublicInbox/LeiMailSync.pm | 41 ++++--
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 79 +++++-----
lib/PublicInbox/LeiStore.pm | 5 +
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 137 ++++++++++++++++++
8 files changed, 231 insertions(+), 49 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (52%)
create mode 100644 t/lei-refresh-mail-sync.t
^ permalink raw reply [relevance 71%]
* [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
@ 2021-09-17 1:56 71% ` Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
Those stderr messages are not useful at all, and harmful with
the noise they cause.
---
lib/PublicInbox/LeiRefreshMailSync.pm | 2 --
1 file changed, 2 deletions(-)
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 3c083965..71fc348c 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -31,14 +31,12 @@ sub prune_mdir { # lms->each_src callback
}
}
# both tries failed
- $self->{lei}->qerr("# maildir:$mdir $$id gone");
$self->{lei}->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
}
sub prune_imap { # lms->each_src callback
my ($oidbin, $uid, $self, $uids, $url) = @_;
return if exists $uids->{$uid};
- $self->{lei}->qerr("# $url $uid gone");
$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
}
^ permalink raw reply related [relevance 71%]
* [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices Eric Wong
@ 2021-09-17 1:56 71% ` Eric Wong
2021-09-17 1:56 61% ` [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders Eric Wong
2021-09-17 1:56 56% ` [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
That option was never wired up, and probably not needed...
---
lib/PublicInbox/LeiRefreshMailSync.pm | 17 +----------------
1 file changed, 1 insertion(+), 16 deletions(-)
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 71fc348c..4cae1536 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -9,27 +9,12 @@ use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
-use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::Import;
-sub eml_match ($$) {
- my ($eml, $oidbin) = @_;
- $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
- $oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
-}
-
sub prune_mdir { # lms->each_src callback
my ($oidbin, $id, $self, $mdir) = @_;
my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur);
- for my $d (@try) {
- my $src = "$mdir/$d/$$id";
- if ($self->{verify}) {
- my $eml = eml_from_path($src) // next;
- return if eml_match($eml, $oidbin);
- } elsif (-f $src) {
- return;
- }
- }
+ for (@try) { return if -f "$mdir/$_/$$id" }
# both tries failed
$self->{lei}->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
}
^ permalink raw reply related [relevance 71%]
* [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
` (2 preceding siblings ...)
2021-09-17 1:56 71% ` [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path Eric Wong
@ 2021-09-17 1:56 61% ` Eric Wong
2021-09-17 1:56 56% ` [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
There's no point in keeping mail_sync.sqlite3 entries around
if the folder is gone. We do keep saved-search configs around,
however, since somebody may decide to blow away a search and
start over.
---
lib/PublicInbox/LeiInput.pm | 11 ++++++++++-
lib/PublicInbox/LeiRefreshMailSync.pm | 5 +++++
t/lei-refresh-mail-sync.t | 10 ++++++++++
3 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 8ce445c8..372e0fe1 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -124,7 +124,11 @@ sub input_path_url {
handle_http_input($self, $input, @args);
return;
}
+
+ # local-only below
+ my $ifmt_pfx = '';
if ($input =~ s!\A([a-z0-9]+):!!i) {
+ $ifmt_pfx = "$1:";
$ifmt = lc($1);
} elsif ($input =~ /\.(?:patch|eml)\z/i) {
$ifmt = 'eml';
@@ -172,11 +176,16 @@ EOM
$self->can('input_maildir_cb'),
$self, @args);
}
+ } elsif ($self->{missing_ok} && !-e $input) { # don't ->fail
+ $self->folder_missing("$ifmt:$input");
} else {
- $lei->fail("$input unsupported (TODO)");
+ $lei->fail("$ifmt_pfx$input unsupported (TODO)");
}
}
+# subclasses should overrride this (see LeiRefreshMailSync)
+sub folder_missing { die "BUG: ->folder_missing undefined for $_[0]" }
+
sub bad_http ($$;$) {
my ($lei, $url, $alt) = @_;
my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 4cae1536..19f64b58 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -11,6 +11,11 @@ use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::Import;
+sub folder_missing {
+ my ($self, $folder) = @_;
+ $self->{lei}->{sto}->ipc_do('lms_forget_folders', $folder);
+}
+
sub prune_mdir { # lms->each_src callback
my ($oidbin, $id, $self, $mdir) = @_;
my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur);
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
index ff558277..d3438011 100644
--- a/t/lei-refresh-mail-sync.t
+++ b/t/lei-refresh-mail-sync.t
@@ -3,6 +3,7 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
require_mods(qw(lei));
+use File::Path qw(remove_tree);
my $stop_daemon = sub { # needed since we don't have inotify
lei_ok qw(daemon-pid);
@@ -62,6 +63,15 @@ test_lei({ daemon_only => 1 }, sub {
lei_ok 'inspect', "blob:$oid";
is_deeply(json_utf8->decode($lei_out), $exp1,
'replaced file noted again');
+
+ $stop_daemon->();
+
+ remove_tree($d);
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, '{}', 'no known locations after "removal"');
+ lei_ok 'ls-mail-sync';
+ is($lei_out, '', 'no sync left when folder is gone');
});
done_testing;
^ permalink raw reply related [relevance 61%]
* [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
@ 2021-09-17 1:56 36% ` Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
Merely pruning mail synchronization information was
insufficient for Maildir: renames are common in Maildir
and we need to detect them after-the-fact when lei-daemon
isn't running.
Running this command could make "lei index" far more
useful...
v2: close R/O mail_sync.sqlite3 dbh before fork
Keeping the DB file handle open across fork can cause bad things
to happen even if we don't use it since sqlite3 itself still knows
about it (but doesn't know Perl code doesn't know about it).
---
MANIFEST | 3 +-
lib/PublicInbox/LEI.pm | 3 +-
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 54 +++++++++------
lib/PublicInbox/LeiStore.pm | 5 ++
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 67 +++++++++++++++++++
6 files changed, 111 insertions(+), 22 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (62%)
create mode 100644 t/lei-refresh-mail-sync.t
diff --git a/MANIFEST b/MANIFEST
index 640eabd1..9f11f2f9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -235,9 +235,9 @@ lib/PublicInbox/LeiNoteEvent.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiPmdir.pm
-lib/PublicInbox/LeiPruneMailSync.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
+lib/PublicInbox/LeiRefreshMailSync.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiRmWatch.pm
@@ -450,6 +450,7 @@ t/lei-q-kw.t
t/lei-q-remote-import.t
t/lei-q-save.t
t/lei-q-thread.t
+t/lei-refresh-mail-sync.t
t/lei-sigpipe.t
t/lei-tag.t
t/lei-up.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ec103231..9794497b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -263,7 +263,7 @@ our %CMD = ( # sorted in order of importance/use:
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
-'prune-mail-sync' => [ 'LOCATION...|--all',
+'refresh-mail-sync' => [ 'LOCATION...|--all',
'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
@@ -616,6 +616,7 @@ sub pkt_ops {
$ops->{x_it} = [ \&x_it, $lei ];
$ops->{child_error} = [ \&child_error, $lei ];
$ops->{incr} = [ \&incr, $lei ];
+ $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
$ops;
}
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
similarity index 62%
rename from lib/PublicInbox/LeiPruneMailSync.pm
rename to lib/PublicInbox/LeiRefreshMailSync.pm
index 3678bd04..3c083965 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -1,16 +1,20 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# "lei prune-mail-sync" drops dangling sync information
-package PublicInbox::LeiPruneMailSync;
+# "lei refresh-mail-sync" drops dangling sync information
+# and attempts to detect moved files
+package PublicInbox::LeiRefreshMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub eml_match ($$) {
my ($eml, $oidbin) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
$oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
}
@@ -20,7 +24,7 @@ sub prune_mdir { # lms->each_src callback
for my $d (@try) {
my $src = "$mdir/$d/$$id";
if ($self->{verify}) {
- my $eml = eml_from_path($src) or next;
+ my $eml = eml_from_path($src) // next;
return if eml_match($eml, $oidbin);
} elsif (-f $src) {
return;
@@ -38,12 +42,27 @@ sub prune_imap { # lms->each_src callback
$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
}
+# detects missed file moves
+sub pmdir_cb { # called via LeiPmdir->each_mdir_fn
+ my ($self, $f, $fl) = @_;
+ my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+ die "BUG: $f was not from a Maildir?";
+ substr($folder, 0, 0) = 'maildir:'; # add prefix
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
+ return if defined($lms->name_oidbin($folder, $bn));
+ my $eml = eml_from_path($f) // return;
+ my $oidbin = $self->{lei}->git_oid($eml)->digest;
+ $self->{lei}->{sto}->ipc_do('lms_set_src', $oidbin, $folder, \$bn);
+}
+
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
- my $mdir = $1;
- $lms->each_src($input, \&prune_mdir, $self, $mdir);
+ $lms->each_src($input, \&prune_mdir, $self, my $mdir = $1);
+ $self->{lse} //= $self->{lei}->{sto}->search;
+ # call pmdir_cb (via maildir_each_file -> each_mdir_fn)
+ PublicInbox::LeiInput::input_path_url($self, $input);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{lei}->{net}->mic_for_folder($uri);
@@ -51,34 +70,31 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
$uids = +{ map { $_ => undef } @$uids };
$lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
} else { die "BUG: $input not supported" }
- my $wait = $self->{lei}->{sto}->ipc_do('done');
+ $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
}
-sub lei_prune_mail_sync {
+sub lei_refresh_mail_sync {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
EOM
- if (my $lms = $lei->lms) {
- if (defined(my $all = $lei->{opt}->{all})) {
- $lms->group2folders($lei, $all, \@folders) or return;
- } else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
- }
- } else {
- return $lei->fail(<<EOM);
+ my $lms = $lei->lms or return $lei->fail(<<EOM);
lei mail_sync.sqlite3 uninitialized, see lei-import(1)
EOM
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
}
+ undef $lms; # must be done before fork
$sto->write_prepare($lei);
my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
- $sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
@@ -89,7 +105,7 @@ EOM
}
no warnings 'once';
-*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e8bcb04e..32f55abd 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -293,6 +293,11 @@ sub set_sync_info {
_lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
}
+sub lms_set_src {
+ my ($self, $oidbin, $folder, $id) = @_;
+ _lms_rw($self)->set_src($oidbin, $folder, $id);
+}
+
sub _remove_if_local { # git->cat_async arg
my ($bref, $oidhex, $type, $size, $self) = @_;
$self->{im}->remove($bref) if $bref;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
index 9531949a..1fe940bb 100644
--- a/t/lei-export-kw.t
+++ b/t/lei-export-kw.t
@@ -6,7 +6,6 @@ use File::Copy qw(cp);
use File::Path qw(make_path);
require_mods(qw(lei -imapd Mail::IMAPClient));
my ($tmpdir, $for_destroy) = tmpdir;
-my ($ro_home, $cfg_path) = setup_public_inboxes;
my $expect = eml_load('t/data/0001.patch');
test_lei({ tmpdir => $tmpdir }, sub {
my $home = $ENV{HOME};
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
new file mode 100644
index 00000000..ff558277
--- /dev/null
+++ b/t/lei-refresh-mail-sync.t
@@ -0,0 +1,67 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+
+my $stop_daemon = sub { # needed since we don't have inotify
+ lei_ok qw(daemon-pid);
+ chomp(my $pid = $lei_out);
+ $pid > 0 or xbail "bad pid: $pid";
+ kill('TERM', $pid) or xbail "kill: $!";
+ for (0..10) {
+ tick;
+ kill(0, $pid) or last;
+ }
+ kill(0, $pid) and xbail "daemon still running (PID:$pid)";
+};
+
+test_lei({ daemon_only => 1 }, sub {
+ my $d = "$ENV{HOME}/d";
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
+ lei_ok qw(daemon-pid);
+ lei_ok qw(add-external), "$ro_home/t2";
+ lei_ok qw(q mid:testmessage@example.com -o), "Maildir:$d";
+ my (@o) = glob("$d/*/*");
+ scalar(@o) == 1 or xbail('multiple results', \@o);
+ my ($bn0) = ($o[0] =~ m!/([^/]+)\z!);
+
+ my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+ lei_ok 'inspect', "blob:$oid";
+ my $before = json_utf8->decode($lei_out);
+ my $exp0 = { 'mail-sync' => { "maildir:$d" => [ $bn0 ] } };
+ is_deeply($before, $exp0, 'inspect shows expected');
+
+ $stop_daemon->();
+ my $dst = $o[0];
+ $dst =~ s/:2,.*\z// and $dst =~ s!/cur/!/new/! and
+ rename($o[0], $dst) or xbail "rename($o[0] => $dst): $!";
+
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out),
+ $before, 'inspect unchanged immediately after restart');
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ my ($bn1) = ($dst =~ m!/([^/]+)\z!);
+ my $exp1 = { 'mail-sync' => { "maildir:$d" => [ $bn1 ] } };
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'refresh-mail-sync updated location');
+
+ $stop_daemon->();
+ rename($dst, "$d/unwatched") or xbail "rename $dst out-of-the-way $!";
+
+ lei_ok 'refresh-mail-sync', $d;
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, '{}', 'no known locations after "removal"');
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+
+ $stop_daemon->();
+ rename("$d/unwatched", $dst) or xbail "rename $dst back";
+
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'replaced file noted again');
+});
+
+done_testing;
^ permalink raw reply related [relevance 36%]
* [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
` (3 preceding siblings ...)
2021-09-17 1:56 61% ` [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders Eric Wong
@ 2021-09-17 1:56 56% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
Like with Maildir, IMAP folders can be deleted entirely.
Ensure they can be eliminated, but don't be fooled into
removing them if they're temporarily unreachable.
---
lib/PublicInbox/LeiRefreshMailSync.pm | 11 +++--
t/lei-refresh-mail-sync.t | 60 +++++++++++++++++++++++++++
2 files changed, 67 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 19f64b58..09a7ead0 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -53,10 +53,13 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
PublicInbox::LeiInput::input_path_url($self, $input);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
- my $mic = $self->{lei}->{net}->mic_for_folder($uri);
- my $uids = $mic->search('UID 1:*');
- $uids = +{ map { $_ => undef } @$uids };
- $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
+ if (my $mic = $self->{lei}->{net}->mic_for_folder($uri)) {
+ my $uids = $mic->search('UID 1:*');
+ $uids = +{ map { $_ => undef } @$uids };
+ $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri)
+ } else {
+ $self->folder_missing($$uri);
+ }
} else { die "BUG: $input not supported" }
$self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
}
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
index d3438011..90356b57 100644
--- a/t/lei-refresh-mail-sync.t
+++ b/t/lei-refresh-mail-sync.t
@@ -72,6 +72,66 @@ test_lei({ daemon_only => 1 }, sub {
is($lei_out, '{}', 'no known locations after "removal"');
lei_ok 'ls-mail-sync';
is($lei_out, '', 'no sync left when folder is gone');
+
+SKIP: {
+ require_mods(qw(-imapd -nntpd Mail::IMAPClient Net::NNTP), 1);
+ require File::Copy; # stdlib
+ my $home = $ENV{HOME};
+ my $srv;
+ my $cfg_path2 = "$home/cfg2";
+ File::Copy::cp($cfg_path, $cfg_path2);
+ my $env = { PI_CONFIG => $cfg_path2 };
+ for my $x (qw(imapd)) {
+ my $s = tcp_server;
+ my $cmd = [ "-$x", '-W0', "--stdout=$home/$x.out",
+ "--stderr=$home/$x.err" ];
+ my $td = start_script($cmd, $env, { 3 => $s}) or xbail("-$x");
+ $srv->{$x} = {
+ addr => (my $scalar = tcp_host_port($s)),
+ td => $td,
+ cmd => $cmd,
+ };
+ }
+ my $url = "imap://$srv->{imapd}->{addr}/t.v1.0";
+ lei_ok 'import', $url, '+L:v1';
+ lei_ok 'inspect', "blob:$oid";
+ $before = json_utf8->decode($lei_out);
+ my @f = grep(m!\Aimap://;AUTH=ANONYMOUS\@\Q$srv->{imapd}->{addr}\E!,
+ keys %{$before->{'mail-sync'}});
+ is(scalar(@f), 1, 'got IMAP folder') or xbail(\@f);
+ xsys([qw(git config), '-f', $cfg_path2,
+ qw(--unset publicinbox.t1.newsgroup)]) and
+ xbail "git config $?";
+ $stop_daemon->(); # drop IMAP IDLE
+ $srv->{imapd}->{td}->kill('HUP');
+ tick; # wait for HUP
+ lei_ok 'refresh-mail-sync', $url;
+ lei_ok 'inspect', "blob:$oid";
+ my $after = json_utf8->decode($lei_out);
+ ok(!$after->{'mail-sync'}, 'no sync info for non-existent mailbox');
+ lei_ok 'ls-mail-sync';
+ unlike $lei_out, qr!^\Q$f[0]\E!, 'IMAP folder gone from mail_sync';
+
+ # simulate server downtime
+ $url = "imap://$srv->{imapd}->{addr}/t.v2.0";
+ lei_ok 'import', $url, '+L:v2';
+
+ lei_ok 'inspect', "blob:$oid";
+ $before = $lei_out;
+ delete $srv->{imapd}->{td}; # kill + join daemon
+
+ ok(!(lei 'refresh-mail-sync', $url), 'URL fails on dead -imapd');
+ ok(!(lei 'refresh-mail-sync', '--all'), '--all fails on dead -imapd');
+
+ # restart server (somewhat dangerous since we released the socket)
+ my $cmd = $srv->{imapd}->{cmd};
+ push @$cmd, '-l', $srv->{imapd}->{addr};
+ $srv->{imapd}->{td} = start_script($cmd, $env) or xbail "@$cmd";
+
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, $before, 'no changes when server was down');
+}; # imapd+nntpd stuff
});
done_testing;
^ permalink raw reply related [relevance 56%]
* Re: [PATCH] lei: sqlite: do not forcibly enable WAL
2021-09-16 11:04 66% [PATCH] lei: sqlite: do not forcibly enable WAL Eric Wong
@ 2021-09-17 2:16 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 2:16 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> It could be a problem with my ancient SSD, improper use on our
> end,
Yes, EDOOFUS :x
I suspect it was fixed in v2 of <20210917015644.32746-2-e@80x24.org>
("lei refresh-mail-sync: replace prune-mail-sync")
So I think lei will keep using WAL (and maybe start using mmap, too)
^ permalink raw reply [relevance 71%]
* [PATCH] script/lei: umask(077) before execve
@ 2021-09-17 12:12 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 12:12 UTC (permalink / raw)
To: meta
While my MUA also runs umask(077) unconditionally, not all
MUAs do. Additionally, pagers may support writing its buffer
to disk, so ensure anything else we spawn has umask(077).
---
script/lei | 1 +
1 file changed, 1 insertion(+)
diff --git a/script/lei b/script/lei
index 2d84487a..591013e3 100755
--- a/script/lei
+++ b/script/lei
@@ -53,6 +53,7 @@ my $exec_cmd = sub {
}
my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
@ENV{keys %env} = values %env;
+ umask 077;
exec(@argv);
warn "exec: @argv: $!\n";
POSIX::_exit(1);
^ permalink raw reply related [relevance 71%]
* [PATCH] doc: add lei-security(7) manpage
@ 2021-09-17 12:38 38% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 12:38 UTC (permalink / raw)
To: meta
It seems like a good idea to have a manpage where somebody
can quickly look up and address their concerns as to what
to put on encrypted device/filesystem.
And I probably would've designed lei around make(1) for
parallelization if I didn't have to keep credentials off
the FS :P
---
Documentation/lei-add-external.pod | 4 +-
Documentation/lei-security.pod | 134 +++++++++++++++++++++++++++++
Documentation/txt2pre | 8 +-
MANIFEST | 1 +
Makefile.PL | 3 +-
5 files changed, 146 insertions(+), 4 deletions(-)
create mode 100644 Documentation/lei-security.pod
diff --git a/Documentation/lei-add-external.pod b/Documentation/lei-add-external.pod
index 71229865..9c8bde0f 100644
--- a/Documentation/lei-add-external.pod
+++ b/Documentation/lei-add-external.pod
@@ -10,8 +10,8 @@ lei add-external [OPTIONS] LOCATION
Configure lei to search against an external (an inbox or external
index). When C<LOCATION> is an existing local path, it should point
-to a directory that is a C<public.<name>.inboxdir> or
-C<extindex.<name>.topdir> value in ~/.public-inbox/config.
+to a directory that is a C<publicinbox.$NAME.inboxdir> or
+C<extindex.$NAME.topdir> value in ~/.public-inbox/config.
=head1 OPTIONS
diff --git a/Documentation/lei-security.pod b/Documentation/lei-security.pod
new file mode 100644
index 00000000..4b712c2d
--- /dev/null
+++ b/Documentation/lei-security.pod
@@ -0,0 +1,134 @@
+=head1 NAME
+
+lei - security information
+
+=head1 SYNOPSIS
+
+L<lei(1)> is intended for use with both publicly-archived
+and "private" mail in personal mailboxes. This document is
+intended to give an overview of security implications and
+lower^Wmanage user expectations.
+
+=head1 DESCRIPTION
+
+lei expects to be run as a regular user on a Unix-like system.
+It expects a case-sensitive filesystem with standard Unix
+permissions support.
+
+It does not use POSIX ACLs, extended attributes, nor any other
+security-related functions which require non-standard Perl modules.
+
+=head1 INTERNAL FILES
+
+lei runs with a umask of 077 to prevent other users on the
+system from accessing each other's mail.
+
+The git storage and Xapian databases are located at
+C<$XDG_DATA_HOME/lei/store> (typically C<~/.local/share/lei/store>).
+Any personal mail imported will reside here, so this should
+be on an encrypted filesystem or block device.
+
+C<$XDG_RUNTIME_DIR/lei> (typically C</run/user/$UID/lei> or
+C</tmp/lei-$UID>) contain the socket used to access the lei
+daemon. It must only be accessible to the owner (mode 0700).
+
+C<$XDG_CACHE_HOME/lei> (typically C<~/.cache/lei>) will
+contain IMAP and Maildir folder names which could leak sensitive
+information as well as git repository names.
+
+C<$XDG_DATA_HOME/lei/saved-searches> (typically
+C<~/.local/share/lei/saved-searches>) will contain aforementioned
+folder names as well as (removable) search history.
+
+The configuration for lei resides at C<$XDG_CONFIG_HOME/lei/config>
+(typically C<~/.config/lei/config>). It may contain sensitive pathnames
+and hostnames in the config if a user chooses to configure them.
+
+lei itself will never write credentials to the
+filesystem. However, L<git-credential(1)> may be
+configured to do so. lei will only read C<~/.netrc> if
+C<--netrc> is used (and it will never write to C<~/.netrc>).
+
+C<$XDG_CACHE_HOME/public-inbox> (typically C<~/.cache/public-inbox>)
+can contain data and L<Inline::C>-built modules which can be
+shared with public-facing L<public-inbox-daemon(8)> instances;
+so no private data should be in "public-inbox" paths.
+
+=head1 EXTERNAL FILES
+
+Locations set by L<lei-add-external(1)> can be shared with
+public-facing L<public-inbox-daemon(8)> processes. They may
+reside on shared storage and may be made world-readable to
+other users on the local system.
+
+=head1 NETWORK ACCESS
+
+lei currently uses the L<curl(1)> and L<git(1)> executables in
+C<$PATH> for HTTP and HTTPS network access. Interactive
+authentication for HTTP and HTTPS is not-yet-supported since all
+currently supported HTTP/HTTPS sources are L<PublicInbox::WWW>
+instances.
+
+The L<Mail::IMAPClient> library is used for IMAP and IMAPS.
+L<Net::NNTP> (standard library) is used for NNTP and NNTPS.
+
+L<Mail::IMAPClient> and L<Net::NNTP> will use L<IO::Socket::SSL>
+for TLS if available. In turn, L<IO::Socket::SSL> uses the
+widely-installed OpenSSL library.
+
+STARTTLS will be attempted if advertised by the server
+unless IMAPS or NNTPS are used. C<-c imap.starttls=0>
+and C<-c nntp.startls=0> may be used to disable STARTTLS.
+
+L<IO::Socket::Socks> will be used if C<-c imap.proxy> or
+C<-c nntp.proxy> point to a C<socks5h://$HOST:$PORT>
+address (common for Tor).
+
+The C<--netrc> switch may be passed to curl and used for
+NNTP/IMAP access (via L<Net::Netrc>).
+
+=head1 CREDENTIAL DATA
+
+lei uses L<git-credential(1)> to prompt users for IMAP and NNTP
+usernames and passwords. These passwords are not encrypted in
+memory and get transferred across processes via anonymous UNIX
+sockets and pipes. They may be exposed via syscall tracing
+tools (e.g. L<strace(1)>).
+
+While credentials are not written to the filesystem by default,
+it is possible for them to end up on disk if processes are
+swapped out. Use of an encrypted swap partition is recommended.
+
+=head1 DENIAL-OF-SERVICE VECTORS
+
+lei uses the same MIME parsing library as L<public-inbox-mda(1)>
+with limits header sizes, parts, nesting and boundary limits
+similar to those found in SpamAssassin and postfix.
+
+Email address parsing is handled by L<Email::Address::XS> if
+available, but may fall back to regular expressions which favor
+speed and predictable execution times over correctness.
+
+=head1 ENCRYPTED EMAILS
+
+Not yet supported, but it should eventually be possible to
+configure decryption and indexing of encrypted messages and
+attachments. When supported, decrypted terms will be stored
+in Xapian DBs under C<$XDG_DATA_HOME/lei/store>.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-overview(7)>, L<lei(1)>
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 6367374d..7bc31d23 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -21,6 +21,7 @@ for (qw[lei(1)
lei-forget-external(1)
lei-forget-search(1)
lei-import(1)
+ lei-index(1)
lei-init(1)
lei-lcat(1)
lei-ls-external(1)
@@ -31,14 +32,19 @@ for (qw[lei(1)
lei-p2q(1)
lei-q(1)
lei-rediff(1)
+ lei-rm(1)
+ lei-store-format(5)
+ lei-security(7)
lei-tag(1)
lei-up(1)
public-inbox.cgi(1)
- public-inbox-compact(1)
+ public-inbox-clone(1)
+ public-inbox-config(5)
public-inbox-config(5)
public-inbox-convert(1)
public-inbox-daemon(8)
public-inbox-edit(1)
+ public-inbox-fetch(1)
public-inbox-glossary(7)
public-inbox-httpd(1)
public-inbox-imapd(1)
diff --git a/MANIFEST b/MANIFEST
index 9f11f2f9..218e20e9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -44,6 +44,7 @@ Documentation/lei-p2q.pod
Documentation/lei-q.pod
Documentation/lei-rediff.pod
Documentation/lei-rm.pod
+Documentation/lei-security.pod
Documentation/lei-store-format.pod
Documentation/lei-tag.pod
Documentation/lei-up.pod
diff --git a/Makefile.PL b/Makefile.PL
index 1e935012..f56445ae 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -54,7 +54,8 @@ $v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
) ];
-$v->{-m7} = [ qw(lei-overview public-inbox-overview public-inbox-tuning
+$v->{-m7} = [ qw(lei-overview lei-security
+ public-inbox-overview public-inbox-tuning
public-inbox-glossary) ];
$v->{-m8} = [ qw(public-inbox-daemon) ];
my @sections = (1, 5, 7, 8);
^ permalink raw reply related [relevance 38%]
* [PATCH 0/9] lei: a bunch of random stuff
@ 2021-09-18 9:33 67% Eric Wong
2021-09-18 9:33 51% ` [PATCH 1/9] lei: lock worker counts Eric Wong
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
The unique timers stuff will be used for "lei up" polling,
as will 9/9 to improve "lei up" usability.
The net_reader changes were noticed while getting imaps://
to work with socks5h:// (not just imap://).
There's still a lot of mail_sync stuff going on, but it's
getting closer...
Eric Wong (9):
lei: lock worker counts
lei_mail_sync: rely on flock(2), avoid IPC
lei_mail_sync: set nodatacow on btrfs
ds: support add unique timers
net_reader: tie SocksDebug to {imap,nntp}.Debug
net_reader: detect IMAP failures earlier
net_reader: support imaps:// w/ socks5h:// proxy
net_reader: set SO_KEEPALIVE on all Net::NNTP sockets
lei up: automatically use dt: for remote externals
Documentation/lei-up.pod | 15 ++++
lib/PublicInbox/DS.pm | 100 +++++++++++++-------------
lib/PublicInbox/LEI.pm | 40 +++++------
lib/PublicInbox/LeiExportKw.pm | 32 ++++-----
lib/PublicInbox/LeiForgetMailSync.pm | 6 +-
lib/PublicInbox/LeiImport.pm | 8 +--
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiInspect.pm | 5 +-
lib/PublicInbox/LeiLsMailSource.pm | 3 +-
lib/PublicInbox/LeiLsMailSync.pm | 3 +-
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMailSync.pm | 51 ++++++++++---
lib/PublicInbox/LeiNoteEvent.pm | 31 ++++----
lib/PublicInbox/LeiRefreshMailSync.pm | 35 ++++-----
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 1 +
lib/PublicInbox/LeiStore.pm | 39 +---------
lib/PublicInbox/LeiTag.pm | 3 +-
lib/PublicInbox/LeiToMail.pm | 10 ++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 50 ++++++++++---
lib/PublicInbox/NetReader.pm | 26 ++++---
t/lei-q-remote-import.t | 4 ++
23 files changed, 259 insertions(+), 211 deletions(-)
^ permalink raw reply [relevance 67%]
* [PATCH 1/9] lei: lock worker counts
2021-09-18 9:33 67% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
@ 2021-09-18 9:33 51% ` Eric Wong
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
It doesn't seem worthwhile to change worker counts dynamically
on a per-command-basis with lei, and I don't know how such an
interface would even work...
---
lib/PublicInbox/LEI.pm | 3 ++-
lib/PublicInbox/LeiExportKw.pm | 1 -
lib/PublicInbox/LeiImport.pm | 1 -
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiRefreshMailSync.pm | 4 +---
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 3 +--
8 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9794497b..41e761f8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -258,7 +258,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
+ qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
@@ -627,6 +627,7 @@ sub workers_start {
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
$flds->{lei} = $lei;
+ $wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d37f3768..8b8aa373 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -124,7 +124,6 @@ EOM
my $ops = {};
$sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7c563bd8..b1cb3940 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -101,7 +101,6 @@ sub do_import_index ($$@) {
}
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
$lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index f012e10e..bcb1838e 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -96,8 +96,7 @@ sub lei_ls_mail_source {
$lei->start_pager if -t $lei->{1};
my $ops = {};
$lei->{auth}->op_merge($ops, $self);
- my $j = $self->{-wq_nr_workers} = 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 70136135..aebf0184 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -71,7 +71,7 @@ sub do_ls_search_long {
sub bg_worker ($$$) {
my ($lei, $pfx, $json) = @_;
- my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
+ my $self = bless { json => $json }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 09a7ead0..cdd99725 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -84,11 +84,9 @@ EOM
my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
- my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 778fa1de..3371f3ed 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -32,7 +32,7 @@ sub lei_rm {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
$lei->{opt}->{'in-format'} //= 'eml';
- my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$self->prepare_inputs($lei, \@inputs) or return;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 44d77b88..c4f5ecff 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -50,8 +50,7 @@ sub lei_tag { # the "lei tag" method
return $lei->fail('no keywords or labels specified');
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- my $j = $self->{-wq_nr_workers} = 1; # locked for now
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
^ permalink raw reply related [relevance 51%]
* [PATCH 9/9] lei up: automatically use dt: for remote externals
2021-09-18 9:33 67% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
2021-09-18 9:33 51% ` [PATCH 1/9] lei: lock worker counts Eric Wong
@ 2021-09-18 9:33 39% ` Eric Wong
2021-09-18 14:31 71% ` Kyle Meyer
1 sibling, 1 reply; 200+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
Since we can't use maxuid for remote externals, automatically
maintaining the last time we got results and appending a dt:
range to the query will prevent HTTP(S) responses from getting
too big.
We could be using "rt:", but no stable release of public-inbox
supports it, yet, so we'll use dt:, instead.
By default, there's a two day fudge factor to account for MTA
downtime and delays; which is hopefully enough. The fudge
factor may be changed per-invocation with the
--remote-fudge-factor=INTERVAL option
Since different externals can have different message transport
routes, "lastresult" entries are stored on a per-external basis.
---
Documentation/lei-up.pod | 15 ++++++++++
lib/PublicInbox/LEI.pm | 7 +++--
lib/PublicInbox/LeiSavedSearch.pm | 1 +
lib/PublicInbox/LeiToMail.pm | 4 ++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 50 ++++++++++++++++++++++++++-----
t/lei-q-remote-import.t | 4 +++
7 files changed, 71 insertions(+), 12 deletions(-)
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index e5d97f43..4dc6d3b8 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -22,6 +22,21 @@ C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
C<--all=local> only updates local mailboxes, C<--all=remote> only
updates remote mailboxes (currently C<imap://> and C<imaps://>).
+=item --remote-fudge-time=INTERVAL
+
+Look for mail older than the time of the last successful query.
+Using a small interval will reduce bandwidth use. A larger
+interval reduces the likelyhood missing a result due to MTA
+delays or downtime.
+
+The time(s) of the last successful queries are the C<lastresult>
+values visible from L<lei-edit-search(1)>.
+
+Date formats understood by L<git-rev-parse(1)> may be used.
+e.g C<1.hour> or C<3.days>
+
+Default: 2.days
+
=back
The following options, described in L<lei-q(1)>, are supported.
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 053b6174..8b0614f2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -183,7 +183,8 @@ our %CMD = ( # sorted in order of importance/use:
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT...|--all', 'update saved search',
- qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
+ qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+
+ remote-fudge-time=s all:s), @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
@@ -420,7 +421,9 @@ my %OPTDESC = (
'remote' => 'limit operations to those requiring network access',
'remote!' => 'prevent operations requiring network access',
-'all:s up' => ['local', 'update all (local) saved searches' ],
+'all:s up' => ['local|remote', 'update all remote or local saved searches' ],
+'remote-fudge-time=s' => [ 'INTERVAL',
+ 'look for mail INTERVAL older than the last successful query' ],
'mid=s' => 'specify the Message-ID of a message',
'oid=s' => 'specify the git object ID of a message',
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 96ff816e..754f8294 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -149,6 +149,7 @@ sub new { # new saved search "lei q --save"
$dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i;
print $fh <<EOM;
; to refresh with new results, run: lei up $sq_dst
+; `maxuid' and `lastresult' lines are maintained by "lei up" for optimization
[lei]
$q
[lei "q"]
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d3253d9b..9f7171fb 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -437,7 +437,9 @@ sub new {
($lei->{opt}->{save} ? 'LeiSavedSearch' : 'LeiDedupe');
eval "require $dd_cls";
die "$dd_cls: $@" if $@;
- $dd_cls->new($lei);
+ my $dd = $dd_cls->new($lei);
+ $lei->{lss} //= $dd if $dd && $dd->can('cfg_set');
+ $dd;
};
$self;
}
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 4637cb46..abb05d46 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -45,7 +45,7 @@ sub up1 ($$) {
ref($v) and return $lei->fail("multiple values of $c in $f");
$lei->{opt}->{$k} = $v;
}
- $lei->{lss} = $lss; # for LeiOverview->new
+ $lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
my $lxs = $lei->lxs_prepare or return;
$lei->ale->refresh_externals($lxs, $lei);
$lei->_start_query;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 50cadb5e..1d49da3d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -18,6 +18,7 @@ use PublicInbox::Smsg;
use PublicInbox::Eml;
use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
use PublicInbox::ContentHash qw(git_sha);
+use POSIX qw(strftime);
sub new {
my ($class) = @_;
@@ -168,8 +169,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $can_kw = !!$ibxish->can('msg_keywords');
my $threads = $lei->{opt}->{threads} // 0;
my $fl = $threads > 1 ? 1 : undef;
- my $lss = $lei->{dedupe};
- $lss = undef unless $lss && $lss->can('cfg_set'); # saved search
+ my $lss = $lei->{lss};
my $maxk = "external.$dir.maxuid";
my $stop_at = $lss ? $lss->{-cfg}->{$maxk} : undef;
if (defined $stop_at) {
@@ -292,6 +292,37 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
$each_smsg->($smsg, undef, $eml);
}
+sub fudge_qstr_time ($$$) {
+ my ($lei, $uri, $qstr) = @_;
+ return ($qstr, undef) unless $lei->{lss};
+ my $cfg = $lei->{lss}->{-cfg} // die 'BUG: no lss->{-cfg}';
+ my $cfg_key = "external.$uri.lastresult";
+ my $lr = $cfg->{$cfg_key} or return ($qstr, $cfg_key);
+ if ($lr !~ /\A\-?[0-9]+\z/) {
+ $lei->child_error(0,
+ "$cfg->{-f}: $cfg_key=$lr not an integer, ignoring");
+ return ($qstr, $cfg_key);
+ }
+ my $rft = $lei->{opt}->{'remote-fudge-time'};
+ if ($rft && $rft !~ /\A-?[0-9]+\z/) {
+ my @t = $lei->{lss}->git->date_parse($rft);
+ my $diff = time - $t[0];
+ $lei->qerr("# $rft => $diff seconds");
+ $rft = $diff;
+ }
+ $lr -= ($rft || (48 * 60 * 60));
+ $lei->qerr("# $uri limiting to ".
+ strftime('%Y-%m-%d %k:%M', gmtime($lr)). ' and newer');
+ # this should really be rt: (received-time), but no stable
+ # public-inbox releases support it, yet.
+ my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..';
+ if ($qstr =~ /\S/) {
+ substr($qstr, 0, 0, '(');
+ $qstr .= ') AND ';
+ }
+ ($qstr .= $dt, $cfg_key);
+}
+
sub query_remote_mboxrd {
my ($self, $uris) = @_;
local $0 = "$0 query_remote_mboxrd";
@@ -300,7 +331,7 @@ sub query_remote_mboxrd {
my $opt = $lei->{opt};
my $qstr = $lei->{mset_opt}->{qstr};
$qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
- my @qform = (q => $qstr, x => 'm');
+ my @qform = (x => 'm');
push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
my ($reap_tail, $reap_curl);
@@ -323,7 +354,9 @@ sub query_remote_mboxrd {
for my $uri (@$uris) {
$lei->{-current_url} = $uri->as_string;
$lei->{-nr_remote_eml} = 0;
- $uri->query_form(@qform);
+ my $start = time;
+ my ($q, $key) = fudge_qstr_time($lei, $uri, $qstr);
+ $uri->query_form(@qform, q => $q);
my $cmd = $curl->for_uri($lei, $uri);
$lei->qerr("# $cmd");
my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
@@ -336,10 +369,11 @@ sub query_remote_mboxrd {
@$reap_curl = (); # cancel OnDestroy
die $err if $err;
my $nr = $lei->{-nr_remote_eml};
- if ($nr && $lei->{sto}) {
- my $wait = $lei->{sto}->ipc_do('done');
- }
+ my $wait = $lei->{sto}->ipc_do('done') if $nr && $lei->{sto};
if ($? == 0) {
+ # don't update if no results, maybe MTA is down
+ $key && $nr and
+ $lei->{lss}->cfg_set($key, $start);
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
}
@@ -353,7 +387,7 @@ sub query_remote_mboxrd {
$lei->err("truncate($cmd stderr): $!");
}
next if (($? >> 8) == 22 && $err =~ /\b404\b/);
- $uri->query_form(q => $lei->{mset_opt}->{qstr});
+ $uri->query_form(q => $qstr);
$lei->child_error($?, "E: <$uri> $err");
}
undef $each_smsg;
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 9131c01b..fdf6a11e 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -99,5 +99,9 @@ EOF
lei_ok('up', "$ENV{HOME}/md");
is_deeply(\@f, [ glob("$ENV{HOME}/md/*/*") ],
'lei up remote dedupe works on maildir');
+ my $edit_env = { VISUAL => 'cat' };
+ lei_ok([qw(edit-search), "$ENV{HOME}/md"], $edit_env);
+ like($lei_out, qr/^\Q[external "$url"]\E\n\s*lastresult = \d+/sm,
+ 'lastresult set');
});
done_testing;
^ permalink raw reply related [relevance 39%]
* Re: [PATCH 9/9] lei up: automatically use dt: for remote externals
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
@ 2021-09-18 14:31 71% ` Kyle Meyer
2021-09-18 20:30 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-18 14:31 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
> index e5d97f43..4dc6d3b8 100644
> --- a/Documentation/lei-up.pod
> +++ b/Documentation/lei-up.pod
> @@ -22,6 +22,21 @@ C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
> C<--all=local> only updates local mailboxes, C<--all=remote> only
> updates remote mailboxes (currently C<imap://> and C<imaps://>).
>
> +=item --remote-fudge-time=INTERVAL
> +
> +Look for mail older than the time of the last successful query.
> +Using a small interval will reduce bandwidth use. A larger
> +interval reduces the likelyhood missing a result due to MTA
> +delays or downtime.
s/likelyhood/likelihood of/
^ permalink raw reply [relevance 71%]
* Re: [PATCH 9/9] lei up: automatically use dt: for remote externals
2021-09-18 14:31 71% ` Kyle Meyer
@ 2021-09-18 20:30 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-18 20:30 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> Eric Wong writes:
> > +interval reduces the likelyhood missing a result due to MTA
> > +delays or downtime.
>
> s/likelyhood/likelihood of/
Thanks, pushed as commit f3d0e746c6a35c8600b91af99958a52cbc114a4b
^ permalink raw reply [relevance 71%]
* [PATCH] t/lei-refresh-mail-sync: improve test reliability
@ 2021-09-18 22:38 66% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-18 22:38 UTC (permalink / raw)
To: meta
We can't assume -imapd will be ready by the time we try to
connect to it after restart when using "-l $ADDR". So recreate
the (closed-for-testing) listen socket in the parent and hand it
off to -imapd as we do normally
---
t/lei-refresh-mail-sync.t | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
index 90356b57..43fbc50a 100644
--- a/t/lei-refresh-mail-sync.t
+++ b/t/lei-refresh-mail-sync.t
@@ -4,6 +4,7 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
require_mods(qw(lei));
use File::Path qw(remove_tree);
+require Socket;
my $stop_daemon = sub { # needed since we don't have inotify
lei_ok qw(daemon-pid);
@@ -81,8 +82,10 @@ SKIP: {
my $cfg_path2 = "$home/cfg2";
File::Copy::cp($cfg_path, $cfg_path2);
my $env = { PI_CONFIG => $cfg_path2 };
+ my $sock_cls;
for my $x (qw(imapd)) {
my $s = tcp_server;
+ $sock_cls //= ref($s);
my $cmd = [ "-$x", '-W0', "--stdout=$home/$x.out",
"--stderr=$home/$x.err" ];
my $td = start_script($cmd, $env, { 3 => $s}) or xbail("-$x");
@@ -124,10 +127,17 @@ SKIP: {
ok(!(lei 'refresh-mail-sync', '--all'), '--all fails on dead -imapd');
# restart server (somewhat dangerous since we released the socket)
+ my $listen = $sock_cls->new(
+ ReuseAddr => 1,
+ Proto => 'tcp',
+ Type => Socket::SOCK_STREAM(),
+ Listen => 1024,
+ Blocking => 0,
+ LocalAddr => $srv->{imapd}->{addr},
+ ) or xbail "$sock_cls->new: $!";
my $cmd = $srv->{imapd}->{cmd};
- push @$cmd, '-l', $srv->{imapd}->{addr};
- $srv->{imapd}->{td} = start_script($cmd, $env) or xbail "@$cmd";
-
+ $srv->{imapd}->{td} = start_script($cmd, $env, { 3 => $listen }) or
+ xbail "@$cmd";
lei_ok 'refresh-mail-sync', '--all';
lei_ok 'inspect', "blob:$oid";
is($lei_out, $before, 'no changes when server was down');
^ permalink raw reply related [relevance 66%]
* [PATCH 00/16] lei IPC overhaul, NNTP fixes
@ 2021-09-19 12:50 61% Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
` (6 more replies)
0 siblings, 7 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
11/16 is a bit worrying for saved search dedupe over HTTP(S),
and I can't seem to reproduce it reliably, either..
ls-mail-source and import use is far nicer, as it provides a
good avenue for doing partial fetches.
lei/store IPC got a massive overhaul, and the sto_done_request
simplification is nice. This will probably simplify automatic
export-kw support to IMAP folders.
I also noticed "lei config --edit" was wonky, so
I made it share code with "lei edit-search".
Starting to document config knobs, too.
Eric Wong (16):
ipc: wq_do: support synchronous waits and responses
ipc: allow disabling broadcast for wq_workers
lei/store: use SOCK_SEQPACKET rather than pipe
lei: simplify sto_done_request
lei_xsearch: drop Data::Dumper use
ipc: drop dynamic WQ process counts
lei: clamp internal worker processes to 4
lei ls-mail-source: use "high"/"low" for NNTP
lei ls-mail-source: pretty JSON support
net_reader: fix single NNTP article fetch, test ranges
xt: add fsck script over over.sqlite3
watch: use net_reader->mic_new wrapper for SOCKS+TLS
net_reader: no STARTTLS for IMAP localhost or onions
lei config --edit: use controlling terminal
net_reader: disallow imap.fetchBatchSize=0
doc: lei-config: document various knobs
Documentation/lei-config.pod | 91 +++++++++++++++++++-
MANIFEST | 2 +
lib/PublicInbox/IPC.pm | 117 +++++++++++---------------
lib/PublicInbox/LEI.pm | 32 +++----
lib/PublicInbox/LeiConfig.pm | 42 +++++++++
lib/PublicInbox/LeiEditSearch.pm | 60 +++++--------
lib/PublicInbox/LeiExternal.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 2 +-
lib/PublicInbox/LeiImportKw.pm | 6 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInit.pm | 4 +-
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 25 +++---
lib/PublicInbox/LeiNoteEvent.pm | 11 +--
lib/PublicInbox/LeiRefreshMailSync.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 4 +-
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 16 +---
lib/PublicInbox/LeiStore.pm | 22 ++---
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 22 ++---
lib/PublicInbox/LeiXSearch.pm | 9 +-
lib/PublicInbox/NetReader.pm | 39 +++++----
lib/PublicInbox/WQWorker.pm | 9 +-
lib/PublicInbox/Watch.pm | 3 +-
t/imapd-tls.t | 11 ++-
t/ipc.t | 19 ++---
t/lei-import-nntp.t | 26 ++++++
t/lei.t | 3 +
t/nntpd-tls.t | 8 ++
t/uri_nntps.t | 3 +
xt/over-fsck.perl | 44 ++++++++++
32 files changed, 403 insertions(+), 239 deletions(-)
create mode 100644 lib/PublicInbox/LeiConfig.pm
create mode 100644 xt/over-fsck.perl
^ permalink raw reply [relevance 61%]
* [PATCH 04/16] lei: simplify sto_done_request
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
@ 2021-09-19 12:50 58% ` Eric Wong
2021-09-19 12:50 68% ` [PATCH 07/16] lei: clamp internal worker processes to 4 Eric Wong
` (4 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
With the switch from pipes to sockets for lei-daemon =>
lei/store IPC, we can send the script/lei client socket to the
lei/store process and rely on reference counting in both Perl
and the kernel to persist the script/lei.
---
lib/PublicInbox/LEI.pm | 13 ++-----------
lib/PublicInbox/LeiRefreshMailSync.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 13 +------------
3 files changed, 4 insertions(+), 24 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 549b855b..f62e82dc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -40,7 +40,6 @@ $GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => [ ino watches ] }
-our %LIVE_SOCK; # "GLOB(0x....)" => $lei->{sock}
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -580,7 +579,6 @@ sub _lei_atfork_child {
$dir_idle->force_close if $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
- %LIVE_SOCK = ();
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
undef $errors_log;
$quit = \&CORE::exit;
@@ -619,7 +617,6 @@ sub pkt_ops {
$ops->{x_it} = [ \&x_it, $lei ];
$ops->{child_error} = [ \&child_error, $lei ];
$ops->{incr} = [ \&incr, $lei ];
- $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
$ops;
}
@@ -1496,12 +1493,11 @@ sub lms {
(-f $f || $rw) ? PublicInbox::LeiMailSync->new($f) : undef;
}
-sub sto_done_request { # only call this from lei-daemon process (not workers)
+sub sto_done_request {
my ($lei, $sock) = @_;
eval {
if ($sock //= $lei->{sock}) { # issue, async wait
- $LIVE_SOCK{"$sock"} = $sock;
- $lei->{sto}->wq_do('done', "$sock");
+ $lei->{sto}->wq_io_do('done', [ $sock ]);
} else { # forcibly wait
my $wait = $lei->{sto}->wq_do('done');
}
@@ -1509,9 +1505,4 @@ sub sto_done_request { # only call this from lei-daemon process (not workers)
$lei->err($@) if $@;
}
-sub sto_done_complete { # called in lei-daemon when LeiStore->done is complete
- my ($sock_str) = @_;
- delete $LIVE_SOCK{$sock_str}; # frees {sock} for waiting lei clients
-}
-
1;
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 72b8fe63..2f105005 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -60,7 +60,7 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
$self->folder_missing($$uri);
}
} else { die "BUG: $input not supported" }
- $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
+ $self->{lei}->sto_done_request;
}
sub lei_refresh_mail_sync {
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 4ec63699..164a9f2d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -534,10 +534,6 @@ sub done {
$self->{priv_eidx}->done; # V2Writable::done
xchg_stderr($self);
die $err if $err;
-
- # notify clients ->done has been issued
- defined($sock_ref) and
- $self->{s2d_op_p}->pkt_do('sto_done_complete', $sock_ref);
}
sub ipc_atfork_child {
@@ -562,9 +558,6 @@ sub write_prepare {
my ($self, $lei) = @_;
$lei // die 'BUG: $lei not passed';
unless ($self->{-ipc_req}) {
- # s2d => store-to-daemon messages
- require PublicInbox::PktOp;
- my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair;
my $dir = $lei->store_path;
substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
@@ -576,14 +569,10 @@ sub write_prepare {
$self->wq_workers_start("lei/store $dir", 1, $lei->oldset, {
lei => $lei,
-err_wr => $w,
- to_close => [ $r, $s2d_op_c->{sock} ],
- s2d_op_p => $s2d_op_p,
+ to_close => [ $r ],
});
require PublicInbox::LeiStoreErr;
PublicInbox::LeiStoreErr->new($r, $lei);
- $s2d_op_c->{ops} = {
- sto_done_complete => [ $lei->can('sto_done_complete') ]
- };
}
$lei->{sto} = $self;
}
^ permalink raw reply related [relevance 58%]
* [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
@ 2021-09-19 12:50 34% ` Eric Wong
2021-09-19 12:50 58% ` [PATCH 04/16] lei: simplify sto_done_request Eric Wong
` (5 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
This has several advantages:
* no need to use ipc.lock to protect a pipe for non-atomic writes
* ability to pass FDs. In another commit, this will let us
simplify lei->sto_done_request and pass newly-created
sockets to lei/store directly.
disadvantages:
- an extra pipe is required for rare messages over several
hundred KB, this is probably a non-issue, though
The performance delta is unknown, but I expect shards
(which remain pipes) to be the primary bottleneck IPC-wise
for lei/store.
---
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiImport.pm | 2 +-
lib/PublicInbox/LeiImportKw.pm | 2 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiNoteEvent.pm | 8 ++++----
lib/PublicInbox/LeiRemote.pm | 4 ++--
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 10 ++++++++--
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 22 ++++++++++++----------
lib/PublicInbox/LeiXSearch.pm | 6 +++---
12 files changed, 37 insertions(+), 29 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8b0614f2..549b855b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1501,9 +1501,9 @@ sub sto_done_request { # only call this from lei-daemon process (not workers)
eval {
if ($sock //= $lei->{sock}) { # issue, async wait
$LIVE_SOCK{"$sock"} = $sock;
- $lei->{sto}->ipc_do('done', "$sock");
+ $lei->{sto}->wq_do('done', "$sock");
} else { # forcibly wait
- my $wait = $lei->{sto}->ipc_do('done');
+ my $wait = $lei->{sto}->wq_do('done');
}
};
$lei->err($@) if $@;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 9084d771..40530914 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -16,7 +16,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
if (my $all_vmd = $self->{all_vmd}) {
@$vmd{keys %$all_vmd} = values %$all_vmd;
}
- $self->{lei}->{sto}->ipc_do('set_eml', $eml, $vmd, $xoids);
+ $self->{lei}->{sto}->wq_do('set_eml', $eml, $vmd, $xoids);
}
sub input_mbox_cb { # MboxReader callback
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 402125cf..2863d17f 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -37,7 +37,7 @@ sub ck_update_kw { # via wq_io_do
$self->{lse}->kw_changed(undef, $kw, \@docids) or return;
$self->{verbose} and
$self->{lei}->qerr('# '.unpack('H*', $oidbin)." => @$kw\n");
- $self->{sto}->ipc_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
+ $self->{sto}->wq_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
}
sub ikw_done_wait {
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 1b327a2c..b3f3e1a0 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -16,7 +16,7 @@ sub input_eml_cb { # used by input_maildir_cb and input_net_cb
if (my $all_vmd = $self->{all_vmd}) {
@$vmd{keys %$all_vmd} = values %$all_vmd;
}
- $self->{lei}->{sto}->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+ $self->{lei}->{sto}->wq_do('index_eml_only', $eml, $vmd, $xoids);
}
sub input_fh { # overrides PublicInbox::LeiInput::input_fh
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index fe736981..22bedba6 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -378,7 +378,7 @@ sub process_inputs {
}
# always commit first, even on error partial work is acceptable for
# lei <import|tag|convert>
- my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto};
+ my $wait = $self->{lei}->{sto}->wq_do('done') if $self->{lei}->{sto};
$self->{lei}->fail($err) if $err;
}
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 18313359..5f692e75 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -36,18 +36,18 @@ sub eml_event ($$$$) {
my ($self, $eml, $vmd, $state) = @_;
my $sto = $self->{lei}->{sto};
if ($state =~ /\Aimport-(?:rw|ro)\z/) {
- $sto->ipc_do('set_eml', $eml, $vmd);
+ $sto->wq_do('set_eml', $eml, $vmd);
} elsif ($state =~ /\Aindex-(?:rw|ro)\z/) {
my $xoids = $self->{lei}->ale->xoids_for($eml);
- $sto->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+ $sto->wq_do('index_eml_only', $eml, $vmd, $xoids);
} elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
my $docids = [];
my $c = $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
if (scalar @$docids) { # already in lei/store
- $sto->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+ $sto->wq_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
# it's in an external, only set kw, here
- $sto->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+ $sto->wq_do('set_xvmd', $xoids, $eml, $vmd);
} # else { totally unknown: ignore
} else {
warn "unknown state: $state (in $self->{lei}->{cfg}->{'-f'})\n";
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 8d4ffed0..346aa6a4 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -28,7 +28,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
my $xoids = $lei->{ale}->xoids_for($eml, 1);
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($lei->{sto} && !$xoids) { # memoize locally
- my $res = $lei->{sto}->ipc_do('add_eml', $eml);
+ my $res = $lei->{sto}->wq_do('add_eml', $eml);
$smsg = $res if ref($res) eq ref($smsg);
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
@@ -56,7 +56,7 @@ sub mset {
my $err = waitpid($pid, 0) == $pid ? undef
: "BUG: waitpid($cmd): $!";
@$reap = (); # cancel OnDestroy
- my $wait = $self->{lei}->{sto}->ipc_do('done');
+ my $wait = $self->{lei}->{sto}->wq_do('done');
die $err if $err;
$self; # we are the mset (and $ibx, and $self)
}
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 3371f3ed..97b1c5c1 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -10,7 +10,7 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
- $self->{lei}->{sto}->ipc_do('remove_eml', $eml);
+ $self->{lei}->{sto}->wq_do('remove_eml', $eml);
}
sub input_mbox_cb { # MboxReader callback
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 08add8f5..4ec63699 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -552,6 +552,12 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
+sub recv_and_run {
+ my ($self, @args) = @_;
+ local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
+ $self->SUPER::recv_and_run(@args);
+}
+
sub write_prepare {
my ($self, $lei) = @_;
$lei // die 'BUG: $lei not passed';
@@ -560,14 +566,14 @@ sub write_prepare {
require PublicInbox::PktOp;
my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair;
my $dir = $lei->store_path;
- $self->ipc_lock_init("$dir/ipc.lock");
substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
$w->autoflush(1);
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
- $self->ipc_worker_spawn("lei/store $dir", $lei->oldset, {
+ $self->{-wq_no_bcast} = 1;
+ $self->wq_workers_start("lei/store $dir", 1, $lei->oldset, {
lei => $lei,
-err_wr => $w,
to_close => [ $r, $s2d_op_c->{sock} ],
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index c4f5ecff..9bbf0d79 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -12,7 +12,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
$self->{lei}->{ale}->xoids_for($eml)) {
- $self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
+ $self->{lei}->{sto}->wq_do('update_xvmd', $xoids, $eml,
$self->{vmd_mod});
} else {
++$self->{unimported};
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 9f7171fb..a419b83f 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -215,14 +215,14 @@ sub update_kw_maybe ($$$$) {
my $c = $lse->kw_changed($eml, $kw, my $docids = []);
my $vmd = { kw => $kw };
if (scalar @$docids) { # already in lei/store
- $lei->{sto}->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+ $lei->{sto}->wq_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $lei->{ale}->xoids_for($eml)) {
# it's in an external, only set kw, here
- $lei->{sto}->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+ $lei->{sto}->wq_do('set_xvmd', $xoids, $eml, $vmd);
} else { # never-before-seen, import the whole thing
# XXX this is critical in protecting against accidental
# data loss without --augment
- $lei->{sto}->ipc_do('set_eml', $eml, $vmd);
+ $lei->{sto}->wq_do('set_eml', $eml, $vmd);
}
}
@@ -296,7 +296,7 @@ sub _maildir_write_cb ($$) {
$lse->xsmsg_vmd($smsg) if $lse;
my $n = _buf2maildir($dst, $bref // \($eml->as_string),
$smsg, $dir);
- $sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
+ $sto->wq_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
++$lei->{-nr_write};
}
}
@@ -326,7 +326,7 @@ sub _imap_write_cb ($$) {
}
# imap_append returns UID if IMAP server has UIDPLUS extension
($sto && $uid =~ /\A[0-9]+\z/) and
- $sto->ipc_do('set_sync_info',
+ $sto->wq_do('set_sync_info',
$smsg->{blob}, $$uri, $uid + 0);
++$lei->{-nr_write};
}
@@ -360,7 +360,7 @@ sub _v2_write_cb ($$) {
my ($bref, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($bref);
return if $dedupe && $dedupe->is_dup($eml, $smsg);
- $lei->{v2w}->ipc_do('add', $eml); # V2Writable->add
+ $lei->{v2w}->wq_do('add', $eml); # V2Writable->add
++$lei->{-nr_write};
}
}
@@ -658,9 +658,10 @@ sub _pre_augment_v2 {
}
PublicInbox::InboxWritable->new($ibx, @creat);
$ibx->init_inbox if @creat;
- my $v2w = $lei->{v2w} = $ibx->importer;
- $v2w->ipc_lock_init("$dir/ipc.lock");
- $v2w->ipc_worker_spawn("lei/v2w $dir", $lei->oldset, { lei => $lei });
+ my $v2w = $ibx->importer;
+ $v2w->{-wq_no_bcast} = 1;
+ $v2w->wq_workers_start("lei/v2w $dir", 1, $lei->oldset, {lei => $lei});
+ $lei->{v2w} = $v2w;
return if !$lei->{opt}->{shared};
my $d = "$lei->{ale}->{git}->{git_dir}/objects";
my $al = "$dir/git/0.git/objects/info/alternates";
@@ -689,7 +690,7 @@ sub do_augment { # slow, runs in wq worker
sub post_augment {
my ($self, $lei, @args) = @_;
my $wait = $lei->{opt}->{'import-before'} ?
- $lei->{sto}->ipc_do('checkpoint', 1) : 0;
+ $lei->{sto}->wq_do('checkpoint', 1) : 0;
# _post_augment_mbox
my $m = $self->can("_post_augment_$self->{base_type}") or return;
$m->($self, $lei, @args);
@@ -774,6 +775,7 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
+ local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
my $lei = $self->{lei};
delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 1d49da3d..4583b067 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -269,7 +269,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
my $xoids = $lei->{ale}->xoids_for($eml, 1);
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+ my $res = $self->{import_sto}->wq_do('add_eml', $eml);
if (ref($res) eq ref($smsg)) { # totally new message
$smsg = $res;
$smsg->{kw} = []; # short-circuit xsmsg_vmd
@@ -369,7 +369,7 @@ sub query_remote_mboxrd {
@$reap_curl = (); # cancel OnDestroy
die $err if $err;
my $nr = $lei->{-nr_remote_eml};
- my $wait = $lei->{sto}->ipc_do('done') if $nr && $lei->{sto};
+ my $wait = $lei->{sto}->wq_do('done') if $nr && $lei->{sto};
if ($? == 0) {
# don't update if no results, maybe MTA is down
$key && $nr and
@@ -413,7 +413,7 @@ sub query_done { # EOF callback for main daemon
warn "BUG: {sto} missing with --mail-sync";
}
$lei->sto_done_request if $lei->{sto};
- my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+ my $wait = $lei->{v2w} ? $lei->{v2w}->wq_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
^ permalink raw reply related [relevance 34%]
* [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (2 preceding siblings ...)
2021-09-19 12:50 68% ` [PATCH 07/16] lei: clamp internal worker processes to 4 Eric Wong
@ 2021-09-19 12:50 71% ` Eric Wong
2021-09-19 12:50 62% ` [PATCH 09/16] lei ls-mail-source: pretty JSON support Eric Wong
` (2 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
The meanings of "hwm" and "lwm" may not be obvious abbreviations
for (high|low) water mark descriptions used by RFC 3977.
"high" and "low" should be obvious to anyone.
---
lib/PublicInbox/LeiLsMailSource.pm | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index bcb1838e..a2e75e94 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -59,10 +59,10 @@ sub input_path_url { # overrides LeiInput version
# <https://rt.cpan.org/Ticket/Display.html?id=129966>
$desc =~ s/\r\z//;
- my ($hwm, $lwm, $status) = @{$all->{$ng}};
+ my ($high, $low, $status) = @{$all->{$ng}};
push @x, { name => $ng, url => "$sec/$ng",
- lwm => $lwm + 0,
- hwm => $hwm + 0, status => $status,
+ low => $low + 0,
+ high => $high + 0, status => $status,
description => $desc };
}
@f = map { "$sec/$_" } keys %$all;
^ permalink raw reply related [relevance 71%]
* [PATCH 07/16] lei: clamp internal worker processes to 4
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
2021-09-19 12:50 58% ` [PATCH 04/16] lei: simplify sto_done_request Eric Wong
@ 2021-09-19 12:50 68% ` Eric Wong
2021-09-19 12:50 71% ` [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP Eric Wong
` (3 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
"All" my CPUs is only 4, but it's probably ridiculous for
somebody with a 16-core system to have 16 processes for
accessing SQLite DBs.
We do the same thing in Pmdir for parallel Maildir access
(and V2Writable).
---
lib/PublicInbox/LeiImportKw.pm | 4 +++-
lib/PublicInbox/LeiNoteEvent.pm | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 2863d17f..379101c2 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -11,7 +11,9 @@ use parent qw(PublicInbox::IPC);
sub new {
my ($cls, $lei) = @_;
my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
- my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
+ my $j = $self->detect_nproc // 4;
+ $j = 4 if $j > 4;
+ my ($op_c, $ops) = $lei->workers_start($self, $j);
$op_c->{ops} = $ops; # for PktOp->event_step
$self->{lei_sock} = $lei->{sock};
$lei->{ikw} = $self;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 5f692e75..a0591a09 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -80,8 +80,9 @@ sub lei_note_event {
my $self = $cfg->{-lei_note_event} //= do {
my $wq = bless { lms => $lms }, __PACKAGE__;
# MUAs such as mutt can trigger massive rename() storms so
- # use all CPU power available:
+ # use some CPU, but don't overwhelm slower storage, either
my $jobs = $wq->detect_nproc // 1;
+ $jobs = 4 if $jobs > 4; # same default as V2Writable
my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
$lei->wait_wq_events($op_c, $ops);
note_event_arm_done($lei);
^ permalink raw reply related [relevance 68%]
* [PATCH 09/16] lei ls-mail-source: pretty JSON support
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (3 preceding siblings ...)
2021-09-19 12:50 71% ` [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP Eric Wong
@ 2021-09-19 12:50 62% ` Eric Wong
2021-09-19 12:50 36% ` [PATCH 14/16] lei config --edit: use controlling terminal Eric Wong
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
As with other commands, we enable pretty JSON by default if
stdout is a terminal or if --pretty is specified. While the
->pretty JSON output has excessive vertical whitespace, too many
lines is preferable to having everything on one line.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 19 ++++++++++---------
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index def85ef1..b468a32c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -214,7 +214,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
- qw(z|0 ascii l url), @c_opt ],
+ qw(z|0 ascii l pretty url), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index a2e75e94..2265969a 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -12,15 +12,9 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_path_url { # overrides LeiInput version
my ($self, $url) = @_;
# TODO: support ndjson and other JSONs we support elsewhere
- my $json;
my $lei = $self->{lei};
- my $ORS = "\n";
- if ($self->{lei}->{opt}->{l}) {
- $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
- $json->ascii(1) if $lei->{opt}->{ascii};
- } elsif ($self->{lei}->{opt}->{z}) {
- $ORS = "\0";
- }
+ my $json = $lei->{json};
+ my $ORS = $self->{lei}->{opt}->{z} ? "\0" : "\n";
my @f;
if ($url =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($url);
@@ -93,7 +87,14 @@ sub lei_ls_mail_source {
my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
$self->{cfg} = $lei->_lei_cfg; # may be undef
$self->prepare_inputs($lei, [ $url ]) or return;
- $lei->start_pager if -t $lei->{1};
+ my $isatty = -t $lei->{1};
+ if ($lei->{opt}->{l}) {
+ my $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
+ $lei->{json} = $json;
+ $json->ascii(1) if $lei->{opt}->{ascii};
+ $json->pretty(1)->indent(2) if $isatty || $lei->{opt}->{pretty};
+ }
+ $lei->start_pager if $isatty;
my $ops = {};
$lei->{auth}->op_merge($ops, $self);
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
^ permalink raw reply related [relevance 62%]
* [PATCH 16/16] doc: lei-config: document various knobs
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (5 preceding siblings ...)
2021-09-19 12:50 36% ` [PATCH 14/16] lei config --edit: use controlling terminal Eric Wong
@ 2021-09-19 12:50 57% ` Eric Wong
2021-09-19 16:14 71% ` Kyle Meyer
6 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
It's still a work-in-progress, but the basic debug knob
comes in handy for new users; as does proxy support.
---
Documentation/lei-config.pod | 91 +++++++++++++++++++++++++++++++++++-
1 file changed, 90 insertions(+), 1 deletion(-)
diff --git a/Documentation/lei-config.pod b/Documentation/lei-config.pod
index a64045ef..daf66fe4 100644
--- a/Documentation/lei-config.pod
+++ b/Documentation/lei-config.pod
@@ -8,10 +8,99 @@ lei config [OPTIONS]
=head1 DESCRIPTION
-Call git-config(1) with C<$XDG_CONFIG_HOME/lei/config> as the
+Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
configuration file. All C<OPTIONS> are passed through, but those that
override the configuration file are not permitted.
+All C<imap> and C<nntp> options may be specified per-host or
+(if using git 2.26+) with wildcards:
+
+ [imap "imap://*.onion"]
+ proxy = socks5h://127.0.0.1:9050
+
+ [nntp "nntp://example.com"]
+ proxy = socks5h://127.0.0.1:1080
+
+=head2 VARIABLES
+
+=over 8
+
+=item external.*
+
+Managed by L<lei-add-external(1)> and L<lei-forget-external(1)>
+
+=item imap.proxy
+
+=item nntp.proxy
+
+The C<socks5h://> proxy address. Older versions of SOCKS may
+be supported if there is user demand.
+
+=item imap.starttls
+
+=item nntp.starttls
+
+Enable or disable STARTTLS on non-imaps:// and non-nntps://
+hosts. By default, STARTTLS is enabled if available unless
+connecting to a Tor .onion or localhost.
+
+=item imap.compress
+
+=item nntp.compress
+
+Enable protocol-level compression, this may be incompatible
+or broken with some servers.
+
+Note: L<Net::NNTP> compression support is pending:
+L<https://rt.cpan.org/Ticket/Display.html?id=129967>
+
+=item imap.debug
+
+=item nntp.debug
+
+Enable debugging output of underlying IMAP and NNTP libraries,
+currently L<Mail::IMAPClient> and L<Net::NNTP>, respectively.
+If using L<imap.proxy> or L<nntp.proxy> point to a SOCKS proxy,
+debugging output for L<IO::Socket::Socks> will be enabled, as
+well.
+
+Disabling L<imap.compress> may be required to improve output.
+
+=item imap.timeout
+
+=item nntp.timeout
+
+The read timeout for responses.
+
+Default: 600 seconds (IMAP); 120 seconds (NNTP)
+
+=item imap.fetchBatchSize
+
+Number of full messages to fetch at once. Larger values reduce
+network round trips at the cost of higher memory use, especially
+when retrieving large messages.
+
+Small responses for IMAP flags are fetched at 10000 times this value.
+
+Default: 1
+
+=item imap.ignoreSizeErrors
+
+Ignore size mismatches from broken IMAP server implementations.
+
+Default: false
+
+=item color.SLOT
+
+C<quoted>, C<hdrdefault>, C<status>, C<attachment> color slots
+are supported for the C<-f text> and C<-f reply> output formats
+of L<lei-lcat(1)> and L<lei-q(1)>.
+
+The any per-project .git/config, and global ~/.gitconfig files
+will also be parsed for diff coloring. git diff color slots
+(C<color.diff.SLOT>) supported are C<new>, C<old>, C<meta>,
+C<frag>, C<func>, and C<context>.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
^ permalink raw reply related [relevance 57%]
* [PATCH 14/16] lei config --edit: use controlling terminal
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (4 preceding siblings ...)
2021-09-19 12:50 62% ` [PATCH 09/16] lei ls-mail-source: pretty JSON support Eric Wong
@ 2021-09-19 12:50 36% ` Eric Wong
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
As with "lei edit-search", "lei config --edit" may
spawn an interactive editor which works best from
the terminal running script/lei.
So implement LeiConfig as a superclass of LeiEditSearch
so the two commands can share the same verification
hooks and retry logic.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 18 +++++-----
lib/PublicInbox/LeiConfig.pm | 42 ++++++++++++++++++++++
lib/PublicInbox/LeiEditSearch.pm | 60 +++++++++++--------------------
lib/PublicInbox/LeiExternal.pm | 2 +-
lib/PublicInbox/LeiInit.pm | 4 +--
lib/PublicInbox/LeiSavedSearch.pm | 16 +++------
t/lei.t | 3 ++
8 files changed, 82 insertions(+), 64 deletions(-)
create mode 100644 lib/PublicInbox/LeiConfig.pm
diff --git a/MANIFEST b/MANIFEST
index 2df743f8..8c2e964b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiALE.pm
lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
lib/PublicInbox/LeiBlob.pm
+lib/PublicInbox/LeiConfig.pm
lib/PublicInbox/LeiConvert.pm
lib/PublicInbox/LeiCurl.pm
lib/PublicInbox/LeiDedupe.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b468a32c..148a5b1e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -278,7 +278,7 @@ our %CMD = ( # sorted in order of importance/use:
'config' => [ '[...]', sub {
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
- qw(c=s@ C=s@), pass_through('git config') ],
+ qw(edit|e c=s@ C=s@), pass_through('git config') ],
'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
qw(stdin| pretty ascii dir=s), @c_opt ],
@@ -870,14 +870,6 @@ sub _config {
waitpid(spawn($cmd, \%env, \%rdr), 0);
}
-sub lei_config {
- my ($self, @argv) = @_;
- $self->{opt}->{'config-file'} and return fail $self,
- "config file switches not supported by `lei config'";
- _config(@_);
- x_it($self, $?) if $?;
-}
-
sub lei_daemon_pid { puts shift, $$ }
sub lei_daemon_kill {
@@ -1504,4 +1496,12 @@ sub sto_done_request {
$lei->err($@) if $@;
}
+sub cfg_dump ($$) {
+ my ($lei, $f) = @_;
+ my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
+ return $ret if !$@;
+ $lei->err($@);
+ undef;
+}
+
1;
diff --git a/lib/PublicInbox/LeiConfig.pm b/lib/PublicInbox/LeiConfig.pm
new file mode 100644
index 00000000..23be9aaf
--- /dev/null
+++ b/lib/PublicInbox/LeiConfig.pm
@@ -0,0 +1,42 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::LeiConfig;
+use strict;
+use v5.10.1;
+use PublicInbox::PktOp;
+
+sub cfg_do_edit ($;$) {
+ my ($self, $reason) = @_;
+ my $lei = $self->{lei};
+ $lei->pgr_err($reason) if defined $reason;
+ my $cmd = [ qw(git config --edit -f), $self->{-f} ];
+ my $env = { GIT_CONFIG => $self->{-f} };
+ $self->cfg_edit_begin if $self->can('cfg_edit_begin');
+ # run in script/lei foreground
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ # $op_p will EOF when $EDITOR is done
+ $op_c->{ops} = { '' => [\&cfg_edit_done, $self] };
+ $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], $cmd, $env);
+}
+
+sub cfg_edit_done { # PktOp
+ my ($self) = @_;
+ eval {
+ my $cfg = $self->{lei}->cfg_dump($self->{-f}, $self->{lei}->{2})
+ // return cfg_do_edit($self, "\n");
+ $self->cfg_verify($cfg) if $self->can('cfg_verify');
+ };
+ $self->{lei}->fail($@) if $@;
+}
+
+sub lei_config {
+ my ($lei, @argv) = @_;
+ $lei->{opt}->{'config-file'} and return $lei->fail(
+ "config file switches not supported by `lei config'");
+ return $lei->_config(@argv) unless $lei->{opt}->{edit};
+ my $f = $lei->_lei_cfg(1)->{-f};
+ my $self = bless { lei => $lei, -f => $f }, __PACKAGE__;
+ cfg_do_edit($self);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 47166ce7..bcf7c105 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -7,48 +7,32 @@ use strict;
use v5.10.1;
use PublicInbox::LeiSavedSearch;
use PublicInbox::LeiUp;
+use parent qw(PublicInbox::LeiConfig);
-sub edit_begin {
- my ($lss, $lei) = @_;
- if (ref($lss->{-cfg}->{'lei.q.output'})) {
- delete $lss->{-cfg}->{'lei.q.output'}; # invalid
- $lei->pgr_err(<<EOM);
-$lss->{-f} has multiple values of lei.q.output
+sub cfg_edit_begin {
+ my ($self) = @_;
+ if (ref($self->{lss}->{-cfg}->{'lei.q.output'})) {
+ delete $self->{lss}->{-cfg}->{'lei.q.output'}; # invalid
+ $self->{lei}->pgr_err(<<EOM);
+$self->{lss}->{-f} has multiple values of lei.q.output
please remove redundant ones
EOM
}
- $lei->{-lss_for_edit} = $lss;
}
-sub do_edit ($$;$) {
- my ($lss, $lei, $reason) = @_;
- $lei->pgr_err($reason) if defined $reason;
- my @cmd = (qw(git config --edit -f), $lss->{'-f'});
- $lei->qerr("# spawning @cmd");
- edit_begin($lss, $lei);
- # run in script/lei foreground
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- # $op_p will EOF when $EDITOR is done
- $op_c->{ops} = { '' => [\&op_edit_done, $lss, $lei] };
- $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], \@cmd, {});
-}
-
-sub _edit_done {
- my ($lss, $lei) = @_;
- my $cfg = $lss->can('cfg_dump')->($lei, $lss->{'-f'}) //
- return do_edit($lss, $lei, <<EOM);
-$lss->{-f} is unparseable
-EOM
+sub cfg_verify {
+ my ($self, $cfg) = @_;
my $new_out = $cfg->{'lei.q.output'} // '';
- return do_edit($lss, $lei, <<EOM) if ref $new_out;
-$lss->{-f} has multiple values of lei.q.output
+ return $self->cfg_do_edit(<<EOM) if ref $new_out;
+$self->{-f} has multiple values of lei.q.output
EOM
- return do_edit($lss, $lei, <<EOM) if $new_out eq '';
-$lss->{-f} needs lei.q.output
+ return $self->cfg_do_edit(<<EOM) if $new_out eq '';
+$self->{-f} needs lei.q.output
EOM
+ my $lss = $self->{lss};
my $old_out = $lss->{-cfg}->{'lei.q.output'} // return;
return if $old_out eq $new_out;
+ my $lei = $self->{lei};
my $old_path = $old_out;
my $new_path = $new_out;
s!$PublicInbox::LeiSavedSearch::LOCAL_PFX!! for ($old_path, $new_path);
@@ -57,10 +41,10 @@ EOM
return if $dir_new eq $dir_old;
($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
- return do_edit($lss, $lei, <<EOM);
+ return $self->cfg_do_edit(<<EOM);
conversions from/to v2 inboxes not supported at this time
EOM
- return do_edit($lss, $lei, <<EOM) if -e $dir_new;
+ return $self->cfg_do_edit(<<EOM) if -e $dir_new;
lei.q.output changed from `$old_out' to `$new_out'
However, $dir_new exists
EOM
@@ -79,16 +63,12 @@ E: rename($dir_old, $dir_new) error: $!
EOM
}
-sub op_edit_done { # PktOp
- my ($lss, $lei) = @_;
- eval { _edit_done($lss, $lei) };
- $lei->fail($@) if $@;
-}
-
sub lei_edit_search {
my ($lei, $out) = @_;
my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
- do_edit($lss, $lei);
+ my $f = $lss->{-f};
+ my $self = bless { lei => $lei, lss => $lss, -f => $f }, __PACKAGE__;
+ $self->cfg_do_edit;
}
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 6fd3efef..d802f0e2 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -139,7 +139,7 @@ sub add_external_finish {
my $key = "external.$location.boost";
my $cur_boost = $cfg->{$key};
return if defined($cur_boost) && $cur_boost == $new_boost; # idempotent
- $self->lei_config($key, $new_boost);
+ $self->_config($key, $new_boost);
}
sub lei_add_external {
diff --git a/lib/PublicInbox/LeiInit.pm b/lib/PublicInbox/LeiInit.pm
index 6558ac0a..27ce8169 100644
--- a/lib/PublicInbox/LeiInit.pm
+++ b/lib/PublicInbox/LeiInit.pm
@@ -23,7 +23,7 @@ sub lei_init {
# some folks like symlinks and bind mounts :P
if (@dir && "@cur[1,0]" eq "@dir[1,0]") {
- $self->lei_config('leistore.dir', $dir);
+ $self->_config('leistore.dir', $dir);
$self->_lei_store(1)->done;
return $self->qerr("$exists (as $cur)");
}
@@ -31,7 +31,7 @@ sub lei_init {
E: leistore.dir=$cur already initialized and it is not $dir
}
- $self->lei_config('leistore.dir', $dir);
+ $self->_config('leistore.dir', $dir);
$self->_lei_store(1)->done;
$exists //= "# leistore.dir=$dir newly initialized";
$self->qerr($exists);
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 754f8294..89f5c359 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -29,14 +29,6 @@ sub BOOL_FIELDS () {
qw(external local remote import-remote import-before threads)
}
-sub cfg_dump ($$) {
- my ($lei, $f) = @_;
- my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
- return $ret if !$@;
- $lei->err($@);
- undef;
-}
-
sub lss_dir_for ($$;$) {
my ($lei, $dstref, $on_fs) = @_;
my $pfx;
@@ -64,7 +56,7 @@ sub lss_dir_for ($$;$) {
for my $g ("$pfx-*", '*') {
my @maybe = glob("$lss_dir$g/lei.saved-search");
for my $f (@maybe) {
- $c = cfg_dump($lei, $f) // next;
+ $c = $lei->cfg_dump($f) // next;
$o = $c->{'lei.q.output'} // next;
$o =~ s!$LOCAL_PFX!! or next;
@st = stat($o) or next;
@@ -88,7 +80,7 @@ sub list {
print $fh "\tpath = ", cquote_val($p), "\n";
}
close $fh or die "close $f: $!";
- my $cfg = cfg_dump($lei, $f);
+ my $cfg = $lei->cfg_dump($f);
unlink($f);
my $out = $cfg ? $cfg->get_all('lei.q.output') : [];
map {;
@@ -113,7 +105,7 @@ sub up { # updating existing saved search via "lei up"
output2lssdir($self, $lei, \$dir, \$f) or
return $lei->fail("--save was not used with $dst cwd=".
$lei->rel2abs('.'));
- $self->{-cfg} = cfg_dump($lei, $f) // return $lei->fail;
+ $self->{-cfg} = $lei->cfg_dump($f) // return $lei->fail;
$self->{-ovf} = "$dir/over.sqlite3";
$self->{'-f'} = $f;
$self->{lock_path} = "$self->{-f}.flock";
@@ -276,7 +268,7 @@ sub output2lssdir {
my $dir = lss_dir_for($lei, \$dst, 1);
my $f = "$dir/lei.saved-search";
if (-f $f && -r _) {
- $self->{-cfg} = cfg_dump($lei, $f) // return;
+ $self->{-cfg} = $lei->cfg_dump($f) // return;
$$dir_ref = $dir;
$$fn_ref = $f;
return 1;
diff --git a/t/lei.t b/t/lei.t
index c8f47576..53fc43fb 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -100,6 +100,9 @@ my $test_config = sub {
is($lei_out, "tr00\n", "-c string value passed as-is");
lei_ok(qw(-c imap.debug=a -c imap.debug=b config --get-all imap.debug));
is($lei_out, "a\nb\n", '-c and --get-all work together');
+
+ lei_ok([qw(config -e)], { VISUAL => 'cat' });
+ is($lei_out, "[a]\n\tb = c\n", '--edit works');
};
my $test_completion = sub {
^ permalink raw reply related [relevance 36%]
* Re: [PATCH 16/16] doc: lei-config: document various knobs
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
@ 2021-09-19 16:14 71% ` Kyle Meyer
2021-09-19 20:00 65% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-19 16:14 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> diff --git a/Documentation/lei-config.pod b/Documentation/lei-config.pod
> index a64045ef..daf66fe4 100644
> --- a/Documentation/lei-config.pod
> +++ b/Documentation/lei-config.pod
> @@ -8,10 +8,99 @@ lei config [OPTIONS]
>
> =head1 DESCRIPTION
>
> -Call git-config(1) with C<$XDG_CONFIG_HOME/lei/config> as the
> +Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
Looks like the ')' was unintentionally dropped.
> +=item imap.compress
> +
> +=item nntp.compress
> +
> +Enable protocol-level compression, this may be incompatible
> +or broken with some servers.
nit: s/, /. This/
> +=item color.SLOT
> +
> +C<quoted>, C<hdrdefault>, C<status>, C<attachment> color slots
> +are supported for the C<-f text> and C<-f reply> output formats
> +of L<lei-lcat(1)> and L<lei-q(1)>.
> +
> +The any per-project .git/config, and global ~/.gitconfig files
s/The any/Any/ ?
> +will also be parsed for diff coloring. git diff color slots
> +(C<color.diff.SLOT>) supported are C<new>, C<old>, C<meta>,
> +C<frag>, C<func>, and C<context>.
^ permalink raw reply [relevance 71%]
* Re: [PATCH 16/16] doc: lei-config: document various knobs
2021-09-19 16:14 71% ` Kyle Meyer
@ 2021-09-19 20:00 65% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 20:00 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> Eric Wong writes:
> > +Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
>
> Looks like the ')' was unintentionally dropped.
> > +Enable protocol-level compression, this may be incompatible
> > +or broken with some servers.
>
> nit: s/, /. This/
> > +The any per-project .git/config, and global ~/.gitconfig files
>
> s/The any/Any/ ?
Thanks, will squash the below. I'll also clarify the sentence
on imap.debug vs imap.compress
diff --git a/Documentation/lei-config.pod b/Documentation/lei-config.pod
index daf66fe4..c91f36ee 100644
--- a/Documentation/lei-config.pod
+++ b/Documentation/lei-config.pod
@@ -8,7 +8,7 @@ lei config [OPTIONS]
=head1 DESCRIPTION
-Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
+Call L<git-config(1)> with C<$XDG_CONFIG_HOME/lei/config> as the
configuration file. All C<OPTIONS> are passed through, but those that
override the configuration file are not permitted.
@@ -48,7 +48,7 @@ connecting to a Tor .onion or localhost.
=item nntp.compress
-Enable protocol-level compression, this may be incompatible
+Enable protocol-level compression. This may be incompatible
or broken with some servers.
Note: L<Net::NNTP> compression support is pending:
@@ -64,7 +64,7 @@ If using L<imap.proxy> or L<nntp.proxy> point to a SOCKS proxy,
debugging output for L<IO::Socket::Socks> will be enabled, as
well.
-Disabling L<imap.compress> may be required to improve output.
+Disabling L<imap.compress> may be required for readability.
=item imap.timeout
@@ -96,7 +96,7 @@ C<quoted>, C<hdrdefault>, C<status>, C<attachment> color slots
are supported for the C<-f text> and C<-f reply> output formats
of L<lei-lcat(1)> and L<lei-q(1)>.
-The any per-project .git/config, and global ~/.gitconfig files
+Any per-project .git/config, and global ~/.gitconfig files
will also be parsed for diff coloring. git diff color slots
(C<color.diff.SLOT>) supported are C<new>, C<old>, C<meta>,
C<frag>, C<func>, and C<context>.
^ permalink raw reply related [relevance 65%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 23:06 57% ` Eric Wong
2021-09-16 17:43 71% ` Konstantin Ryabitsev
@ 2021-09-19 21:21 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 21:21 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Eric Wong <e@80x24.org> wrote:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
> # populate ~/.cache/lei/uri_folder/.sqlite3
> lei ls-mail-source nntp://nntp.lore.kernel.org
lei ls-mail-source -l nntp://nntp.lore.kernel.org
I just made "-l" show pretty JSON output in the terminal
to help find NNTP article ranges.
> # if you're using bash:
> . contrib/completion/lei-completion.bash
>
> lei import <TAB><TAB>
>
> lei import will download the entire newsgroup, which may get
> huge, unfortunately. It should support ranges (e.g.
> nntp://example.com/inbox.foo.example/900-1000) but the URI
> package doesn't support it, yet...
I was wrong about the URI package not supporting ranges,
it does; and forgot lei even seemed to have (untested)
support for it. It should be working, now, just pushed out
commit 67fe4d8d90ac77419c8fc41457c849aa7d366a9d
("net_reader: fix single NNTP article fetch, test ranges")
<https://public-inbox.org/meta/20210919125035.6331-11-e@80x24.org/>
Neither Net::NNTP nor Mail::IMAPClient support pipelining
out-of-the-box, though; so network latency is still a problem.
I'm still not sure about doing an interactive UI (my brain
tends to struggle with that stuff :x); but I think extending
tab-completion for epochs and NNTP article ranges would be
good (I'm not particularly good with tab-completion, either :x)
^ permalink raw reply [relevance 71%]
* [PATCH 06/12] doc: lei-security: section for WIP auth methods
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (3 preceding siblings ...)
2021-09-21 7:41 61% ` [PATCH 05/12] lei lcat: use single queue for ordering Eric Wong
@ 2021-09-21 7:41 71% ` Eric Wong
2021-09-21 7:41 44% ` [PATCH 07/12] lei lcat: support NNTP URLs Eric Wong
` (5 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Lots of stuff out there that becomes a pain to setup
configuration for and test...
---
Documentation/lei-security.pod | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/Documentation/lei-security.pod b/Documentation/lei-security.pod
index 4b712c2d..02305b90 100644
--- a/Documentation/lei-security.pod
+++ b/Documentation/lei-security.pod
@@ -99,6 +99,14 @@ While credentials are not written to the filesystem by default,
it is possible for them to end up on disk if processes are
swapped out. Use of an encrypted swap partition is recommended.
+=head1 AUTHENTICATION METHODS
+
+LOGIN (username + password) is known to work over IMAP(S),
+as does AUTH=ANONYMOUS (which is used by L<public-inbox-imapd(1)>
+as part of our test suite). AUTHINFO may work for NNTP, but
+is untested. Testers will be needed for other authentication
+methods.
+
=head1 DENIAL-OF-SERVICE VECTORS
lei uses the same MIME parsing library as L<public-inbox-mda(1)>
^ permalink raw reply related [relevance 71%]
* [PATCH 00/12] lei: fix various annoyances
@ 2021-09-21 7:41 68% Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
` (10 more replies)
0 siblings, 11 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Eric Wong (12):
lei inspect: convert to WQ worker
lei inspect: support NNTP URLs
lei_mail_sync: account for non-unique cases
lei: simplify internal arg2folder usage
lei lcat: use single queue for ordering
doc: lei-security: section for WIP auth methods
lei lcat: support NNTP URLs
lei: various completion improvements
lei q: show progress on >1s preparation phase
search: drop reopen retry message
lei q: update messages to reflect --save default
lei q: improve --limit behavior and progress
Documentation/lei-q.pod | 6 +-
Documentation/lei-security.pod | 8 ++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 12 +--
lib/PublicInbox/LeiExternal.pm | 5 ++
lib/PublicInbox/LeiForgetMailSync.pm | 9 +--
lib/PublicInbox/LeiImport.pm | 20 +++--
lib/PublicInbox/LeiImportKw.pm | 12 ++-
lib/PublicInbox/LeiInspect.pm | 103 +++++++++++++++++---------
lib/PublicInbox/LeiLcat.pm | 91 +++++++++++++----------
lib/PublicInbox/LeiLsMailSource.pm | 11 +--
lib/PublicInbox/LeiMailSync.pm | 102 +++++++++++++++++--------
lib/PublicInbox/LeiNoteEvent.pm | 5 +-
lib/PublicInbox/LeiQuery.pm | 9 ++-
lib/PublicInbox/LeiRefreshMailSync.pm | 18 +++--
lib/PublicInbox/LeiSavedSearch.pm | 22 +++---
lib/PublicInbox/LeiTag.pm | 14 ++--
lib/PublicInbox/LeiToMail.pm | 23 +++++-
lib/PublicInbox/LeiUp.pm | 25 +++----
lib/PublicInbox/LeiXSearch.pm | 36 ++++++---
lib/PublicInbox/Search.pm | 1 -
lib/PublicInbox/SharedKV.pm | 19 +++--
lib/PublicInbox/TestCommon.pm | 11 ++-
t/lei-import-nntp.t | 44 +++++++++++
t/lei-q-save.t | 6 +-
t/lei-watch.t | 2 +-
26 files changed, 414 insertions(+), 202 deletions(-)
^ permalink raw reply [relevance 68%]
* [PATCH 01/12] lei inspect: convert to WQ worker
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
@ 2021-09-21 7:41 64% ` Eric Wong
2021-09-21 7:41 42% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
` (9 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Xapian and SQLite access can be slow when a DB is large and/or
on high-latency storage.
---
lib/PublicInbox/LeiInspect.pm | 31 +++++++++++++++++++++----------
1 file changed, 21 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index f06cea61..48da826b 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -8,6 +8,7 @@
package PublicInbox::LeiInspect;
use strict;
use v5.10.1;
+use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
use PublicInbox::MID qw(mids);
@@ -184,9 +185,11 @@ sub inspect1 ($$$) {
1;
}
-sub _inspect_argv ($$) {
- my ($lei, $argv) = @_;
+sub inspect_argv { # via wq_do
+ my ($self) = @_;
+ my ($lei, $argv) = delete @$self{qw(lei argv)};
my $multi = scalar(@$argv) > 1;
+ $lei->{1}->autoflush(0);
$lei->out('[') if $multi;
while (defined(my $x = shift @$argv)) {
inspect1($lei, $x, scalar(@$argv)) or return;
@@ -194,6 +197,16 @@ sub _inspect_argv ($$) {
$lei->out(']') if $multi;
}
+sub inspect_start ($$) {
+ my ($lei, $argv) = @_;
+ my $self = bless { lei => $lei, argv => $argv }, __PACKAGE__;
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
+ $lei->{wq1} = $self;
+ $lei->wait_wq_events($op_c, $ops);
+ $self->wq_do('inspect_argv');
+ $self->wq_close(1);
+}
+
sub ins_add { # InputPipe->consume callback
my ($lei) = @_; # $_[1] = $rbuf
if (defined $_[1]) {
@@ -201,7 +214,7 @@ sub ins_add { # InputPipe->consume callback
my $str = delete $lei->{istr};
$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
my $eml = PublicInbox::Eml->new(\$str);
- _inspect_argv($lei, [
+ inspect_start($lei, [
'blob:'.$lei->git_oid($eml)->hexdigest,
map { "mid:$_" } @{mids($eml)} ]);
};
@@ -218,20 +231,18 @@ sub lei_inspect {
my $sto = $lei->_lei_store;
$sto ? $sto->search : undef;
} : undef;
- if ($lei->{opt}->{pretty} || -t $lei->{1}) {
- $lei->{json}->pretty(1)->indent(2);
- }
- $lei->start_pager if -t $lei->{1};
- $lei->{1}->autoflush(0);
+ my $isatty = -t $lei->{1};
+ $lei->{json}->pretty(1)->indent(2) if $lei->{opt}->{pretty} || $isatty;
+ $lei->start_pager if $isatty;
if ($lei->{opt}->{stdin}) {
return $lei->fail(<<'') if @argv;
no args allowed on command-line with --stdin
require PublicInbox::InputPipe;
PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
- return;
+ } else {
+ inspect_start($lei, \@argv);
}
- _inspect_argv($lei, \@argv);
}
sub _complete_inspect {
^ permalink raw reply related [relevance 64%]
* [PATCH 09/12] lei q: show progress on >1s preparation phase
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (6 preceding siblings ...)
2021-09-21 7:41 39% ` [PATCH 08/12] lei: various completion improvements Eric Wong
@ 2021-09-21 7:41 63% ` Eric Wong
2021-09-21 7:41 61% ` [PATCH 11/12] lei q: update messages to reflect --save default Eric Wong
` (2 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Overwriting existing destinations safe (but slow) by default,
so show a progress message noting what we're doing while
a user waits.
---
lib/PublicInbox/LeiToMail.pm | 23 ++++++++++++++++++++++-
lib/PublicInbox/LeiXSearch.pm | 2 +-
t/lei-watch.t | 2 +-
3 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index a419b83f..ed609081 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -689,6 +689,8 @@ sub do_augment { # slow, runs in wq worker
# fast (spawn compressor or mkdir), runs in same process as pre_augment
sub post_augment {
my ($self, $lei, @args) = @_;
+ $self->{-au_noted}++ and $lei->qerr("# writing to $self->{dst} ...");
+
my $wait = $lei->{opt}->{'import-before'} ?
$lei->{sto}->wq_do('checkpoint', 1) : 0;
# _post_augment_mbox
@@ -784,9 +786,28 @@ sub wq_atexit_child {
$lei->{pkt_op_p}->pkt_do('l2m_progress', $nr);
}
+# runs on a 1s timer in lei-daemon
+sub augment_inprogress {
+ my ($err, $opt, $dst, $au_noted) = @_;
+ $$au_noted++ and return;
+ print $err '# '.($opt->{'import-before'} ?
+ "importing non-external contents of $dst" : (
+ ($opt->{dedupe} // 'content') ne 'none') ?
+ "scanning old contents of $dst for dedupe" :
+ "removing old contents of $dst")." ...\n";
+}
+
# called in top-level lei-daemon when LeiAuth is done
sub net_merge_all_done {
- my ($self) = @_;
+ my ($self, $lei) = @_;
+ if ($PublicInbox::DS::in_loop &&
+ $self->can("_do_augment_$self->{base_type}") &&
+ !$lei->{opt}->{quiet}) {
+ $self->{-au_noted} = 0;
+ PublicInbox::DS::add_timer(1, \&augment_inprogress,
+ $lei->{2}, $lei->{opt},
+ $self->{dst}, \$self->{-au_noted});
+ }
$self->wq_broadcast('do_post_auth');
$self->wq_close(1);
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 3ce8f32d..2227c2ac 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -572,7 +572,7 @@ sub do_query {
$self->{opt_sort} = $lei->{opt}->{'sort'};
$self->{-do_lcat} = !!(delete $lei->{lcat_todo});
if ($l2m) {
- $l2m->net_merge_all_done unless $lei->{auth};
+ $l2m->net_merge_all_done($lei) unless $lei->{auth};
} else {
start_query($self);
}
diff --git a/t/lei-watch.t b/t/lei-watch.t
index a881fbb9..df887a03 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -91,7 +91,7 @@ test_lei(sub {
$ino_fdinfo or skip 'Linux/inotify-only removal removal', 1;
open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
my $cmp = [ <$fh> ];
- is_deeply($cmp, $ino_contents, 'inotify Maildir watches gone');
+ is_xdeeply($cmp, $ino_contents, 'inotify Maildir watches gone');
};
});
^ permalink raw reply related [relevance 63%]
* [PATCH 05/12] lei lcat: use single queue for ordering
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (2 preceding siblings ...)
2021-09-21 7:41 45% ` [PATCH 04/12] lei: simplify internal arg2folder usage Eric Wong
@ 2021-09-21 7:41 61% ` Eric Wong
2021-09-21 7:41 71% ` [PATCH 06/12] doc: lei-security: section for WIP auth methods Eric Wong
` (6 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
If lcat-ing multiple argument types (blobs vs folders),
maintain the original order of the arguments instead of
dumping all blobs before folder contents.
---
lib/PublicInbox/LeiLcat.pm | 13 ++++++-------
lib/PublicInbox/LeiXSearch.pm | 16 ++++++++--------
2 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index ccb1823d..1a4a988e 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -21,7 +21,7 @@ sub lcat_folder ($$$) {
} else {
for my $f (@$folders) {
my $fid = $lms->fid_for($f);
- push @{$lei->{lcat_fid}}, $fid;
+ push @{$lei->{lcat_todo}}, { fid => $fid };
}
}
}
@@ -31,10 +31,9 @@ sub lcat_imap_uri ($$) {
my $lms = $lei->lms or return;
# cf. LeiXsearch->lcat_dump
if (defined $uri->uid) {
- my @oidhex = $lms->imap_oidhex($lei, $uri);
- push @{$lei->{lcat_blob}}, @oidhex;
+ push @{$lei->{lcat_todo}}, $lms->imap_oidhex($lei, $uri);
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
- push @{$lei->{lcat_fid}}, $fid;
+ push @{$lei->{lcat_todo}}, { fid => $fid };
} else {
lcat_folder($lei, $lms, $$uri);
}
@@ -46,10 +45,10 @@ sub extract_1 ($$) {
my $u = $1;
require PublicInbox::URIimap;
lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
- '""'; # blank query, using {lcat_blob} or {lcat_fid}
+ '""'; # blank query, using {lcat_todo}
} elsif ($x =~ m!\b(maildir:.+)!i) {
lcat_folder($lei, undef, $1);
- '""'; # blank query, using {lcat_blob} or {lcat_fid}
+ '""'; # blank query, using {lcat_todo}
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
@@ -82,7 +81,7 @@ sub extract_1 ($$) {
} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
"mid:$1";
} elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
- push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+ push @{$lei->{lcat_todo}}, $1; # cf. LeiToMail->wq_atexit_child
'""'; # blank query
} else {
undef;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 756183a9..3ce8f32d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -570,7 +570,7 @@ sub do_query {
@$end = ();
$self->{opt_threads} = $lei->{opt}->{threads};
$self->{opt_sort} = $lei->{opt}->{'sort'};
- $self->{-do_lcat} = $lei->{lcat_blob} // $lei->{lcat_fid};
+ $self->{-do_lcat} = !!(delete $lei->{lcat_todo});
if ($l2m) {
$l2m->net_merge_all_done unless $lei->{auth};
} else {
@@ -646,13 +646,13 @@ sub lcat_dump { # via wq_io_do
$git->cat_async($smsg->{blob}, \&_lcat2smsg, $smsg);
};
}
- for my $oid (@{$lei->{lcat_blob} // []}) {
- $each_smsg->({ blob => $oid, pct => 100 });
- }
- if (my $fids = delete $lei->{lcat_fid}) {
- my $lms = $lei->{lse}->lms;
- for my $fid (@$fids) {
- $lms->each_src({fid => $fid}, \&_lcat_i, $each_smsg);
+ my $lms;
+ for my $ent (@{$lei->{lcat_todo}}) {
+ if (ref $ent eq 'HASH') { # { fid => $fid ,.. }
+ $lms //= $lei->{lse}->lms;
+ $lms->each_src($ent, \&_lcat_i, $each_smsg);
+ } else { # oidhex
+ $each_smsg->({ blob => $ent, pct => 100 });
}
}
$git->async_wait_all;
^ permalink raw reply related [relevance 61%]
* [PATCH 04/12] lei: simplify internal arg2folder usage
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
2021-09-21 7:41 42% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
@ 2021-09-21 7:41 45% ` Eric Wong
2021-09-21 7:41 61% ` [PATCH 05/12] lei lcat: use single queue for ordering Eric Wong
` (7 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
We can set opt->{quiet} for (internal) 'note-event' command
to quiet ->qerr, since we use ->qerr everywhere else. And
we'll just die() instead of setting a ->{fail} message, since
eval + die are more inline with the rest of our Perl code.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 4 +---
lib/PublicInbox/LeiForgetMailSync.pm | 4 +---
lib/PublicInbox/LeiInspect.pm | 17 +++++------------
lib/PublicInbox/LeiLcat.pm | 5 ++---
lib/PublicInbox/LeiMailSync.pm | 17 ++++++-----------
lib/PublicInbox/LeiNoteEvent.pm | 5 +++--
lib/PublicInbox/LeiRefreshMailSync.pm | 4 +---
8 files changed, 20 insertions(+), 38 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 148a5b1e..f94bfa45 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1194,7 +1194,7 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
$lei->dispatch('note-event',
"maildir:$mdir", $nc, $bn, $fn);
};
- warn "E note-event $f: $@\n" if $@;
+ warn "E: note-event $f: $@\n" if $@;
}
}
if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 8c5fbc13..d5533a2a 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -92,9 +92,7 @@ EOM
$lms->group2folders($lei, $all, \@folders) or return;
@folders = grep(/\A(?:maildir|imaps?):/i, @folders);
} else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ $lms->arg2folder($lei, \@folders); # may die
}
$lms->lms_pause;
my $self = bless { lse => $sto->search, lms => $lms }, __PACKAGE__;
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 701f48d2..d85616cc 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -16,9 +16,7 @@ sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
my $lms = $lei->lms or return;
$lms->lms_write_prepare;
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ $lms->arg2folder($lei, \@folders); # may die
$lms->forget_folders(@folders);
}
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 722ba5b2..8e128580 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -46,10 +46,9 @@ sub inspect_nntp_range {
my $ent = {};
my $ret = { "$uri" => $ent };
my $lms = $lei->lms or return $ret;
- my $err = $lms->arg2folder($lei, my $folders = [ $$uri ]);
- if ($err) {
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- }
+ my $folders = [ $$uri ];
+ eval { $lms->arg2folder($lei, $folders) };
+ $lei->qerr("# no folders match $$uri (non-fatal)") if $@;
$end //= $beg;
for my $art ($beg..$end) {
my @oidhex = map { unpack('H*', $_) }
@@ -65,14 +64,8 @@ sub inspect_sync_folder ($$) {
my $ent = {};
my $lms = $lei->lms or return $ent;
my $folders = [ $folder ];
- my $err = $lms->arg2folder($lei, $folders);
- if ($err) {
- if ($err->{fail}) {
- $lei->qerr("# no folders match $folder (non-fatal)");
- @$folders = ();
- }
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- }
+ eval { $lms->arg2folder($lei, $folders) };
+ $lei->qerr("# no folders match $folder (non-fatal)") if $@;
for my $f (@$folders) {
$ent->{$f} = $lms->location_stats($f); # may be undef
}
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 8f8e83bc..ccb1823d 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -15,9 +15,8 @@ sub lcat_folder ($$$) {
my ($lei, $lms, $folder) = @_;
$lms //= $lei->lms or return;
my $folders = [ $folder];
- my $err = $lms->arg2folder($lei, $folders);
- $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
- if ($err && $err->{fail}) {
+ eval { $lms->arg2folder($lei, $folders) };
+ if ($@) {
$lei->child_error(0, "# unknown folder: $folder");
} else {
for my $f (@$folders) {
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 3e725d30..f83c7de2 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -437,7 +437,7 @@ sub arg2folder {
my ($self, $lei, $folders) = @_;
my @all = $self->folders;
my %all = map { $_ => 1 } @all;
- my ($err, @no);
+ my @no;
for (@$folders) {
next if $all{$_}; # ok
if (m!\A(maildir|mh):(.+)!i) {
@@ -454,7 +454,7 @@ sub arg2folder {
my $res = match_imap_url($self, $orig, \@all);
if (ref $res) {
$_ = $$res;
- push(@{$err->{qerr}}, <<EOM);
+ $lei->qerr(<<EOM);
# using `$res' instead of `$orig'
EOM
} else {
@@ -466,7 +466,7 @@ EOM
my $res = match_nntp_url($self, $orig, \@all);
if (ref $res) {
$_ = $$res;
- push(@{$err->{qerr}}, <<EOM);
+ $lei->qerr(<<EOM);
# using `$res' instead of `$orig'
EOM
} else {
@@ -479,12 +479,11 @@ EOM
}
if (@no) {
my $no = join("\n\t", @no);
- $err->{fail} = <<EOF;
+ die <<EOF;
No sync information for: $no
Run `lei ls-mail-sync' to display valid choices
EOF
}
- $err;
}
sub forget_folders {
@@ -549,12 +548,8 @@ sub imap_oidhex {
my $mailbox_uri = $uid_uri->clone;
$mailbox_uri->uid(undef);
my $folders = [ $$mailbox_uri ];
- if (my $err = $self->arg2folder($lei, $folders)) {
- if ($err->{fail}) {
- $lei->qerr("# no sync information for $mailbox_uri");
- }
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- }
+ eval { $self->arg2folder($lei, $folders) };
+ $lei->qerr("# no sync information for $mailbox_uri") if $@;
map { unpack('H*',$_) } num_oidbin($self, $folders->[0], $uid_uri->uid)
}
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index a0591a09..43d5ed0f 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -68,8 +68,9 @@ sub lei_note_event {
return flush_lei($lei) if $folder eq 'done'; # special case
my $lms = $lei->lms or return;
$lms->lms_write_prepare if $new_cur eq ''; # for ->clear_src below
- my $err = $lms->arg2folder($lei, [ $folder ]);
- return if $err->{fail};
+ $lei->{opt}->{quiet} = 1;
+ eval { $lms->arg2folder($lei, [ $folder ]) };
+ return if $@;
my $state = $cfg->get_1("watch.$folder", 'state') // 'tag-rw';
return if $state eq 'pause';
return $lms->clear_src($folder, \$bn) if $new_cur eq '';
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 92673492..51e89b23 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -74,9 +74,7 @@ EOM
if (defined(my $all = $lei->{opt}->{all})) {
$lms->group2folders($lei, $all, \@folders) or return;
} else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ $lms->arg2folder($lei, \@folders); # may die
}
$lms->lms_pause; # must be done before fork
$sto->write_prepare($lei);
^ permalink raw reply related [relevance 45%]
* [PATCH 07/12] lei lcat: support NNTP URLs
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (4 preceding siblings ...)
2021-09-21 7:41 71% ` [PATCH 06/12] doc: lei-security: section for WIP auth methods Eric Wong
@ 2021-09-21 7:41 44% ` Eric Wong
2021-09-21 7:41 39% ` [PATCH 08/12] lei: various completion improvements Eric Wong
` (4 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
NNTP URLs are probably more prevalent in public message archives
than IMAP URLs.
---
lib/PublicInbox/LeiLcat.pm | 66 +++++++++++++++++++++-------------
lib/PublicInbox/LeiMailSync.pm | 14 +++++---
t/lei-import-nntp.t | 23 ++++++++++++
3 files changed, 74 insertions(+), 29 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 1a4a988e..0902c213 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -11,47 +11,64 @@ use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_folder ($$$) {
- my ($lei, $lms, $folder) = @_;
- $lms //= $lei->lms or return;
- my $folders = [ $folder];
+sub lcat_folder ($$;$$) {
+ my ($lei, $folder, $beg, $end) = @_;
+ my $lms = $lei->{-lms_ro} //= $lei->lms // return;
+ my $folders = [ $folder ];
eval { $lms->arg2folder($lei, $folders) };
- if ($@) {
- $lei->child_error(0, "# unknown folder: $folder");
- } else {
- for my $f (@$folders) {
- my $fid = $lms->fid_for($f);
- push @{$lei->{lcat_todo}}, { fid => $fid };
- }
+ return $lei->child_error(0, "# unknown folder: $folder") if $@;
+ my %range;
+ if (defined($beg)) { # NNTP article range
+ $range{min} = $beg;
+ $range{max} = $end // $beg;
+ }
+ for my $f (@$folders) {
+ my $fid = $lms->fid_for($f);
+ push @{$lei->{lcat_todo}}, { fid => $fid, %range };
}
}
sub lcat_imap_uri ($$) {
my ($lei, $uri) = @_;
- my $lms = $lei->lms or return;
- # cf. LeiXsearch->lcat_dump
+ # cf. LeiXSearch->lcat_dump
+ my $lms = $lei->{-lms_ro} //= $lei->lms // return;
if (defined $uri->uid) {
push @{$lei->{lcat_todo}}, $lms->imap_oidhex($lei, $uri);
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
push @{$lei->{lcat_todo}}, { fid => $fid };
} else {
- lcat_folder($lei, $lms, $$uri);
+ lcat_folder($lei, $$uri);
}
}
+sub lcat_nntp_uri ($$) {
+ my ($lei, $uri) = @_;
+ my $mid = $uri->message; # already unescaped by URI::news
+ return "mid:$mid" if defined($mid);
+ my $lms = $lei->{-lms_ro} //= $lei->lms // return;
+ my ($ng, $beg, $end) = $uri->group;
+ $uri->group($ng);
+ lcat_folder($lei, $$uri, $beg, $end);
+ '""';
+}
+
sub extract_1 ($$) {
my ($lei, $x) = @_;
- if ($x =~ m!\b(imaps?://[^>]+)!i) {
- my $u = $1;
- require PublicInbox::URIimap;
- lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
- '""'; # blank query, using {lcat_todo}
- } elsif ($x =~ m!\b(maildir:.+)!i) {
- lcat_folder($lei, undef, $1);
+ if ($x =~ m!\b(maildir:.+)!i) {
+ lcat_folder($lei, $1);
'""'; # blank query, using {lcat_todo}
- } elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
- my $u = $1;
+ } elsif ($x =~ m!\b(([a-z]+)://\S+)!i) {
+ my ($u, $scheme) = ($1, $2);
$u =~ s/[\>\]\)\,\.\;]+\z//;
+ if ($scheme =~ m!\A(imaps?)\z!i) {
+ require PublicInbox::URIimap;
+ lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
+ return '""'; # blank query, using {lcat_todo}
+ } elsif ($scheme =~ m!\A(?:nntps?|s?news)\z!i) {
+ require PublicInbox::URInntps;
+ $u = PublicInbox::URInntps->new($u);
+ return lcat_nntp_uri($lei, $u);
+ } # http, or something else:
require URI;
$u = URI->new($u);
my $p = $u->path;
@@ -93,7 +110,7 @@ sub extract_all {
my $strict = !$lei->{opt}->{stdin};
my @q;
for my $x (@argv) {
- if (my $term = extract_1($lei,$x)) {
+ if (my $term = extract_1($lei, $x)) {
push @q, $term;
} elsif ($strict) {
return $lei->fail(<<"");
@@ -101,6 +118,7 @@ could not extract Message-ID from $x
}
}
+ delete $lei->{-lms_ro};
@q ? join(' OR ', @q) : $lei->fail("no Message-ID in: @argv");
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index f83c7de2..522a5ebc 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -197,9 +197,12 @@ INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?)
sub each_src {
my ($self, $folder, $cb, @args) = @_;
my $dbh = $self->{dbh} //= dbh_new($self);
- my $fid;
+ my ($fid, @rng);
+ my $and_ge_le = '';
if (ref($folder) eq 'HASH') {
$fid = $folder->{fid} // die "BUG: no `fid'";
+ @rng = grep(defined, @$folder{qw(min max)});
+ $and_ge_le = 'AND uid >= ? AND uid <= ?' if @rng;
} else {
$fid = $self->{fmap}->{$folder} //=
fid_for($self, $folder) // return;
@@ -208,16 +211,17 @@ sub each_src {
# minimize implicit txn time to avoid blocking writers by
# batching SELECTs. This looks wonky but is necessary since
# $cb-> may access the DB on its own.
- my $ary = $dbh->selectall_arrayref(<<'', undef, $fid);
-SELECT _rowid_,oidbin,uid FROM blob2num WHERE fid = ?
+ my $ary = $dbh->selectall_arrayref(<<"", undef, $fid, @rng);
+SELECT _rowid_,oidbin,uid FROM blob2num WHERE fid = ? $and_ge_le
ORDER BY _rowid_ ASC LIMIT 1000
my $min = @$ary ? $ary->[-1]->[0] : undef;
while (defined $min) {
for my $row (@$ary) { $cb->($row->[1], $row->[2], @args) }
- $ary = $dbh->selectall_arrayref(<<'', undef, $fid, $min);
-SELECT _rowid_,oidbin,uid FROM blob2num WHERE fid = ? AND _rowid_ > ?
+ $ary = $dbh->selectall_arrayref(<<"", undef, $fid, @rng, $min);
+SELECT _rowid_,oidbin,uid FROM blob2num
+WHERE fid = ? $and_ge_le AND _rowid_ > ?
ORDER BY _rowid_ ASC LIMIT 1000
$min = @$ary ? $ary->[-1]->[0] : undef;
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index 0b080781..eb1ae312 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -25,6 +25,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON');
lei_ok('import', $url);
+ lei_ok "lcat", "nntp://$host_port/testmessage\@example.com";
+ my $local = $lei_out;
+ lei_ok "lcat", "nntp://example.com/testmessage\@example.com";
+ my $remote = $lei_out;
+ is($local, $remote, 'Message-ID used even from unknown host');
lei_ok(qw(q z:1..));
$out = json_utf8->decode($lei_out);
ok(scalar(@$out) > 1, 'got imported messages');
@@ -57,6 +62,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('inspect', "$url/$high");
my $x = json_utf8->decode($lei_out);
like($x->{$url}->{$high}, qr/\A[a-f0-9]{40,}\z/, 'inspect shows blob');
+ lei_ok qw(lcat -f json), "$url/$high";
+ my $lcat = json_utf8->decode($lei_out);
+ is($lcat->[1], undef, 'only one result for lcat');
+ is($lcat->[0]->{blob}, $x->{$url}->{$high},
+ 'lcat showed correct blob');
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article number not stored as folder');
@@ -78,6 +88,19 @@ test_lei({ tmpdir => $tmpdir }, sub {
is(scalar(grep(/\A[a-f0-9]{40,}\z/, values %{$x->{$url}})),
$end - $low + 1, 'all values are git blobs');
+ lei_ok qw(lcat -f json), "$url/$low";
+ $lcat = json_utf8->decode($lei_out);
+ is($lcat->[1], undef, 'only one result for lcat');
+ is($lcat->[0]->{blob}, $x->{$url}->{$low},
+ 'lcat showed correct blob');
+ lei_ok qw(lcat -f json), "$url/$low-$end";
+ $lcat = json_utf8->decode($lei_out);
+ pop @$lcat;
+ for ($low..$end) {
+ my $tip = shift @$lcat;
+ is($x->{$url}->{$_}, $tip->{blob}, "blob matches art #$_");
+ }
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article range not stored as folder');
lei_ok qw(q z:0..); my $start = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 44%]
* [PATCH 02/12] lei inspect: support NNTP URLs
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
@ 2021-09-21 7:41 42% ` Eric Wong
2021-09-21 7:41 45% ` [PATCH 04/12] lei: simplify internal arg2folder usage Eric Wong
` (8 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
No reason not to support them, since there's more
public-inbox-nntpd instances than -imapd instances,
currently.
---
lib/PublicInbox/LeiInspect.pm | 48 +++++++++++++++++++++++++++-------
lib/PublicInbox/LeiMailSync.pm | 40 +++++++++++++++++++++++++++-
lib/PublicInbox/TestCommon.pm | 11 ++++++--
t/lei-import-nntp.t | 21 +++++++++++++++
4 files changed, 108 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 48da826b..ab2c98d9 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -11,6 +11,7 @@ use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
use PublicInbox::MID qw(mids);
+use PublicInbox::NetReader qw(imap_uri nntp_uri);
sub inspect_blob ($$) {
my ($lei, $oidhex) = @_;
@@ -32,13 +33,33 @@ sub inspect_imap_uid ($$) {
my $ent = {};
my $lms = $lei->lms or return $ent;
my $oidhex = $lms->imap_oid($lei, $uid_uri);
- if (ref(my $err = $oidhex)) { # art2folder error
+ if (ref(my $err = $oidhex)) { # arg2folder error
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
$ent->{$$uid_uri} = $oidhex;
$ent;
}
+sub inspect_nntp_range {
+ my ($lei, $uri) = @_;
+ my ($ng, $beg, $end) = $uri->group;
+ $uri = $uri->clone;
+ $uri->group($ng);
+ my $ent = {};
+ my $ret = { "$uri" => $ent };
+ my $lms = $lei->lms or return $ret;
+ my $err = $lms->arg2folder($lei, my $folders = [ $$uri ]);
+ if ($err) {
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ $end //= $beg;
+ for my $art ($beg..$end) {
+ my $oidbin = $lms->imap_oidbin($folders->[0], $art);
+ $ent->{$art} = $oidbin ? unpack('H*', $oidbin) : undef;
+ }
+ $ret;
+}
+
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
@@ -161,14 +182,6 @@ sub inspect1 ($$$) {
my $ent;
if ($item =~ /\Ablob:(.+)/) {
$ent = inspect_blob($lei, $1);
- } elsif ($item =~ m!\Aimaps?://!i) {
- require PublicInbox::URIimap;
- my $uri = PublicInbox::URIimap->new($item);
- if (defined($uri->uid)) {
- $ent = inspect_imap_uid($lei, $uri);
- } else {
- $ent = inspect_sync_folder($lei, $item);
- }
} elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
} elsif ($item =~ m!\Adocid:([0-9]+)\z!) {
@@ -177,6 +190,23 @@ sub inspect1 ($$$) {
$ent = inspect_num($lei, $1 + 0);
} elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
$ent = inspect_mid($lei, $1);
+ } elsif (my $iuri = imap_uri($item)) {
+ if (defined($iuri->uid)) {
+ $ent = inspect_imap_uid($lei, $iuri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ } elsif (my $nuri = nntp_uri($item)) {
+ if (defined(my $mid = $nuri->message)) {
+ $ent = inspect_mid($lei, $mid);
+ } else {
+ my ($group, $beg, $end) = $nuri->group;
+ if (defined($beg)) {
+ $ent = inspect_nntp_range($lei, $nuri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ }
} else { # TODO: more things
return $lei->fail("$item not understood");
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index f185b585..d9b9e117 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -247,12 +247,14 @@ sub location_stats {
SELECT COUNT(name) FROM blob2name WHERE fid = ?
$ret->{'name.count'} = $row if $row;
+ my $ntype = ($folder =~ m!\A(?:nntps?|s?news)://!i) ? 'article' :
+ (($folder =~ m!\Aimaps?://!i) ? 'uid' : "TODO<$folder>");
for my $op (qw(count min max)) {
($row) = $dbh->selectrow_array(<<"", undef, $fid);
SELECT $op(uid) FROM blob2num WHERE fid = ?
$row or last;
- $ret->{"uid.$op"} = $row;
+ $ret->{"$ntype.$op"} = $row;
}
$ret;
}
@@ -369,6 +371,30 @@ sub match_imap_url {
"E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
}
+sub match_nntp_url ($$$) {
+ my ($self, $url, $all) = @_; # $all = [ $lms->folders ];
+ $all //= [ $self->folders ];
+ require PublicInbox::URInntps;
+ my $want = PublicInbox::URInntps->new($url)->canonical;
+ my ($s, $h, $p) = ($want->scheme, $want->host, $want->port);
+ my $ng = $want->group; # force scalar (no article ranges)
+ my @uri = map { PublicInbox::URInntps->new($_)->canonical }
+ grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$ng\E\b!, @$all);
+ my @match;
+ for my $x (@uri) {
+ next if $x->group ne $ng || $x->host ne $h || $x->port != $p;
+ # maybe user was forgotten on CLI:
+ if (defined($x->userinfo) && !defined($want->userinfo)) {
+ push @match, $x;
+ } elsif (($x->userinfo//"\0") eq ($want->userinfo//"\0")) {
+ push @match, $x;
+ }
+ }
+ return @match if wantarray;
+ scalar(@match) <= 1 ? $match[0] :
+ "E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
+}
+
# returns undef on failure, number on success
sub group2folders {
my ($self, $lei, $all, $folders) = @_;
@@ -428,6 +454,18 @@ sub arg2folder {
$_ = $$res;
push(@{$err->{qerr}}, <<EOM);
# using `$res' instead of `$orig'
+EOM
+ } else {
+ $lei->err($res) if defined $res;
+ push @no, $orig;
+ }
+ } elsif (m!\A(?:nntps?|s?news)://!i) {
+ my $orig = $_;
+ my $res = match_nntp_url($self, $orig, \@all);
+ if (ref $res) {
+ $_ = $$res;
+ push(@{$err->{qerr}}, <<EOM);
+# using `$res' instead of `$orig'
EOM
} else {
$lei->err($res) if defined $res;
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 0ee4b228..9e152394 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -18,7 +18,7 @@ BEGIN {
run_script start_script key2sub xsys xsys_e xqx eml_load tick
have_xapian_compact json_utf8 setup_public_inboxes create_inbox
tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt
- test_httpd xbail require_cmd);
+ test_httpd xbail require_cmd is_xdeeply);
require Test::More;
my @methods = grep(!/\W/, @Test::More::EXPORT);
eval(join('', map { "*$_=\\&Test::More::$_;" } @methods));
@@ -520,6 +520,13 @@ sub json_utf8 () {
state $x = ref(PublicInbox::Config->json)->new->utf8->canonical;
}
+sub is_xdeeply ($$$) {
+ my ($x, $y, $desc) = @_;
+ my $ok = is_deeply($x, $y, $desc);
+ diag explain([$x, '!=', $y]) if !$ok;
+ $ok;
+}
+
sub test_lei {
SKIP: {
my ($cb) = pop @_;
@@ -590,7 +597,7 @@ SKIP: {
my $f = "$daemon_xrd/lei/errors.log";
open my $fh, '<', $f or BAIL_OUT "$f: $!";
my @l = <$fh>;
- is_deeply(\@l, [],
+ is_xdeeply(\@l, [],
"$t daemon XDG_RUNTIME_DIR/lei/errors.log empty");
}
}; # SKIP if missing git 2.6+ || Xapian || SQLite || json
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index df0594d4..0b080781 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -49,6 +49,15 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $end = $high - 1;
lei_ok qw(import), "$url/$high";
+ lei_ok('inspect', $url); is_xdeeply(json_utf8->decode($lei_out), {
+ $url => { 'article.count' => 1,
+ 'article.min' => $high,
+ 'article.max' => $high, }
+ }, 'inspect output for URL after single message') or diag $lei_out;
+ lei_ok('inspect', "$url/$high");
+ my $x = json_utf8->decode($lei_out);
+ like($x->{$url}->{$high}, qr/\A[a-f0-9]{40,}\z/, 'inspect shows blob');
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article number not stored as folder');
lei_ok qw(q z:0..); my $one = json_utf8->decode($lei_out);
@@ -57,6 +66,18 @@ test_lei({ tmpdir => $tmpdir }, sub {
local $ENV{HOME} = "$tmpdir/h3";
lei_ok qw(import), "$url/$low-$end";
+ lei_ok('inspect', $url); is_xdeeply(json_utf8->decode($lei_out), {
+ $url => { 'article.count' => $end - $low + 1,
+ 'article.min' => $low,
+ 'article.max' => $end, }
+ }, 'inspect output for URL after range') or diag $lei_out;
+ lei_ok('inspect', "$url/$low-$end");
+ $x = json_utf8->decode($lei_out);
+ is_deeply([ ($low..$end) ], [ sort { $a <=> $b } keys %{$x->{$url}} ],
+ 'inspect range shows range');
+ is(scalar(grep(/\A[a-f0-9]{40,}\z/, values %{$x->{$url}})),
+ $end - $low + 1, 'all values are git blobs');
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article range not stored as folder');
lei_ok qw(q z:0..); my $start = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 42%]
* [PATCH 08/12] lei: various completion improvements
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (5 preceding siblings ...)
2021-09-21 7:41 44% ` [PATCH 07/12] lei lcat: support NNTP URLs Eric Wong
@ 2021-09-21 7:41 39% ` Eric Wong
2021-09-21 7:41 63% ` [PATCH 09/12] lei q: show progress on >1s preparation phase Eric Wong
` (3 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
"lei export-kw" no longer completes for anonymous sources.
More commands use "lei refresh-mail-sync" as a basis for their
completion work, as well.
";AUTH=ANONYMOUS@" is stripped from completions since it was
preventing bash completion from working on AUTH=ANONYMOUS IMAP
URLs. I'm not sure if there's a better way, but all of our code
works fine without specifying AUTH=ANONYMOUS as a command-line
arg.
Finally, we fallback to using more candidates if none can
be found, allowing multiple URLs to be completed.
---
lib/PublicInbox/LeiExportKw.pm | 8 ++++++--
lib/PublicInbox/LeiExternal.pm | 5 +++++
lib/PublicInbox/LeiForgetMailSync.pm | 5 +++--
lib/PublicInbox/LeiImport.pm | 12 +++++++-----
lib/PublicInbox/LeiInspect.pm | 7 +++----
lib/PublicInbox/LeiLcat.pm | 7 +++----
lib/PublicInbox/LeiLsMailSource.pm | 11 ++++++-----
lib/PublicInbox/LeiMailSync.pm | 13 ++++++++-----
lib/PublicInbox/LeiRefreshMailSync.pm | 12 ++++++++++--
lib/PublicInbox/LeiTag.pm | 14 ++++++++------
lib/PublicInbox/SharedKV.pm | 19 +++++++++++++------
11 files changed, 72 insertions(+), 41 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d5533a2a..cea9beeb 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -127,9 +127,13 @@ EOM
sub _complete_export_kw {
my ($lei, @argv) = @_;
- my $lms = $lei->lms or return;
+ my $lms = $lei->lms or return ();
my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ # filter-out read-only sources:
+ my @k = grep(!m!(?://;AUTH=ANONYMOUS\@|\A(?:nntps?|s?news)://)!,
+ $lms->folders($argv[-1], 1));
+ my @m = map { $match_cb->($_) } @k;
+ @m ? @m : @k;
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index d802f0e2..f8e610ca 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -241,6 +241,11 @@ sub complete_url_prepare {
$re = quotemeta($re);
}
my $match_cb = sub {
+ # the "//;" here (for AUTH=ANONYMOUS) interacts badly with
+ # bash tab completion, strip it out for now since our commands
+ # work w/o it. Not sure if there's a better solution...
+ $_[0] =~ s!//;AUTH=ANONYMOUS\@!//!i;
+ $_[0] =~ s!;!\\;!g;
# only return the part specified on the CLI
# don't duplicate if already 100% completed
$_[0] =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ()
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index d85616cc..762910ed 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -10,7 +10,7 @@
package PublicInbox::LeiForgetMailSync;
use strict;
use v5.10.1;
-use PublicInbox::LeiExportKw;
+use PublicInbox::LeiRefreshMailSync;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
@@ -20,6 +20,7 @@ sub lei_forget_mail_sync {
$lms->forget_folders(@folders);
}
-*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_forget_mail_sync =
+ \&PublicInbox::LeiRefreshMailSync::_complete_refresh_mail_sync;
1;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 3c30db8d..397292d4 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -121,12 +121,14 @@ sub lei_import { # the main "lei import" method
sub _complete_import {
my ($lei, @argv) = @_;
- my $match_cb = $lei->complete_url_prepare(\@argv);
- my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
- my %f = map { $_ => 1 } @m;
+ my ($re, $cur, $match_cb) = $lei->complete_url_prepare(\@argv);
+ my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @m = map { $match_cb->($_) } @k;
+ my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @m = map { $match_cb->($_) } $lms->folders;
- @f{@m} = @m;
+ @k = $lms->folders($argv[-1], 1);
+ @m = map { $match_cb->($_) } @k;
+ if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
keys %f;
}
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 8e128580..2158b996 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -269,10 +269,9 @@ no args allowed on command-line with --stdin
}
sub _complete_inspect {
- my ($lei, @argv) = @_;
- my $lms = $lei->lms or return;
- my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ require PublicInbox::LeiRefreshMailSync;
+ PublicInbox::LeiRefreshMailSync::_complete_refresh_mail_sync(@_);
+ # TODO: message-ids?, blobs? could get expensive...
}
1;
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 0902c213..c13e2153 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -164,10 +164,9 @@ no args allowed on command-line with --stdin
}
sub _complete_lcat {
- my ($lei, @argv) = @_;
- my $lms = $lei->lms or return;
- my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ require PublicInbox::LeiRefreshMailSync;
+ PublicInbox::LeiRefreshMailSync::_complete_refresh_mail_sync(@_);
+ # TODO: message-ids?, blobs? could get expensive...
}
1;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 2265969a..1db15d57 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -107,12 +107,13 @@ sub lei_ls_mail_source {
sub _complete_ls_mail_source {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
- my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
- my %f = map { $_ => 1 } @m;
+ my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @m = map { $match_cb->($_) } @k;
+ my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @m = map { $match_cb->($_) } grep(
- m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
- @f{@m} = @m;
+ @k = $lms->folders($argv[-1], 1);
+ @m = map { $match_cb->($_) } grep(m!\A[a-z]+://!, @k);
+ if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
keys %f;
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 522a5ebc..91cd1c93 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -299,16 +299,19 @@ sub locations_for {
# returns a list of folders used for completion
sub folders {
- my ($self, $pfx) = @_;
- my $dbh = $self->{dbh} //= dbh_new($self);
+ my ($self, @pfx) = @_;
my $sql = 'SELECT loc FROM folders';
- my @pfx;
- if (defined $pfx) {
+ if (defined($pfx[0])) {
$sql .= ' WHERE loc LIKE ? ESCAPE ?';
- @pfx = ($pfx, '\\');
+ my $anywhere = !!$pfx[1];
+ $pfx[1] = '\\';
$pfx[0] =~ s/([%_\\])/\\$1/g; # glob chars
$pfx[0] .= '%';
+ substr($pfx[0], 0, 0, '%') if $anywhere;
+ } else {
+ @pfx = (); # [0] may've been undef
}
+ my $dbh = $self->{dbh} //= dbh_new($self);
map { $_->[0] } @{$dbh->selectall_arrayref($sql, undef, @pfx)};
}
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 51e89b23..eb842843 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -7,7 +7,7 @@ package PublicInbox::LeiRefreshMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
-use PublicInbox::LeiExportKw;
+use PublicInbox::LeiImport;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::Import;
@@ -97,8 +97,16 @@ sub ipc_atfork_child { # needed for PublicInbox::LeiPmdir
undef;
}
+sub _complete_refresh_mail_sync {
+ my ($lei, @argv) = @_;
+ my $lms = $lei->lms or return ();
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ my @k = $lms->folders($argv[-1], 1);
+ my @m = map { $match_cb->($_) } @k;
+ @m ? @m : @k
+}
+
no warnings 'once';
-*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
1;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 9bbf0d79..817b87f8 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -74,7 +74,7 @@ sub ipc_atfork_child {
# Workaround bash word-splitting s to ['kw', ':', 'keyword' ...]
# Maybe there's a better way to go about this in
# contrib/completion/lei-completion.bash
-sub _complete_mark_common ($) {
+sub _complete_tag_common ($) {
my ($argv) = @_;
# Workaround bash word-splitting URLs to ['https', ':', '//' ...]
# Maybe there's a better way to go about this in
@@ -104,16 +104,18 @@ sub _complete_mark_common ($) {
# FIXME: same problems as _complete_forget_external and similar
sub _complete_tag {
my ($self, @argv) = @_;
+ require PublicInbox::LeiImport;
+ my @in = PublicInbox::LeiImport::_complete_import(@_);
my @L = eval { $self->_lei_store->search->all_terms('L') };
- my @all = ((map { ("+kw:$_", "-kw:$_") } @PublicInbox::LeiInput::KW),
+ my @kwL = ((map { ("+kw:$_", "-kw:$_") } @PublicInbox::LeiInput::KW),
(map { ("+L:$_", "-L:$_") } @L));
- return @all if !@argv;
- my ($cur, $re) = _complete_mark_common(\@argv);
- map {
+ my ($cur, $re) = _complete_tag_common(\@argv);
+ my @m = map {
# only return the part specified on the CLI
# don't duplicate if already 100% completed
/\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ();
- } grep(/$re\Q$cur/, @all);
+ } grep(/$re\Q$cur/, @kwL);
+ (@in, (@m ? @m : @kwL));
}
no warnings 'once'; # the following works even when LeiAuth is lazy-loaded
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index 3487e820..645bb57c 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -84,12 +84,19 @@ SELECT k,v FROM kv
}
sub keys {
- my ($self) = @_;
- my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
-SELECT k FROM kv
-
- $sth->execute;
- map { $_->[0] } @{$sth->fetchall_arrayref};
+ my ($self, @pfx) = @_;
+ my $sql = 'SELECT k FROM kv';
+ if (defined $pfx[0]) {
+ $sql .= ' WHERE k LIKE ? ESCAPE ?';
+ my $anywhere = !!$pfx[1];
+ $pfx[1] = '\\';
+ $pfx[0] =~ s/([%_\\])/\\$1/g; # glob chars
+ $pfx[0] .= '%';
+ substr($pfx[0], 0, 0, '%') if $anywhere;
+ } else {
+ @pfx = (); # [0] may've been undef
+ }
+ map { $_->[0] } @{$self->dbh->selectall_arrayref($sql, undef, @pfx)};
}
sub delete_by_val {
^ permalink raw reply related [relevance 39%]
* [PATCH 11/12] lei q: update messages to reflect --save default
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (7 preceding siblings ...)
2021-09-21 7:41 63% ` [PATCH 09/12] lei q: show progress on >1s preparation phase Eric Wong
@ 2021-09-21 7:41 61% ` Eric Wong
2021-09-21 7:41 39% ` [PATCH 12/12] lei q: improve --limit behavior and progress Eric Wong
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
I wanted to try --dedupe=none for something, but it failed
since I forgot --no-save :x So hint users towards --no-save
if necessary.
---
lib/PublicInbox/LeiSavedSearch.pm | 4 ++--
t/lei-q-save.t | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 89f5c359..637456e4 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -94,7 +94,7 @@ sub translate_dedupe ($$$) {
$dd //= 'content';
return 1 if $dd eq 'content'; # the default
return $self->{"-dedupe_$dd"} = 1 if ($dd eq 'oid' || $dd eq 'mid');
- $lei->fail("--dedupe=$dd unsupported with --save");
+ $lei->fail("--dedupe=$dd requires --no-save");
}
sub up { # updating existing saved search via "lei up"
@@ -103,7 +103,7 @@ sub up { # updating existing saved search via "lei up"
my $self = bless { ale => $lei->ale }, $cls;
my $dir = $dst;
output2lssdir($self, $lei, \$dir, \$f) or
- return $lei->fail("--save was not used with $dst cwd=".
+ return $lei->fail("--no-save was used with $dst cwd=".
$lei->rel2abs('.'));
$self->{-cfg} = $lei->cfg_dump($f) // return $lei->fail;
$self->{-ovf} = "$dir/over.sqlite3";
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 9c17a011..5940018c 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -69,11 +69,11 @@ test_lei(sub {
ok(-s "$home/mbcl2" > $size, 'size increased after up');
ok(!lei(qw(up -q), $home), 'up fails on unknown dir');
- like($lei_err, qr/--save was not used/, 'error noted --save');
+ like($lei_err, qr/--no-save was used/, 'error noted --no-save');
lei_ok(qw(q --no-save d:last.week.. -q -o), "$home/no-save");
ok(!lei(qw(up -q), "$home/no-save"), 'up fails on --no-save');
- like($lei_err, qr/--save was not used/, 'error noted --save');
+ like($lei_err, qr/--no-save was used/, 'error noted --no-save');
lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
lei_ok qw(ls-search -z); my @z = split(/\0/, $lei_out);
@@ -131,7 +131,7 @@ test_lei(sub {
unlike($lei_out, qr/mbrd-aug/,
'forget-search completion cleared after forget');
ok(!lei('up', "$home/mbrd-aug"), 'lei up fails after forget');
- like($lei_err, qr/--save was not used/, 'error noted --save');
+ like($lei_err, qr/--no-save was used/, 'error noted --no-save');
# dedupe=mid
my $o = "$home/dd-mid";
^ permalink raw reply related [relevance 61%]
* [PATCH 12/12] lei q: improve --limit behavior and progress
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (8 preceding siblings ...)
2021-09-21 7:41 61% ` [PATCH 11/12] lei q: update messages to reflect --save default Eric Wong
@ 2021-09-21 7:41 39% ` Eric Wong
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Avoid slurping gigantic (e.g. 100000) result sets into a single
response if a giant limit is specified, and instead use 10000
as a window for the mset with a given offset. We'll also warn
and hint towards about the --limit= switch when the estimated
result set is larger than the default limit.
---
Documentation/lei-q.pod | 6 ++++--
lib/PublicInbox/LeiLcat.pm | 3 +--
lib/PublicInbox/LeiQuery.pm | 9 +++++++--
lib/PublicInbox/LeiSavedSearch.pm | 18 +++++++++++-------
lib/PublicInbox/LeiUp.pm | 25 +++++++++----------------
lib/PublicInbox/LeiXSearch.pm | 18 +++++++++++++++---
6 files changed, 47 insertions(+), 32 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 2823ced8..e1e3666d 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -10,7 +10,7 @@ lei q [OPTIONS] (--stdin|-)
=head1 DESCRIPTION
-Search for messages across the lei store and externals.
+Search for messages across the lei/store and externals.
=for comment
TODO: Give common prefixes, or at least a description/reference.
@@ -192,7 +192,9 @@ Default: fcntl,dotlock
=item -n NUMBER
-Limit the number of matches.
+Fuzzy limit the number of matches per-local external and lei/store.
+Messages added by the L<--threads> switch do not count towards this
+limit, and there is no limit on remote externals.
Default: 10000
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index c13e2153..d553b187 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -144,9 +144,8 @@ sub lei_lcat {
$lei->ale->refresh_externals($lxs, $lei);
$lei->_lei_store(1);
my $opt = $lei->{opt};
- my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
+ my %mset_opt;
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
- $mset_opt{limit} //= 10000;
$opt->{sort} //= 'relevance';
$mset_opt{relevance} = 1;
$lei->{mset_opt} = \%mset_opt;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index d5f132f1..cb5ac8fb 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -41,6 +41,12 @@ sub _start_query { # used by "lei q" and "lei up"
# descending docid order is cheapest, MUA controls sorting order
$self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads};
+
+ my $tot = $self->{mset_opt}->{total} //= $self->{opt}->{limit} // 10000;
+ $self->{mset_opt}->{limit} = $tot > 10000 ? 10000 : $tot;
+ $self->{mset_opt}->{offset} //= 0;
+ $self->{mset_opt}->{threads} //= $opt->{threads};
+
if ($self->{net}) {
require PublicInbox::LeiAuth;
$self->{auth} = PublicInbox::LeiAuth->new
@@ -118,9 +124,8 @@ sub lei_q {
my $lxs = lxs_prepare($self) or return;
$self->ale->refresh_externals($lxs, $self);
my $opt = $self->{opt};
- my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
+ my %mset_opt;
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
- $mset_opt{limit} //= 10000;
if (defined(my $sort = $opt->{'sort'})) {
if ($sort eq 'relevance') {
$mset_opt{relevance} = 1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 637456e4..3e10f780 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -29,6 +29,8 @@ sub BOOL_FIELDS () {
qw(external local remote import-remote import-before threads)
}
+sub SINGLE_FIELDS () { qw(limit dedupe output) }
+
sub lss_dir_for ($$;$) {
my ($lei, $dstref, $on_fs) = @_;
my $pfx;
@@ -89,9 +91,9 @@ sub list {
} @$out
}
-sub translate_dedupe ($$$) {
- my ($self, $lei, $dd) = @_;
- $dd //= 'content';
+sub translate_dedupe ($$) {
+ my ($self, $lei) = @_;
+ my $dd = $lei->{opt}->{dedupe} // 'content';
return 1 if $dd eq 'content'; # the default
return $self->{"-dedupe_$dd"} = 1 if ($dd eq 'oid' || $dd eq 'mid');
$lei->fail("--dedupe=$dd requires --no-save");
@@ -128,8 +130,7 @@ sub new { # new saved search "lei q --save"
File::Path::make_path($dir); # raises on error
$self->{-cfg} = {};
my $f = $self->{'-f'} = "$dir/lei.saved-search";
- my $dd = $lei->{opt}->{dedupe};
- translate_dedupe($self, $lei, $dd) or return;
+ translate_dedupe($self, $lei) or return;
open my $fh, '>', $f or return $lei->fail("open $f: $!");
my $sq_dst = PublicInbox::Config::squote_maybe($dst);
my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
@@ -139,15 +140,14 @@ sub new { # new saved search "lei q --save"
$q = "\tq = ".cquote_val($q);
}
$dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i;
+ $lei->{opt}->{output} = $dst;
print $fh <<EOM;
; to refresh with new results, run: lei up $sq_dst
; `maxuid' and `lastresult' lines are maintained by "lei up" for optimization
[lei]
$q
[lei "q"]
- output = $dst
EOM
- print $fh "\tdedupe = $dd\n" if $dd;
for my $k (ARRAY_FIELDS) {
my $ary = $lei->{opt}->{$k} // next;
for my $x (@$ary) {
@@ -158,6 +158,10 @@ EOM
my $val = $lei->{opt}->{$k} // next;
print $fh "\t$k = ".($val ? 1 : 0)."\n";
}
+ for my $k (SINGLE_FIELDS) {
+ my $val = $lei->{opt}->{$k} // next;
+ print $fh "\t$k = $val\n";
+ }
close($fh) or return $lei->fail("close $f: $!");
$self->{lock_path} = "$self->{-f}.flock";
$self->{-ovf} = "$dir/over.sqlite3";
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index abb05d46..89cf0112 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -18,7 +18,6 @@ sub up1 ($$) {
my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
my $f = $lss->{'-f'};
my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
- $mset_opt->{limit} = $lei->{opt}->{limit} // 10000;
my $q = $mset_opt->{q_raw} = $lss->{-cfg}->{'lei.q'} //
return $lei->fail("lei.q unset in $f");
my $lse = $lei->{lse} // die 'BUG: {lse} missing';
@@ -27,24 +26,18 @@ sub up1 ($$) {
} else {
$lse->query_approxidate($lse->git, $mset_opt->{qstr} = $q);
}
- my $o = $lei->{opt}->{output} = $lss->{-cfg}->{'lei.q.output'} //
- return $lei->fail("lei.q.output unset in $f");
- ref($o) and return $lei->fail("multiple values of lei.q.output in $f");
- if (defined(my $dd = $lss->{-cfg}->{'lei.q.dedupe'})) {
- $lss->translate_dedupe($lei, $dd) or return;
- $lei->{opt}->{dedupe} = $dd;
- }
- for my $k (qw(only include exclude)) {
+ # n.b. only a few CLI args are accepted for "up", so //= usually sets
+ for my $k ($lss->ARRAY_FIELDS) {
my $v = $lss->{-cfg}->get_all("lei.q.$k") // next;
- $lei->{opt}->{$k} = $v;
+ $lei->{opt}->{$k} //= $v;
}
- for my $k (qw(external local remote
- import-remote import-before threads)) {
- my $c = "lei.q.$k";
- my $v = $lss->{-cfg}->{$c} // next;
- ref($v) and return $lei->fail("multiple values of $c in $f");
- $lei->{opt}->{$k} = $v;
+ for my $k ($lss->BOOL_FIELDS, $lss->SINGLE_FIELDS) {
+ my $v = $lss->{-cfg}->get_1('lei.q', $k) // next;
+ $lei->{opt}->{$k} //= $v;
}
+ my $o = $lei->{opt}->{output} // '';
+ return $lei->fail("lei.q.output unset in $f") if $o eq '';
+ $lss->translate_dedupe($lei) or return;
$lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
my $lxs = $lei->lxs_prepare or return;
$lei->ale->refresh_externals($lxs, $lei);
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 2227c2ac..584ffde9 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -110,10 +110,20 @@ sub recent {
sub over {}
+sub _check_mset_limit ($$$) {
+ my ($lei, $desc, $mset) = @_;
+ return if defined($lei->{opt}->{limit}); # user requested limit
+ my $est = $mset->get_matches_estimated;
+ my $tot = $lei->{mset_opt}->{total};
+ $est > $tot and $lei->qerr(<<"");
+# $desc estimated matches ($est) exceeds default --limit=$tot
+
+}
+
sub _mset_more ($$) {
my ($mset, $mo) = @_;
my $size = $mset->size;
- $size >= $mo->{limit} && (($mo->{offset} += $size) < $mo->{limit});
+ $size >= $mo->{limit} && (($mo->{offset} += $size) < $mo->{total});
}
# $startq will EOF when do_augment is done augmenting and allow
@@ -182,7 +192,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $first_ids;
do {
$mset = $srch->mset($mo->{qstr}, $mo);
- mset_progress($lei, $dir, $mset->size,
+ mset_progress($lei, $dir, $mo->{offset} + $mset->size,
$mset->get_matches_estimated);
wait_startq($lei); # wait for keyword updates
my $ids = $srch->mset_to_artnums($mset, $mo);
@@ -222,6 +232,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
}
}
} while (_mset_more($mset, $mo));
+ _check_mset_limit($lei, $dir, $mset);
if ($lss && scalar(@$first_ids)) {
undef $stop_at;
my $max = $first_ids->[0];
@@ -244,7 +255,7 @@ sub query_combined_mset { # non-parallel for non-"--threads" users
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
do {
$mset = $self->mset($mo->{qstr}, $mo);
- mset_progress($lei, 'xsearch', $mset->size,
+ mset_progress($lei, 'xsearch', $mo->{offset} + $mset->size,
$mset->get_matches_estimated);
wait_startq($lei); # wait for keyword updates
for my $mitem ($mset->items) {
@@ -252,6 +263,7 @@ sub query_combined_mset { # non-parallel for non-"--threads" users
$each_smsg->($smsg, $mitem);
}
} while (_mset_more($mset, $mo));
+ _check_mset_limit($lei, 'xsearch', $mset);
undef $each_smsg; # may commit
$lei->{ovv}->ovv_atexit_child($lei);
}
^ permalink raw reply related [relevance 39%]
* [PATCH 0/3] lei: a few more annoyances fixed
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (9 preceding siblings ...)
2021-09-21 7:41 39% ` [PATCH 12/12] lei q: improve --limit behavior and progress Eric Wong
@ 2021-09-21 9:29 71% ` Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
` (2 more replies)
10 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
The more I use it, the more I find wrong...
Eric Wong (3):
t/lei-up: use '-q' to silence non-redirected test
script/lei: handle SIGTSTP and SIGCONT
lei: umask(077) before opening errors.log
lib/PublicInbox/LEI.pm | 20 +++++++++++++-------
script/lei | 9 ++++-----
t/lei-up.t | 2 +-
3 files changed, 18 insertions(+), 13 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
@ 2021-09-21 9:29 71% ` Eric Wong
2021-09-21 9:29 62% ` [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT Eric Wong
2021-09-21 9:29 71% ` [PATCH 3/3] lei: umask(077) before opening errors.log Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
We could redirect, too, but just use -q since we don't care
for the output with run_mode => 0.
---
t/lei-up.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-up.t b/t/lei-up.t
index 8937cfb1..fc369156 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -35,7 +35,7 @@ test_lei(sub {
$uc = do { local $/; <$fh> };
is($uc, $exp, 'uncompressed both match');
- lei_ok [ 'up', "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
+ lei_ok [ qw(up -q), "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
undef, { run_mode => 0 };
ok(-f "$ENV{HOME}/c", '--mua works with single output');
});
^ permalink raw reply related [relevance 71%]
* [PATCH 3/3] lei: umask(077) before opening errors.log
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
2021-09-21 9:29 62% ` [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT Eric Wong
@ 2021-09-21 9:29 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
There's a chance some sensitive information (e.g. folder names)
can end up in errors.log, though $XDG_RUNTIME_DIR or
/tmp/lei-$UID/ will have 0700 permissions, anyways.
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 2df1f326..29293e6c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1225,6 +1225,7 @@ sub lazy_start {
$errors_log = "$sock_dir/errors.log";
my $addr = pack_sockaddr_un($path);
my $lk = bless { lock_path => $errors_log }, 'PublicInbox::Lock';
+ umask(077) // die("umask(077): $!");
$lk->lock_acquire;
socket($listener, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
if ($errno == ECONNREFUSED || $errno == ENOENT) {
@@ -1236,7 +1237,6 @@ sub lazy_start {
$! = $errno; # allow interpolation to stringify in die
die "connect($path): $!";
}
- umask(077) // die("umask(077): $!");
bind($listener, $addr) or die "bind($path): $!";
$lk->lock_release;
undef $lk;
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
@ 2021-09-21 9:29 62% ` Eric Wong
2021-09-21 9:29 71% ` [PATCH 3/3] lei: umask(077) before opening errors.log Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
Sometimes it's useful to pause an expensive query or
refresh-mail-sync to do something else. While lei-daemon and
lei/store can't be paused since they're shared across clients,
per-invocation WQ workers can be paused safely using the
unblockable SIGSTOP.
While we're at it, drop the ETOOMANYREFS hint since it
hasn't been a problem since we drastically reduced FD passing
early in development.
---
lib/PublicInbox/LEI.pm | 18 ++++++++++++------
script/lei | 9 ++++-----
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f94bfa45..2df1f326 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1118,23 +1118,28 @@ sub dclose {
sub event_step {
my ($self) = @_;
local %ENV = %{$self->{env}};
- my $sock = $self->{sock};
local $current_lei = $self;
eval {
- while (my @fds = $recv_cmd->($sock, my $buf, 4096)) {
+ my $buf;
+ while (my @fds = $recv_cmd->($self->{sock}, $buf, 4096)) {
if (scalar(@fds) == 1 && !defined($fds[0])) {
return if $! == EAGAIN;
next if $! == EINTR;
last if $! == ECONNRESET;
die "recvmsg: $!";
}
- for my $fd (@fds) {
- open my $rfh, '+<&=', $fd;
+ for (@fds) { open my $rfh, '+<&=', $_ }
+ }
+ if ($buf eq '') {
+ _drop_wq($self); # EOF, client disconnected
+ dclose($self);
+ } elsif ($buf =~ /\A(STOP|CONT)\z/) {
+ for my $wq (grep(defined, @$self{@WQ_KEYS})) {
+ $wq->wq_kill($buf) or $wq->wq_kill_old($buf);
}
+ } else {
die "unrecognized client signal: $buf";
}
- _drop_wq($self); # EOF, client disconnected
- dclose($self);
};
if (my $err = $@) {
eval { $self->fail($err) };
@@ -1146,6 +1151,7 @@ sub event_step_init {
my ($self) = @_;
my $sock = $self->{sock} or return;
$self->{-event_init_done} //= do { # persist til $ops done
+ $sock->blocking(0);
$self->SUPER::new($sock, EPOLLIN|EPOLLET);
$sock;
};
diff --git a/script/lei b/script/lei
index 591013e3..399296ba 100755
--- a/script/lei
+++ b/script/lei
@@ -106,11 +106,10 @@ open my $dh, '<', '.' or die "open(.) $!";
my $buf = join("\0", scalar(@ARGV), @ARGV);
while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
$buf .= "\0\0";
-my $n = $send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR);
-if (!$n) {
- die "sendmsg: $! (check RLIMIT_NOFILE)\n" if $!{ETOOMANYREFS};
- die "sendmsg: $!\n";
-}
+$send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR) or die "sendmsg: $!";
+$SIG{TSTP} = sub { $send_cmd->($sock, [], 'STOP', MSG_EOR); kill 'STOP', $$ };
+$SIG{CONT} = sub { $send_cmd->($sock, [], 'CONT', MSG_EOR) };
+
my $x_it_code = 0;
while (1) {
my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
^ permalink raw reply related [relevance 62%]
* [PATCH 3/7] script/lei: describe purpose of sleep loop
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
@ 2021-09-22 2:24 71% ` Eric Wong
2021-09-22 2:24 71% ` [PATCH 4/7] lei: dclose: do not close unnecessarily Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
It looks dumb, but I'm not about to take a runtime penalty to
use signalfd|EVFILT_SIGNAL, here, either.
---
script/lei | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/script/lei b/script/lei
index 399296ba..bc437798 100755
--- a/script/lei
+++ b/script/lei
@@ -137,6 +137,6 @@ while (1) {
$sigchld->();
if (my $sig = ($x_it_code & 127)) {
kill $sig, $$;
- sleep(1) while 1;
+ sleep(1) while 1; # no self-pipe/signalfd, here, so we loop
}
exit($x_it_code >> 8);
^ permalink raw reply related [relevance 71%]
* [PATCH 4/7] lei: dclose: do not close unnecessarily
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
@ 2021-09-22 2:24 71% ` Eric Wong
2021-09-22 2:24 49% ` [PATCH 6/7] lei up: avoid excessively parallel --all Eric Wong
2021-09-22 2:24 58% ` [PATCH 7/7] lei: drop redundant WQ EOF callbacks Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
The bit about reap_compress is no longer true since
LeiXSearch->query_done triggers it, instead. I only noticed
this while working on "lei up".
---
lib/PublicInbox/LEI.pm | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 29293e6c..a1cab55a 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1110,7 +1110,6 @@ sub dclose {
my ($self) = @_;
delete $self->{-progress};
_drop_wq($self) if $self->{failed};
- close(delete $self->{1}) if $self->{1}; # may reap_compress
$self->close if $self->{-event_init_done}; # PublicInbox::DS::close
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/7] lei bugfixes and other fixes
@ 2021-09-22 2:24 70% Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
How did I forget 2/7 for all these months? :<
6/7 took a while for me to figure out, I only noticed it
because of background processes running after MUA exit
(from "lei up --mua=... MFOLDER"), but I got 4/7 and 7/7
fixes out of it.
Eric Wong (7):
ipc: do not add "0" to $0 of solo workers
treewide: fix %SIG localization, harder
script/lei: describe purpose of sleep loop
lei: dclose: do not close unnecessarily
inbox: do not waste hash slot on httpbackend_limiter
lei up: avoid excessively parallel --all
lei: drop redundant WQ EOF callbacks
lib/PublicInbox/Admin.pm | 2 +-
lib/PublicInbox/ExtSearchIdx.pm | 4 +-
lib/PublicInbox/IPC.pm | 7 +--
lib/PublicInbox/Inbox.pm | 10 ++--
lib/PublicInbox/LEI.pm | 3 +-
lib/PublicInbox/LeiImportKw.pm | 8 +--
lib/PublicInbox/LeiMirror.pm | 10 +---
lib/PublicInbox/LeiNoteEvent.pm | 8 +--
lib/PublicInbox/LeiPmdir.pm | 8 +--
lib/PublicInbox/LeiUp.pm | 86 +++++++++++++++++++++------------
lib/PublicInbox/TestCommon.pm | 3 +-
lib/PublicInbox/Watch.pm | 4 +-
lib/PublicInbox/Xapcmd.pm | 8 +--
script/lei | 2 +-
script/public-inbox-clone | 2 +-
t/run.perl | 1 +
16 files changed, 86 insertions(+), 80 deletions(-)
^ permalink raw reply [relevance 70%]
* [PATCH 7/7] lei: drop redundant WQ EOF callbacks
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
` (2 preceding siblings ...)
2021-09-22 2:24 49% ` [PATCH 6/7] lei up: avoid excessively parallel --all Eric Wong
@ 2021-09-22 2:24 58% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
Redundant code is noise and therefore confusing :<
---
lib/PublicInbox/LeiImportKw.pm | 8 +-------
lib/PublicInbox/LeiMirror.pm | 10 ++--------
lib/PublicInbox/LeiNoteEvent.pm | 8 +-------
lib/PublicInbox/LeiPmdir.pm | 8 +-------
script/public-inbox-clone | 2 +-
5 files changed, 6 insertions(+), 30 deletions(-)
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 21c93515..c35c5c26 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -46,17 +46,11 @@ sub ck_update_kw { # via wq_io_do
$self->{sto}->wq_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
}
-sub ikw_done_wait {
- my ($arg, $pid) = @_;
- my ($self, $lei) = @$arg;
- $lei->can('wq_done_wait')->($arg, $pid);
-}
-
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $ikw = delete $lei->{ikw} or return $lei->fail;
$lei->sto_done_request($ikw->{lei_sock});
- $ikw->wq_wait_old(\&ikw_done_wait, $lei);
+ $ikw->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index d9c13f05..6bfa4b6f 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -11,7 +11,7 @@ use PublicInbox::Spawn qw(popen_rd spawn run_die);
use File::Temp ();
use Fcntl qw(SEEK_SET O_CREAT O_EXCL O_WRONLY);
-sub do_finish_mirror { # dwaitpid callback
+sub _wq_done_wait { # dwaitpid callback (via wq_eof)
my ($arg, $pid) = @_;
my ($mrr, $lei) = @$arg;
my $f = "$mrr->{dst}/mirror.done";
@@ -28,12 +28,6 @@ sub do_finish_mirror { # dwaitpid callback
$lei->dclose;
}
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $mrr = delete $lei->{wq1} or return $lei->fail;
- $mrr->wq_wait_old(\&do_finish_mirror, $lei);
-}
-
# for old installations without manifest.js.gz
sub try_scrape {
my ($self) = @_;
@@ -176,7 +170,7 @@ sub index_cloned_inbox {
PublicInbox::Admin::progress_prepare($opt, $lei->{2});
PublicInbox::Admin::index_inbox($ibx, undef, $opt);
}
- open my $x, '>', "$self->{dst}/mirror.done"; # for do_finish_mirror
+ open my $x, '>', "$self->{dst}/mirror.done"; # for _wq_done_wait
}
sub run_reap {
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 43d5ed0f..d2429485 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -107,17 +107,11 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
-sub lne_done_wait {
- my ($arg, $pid) = @_;
- my ($self, $lei) = @$arg;
- $lei->can('wq_done_wait')->($arg, $pid);
-}
-
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $lne = delete $lei->{lne} or return $lei->fail;
$lei->sto_done_request;
- $lne->wq_wait_old(\&lne_done_wait, $lei);
+ $lne->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index 23bccb4f..2d3b9755 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -47,17 +47,11 @@ sub mdir_iter { # via wq_io_do
$self->{ipt}->pmdir_cb($f, $fl, @args);
}
-sub pmd_done_wait {
- my ($arg, $pid) = @_;
- my ($self, $lei) = @$arg;
- $lei->can('wq_done_wait')->($arg, $pid);
-}
-
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $pmd = delete $lei->{pmd} or return $lei->fail;
$lei->sto_done_request($pmd->{lei_sock});
- $pmd->wq_wait_old(\&pmd_done_wait, $lei);
+ $pmd->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/script/public-inbox-clone b/script/public-inbox-clone
index 2b18969f..0efde1a8 100755
--- a/script/public-inbox-clone
+++ b/script/public-inbox-clone
@@ -54,5 +54,5 @@ my $mrr = bless {
dst => $dst,
}, 'PublicInbox::LeiMirror';
$mrr->do_mirror;
-$mrr->can('do_finish_mirror')->([$mrr, $lei], $$);
+$mrr->can('_wq_done_wait')->([$mrr, $lei], $$);
exit(($lei->{child_error} // 0) >> 8);
^ permalink raw reply related [relevance 58%]
* [PATCH 6/7] lei up: avoid excessively parallel --all
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
2021-09-22 2:24 71% ` [PATCH 4/7] lei: dclose: do not close unnecessarily Eric Wong
@ 2021-09-22 2:24 49% ` Eric Wong
2021-09-22 2:24 58% ` [PATCH 7/7] lei: drop redundant WQ EOF callbacks Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
We shouldn't dispatch all outputs right away since they
can be expensive CPU-wise. Instead, rely on DESTROY to
trigger further redispatches.
This also fixes a circular reference bug for the single-output
case that could lead to a leftover script/lei after MUA exit.
I'm not sure how --jobs/-j should work when the actual xsearch
and lei2mail has it's own parallelism ("--jobs=$X,$M"), but
it's better than having thousands of subtasks running.
Fixes: b34a267efff7b831 ("lei up: fix --mua with single output")
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 86 +++++++++++++++++++++++++---------------
2 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a1cab55a..1305dfb8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1384,7 +1384,7 @@ sub fchdir {
sub wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
- $wq1->wq_wait_old(\&wq_done_wait, $lei);
+ $wq1->wq_wait_old($wq1->can('_wq_done_wait') // \&wq_done_wait, $lei);
}
sub watch_state_ok ($) {
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 89cf0112..377a720e 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -36,7 +36,7 @@ sub up1 ($$) {
$lei->{opt}->{$k} //= $v;
}
my $o = $lei->{opt}->{output} // '';
- return $lei->fail("lei.q.output unset in $f") if $o eq '';
+ return $lei->fail("lei.q.output unset in $f (out=$out)") if $o eq '';
$lss->translate_dedupe($lei) or return;
$lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
my $lxs = $lei->lxs_prepare or return;
@@ -44,39 +44,30 @@ sub up1 ($$) {
$lei->_start_query;
}
-sub up1_redispatch {
- my ($lei, $out, $op_p) = @_;
- my $l;
- if (defined($lei->{opt}->{mua})) { # single output
- $l = $lei;
- } else { # multiple outputs
- $l = bless { %$lei }, ref($lei);
- $l->{opt} = { %{$l->{opt}} }; # deep copy
- delete $l->{opt}->{all};
- delete $l->{sock}; # do not close
- # make close($l->{1}) happy in lei->dclose
- open my $fh, '>&', $l->{1} or
- return $l->child_error(0, "dup: $!");
- $l->{1} = $fh;
- $l->qerr("# updating $out");
- }
- $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
- eval { $l->dispatch('up', $out) };
- $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
-}
-
sub redispatch_all ($$) {
my ($self, $lei) = @_;
+ my $upq = [ (@{$self->{local} // []}, @{$self->{remote} // []}) ];
+ return up1($lei, $upq->[0]) if @$upq == 1; # just one, may start MUA
+
+ # FIXME: this is also used per-query, see lei->_start_query
+ my $j = $lei->{opt}->{jobs} || do {
+ my $n = $self->detect_nproc // 1;
+ $n > 4 ? 4 : $n;
+ };
+ $j = ($j =~ /\A([0-9]+)/) ? $1 + 0 : 1; # may be --jobs=$x,$m on CLI
# re-dispatch into our event loop w/o creating an extra fork-level
+ # $upq will be drained via DESTROY as each query finishes
$lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei);
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
+ # call lei->dclose when upq is done processing:
+ $op_c->{ops} = { '' => [ $lei->can('dclose'), $lei ] };
+ my @first_batch = splice(@$upq, 0, $j); # initial parallelism
+ $lei->{-upq} = $upq;
+ $lei->event_step_init; # wait for client disconnects
+ for my $out (@first_batch) {
+ PublicInbox::DS::requeue(
+ PublicInbox::LeiUp1::nxt($lei, $out, $op_p));
}
- $lei->event_step_init;
- $lei->pkt_ops($op_c->{ops} = { '' => [$lei->can('dclose'), $lei] });
}
sub lei_up {
@@ -98,7 +89,7 @@ sub lei_up {
} else {
$lei->fail("only --all=$all not understood");
}
- } elsif ($lei->{lse}) {
+ } elsif ($lei->{lse}) { # redispatched
scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
return up1($lei, $outs[0]);
} else {
@@ -131,16 +122,49 @@ sub net_merge_all_done {
my ($self, $lei) = @_;
$lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
$self->wq_close(1);
- redispatch_all($self, $lei);
+ eval { redispatch_all($self, $lei) };
+ warn "E: $@" if $@;
}
-sub _complete_up {
+sub _complete_up { # lei__complete hook
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
}
+sub _wq_done_wait { # dwaitpid callback
+ my ($arg, $pid) = @_;
+ my ($wq, $lei) = @$arg;
+ $lei->child_error($?, 'auth failure') if $?
+}
+
no warnings 'once';
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+package PublicInbox::LeiUp1; # for redispatch_all
+use strict;
+use v5.10.1;
+
+sub nxt ($$$) {
+ my ($lei, $out, $op_p) = @_;
+ bless { lei => $lei, out => $out, op_p => $op_p }, __PACKAGE__;
+}
+
+sub event_step { # runs via PublicInbox::DS::requeue
+ my ($self) = @_;
+ my $lei = $self->{lei}; # the original, from lei_up
+ my $l = bless { %$lei }, ref($lei); # per-output copy
+ delete($l->{sock}) or return; # client disconnected if {sock} is gone
+ $l->{opt} = { %{$l->{opt}} }; # deep copy
+ delete $l->{opt}->{all};
+ $l->qerr("# updating $self->{out}");
+ $l->{up_op_p} = $self->{op_p}; # ($l => $lei => script/lei)
+ eval { $l->dispatch('up', $self->{out}) };
+ $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
+
+ # onto the next:
+ my $out = shift(@{$lei->{-upq}}) or return;
+ PublicInbox::DS::requeue(nxt($lei, $out, $self->{op_p}));
+}
+
1;
^ permalink raw reply related [relevance 49%]
* [PATCH] lei: common --all[=remote|local] help message
@ 2021-09-23 10:36 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-23 10:36 UTC (permalink / raw)
To: meta
It helps to be consistent and reduce the learning curve, here.
---
lib/PublicInbox/LEI.pm | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 1305dfb8..96f63805 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -420,7 +420,9 @@ my %OPTDESC = (
'remote' => 'limit operations to those requiring network access',
'remote!' => 'prevent operations requiring network access',
-'all:s up' => ['local|remote', 'update all remote or local saved searches' ],
+# up, refresh-mail-sync, export-kw
+'all:s' => ['TYPE|local|remote', 'all remote or local folders' ],
+
'remote-fudge-time=s' => [ 'INTERVAL',
'look for mail INTERVAL older than the last successful query' ],
^ permalink raw reply related [relevance 71%]
* [PATCH 0/4] doc: lei: some manpage updaets
@ 2021-09-24 12:51 71% Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Tring ot ut somthing mildly coharent. Insomnia :<
Eric Wong (4):
doc: lei blob+rediff+p2q: add notes about git directory
doc: lei-overview: implicit stdin, correct Inline::C notes
doc: lei-index: remove --stdin, reword -F
doc: lei: manpages for export-kw and refresh-mail-sync
Documentation/lei-blob.pod | 6 ++-
Documentation/lei-export-kw.pod | 52 ++++++++++++++++++++++
Documentation/lei-index.pod | 18 +++-----
Documentation/lei-overview.pod | 22 +++++-----
Documentation/lei-p2q.pod | 2 +-
Documentation/lei-rediff.pod | 23 +++++++++-
Documentation/lei-refresh-mail-sync.pod | 57 +++++++++++++++++++++++++
MANIFEST | 2 +
Makefile.PL | 5 ++-
9 files changed, 158 insertions(+), 29 deletions(-)
create mode 100644 Documentation/lei-export-kw.pod
create mode 100644 Documentation/lei-refresh-mail-sync.pod
^ permalink raw reply [relevance 71%]
* [PATCH 3/4] doc: lei-index: remove --stdin, reword -F
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
2021-09-24 12:51 56% ` [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes Eric Wong
@ 2021-09-24 12:51 68% ` Eric Wong
2021-09-24 12:51 49% ` [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
lei-index really only works for Maildir, at the moment.
---
Documentation/lei-index.pod | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
diff --git a/Documentation/lei-index.pod b/Documentation/lei-index.pod
index bd125bcceb63..9e72026351a1 100644
--- a/Documentation/lei-index.pod
+++ b/Documentation/lei-index.pod
@@ -6,8 +6,6 @@ lei-index - index messages without importing them into lei/store
lei index [OPTIONS] FOLDER
-lei index [OPTIONS] --stdin
-
=head1 DESCRIPTION
Similar to L<lei-import(1)>, but does not store a copy of
@@ -26,22 +24,16 @@ messages into C<lei/store>.
=over
-=item -
-
-=item --stdin
-
-Read input from standard input. This is the default if standard
-input is a pipe or regular file and there are no arguments on
-the command-line.
-
=item -F MAIL_FORMAT
=item --in-format=MAIL_FORMAT
-Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
-C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+There is currently no need for this option. It will support C<mh>,
+eventually. For now, the default (and only supported) format is
+C<maildir>. When IMAP and NNTP support are fleshed out, those
+formats will be inferred from their URLs.
-Default: C<eml> when reading from stdin
+Default: C<maildir>
=item -q
^ permalink raw reply related [relevance 68%]
* [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
@ 2021-09-24 12:51 56% ` Eric Wong
2021-09-24 12:51 68% ` [PATCH 3/4] doc: lei-index: remove --stdin, reword -F Eric Wong
2021-09-24 12:51 49% ` [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Implicit stdin based on standard input being a pipe or regular
file is here to stay, so save users the trouble of typing '-'
or '--stdin'.
Inline::C is required as of commit 1d6e1f9a6a66 (lei: require
Socket::MsgHdr or Inline::C, drop oneshot, 2021-05-26); but
Socket::MsgHdr still gives a noticeable improvement in bash
completion speed.
Also, spell-out "MESSAGE-ID" since "MID" is actually not a
common abbreviation ("MSGID" is used by RFC 3977 and several
other RFCs, I recall).
---
Documentation/lei-overview.pod | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index e80cb094cecc..40a7b0aadd04 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -33,7 +33,7 @@ Show message with the git blob OID of 59ec517f9. If a message with
that OID isn't found, check if the current git repository has the
blob, trying to reconstruct it from a message if needed.
-=item $ lei blob 59ec517f9 | lei tag - -F eml +kw:flagged +L:next
+=item $ lei blob 59ec517f9 | lei tag -F eml +kw:flagged +L:next
Set the "flagged" keyword and "next" label on the message with the
blob OID of 59ec517f9.
@@ -92,18 +92,18 @@ and written.
Search for all flagged messages that also have a "next" label.
-=item $ lei p2q HEAD | lei q --stdin -tt -o /tmp/mdir
+=item $ lei p2q HEAD | lei q -tt -o /tmp/mdir
Search for messages that have post-image git blob IDs that match those
of the current repository's HEAD commit, writing them to the Maildir
directory "mdir" and flagging the messages that were an exact match.
-=item $ git show -s HEAD | lei lcat -
+=item $ git show -s HEAD | lei lcat
Display a local message for the public-inbox link contained in a
commit message.
-=item $ lei q -f text m:MID | lei rediff -U5 -
+=item $ lei q -f text m:MESSAGE-ID | lei rediff -U5
Feed a message containing a diff to L<lei-rediff(1)> to regenerate its
diff with five context lines. Unless C<--git-dir> is specified, this
@@ -114,13 +114,15 @@ code repository.
=head1 PERFORMANCE NOTES
-L<Inline::C> is recommended for performance. To enable it, create
-C<~/.cache/public-inbox/inline-c/>.
+L<Inline::C> is required, lei runs as a background daemon to reduce
+startup costs and can provide real-time L<kqueue(2)>/L<inotify(7)>
+Maildir monitoring. L<IO::KQueue> (p5-IO-KQueue on FreeBSD) and
+L<Linux::Inotify2> (liblinux-inotify2-perl and perl-Linux-Inotify2 in
+.deb and .rpm-based distros, respectively) are recommended.
-If Socket::MsgHdr is installed (libsocket-msghdr-perl in Debian), the
-first invocation of lei starts a daemon, reducing the startup cost of
-for future invocations (which is particularly important for Bash
-completion).
+L<Socket::MsgHdr> is optional (libsocket-msghdr-perl in Debian),
+and further improves startup performance. Its effect is most felt
+when using shell completion.
=head1 BASH COMPLETION
^ permalink raw reply related [relevance 56%]
* [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
@ 2021-09-24 12:51 53% ` Eric Wong
2021-09-24 12:51 56% ` [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Try to clarify these commands are intended to be useful for
git-using (usually software) projects (and not the bare git
repos we use internally).
We'll also document some commonly useful git-diff switches
in the lei-rediff man page to highlight the usefulness
of the command.
---
Documentation/lei-blob.pod | 6 ++++--
Documentation/lei-p2q.pod | 2 +-
Documentation/lei-rediff.pod | 23 +++++++++++++++++++++--
3 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index 5b611d11ba1f..429d206e358e 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -10,8 +10,8 @@ lei blob [OPTIONS] OID
Display a git blob. The blob may correspond to a message from the
local store, an existing blob in the current repository, or a
-not-yet-created blob in the current repository that can be
-reconstructed from a message.
+not-yet-created blob in the current git project repository (if any)
+that can be reconstructed from a message.
=head1 OPTIONS
@@ -22,6 +22,8 @@ reconstructed from a message.
Specify an additional .git/ directory to scan. This option may be
given multiple times.
+Default: the output of C<git rev-parse --git-dir>
+
=item --no-cwd
Do not look in the git repository of the current working directory.
diff --git a/Documentation/lei-p2q.pod b/Documentation/lei-p2q.pod
index 44798ac3d2a0..1068ff0beb27 100644
--- a/Documentation/lei-p2q.pod
+++ b/Documentation/lei-p2q.pod
@@ -16,7 +16,7 @@ inbox.
The patch can be provided on stdin or as a file. Alternatively, when
an argument is given that does not point to an existing file, it is
-taken as a reference to a commit in the current repository, and
+taken as a reference to a commit in the current git repository, and
L<git-format-patch(1)> is used to generate the patch.
=head1 OPTIONS
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index c7db6c1e20b6..f34e946ffe52 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -15,10 +15,15 @@ the specified L<git-diff(1)> options. This is useful if you want to
change the display of the original patch (e.g., increasing context,
coloring moved lines differently, or using an external diff viewer).
+It relies on the contents of the .git directory of your current
+project working tree. In other words, it works anywhere
+L<git-am(1)> works. Otherwise, C<--git-dir=> may be specified
+any number of times to add repositories to build blob data from.
+
=head1 OPTIONS
-In addition to many L<git-diff(1)>, the following options are
-supported.
+In addition to many L<git-diff(1)> options (e.g. C<-W>, C<-w>,
+C<-U $LINES>) the following options are supported:
=over
@@ -27,15 +32,29 @@ supported.
Read message from stdin. This is implicit if no arguments are given
and stdin is a pipe or regular file.
+For users of text editors and pagers capable of piping its
+buffer to arbitrary commands, it is useful to pipe a patch email
+to C<lei rediff> before piping it to L<git-am(1)>. The output
+of C<lei rediff> is compatible with C<git am> if its input was a
+patch email.
+
=item --git-dir=DIR
Specify an additional .git/ directory to scan. This option may be
given multiple times.
+Default: the output of C<git rev-parse --git-dir>
+
=item --no-cwd
Do not look in the git repository of the current working directory.
+=item -q
+
+=item --quiet
+
+Suppress progress output.
+
=item -v
=item --verbose
^ permalink raw reply related [relevance 53%]
* [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
` (2 preceding siblings ...)
2021-09-24 12:51 68% ` [PATCH 3/4] doc: lei-index: remove --stdin, reword -F Eric Wong
@ 2021-09-24 12:51 49% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Something is better than nothing.
---
Documentation/lei-export-kw.pod | 52 ++++++++++++++++++++++
Documentation/lei-refresh-mail-sync.pod | 57 +++++++++++++++++++++++++
MANIFEST | 2 +
Makefile.PL | 5 ++-
4 files changed, 115 insertions(+), 1 deletion(-)
create mode 100644 Documentation/lei-export-kw.pod
create mode 100644 Documentation/lei-refresh-mail-sync.pod
diff --git a/Documentation/lei-export-kw.pod b/Documentation/lei-export-kw.pod
new file mode 100644
index 000000000000..cf482ca0d307
--- /dev/null
+++ b/Documentation/lei-export-kw.pod
@@ -0,0 +1,52 @@
+=head1 NAME
+
+lei-export-kw - export keywords (flags) to Maildir and IMAP folders
+
+=head1 SYNOPSIS
+
+lei export-kw --all=[<remote|local>]
+
+lei export-kw MFOLDER [MFOLDER...]
+
+=head1 DESCRIPTION
+
+C<lei export-kw> propagates keywords (e.g. C<seen>, C<answered>,
+C<flagged>, etc.) from lei/store to IMAP folders and/or Maildirs.
+
+It does not delete, write, nor modify messages themselves;
+it only sets metadata on Maildirs and IMAP folders.
+
+=head1 OPTIONS
+
+=over
+
+=item --all
+
+Export to all local Maildirs and remote IMAP folders
+
+=item --all=local
+
+Export all local Maildirs
+
+=item --all=remote
+
+Export all remote IMAP folders
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-refresh-mail-sync(1)>, L<lei-tag(1)>
diff --git a/Documentation/lei-refresh-mail-sync.pod b/Documentation/lei-refresh-mail-sync.pod
new file mode 100644
index 000000000000..aefd516fad93
--- /dev/null
+++ b/Documentation/lei-refresh-mail-sync.pod
@@ -0,0 +1,57 @@
+=head1 NAME
+
+lei-refresh-mail-sync - refresh sync info with Maildir, IMAP
+
+=head1 SYNOPSIS
+
+lei refresh-mail-sync --all[=<remote|local>]
+
+lei refresh-mail-sync MFOLDER [MFOLDER...]
+
+=head1 DESCRIPTION
+
+C<lei refresh-mail-sync> is intended to to keep old messages
+indexed with L<lei-index(1)> retrievable if Maildir flags change
+a filename. It will prune invalid entries for messages which no
+longer exist in a Maildir.
+
+It is also useful for ensuring L<lei-export-kw(1)> can propagate
+keyword (flag) changes to Maildirs and IMAP folders.
+
+It only needs read-only access to Maildirs and IMAP folders
+and will not attempt to write to them at all.
+
+=head1 OPTIONS
+
+=over
+
+=item --all
+
+Refresh all local Maildirs and remote IMAP folders
+
+=item --all=local
+
+Refresh all local Maildirs
+
+=item --all=remote
+
+Refresh all remote IMAP folders
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-index(1)>, L<lei-export-kw(1)>
diff --git a/MANIFEST b/MANIFEST
index 8c2e964b3dce..3595195a6996 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -28,6 +28,7 @@ Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
Documentation/lei-edit-search.pod
+Documentation/lei-export-kw.pod
Documentation/lei-forget-external.pod
Documentation/lei-forget-search.pod
Documentation/lei-import.pod
@@ -43,6 +44,7 @@ Documentation/lei-overview.pod
Documentation/lei-p2q.pod
Documentation/lei-q.pod
Documentation/lei-rediff.pod
+Documentation/lei-refresh-mail-sync.pod
Documentation/lei-rm.pod
Documentation/lei-security.pod
Documentation/lei-store-format.pod
diff --git a/Makefile.PL b/Makefile.PL
index f56445ae422d..00a558d12472 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -46,9 +46,12 @@ $v->{-m1} = [ map {
} @EXE_FILES,
qw(
lei-add-external lei-blob lei-config lei-convert lei-edit-search
+ lei-export-kw
lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
lei-import lei-index lei-init lei-lcat lei-ls-external lei-ls-label
- lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-rm lei-tag
+ lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff
+ lei-refresh-mail-sync
+ lei-rm lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
^ permalink raw reply related [relevance 49%]
* [PATCH 1/3] lei up: show timezone offset with localtime
2021-09-25 5:49 71% [PATCH 0/3] lei: some robustness fixes Eric Wong
@ 2021-09-25 5:49 71% ` Eric Wong
2021-09-25 5:49 71% ` [PATCH 2/3] lei: restore old sigmask before daemon exit Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 5:49 UTC (permalink / raw)
To: meta
Sometimes a user (e.g. me) isn't really sure what timezone
they're in...
---
lib/PublicInbox/LeiXSearch.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 88540bc22367..99f887ce730f 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -324,7 +324,7 @@ sub fudge_qstr_time ($$$) {
}
$lr -= ($rft || (48 * 60 * 60));
$lei->qerr("# $uri limiting to ".
- strftime('%Y-%m-%d %k:%M', localtime($lr)). ' and newer');
+ strftime('%Y-%m-%d %k:%M %z', localtime($lr)). ' and newer');
# this should really be rt: (received-time), but no stable
# public-inbox releases support it, yet.
my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..';
^ permalink raw reply related [relevance 71%]
* [PATCH 0/3] lei: some robustness fixes
@ 2021-09-25 5:49 71% Eric Wong
2021-09-25 5:49 71% ` [PATCH 1/3] lei up: show timezone offset with localtime Eric Wong
2021-09-25 5:49 71% ` [PATCH 2/3] lei: restore old sigmask before daemon exit Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-25 5:49 UTC (permalink / raw)
To: meta
It out I can still fine localtime confusing if I'm constantly
switching timezones to fit wherever insomnia puts me.
I also noticed an odd failure triggered by augment_inprogress
which I can't reproduce. In any case, a progress message
shouldn't cause catastrophic failure.
Eric Wong (3):
lei up: show timezone offset with localtime
lei: restore old sigmask before daemon exit
lei2mail: augment_inprogress: guard against closed FDs
lib/PublicInbox/LEI.pm | 2 ++
lib/PublicInbox/LeiToMail.pm | 15 +++++++++------
lib/PublicInbox/LeiXSearch.pm | 2 +-
3 files changed, 12 insertions(+), 7 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei: restore old sigmask before daemon exit
2021-09-25 5:49 71% [PATCH 0/3] lei: some robustness fixes Eric Wong
2021-09-25 5:49 71% ` [PATCH 1/3] lei up: show timezone offset with localtime Eric Wong
@ 2021-09-25 5:49 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 5:49 UTC (permalink / raw)
To: meta
If the event loop fails, we want blocking waitpid (wait4) calls
to be interruptible with SIGTERM via "kill $PID" rather than
SIGKILL. Though a failing event loop is something we should
avoid...
---
lib/PublicInbox/LEI.pm | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9d5a5a46b398..3ff8a347af8b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1346,6 +1346,8 @@ sub lazy_start {
# $daemon pipe to `lei' closed, main loop begins:
eval { PublicInbox::DS->EventLoop };
warn "event loop error: $@\n" if $@;
+ # exit() may trigger waitpid via various DESTROY, ensure interruptible
+ PublicInbox::DS::sig_setmask($oldset);
dump_and_clear_log();
exit($exit_code // 0);
}
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: make pkt_op easier-to-use and understand
@ 2021-09-25 6:17 55% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 6:17 UTC (permalink / raw)
To: meta
Since switching to SOCK_SEQUENTIAL, we no longer have to use
fixed-width records to guarantee atomic reads. Thus we can
maintain more human-readable/searchable PktOp opcodes.
Furthermore, we can infer the subroutine name in many cases
to avoid repeating ourselves by specifying a command-name
twice (e.g. $ops->{CMD} => [ \&CMD, $obj ]; can now simply be
written as: $ops->{CMD} => [ $obj ] if CMD is a method of
$obj.
---
lib/PublicInbox/LEI.pm | 15 +++++++--------
lib/PublicInbox/LeiToMail.pm | 7 +++----
lib/PublicInbox/LeiXSearch.pm | 14 +++++++-------
lib/PublicInbox/PktOp.pm | 5 +++--
4 files changed, 20 insertions(+), 21 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3ff8a347af8b..a337fb0d80c2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -519,8 +519,7 @@ sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
$self->{failed}++;
err($self, $buf) if defined $buf;
- # calls fail_handler
- $self->{pkt_op_p}->pkt_do('!') if $self->{pkt_op_p};
+ $self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
x_it($self, ($exit_code // 1) << 8);
undef;
}
@@ -552,7 +551,7 @@ sub child_error { # passes non-fatal curl exit codes to user
sub note_sigpipe { # triggers sigpipe_handler
my ($self, $fd) = @_;
close(delete($self->{$fd})); # explicit close silences Perl warning
- $self->{pkt_op_p}->pkt_do('|') if $self->{pkt_op_p};
+ $self->{pkt_op_p}->pkt_do('sigpipe_handler') if $self->{pkt_op_p};
x_it($self, 13);
}
@@ -614,11 +613,11 @@ sub incr {
sub pkt_ops {
my ($lei, $ops) = @_;
- $ops->{'!'} = [ \&fail_handler, $lei ];
- $ops->{'|'} = [ \&sigpipe_handler, $lei ];
- $ops->{x_it} = [ \&x_it, $lei ];
- $ops->{child_error} = [ \&child_error, $lei ];
- $ops->{incr} = [ \&incr, $lei ];
+ $ops->{fail_handler} = [ $lei ];
+ $ops->{sigpipe_handler} = [ $lei ];
+ $ops->{x_it} = [ $lei ];
+ $ops->{child_error} = [ $lei ];
+ $ops->{incr} = [ $lei ];
$ops;
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 467b27bf275d..d42759cf71d4 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -714,16 +714,15 @@ sub do_post_auth {
} else { # Maildir
$self->{shard_info} = [ $mod, $shard ];
}
- $aug = '+'; # incr_post_augment
+ $aug = 'incr_post_augment';
} elsif ($self->{-wq_worker_nr} == 0) { # 1st worker do_augment
- $aug = '.'; # do_post_augment
+ $aug = 'do_post_augment';
}
if ($aug) {
local $0 = 'do_augment';
eval { do_augment($self, $lei) };
$lei->fail($@) if $@;
- $lei->{pkt_op_p}->pkt_do($aug) == 1 or
- die "do_post_augment trigger: $!";
+ $lei->{pkt_op_p}->pkt_do($aug) or die "pkt_do($aug): $!";
}
# done augmenting, connect the compressor pipe for each worker
if (my $zpipe = delete $lei->{zpipe}) {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 99f887ce730f..ae9f5881676d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -536,16 +536,16 @@ sub do_query {
my ($self, $lei) = @_;
my $l2m = $lei->{l2m};
my $ops = {
- '|' => [ $lei->can('sigpipe_handler'), $lei ],
- '!' => [ $lei->can('fail_handler'), $lei ],
- '.' => [ \&do_post_augment, $lei ],
- '+' => [ \&incr_post_augment, $lei ],
+ 'sigpipe_handler' => [ $lei ],
+ 'fail_handler' => [ $lei ],
+ 'do_post_augment' => [ \&do_post_augment, $lei ],
+ 'incr_post_augment' => [ \&incr_post_augment, $lei ],
'' => [ \&query_done, $lei ],
'mset_progress' => [ \&mset_progress, $lei ],
'l2m_progress' => [ \&l2m_progress, $lei ],
- 'x_it' => [ $lei->can('x_it'), $lei ],
- 'child_error' => [ $lei->can('child_error'), $lei ],
- 'incr_start_query' => [ \&incr_start_query, $self, $l2m ],
+ 'x_it' => [ $lei ],
+ 'child_error' => [ $lei ],
+ 'incr_start_query' => [ $self, $l2m ],
};
$lei->{auth}->op_merge($ops, $l2m) if $l2m && $lei->{auth};
my $end = $lei->pkt_op_pair;
diff --git a/lib/PublicInbox/PktOp.pm b/lib/PublicInbox/PktOp.pm
index 10942dd19b68..fd2569badd74 100644
--- a/lib/PublicInbox/PktOp.pm
+++ b/lib/PublicInbox/PktOp.pm
@@ -13,6 +13,7 @@ use Errno qw(EAGAIN EINTR);
use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
use Socket qw(AF_UNIX MSG_EOR SOCK_SEQPACKET);
use PublicInbox::IPC qw(ipc_freeze ipc_thaw);
+use Scalar::Util qw(blessed);
sub new {
my ($cls, $r) = @_;
@@ -57,8 +58,8 @@ sub event_step {
}
my $op = $self->{ops}->{$cmd //= $msg};
if ($op) {
- my ($sub, @args) = @$op;
- $sub->(@args, @pargs);
+ my ($obj, @args) = (@$op, @pargs);
+ blessed($obj) ? $obj->$cmd(@args) : $obj->(@args);
} elsif ($msg ne '') {
die "BUG: unknown message: `$cmd'";
}
^ permalink raw reply related [relevance 55%]
* [PATCH] doc: lei-rm: remove unnecessary -F values
@ 2021-09-25 7:08 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 7:08 UTC (permalink / raw)
To: meta
-F is really only useful for distinguishing between mbox
variants and single message/rfc822 files. URLs and
directory-based formats can be auto-detected easily enough.
---
Documentation/lei-rm.pod | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/Documentation/lei-rm.pod b/Documentation/lei-rm.pod
index f2a0c0f046e1..67bfb5510d63 100644
--- a/Documentation/lei-rm.pod
+++ b/Documentation/lei-rm.pod
@@ -34,10 +34,13 @@ the command-line.
=item --in-format=MAIL_FORMAT
-Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
-C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+Message input format: C<eml>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>
+when reading from stdin or using one of the mbox variants.
-Default: C<eml> when reading from stdin
+Not necessary when using an IMAP URL, NNTP URL or Maildir.
+
+Default: C<eml> when reading from stdin or if the file suffix
+ends in C<.patch> or C<.eml>.
=item --lock=METHOD
^ permalink raw reply related [relevance 71%]
* [PATCH 0/3] lei *-external: split off into separate files
@ 2021-09-25 8:49 71% Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
Some of the early parts of lei predated the current autoloading
using lazy_cb. None of these commands are expected to be
frequently used, so wasting memory on them is unnecessary.
Eric Wong (3):
lei forget-external: split into separate file
lei add-external: split into separate file
lei ls-external: split into separate file
MANIFEST | 3 +
lib/PublicInbox/LeiAddExternal.pm | 72 ++++++++++++++
lib/PublicInbox/LeiExternal.pm | 140 +--------------------------
lib/PublicInbox/LeiForgetExternal.pm | 46 +++++++++
lib/PublicInbox/LeiLsExternal.pm | 32 ++++++
lib/PublicInbox/LeiMirror.pm | 3 +-
lib/PublicInbox/LeiQuery.pm | 6 +-
7 files changed, 161 insertions(+), 141 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddExternal.pm
create mode 100644 lib/PublicInbox/LeiForgetExternal.pm
create mode 100644 lib/PublicInbox/LeiLsExternal.pm
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei ls-external: split into separate file
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
2021-09-25 8:49 48% ` [PATCH 2/3] lei add-external: " Eric Wong
@ 2021-09-25 8:49 58% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
This was written before we had auto-loading and rarely used.
---
MANIFEST | 1 +
lib/PublicInbox/LeiExternal.pm | 26 +-------------------------
lib/PublicInbox/LeiLsExternal.pm | 32 ++++++++++++++++++++++++++++++++
3 files changed, 34 insertions(+), 25 deletions(-)
create mode 100644 lib/PublicInbox/LeiLsExternal.pm
diff --git a/MANIFEST b/MANIFEST
index 8db9cdc791bb..3e942855127f 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -230,6 +230,7 @@ lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
lib/PublicInbox/LeiLcat.pm
+lib/PublicInbox/LeiLsExternal.pm
lib/PublicInbox/LeiLsLabel.pm
lib/PublicInbox/LeiLsMailSource.pm
lib/PublicInbox/LeiLsMailSync.pm
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 5a54cc19281e..701d1ad53adf 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -50,7 +50,7 @@ my %re_map = ( '*' => '[^/]*?', '?' => '[^/]',
'[' => '[', ']' => ']', ',' => ',' );
sub glob2re {
- my $re = $_[-1];
+ my $re = $_[-1]; # $_[0] may be $lei
my $p = '';
my $in_bracket = 0;
my $qm = 0;
@@ -108,30 +108,6 @@ sub get_externals {
();
}
-# TODO: does this need JSON output?
-sub lei_ls_external {
- my ($self, $filter) = @_;
- my $opt = $self->{opt};
- my $do_glob = !$opt->{globoff}; # glob by default
- my ($OFS, $ORS) = $opt->{z} ? ("\0", "\0\0") : (" ", "\n");
- $filter //= '*';
- my $re = $do_glob ? glob2re($filter) : undef;
- $re //= index($filter, '/') < 0 ?
- qr!/\Q$filter\E/?\z! : # exact basename match
- qr/\Q$filter\E/; # grep -F semantics
- my @ext = externals_each($self, my $boost = {});
- @ext = $opt->{'invert-match'} ? grep(!/$re/, @ext)
- : grep(/$re/, @ext);
- if ($opt->{'local'} && !$opt->{remote}) {
- @ext = grep(!m!\A[a-z\+]+://!, @ext);
- } elsif ($opt->{remote} && !$opt->{'local'}) {
- @ext = grep(m!\A[a-z\+]+://!, @ext);
- }
- for my $loc (@ext) {
- $self->out($loc, $OFS, 'boost=', $boost->{$loc}, $ORS);
- }
-}
-
# returns an anonymous sub which returns an array of potential results
sub complete_url_prepare {
my $argv = $_[-1]; # $_[0] may be $lei
diff --git a/lib/PublicInbox/LeiLsExternal.pm b/lib/PublicInbox/LeiLsExternal.pm
new file mode 100644
index 000000000000..dd2eb2e7d16e
--- /dev/null
+++ b/lib/PublicInbox/LeiLsExternal.pm
@@ -0,0 +1,32 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei ls-external" command
+package PublicInbox::LeiLsExternal;
+use strict;
+use v5.10.1;
+
+# TODO: does this need JSON output?
+sub lei_ls_external {
+ my ($lei, $filter) = @_;
+ my $do_glob = !$lei->{opt}->{globoff}; # glob by default
+ my ($OFS, $ORS) = $lei->{opt}->{z} ? ("\0", "\0\0") : (" ", "\n");
+ $filter //= '*';
+ my $re = $do_glob ? $lei->glob2re($filter) : undef;
+ $re //= index($filter, '/') < 0 ?
+ qr!/\Q$filter\E/?\z! : # exact basename match
+ qr/\Q$filter\E/; # grep -F semantics
+ my @ext = $lei->externals_each(my $boost = {});
+ @ext = $lei->{opt}->{'invert-match'} ? grep(!/$re/, @ext)
+ : grep(/$re/, @ext);
+ if ($lei->{opt}->{'local'} && !$lei->{opt}->{remote}) {
+ @ext = grep(!m!\A[a-z\+]+://!, @ext);
+ } elsif ($lei->{opt}->{remote} && !$lei->{opt}->{'local'}) {
+ @ext = grep(m!\A[a-z\+]+://!, @ext);
+ }
+ for my $loc (@ext) {
+ $lei->out($loc, $OFS, 'boost=', $boost->{$loc}, $ORS);
+ }
+}
+
+1;
^ permalink raw reply related [relevance 58%]
* [PATCH 1/3] lei forget-external: split into separate file
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
@ 2021-09-25 8:49 50% ` Eric Wong
2021-09-25 8:49 48% ` [PATCH 2/3] lei add-external: " Eric Wong
2021-09-25 8:49 58% ` [PATCH 3/3] lei ls-external: " Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
This was written before we had auto-loading, and forget-external
should be a rarely-used command that's not worth loading at
startup. Do some golfing while we're in the area, too.
---
MANIFEST | 1 +
lib/PublicInbox/LeiExternal.pm | 48 ++--------------------------
lib/PublicInbox/LeiForgetExternal.pm | 46 ++++++++++++++++++++++++++
lib/PublicInbox/LeiQuery.pm | 6 ++--
4 files changed, 52 insertions(+), 49 deletions(-)
create mode 100644 lib/PublicInbox/LeiForgetExternal.pm
diff --git a/MANIFEST b/MANIFEST
index 3595195a6996..a39afe24aa87 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -218,6 +218,7 @@ lib/PublicInbox/LeiEditSearch.pm
lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
lib/PublicInbox/LeiFinmsg.pm
+lib/PublicInbox/LeiForgetExternal.pm
lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index f8e610cacb21..35a7d68a17b5 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -31,7 +31,7 @@ sub externals_each {
}
sub ext_canonicalize {
- my ($location) = @_;
+ my $location = $_[-1]; # $_[0] may be $lei
if ($location !~ m!\Ahttps?://!) {
PublicInbox::Config::rel2abs_collapsed($location);
} else {
@@ -185,39 +185,9 @@ sub lei_add_external {
}
}
-sub lei_forget_external {
- my ($self, @locations) = @_;
- my $cfg = $self->_lei_cfg(1);
- my $quiet = $self->{opt}->{quiet};
- my %seen;
- for my $loc (@locations) {
- my (@unset, @not_found);
- for my $l ($loc, ext_canonicalize($loc)) {
- next if $seen{$l}++;
- my $key = "external.$l.boost";
- delete($cfg->{$key});
- $self->_config('--unset', $key);
- if ($? == 0) {
- push @unset, $l;
- } elsif (($? >> 8) == 5) {
- push @not_found, $l;
- } else {
- $self->err("# --unset $key error");
- return $self->x_it($?);
- }
- }
- if (@unset) {
- next if $quiet;
- $self->err("# $_ gone") for @unset;
- } elsif (@not_found) {
- $self->err("# $_ not found") for @not_found;
- } # else { already exited
- }
-}
-
# returns an anonymous sub which returns an array of potential results
sub complete_url_prepare {
- my $argv = $_[-1];
+ my $argv = $_[-1]; # $_[0] may be $lei
# Workaround bash word-splitting URLs to ['https', ':', '//' ...]
# Maybe there's a better way to go about this in
# contrib/completion/lei-completion.bash
@@ -253,20 +223,6 @@ sub complete_url_prepare {
wantarray ? ($re, $cur, $match_cb) : $match_cb;
}
-# shell completion helper called by lei__complete
-sub _complete_forget_external {
- my ($self, @argv) = @_;
- my $cfg = $self->_lei_cfg;
- my ($cur, $re, $match_cb) = complete_url_prepare(\@argv);
- # FIXME: bash completion off "http:" or "https:" when the last
- # character is a colon doesn't work properly even if we're
- # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
- # be a bash issue.
- map {
- $match_cb->(substr($_, length('external.')));
- } grep(/\Aexternal\.$re\Q$cur/, @{$cfg->{-section_order}});
-}
-
sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
my ($self, @argv) = @_;
my $cfg = $self->_lei_cfg;
diff --git a/lib/PublicInbox/LeiForgetExternal.pm b/lib/PublicInbox/LeiForgetExternal.pm
new file mode 100644
index 000000000000..7a4bbcf80a3a
--- /dev/null
+++ b/lib/PublicInbox/LeiForgetExternal.pm
@@ -0,0 +1,46 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei forget-external" command
+package PublicInbox::LeiForgetExternal;
+use strict;
+use v5.10.1;
+
+sub lei_forget_external {
+ my ($lei, @locations) = @_;
+ my $cfg = $lei->_lei_cfg or
+ return $lei->fail('no externals configured');
+ my %seen;
+ for my $loc (@locations) {
+ for my $l ($loc, $lei->ext_canonicalize($loc)) {
+ next if $seen{$l}++;
+ my $key = "external.$l.boost";
+ delete($cfg->{$key});
+ $lei->_config('--unset', $key);
+ if ($? == 0) {
+ $lei->qerr("# $l forgotten ");
+ } elsif (($? >> 8) == 5) {
+ $lei->err("# $l not found");
+ } else {
+ $lei->err("# --unset $key error");
+ return $lei->x_it($?);
+ }
+ }
+ }
+}
+
+# shell completion helper called by lei__complete
+sub _complete_forget_external {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg or return ();
+ my ($cur, $re, $match_cb) = $lei->complete_url_prepare(\@argv);
+ # FIXME: bash completion off "http:" or "https:" when the last
+ # character is a colon doesn't work properly even if we're
+ # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
+ # be a bash issue.
+ map {
+ $match_cb->(substr($_, length('external.')));
+ } grep(/\Aexternal\.$re\Q$cur/, @{$cfg->{-section_order}});
+}
+
+1;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index cb5ac8fb84a7..c65b00ca0986 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -158,11 +158,11 @@ no query allowed on command-line with --stdin
# shell completion helper called by lei__complete
sub _complete_q {
my ($self, @argv) = @_;
- my $ext = qr/\A(?:-I|(?:--(?:include|exclude|only)))\z/;
my @cur;
+ my $cb = $self->lazy_cb(qw(forget-external _complete_));
while (@argv) {
- if ($argv[-1] =~ $ext) {
- my @c = $self->_complete_forget_external(@cur);
+ if ($argv[-1] =~ /\A(?:-I|(?:--(?:include|exclude|only)))\z/) {
+ my @c = $cb->($self, @cur);
# try basename match:
if (scalar(@cur) == 1 && index($cur[0], '/') < 0) {
my $all = $self->externals_each;
^ permalink raw reply related [relevance 50%]
* [PATCH 2/3] lei add-external: split into separate file
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
@ 2021-09-25 8:49 48% ` Eric Wong
2021-09-25 8:49 58% ` [PATCH 3/3] lei ls-external: " Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
Also was written before we had auto-loading and rarely used.
---
MANIFEST | 1 +
lib/PublicInbox/LeiAddExternal.pm | 72 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiExternal.pm | 66 ----------------------------
lib/PublicInbox/LeiMirror.pm | 3 +-
4 files changed, 75 insertions(+), 67 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddExternal.pm
diff --git a/MANIFEST b/MANIFEST
index a39afe24aa87..8db9cdc791bb 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -207,6 +207,7 @@ lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
lib/PublicInbox/LI2Wrap.pm
lib/PublicInbox/LeiALE.pm
+lib/PublicInbox/LeiAddExternal.pm
lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
lib/PublicInbox/LeiBlob.pm
diff --git a/lib/PublicInbox/LeiAddExternal.pm b/lib/PublicInbox/LeiAddExternal.pm
new file mode 100644
index 000000000000..5eef206c761c
--- /dev/null
+++ b/lib/PublicInbox/LeiAddExternal.pm
@@ -0,0 +1,72 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei add-external" command
+package PublicInbox::LeiAddExternal;
+use strict;
+use v5.10.1;
+
+sub _finish_add_external {
+ my ($lei, $location) = @_;
+ my $new_boost = $lei->{opt}->{boost} // 0;
+ my $key = "external.$location.boost";
+ my $cur_boost = $lei->_lei_cfg(1)->{$key};
+ return if defined($cur_boost) && $cur_boost == $new_boost; # idempotent
+ $lei->_config($key, $new_boost);
+}
+
+sub lei_add_external {
+ my ($lei, $location) = @_;
+ my $mirror = $lei->{opt}->{mirror} // do {
+ my @fail;
+ for my $sw ($lei->index_opt, $lei->curl_opt,
+ qw(no-torsocks torsocks inbox-version)) {
+ my ($f) = (split(/|/, $sw, 2))[0];
+ next unless defined $lei->{opt}->{$f};
+ $f = length($f) == 1 ? "-$f" : "--$f";
+ push @fail, $f;
+ }
+ if (scalar(@fail) == 1) {
+ return $lei->("@fail requires --mirror");
+ } elsif (@fail) {
+ my $last = pop @fail;
+ my $fail = join(', ', @fail);
+ return $lei->("@fail and $last require --mirror");
+ }
+ undef;
+ };
+ $location = $lei->ext_canonicalize($location);
+ if (defined($mirror) && -d $location) {
+ $lei->fail(<<""); # TODO: did you mean "update-external?"
+--mirror destination `$location' already exists
+
+ } elsif (-d $location) {
+ index($location, "\n") >= 0 and
+ return $lei->fail("`\\n' not allowed in `$location'");
+ }
+ if ($location !~ m!\Ahttps?://! && !-d $location) {
+ $mirror // return $lei->fail("$location not a directory");
+ index($location, "\n") >= 0 and
+ return $lei->fail("`\\n' not allowed in `$location'");
+ $mirror = $lei->ext_canonicalize($mirror);
+ require PublicInbox::LeiMirror;
+ PublicInbox::LeiMirror->start($lei, $mirror => $location);
+ } else {
+ _finish_add_external($lei, $location);
+ }
+}
+
+sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg or return ();
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ require URI;
+ map {
+ my $u = URI->new(substr($_, length('external.')));
+ my ($base) = ($u->path =~ m!((?:/?.*)?/)[^/]+/?\z!);
+ $u->path($base);
+ $match_cb->($u->as_string);
+ } grep(m!\Aexternal\.https?://!, @{$cfg->{-section_order}});
+}
+
+1;
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 35a7d68a17b5..5a54cc19281e 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -132,59 +132,6 @@ sub lei_ls_external {
}
}
-sub add_external_finish {
- my ($self, $location) = @_;
- my $cfg = $self->_lei_cfg(1);
- my $new_boost = $self->{opt}->{boost} // 0;
- my $key = "external.$location.boost";
- my $cur_boost = $cfg->{$key};
- return if defined($cur_boost) && $cur_boost == $new_boost; # idempotent
- $self->_config($key, $new_boost);
-}
-
-sub lei_add_external {
- my ($self, $location) = @_;
- my $opt = $self->{opt};
- my $mirror = $opt->{mirror} // do {
- my @fail;
- for my $sw ($self->index_opt, $self->curl_opt,
- qw(no-torsocks torsocks inbox-version)) {
- my ($f) = (split(/|/, $sw, 2))[0];
- next unless defined $opt->{$f};
- $f = length($f) == 1 ? "-$f" : "--$f";
- push @fail, $f;
- }
- if (scalar(@fail) == 1) {
- return $self->("@fail requires --mirror");
- } elsif (@fail) {
- my $last = pop @fail;
- my $fail = join(', ', @fail);
- return $self->("@fail and $last require --mirror");
- }
- undef;
- };
- my $new_boost = $opt->{boost} // 0;
- $location = ext_canonicalize($location);
- if (defined($mirror) && -d $location) {
- $self->fail(<<""); # TODO: did you mean "update-external?"
---mirror destination `$location' already exists
-
- } elsif (-d $location) {
- index($location, "\n") >= 0 and
- return $self->fail("`\\n' not allowed in `$location'");
- }
- if ($location !~ m!\Ahttps?://! && !-d $location) {
- $mirror // return $self->fail("$location not a directory");
- index($location, "\n") >= 0 and
- return $self->fail("`\\n' not allowed in `$location'");
- $mirror = ext_canonicalize($mirror);
- require PublicInbox::LeiMirror;
- PublicInbox::LeiMirror->start($self, $mirror => $location);
- } else {
- add_external_finish($self, $location);
- }
-}
-
# returns an anonymous sub which returns an array of potential results
sub complete_url_prepare {
my $argv = $_[-1]; # $_[0] may be $lei
@@ -223,17 +170,4 @@ sub complete_url_prepare {
wantarray ? ($re, $cur, $match_cb) : $match_cb;
}
-sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
- my ($self, @argv) = @_;
- my $cfg = $self->_lei_cfg;
- my $match_cb = complete_url_prepare(\@argv);
- require URI;
- map {
- my $u = URI->new(substr($_, length('external.')));
- my ($base) = ($u->path =~ m!((?:/?.*)?/)[^/]+/?\z!);
- $u->path($base);
- $match_cb->($u->as_string);
- } grep(m!\Aexternal\.https?://!, @{$cfg->{-section_order}});
-}
-
1;
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 1ab5e0d898d2..5cfa6fea851c 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -23,7 +23,8 @@ sub _wq_done_wait { # dwaitpid callback (via wq_eof)
$lei->err("unlink($f): $!") unless $!{ENOENT};
} else {
if ($lei->{cmd} ne 'public-inbox-clone') {
- $lei->add_external_finish($mrr->{dst});
+ $lei->lazy_cb('add-external', '_finish_'
+ )->($lei, $mrr->{dst});
}
$lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
}
^ permalink raw reply related [relevance 48%]
* [PATCH] lei note-event: ignore kw_changed exceptions
@ 2021-09-26 0:02 62% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-26 0:02 UTC (permalink / raw)
To: meta
The note-event worker may see changes before a Xapian shard
commit happens, meaning keyword lookups fail as a result.
Just emit the request to the lei/store worker since it's a
fairly cheap operation at this point.
We'll try harder to look for kw changes, too, since
deduplication changes may lead to multiple docids being
resolved for a single message.
---
lib/PublicInbox/LeiNoteEvent.pm | 4 +++-
lib/PublicInbox/LeiSearch.pm | 12 +++++++++---
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index d24294850fd2..2a7b52c13487 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -42,7 +42,9 @@ sub eml_event ($$$$) {
$sto->wq_do('index_eml_only', $eml, $vmd, $xoids);
} elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
my $docids = [];
- my $c = $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
+ my $c = eval {
+ $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
+ } // 1; # too new, assume changed since still to-be-committed.
if (scalar @$docids) { # already in lei/store
$sto->wq_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index a10e6e176775..568277a6f404 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -9,6 +9,7 @@ use parent qw(PublicInbox::ExtSearch); # PublicInbox::Search->reopen
use PublicInbox::Search qw(xap_terms);
use PublicInbox::ContentHash qw(content_digest content_hash);
use PublicInbox::MID qw(mids mids_for_index);
+use Carp qw(croak);
# get combined docid from over.num:
# (not generic Xapian, only works with our sharding scheme)
@@ -131,14 +132,19 @@ sub xoids_for {
# returns true if $eml is indexed by lei/store and keywords don't match
sub kw_changed {
my ($self, $eml, $new_kw_sorted, $docids) = @_;
+ my $cur_kw;
if ($eml) {
my $xoids = xoids_for($self, $eml) // return;
$docids //= [];
@$docids = sort { $a <=> $b } values %$xoids;
}
- my $cur_kw = eval { msg_keywords($self, $docids->[0]) };
- die "E: #$docids->[0] keyword lookup failure: $@\n" if $@;
-
+ for my $id (@$docids) {
+ $cur_kw = eval { msg_keywords($self, $id) } and last;
+ }
+ if (!defined($cur_kw) && $@) {
+ $docids = join(', num:', @$docids);
+ croak "E: num:$docids keyword lookup failure: $@";
+ }
# RFC 5550 sec 5.9 on the $Forwarded keyword states:
# "Once set, the flag SHOULD NOT be cleared"
if (exists($cur_kw->{forwarded}) &&
^ permalink raw reply related [relevance 62%]
* [PATCH] lei -f reply: fix Cc: header combining
@ 2021-09-26 1:42 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-26 1:42 UTC (permalink / raw)
To: meta
When combining lines from To: and Cc: headers, ", " needs to be
used to separate them.
---
lib/PublicInbox/LeiViewText.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 34612711..1f002ccd 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -96,10 +96,11 @@ sub quote_hdr_buf ($$) {
for my $f (qw(To Cc)) {
for my $v ($eml->header_raw($f)) {
next if $v !~ /\S/;
- $cc .= $v;
+ $cc .= ", $v";
$to //= $v;
}
}
+ substr($cc, 0, 2, ''); # s/^, //;
PublicInbox::View::fold_addresses($to);
PublicInbox::View::fold_addresses($cc);
_xs($to);
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: ensure refresh_watches isn't called from workers
@ 2021-09-26 5:38 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-26 5:38 UTC (permalink / raw)
To: meta
Only the top-level lei-daemon will do inotify/kevent.
---
lib/PublicInbox/LEI.pm | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a337fb0d80c2..8faf74a29466 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -578,6 +578,7 @@ sub _lei_atfork_child {
close $listener if $listener;
undef $listener;
$dir_idle->force_close if $dir_idle;
+ undef $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
@@ -1413,6 +1414,7 @@ sub add_maildir_watch ($$) {
sub refresh_watches {
my ($lei) = @_;
+ $dir_idle or return;
my $cfg = _lei_cfg($lei) or return;
my $old = $cfg->{-watches};
my $watches = $cfg->{-watches} //= {};
^ permalink raw reply related [relevance 71%]
* [PATCH 0/2] lei rediff: --drq and --dequote-only
@ 2021-09-27 4:59 71% Eric Wong
2021-09-27 4:59 69% ` [PATCH 1/2] lei rediff: quiet warnings from Import and Eml Eric Wong
2021-09-27 4:59 42% ` [PATCH 2/2] lei rediff: add --drq and --dequote-only Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-27 4:59 UTC (permalink / raw)
To: meta
Things to make lei-rediff more useful inside a text editor
while writing replies to patch emails.
Eric Wong (2):
lei rediff: quiet warnings from Import and Eml
lei rediff: add --drq and --dequote-only
Documentation/lei-rediff.pod | 27 +++++++++++++++++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 58 ++++++++++++++++++++++++++++++++++--
t/solver_git.t | 41 +++++++++++++++++++++++--
4 files changed, 123 insertions(+), 5 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei rediff: quiet warnings from Import and Eml
2021-09-27 4:59 71% [PATCH 0/2] lei rediff: --drq and --dequote-only Eric Wong
@ 2021-09-27 4:59 69% ` Eric Wong
2021-09-27 4:59 42% ` [PATCH 2/2] lei rediff: add --drq and --dequote-only Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 4:59 UTC (permalink / raw)
To: meta
lei rediff is expected to see partial patch fragments and such,
so silence warnings when something isn't exactly a valid email
message.
---
lib/PublicInbox/LeiRediff.pm | 11 +++++++++--
t/solver_git.t | 1 +
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index ea9b2a643f9a..f6960560f1a2 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -193,8 +193,15 @@ sub extract_oids { # Eml each_part callback
sub input_eml_cb { # callback for all emails
my ($self, $eml) = @_;
- $self->{tmp_sto}->add_eml($eml);
- $self->{tmp_sto}->done;
+ {
+ local $SIG{__WARN__} = sub {
+ return if "@_" =~ /^no email in From: .*? or Sender:/;
+ return if PublicInbox::Eml::warn_ignore(@_);
+ warn @_;
+ };
+ $self->{tmp_sto}->add_eml($eml);
+ $self->{tmp_sto}->done;
+ }
$eml->each_part(\&extract_oids, $self, 1);
}
diff --git a/t/solver_git.t b/t/solver_git.t
index d5f4c4a02758..3c0b7f6524c6 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -93,6 +93,7 @@ index 15ac20eb..771486c4
EOM
like($lei_out, qr/\Q$exp\E/,
'preserve mode, regen header + context from -U0 patch');
+ is($lei_err, '', 'no warnings from bare patch');
my $e = { GIT_DIR => "$ENV{HOME}/.local/share/lei/store/ALL.git" };
my @x = xqx([qw(git cat-file --batch-all-objects --batch-check)], $e);
is_deeply(\@x, [], 'no objects stored') or diag explain(\@x);
^ permalink raw reply related [relevance 69%]
* [PATCH 2/2] lei rediff: add --drq and --dequote-only
2021-09-27 4:59 71% [PATCH 0/2] lei rediff: --drq and --dequote-only Eric Wong
2021-09-27 4:59 69% ` [PATCH 1/2] lei rediff: quiet warnings from Import and Eml Eric Wong
@ 2021-09-27 4:59 42% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 4:59 UTC (permalink / raw)
To: meta
More switches which can be useful for users who pipe from text
editors. --drq can be helpful while writing patch review email
replies, and perhaps --dequote-only, too.
---
Documentation/lei-rediff.pod | 27 +++++++++++++++++++++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 47 ++++++++++++++++++++++++++++++++++++
t/solver_git.t | 40 ++++++++++++++++++++++++++++--
4 files changed, 113 insertions(+), 3 deletions(-)
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index f34e946ffe52..e711de97015c 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -38,6 +38,33 @@ to C<lei rediff> before piping it to L<git-am(1)>. The output
of C<lei rediff> is compatible with C<git am> if its input was a
patch email.
+=item --drq[=COUNT]
+
+De-Re-Quote. De-quote the input and re-quotes (the output).
+Removes COUNT levels of C<E<gt> > email reply prefixes and
+re-adds them upon regenerating the diff.
+
+This switch is intended as a convenience for running inside a
+pipe-capable text editor when writing replies to a patch email.
+Note: this may over-add C<E<gt> > prefixes if some input lines
+are missing C<E<gt> > prefixes.
+
+COUNT is 1 if unspecified; in other words, C<--drq=1> and
+C<--drq> are equivalent.
+
+It implies L</--quiet> unless L</--verbose> is specified
+since text editors tend to combine stderr with stdout.
+
+=item --dequote-only[=COUNT]
+
+Like L</--drq>, but does not re-add quote prefixes to the output.
+
+This can be useful for feeding a hunk to L<git-apply(1)>
+or L<patch(1)> while writing a reply or further processing
+by another diff viewer.
+
+Unlike L</--drq>, it does NOT implies L</--quiet>.
+
=item --git-dir=DIR
Specify an additional .git/ directory to scan. This option may be
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8faf74a29466..b8159cba2922 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -201,7 +201,7 @@ our %CMD = ( # sorted in order of importance/use:
'rediff' => [ '--stdin|LOCATION...',
'regenerate a diff with different options',
'stdin|', # /|\z/ must be first for lone dash
- qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
+ qw(git-dir=s@ cwd! verbose|v+ color:s no-color drq:1 dequote-only:1),
@diff_opt, @lxs_opt, @net_opt, @c_opt ],
'add-external' => [ 'LOCATION',
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index f6960560f1a2..1e95e55ac1cc 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -134,6 +134,27 @@ EOM
$pid = spawn(\@cmd, $lei->{env}, { 2 => $lei->{2}, 1 => $lei->{1} });
waitpid($pid, 0);
$lei->child_error($?) if $?; # for git diff --exit-code
+ undef;
+}
+
+sub wait_requote ($$$) { # OnDestroy callback
+ my ($lei, $pid, $old_1) = @_;
+ $lei->{1} = $old_1; # closes stdin of `perl -pE 's/^/> /'`
+ waitpid($pid, 0) == $pid or die "BUG(?) waitpid: \$!=$! \$?=$?";
+ $lei->child_error($?) if $?;
+}
+
+sub requote ($$) {
+ my ($lei, $pfx) = @_;
+ pipe(my($r, $w)) or die "pipe: $!";
+ my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
+ # $^X (perl) is overkill, but maybe there's a weird system w/o sed
+ my $pid = spawn([$^X, '-pE', "s/^/$pfx/"], $lei->{env}, $rdr);
+ my $old_1 = $lei->{1};
+ $w->autoflush(1);
+ binmode $w, ':utf8';
+ $lei->{1} = $w;
+ PublicInbox::OnDestroy->new(\&wait_requote, $lei, $pid, $old_1);
}
sub extract_oids { # Eml each_part callback
@@ -142,6 +163,10 @@ sub extract_oids { # Eml each_part callback
$self->{lei}->out($p->header_obj->as_string, "\n");
my ($s, undef) = msg_part_text($p, $p->content_type || 'text/plain');
defined $s or return;
+ my $rq;
+ if ($self->{dqre} && $s =~ s/$self->{dqre}//g) { # '> ' prefix(es)
+ $rq = requote($self->{lei}, $1) if $self->{lei}->{opt}->{drq};
+ }
my @top = split($PublicInbox::ViewDiff::EXTRACT_DIFFS, $s);
undef $s;
my $blobs = $self->{blobs}; # blobs to resolve
@@ -191,6 +216,19 @@ sub extract_oids { # Eml each_part callback
$ctxq = diff_ctxq($self, $ctxq);
}
+# ensure dequoted parts are available for rebuilding patches:
+sub dequote_add { # Eml each_part callback
+ my ($ary, $self) = @_;
+ my ($p, undef, $idx) = @$ary;
+ my ($s, undef) = msg_part_text($p, $p->content_type || 'text/plain');
+ defined $s or return;
+ if ($s =~ s/$self->{dqre}//g) { # remove '> ' prefix(es)
+ substr($s, 0, 0, "part-dequoted: $idx\n\n");
+ utf8::encode($s);
+ $self->{tmp_sto}->add_eml(PublicInbox::Eml->new(\$s));
+ }
+}
+
sub input_eml_cb { # callback for all emails
my ($self, $eml) = @_;
{
@@ -200,6 +238,7 @@ sub input_eml_cb { # callback for all emails
warn @_;
};
$self->{tmp_sto}->add_eml($eml);
+ $eml->each_part(\&dequote_add, $self) if $self->{dqre};
$self->{tmp_sto}->done;
}
$eml->each_part(\&extract_oids, $self, 1);
@@ -207,6 +246,10 @@ sub input_eml_cb { # callback for all emails
sub lei_rediff {
my ($lei, @inputs) = @_;
+ ($lei->{opt}->{drq} && $lei->{opt}->{'dequote-only'}) and return
+ $lei->fail('--drq and --dequote-only are mutually exclusive');
+ ($lei->{opt}->{drq} && !$lei->{opt}->{verbose}) and
+ $lei->{opt}->{quiet} //= 1;
$lei->_lei_store(1)->write_prepare($lei);
$lei->{opt}->{'in-format'} //= 'eml';
# maybe it's a non-email (code) blob from a coderepo
@@ -257,6 +300,10 @@ sub ipc_atfork_child {
} @{$self->{lei}->{opt}->{'git-dir'}} ];
$lei->{env}->{'psgi.errors'} = $lei->{2}; # ugh...
$lei->{env}->{TMPDIR} = $self->{rdtmp}->dirname;
+ if (my $nr = ($lei->{opt}->{drq} || $lei->{opt}->{'dequote-only'})) {
+ my $re = '\s*> ' x $nr;
+ $self->{dqre} = qr/^($re)/ms;
+ }
undef;
}
diff --git a/t/solver_git.t b/t/solver_git.t
index 3c0b7f6524c6..cf450e2459e5 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -80,12 +80,48 @@ test_lei({tmpdir => "$tmpdir/rediff"}, sub {
lei_ok(qw(rediff -q -U9 t/solve/0001-simple-mod.patch));
like($lei_out, qr!^\Q+++\E b/TODO\n@@ -103,9 \+103,11 @@!sm,
'got more context with -U9');
- lei_ok(qw(rediff -q -U9 t/solve/bare.patch));
+
+ my (undef, $re) = split(/\n\n/, $lei_out, 2);
+ $re =~ s/^/> /sgm;
+ substr($re, 0, 0, <<EOM);
+From: me\@example.com
+Subject: Re: awesome advice
+
+WEB DESIGN EXPERT wrote:
+EOM
+ lei_ok([qw(rediff --abbrev=40 -U16 --drq)], undef,
+ { 0 => \$re, %$lei_opt });
my $exp = <<'EOM';
+From: me@example.com
+Subject: Re: awesome advice
+
+EOM
+ like($lei_out, qr/\Q$exp\E/, '--drq preserved header');
+
+ # n.b. --drq can requote the attribution line ("So-and-so wrote:"),
+ # but it's probably not worth preventing...
+
+ $exp = <<'EOM';
+> ---
+> TODO | 2 ++
+> Ω | 5 --
+> 1 file changed, 2 insertions(+)
+>
+> diff --git a/TODO b/TODO
+> index 605013e4904baabecd4a0a55997aebd8e8477a8f..69df7d565d49fbaaeb0a067910f03dc22cd52bd0 100644
+> --- a/TODO
+> +++ b/TODO
+> @@ -96,16 +96,18 @@ all need to be considered for everything we introduce)
+EOM
+ $exp =~ s/^>$/> /sgm; # re-add trailing white space
+ like($lei_out, qr/\Q$exp\E/, '--drq diffstat + context');
+
+ lei_ok(qw(rediff -q --abbrev=40 -U9 t/solve/bare.patch));
+ $exp = <<'EOM';
diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
old mode 100644
new mode 100755
-index 15ac20eb..771486c4
+index 15ac20eb871bf47697377e58a27db23102a38fca..771486c425b315bae70fd8a82d62ab0331e0a827
--- a/script/public-inbox-extindex
+++ b/script/public-inbox-extindex
@@ -1,13 +1,12 @@
^ permalink raw reply related [relevance 42%]
* [PATCH 2/3] t/lei-index: IMAP and NNTP dependencies are optional
@ 2021-09-27 21:05 63% ` Eric Wong
2021-09-27 21:05 58% ` [PATCH 3/3] lei completion: workaround old Perl bug Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 21:05 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
"lei index" support for IMAP and NNTP is incomplete, so there's
no point in requiring them.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210927124056.kj5okiefvs4ztk27@meerkat.local/
---
t/lei-index.t | 35 +++++++++++++++++++++++------------
1 file changed, 23 insertions(+), 12 deletions(-)
diff --git a/t/lei-index.t b/t/lei-index.t
index eeda5196f411..aab8f7e63f06 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -3,21 +3,30 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Spec;
-require_mods(qw(lei -nntpd));
+require_mods(qw(lei));
my ($ro_home, $cfg_path) = setup_public_inboxes;
my ($tmpdir, $for_destroy) = tmpdir;
my $env = { PI_CONFIG => $cfg_path };
+my $srv = {};
-my $sock = tcp_server;
-my $cmd = [ '-nntpd', '-W0', "--stdout=$tmpdir/n1", "--stderr=$tmpdir/n2" ];
-my $nntpd = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-nntpd $?");
-my $nntp_host_port = tcp_host_port($sock);
+SKIP: {
+ require_mods(qw(-nntpd Net::NNTP), 1);
+ my $rdr = { 3 => tcp_server };
+ $srv->{nntpd} = start_script(
+ [qw(-nntpd -W0), "--stdout=$tmpdir/n1", "--stderr=$tmpdir/n2"],
+ $env, $rdr) or xbail "nntpd: $?";
+ $srv->{nntp_host_port} = tcp_host_port($rdr->{3});
+}
+
+SKIP: {
+ require_mods(qw(-imapd Mail::IMAPClient), 1);
+ my $rdr = { 3 => tcp_server };
+ $srv->{imapd} = start_script(
+ [qw(-imapd -W0), "--stdout=$tmpdir/i1", "--stderr=$tmpdir/i2"],
+ $env, $rdr) or xbail("-imapd $?");
+ $srv->{imap_host_port} = tcp_host_port($rdr->{3});
+}
-$sock = tcp_server;
-$cmd = [ '-imapd', '-W0', "--stdout=$tmpdir/i1", "--stderr=$tmpdir/i2" ];
-my $imapd = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-imapd $?");
-my $imap_host_port = tcp_host_port($sock);
-undef $sock;
for ('', qw(cur new)) {
mkdir "$tmpdir/md/$_" or xbail "mkdir: $!";
mkdir "$tmpdir/md1/$_" or xbail "mkdir: $!";
@@ -77,8 +86,10 @@ test_lei({ tmpdir => $tmpdir }, sub {
is_deeply(json_utf8->decode($lei_out)->[0]->{'kw'},
['seen'], 'keyword set');
- lei_ok('index', "nntp://$nntp_host_port/t.v2");
- lei_ok('index', "imap://$imap_host_port/t.v2.0");
+ $srv->{nntpd} and
+ lei_ok('index', "nntp://$srv->{nntp_host_port}/t.v2");
+ $srv->{imapd} and
+ lei_ok('index', "imap://$srv->{imap_host_port}/t.v2.0");
is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
lei_ok qw(q m:multipart-html-sucks@11);
^ permalink raw reply related [relevance 63%]
* [PATCH 3/3] lei completion: workaround old Perl bug
2021-09-27 21:05 63% ` [PATCH 2/3] t/lei-index: IMAP and NNTP dependencies are optional Eric Wong
@ 2021-09-27 21:05 58% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 21:05 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
While `$argv[-1]' is `undef' on an empty @argv, using `$argv[-1]'
as a subroutine argument would fail incorrectly with:
Modification of non-creatable array value attempted, subscript -1 at ...
...even though we'd never attempt to modify @_ itself in the
subroutines being called. Work around the bug (tested on
5.16.3) by passing `undef' explicitly when `$argv[-1]' is
already `undef'.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210927124056.kj5okiefvs4ztk27@meerkat.local/
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 4 ++--
lib/PublicInbox/LeiLsMailSource.pm | 4 ++--
lib/PublicInbox/LeiRefreshMailSync.pm | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index cea9beeb5694..0b65c2762633 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -131,7 +131,7 @@ sub _complete_export_kw {
my $match_cb = $lei->complete_url_prepare(\@argv);
# filter-out read-only sources:
my @k = grep(!m!(?://;AUTH=ANONYMOUS\@|\A(?:nntps?|s?news)://)!,
- $lms->folders($argv[-1], 1));
+ $lms->folders($argv[-1] // undef, 1));
my @m = map { $match_cb->($_) } @k;
@m ? @m : @k;
}
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 397292d4c787..69d63ab6b397 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -122,11 +122,11 @@ sub lei_import { # the main "lei import" method
sub _complete_import {
my ($lei, @argv) = @_;
my ($re, $cur, $match_cb) = $lei->complete_url_prepare(\@argv);
- my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @k = $lei->url_folder_cache->keys($argv[-1] // undef, 1);
my @m = map { $match_cb->($_) } @k;
my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @k = $lms->folders($argv[-1], 1);
+ @k = $lms->folders($argv[-1] // undef, 1);
@m = map { $match_cb->($_) } @k;
if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 1db15d5742b5..7c3c0cda18d6 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -107,11 +107,11 @@ sub lei_ls_mail_source {
sub _complete_ls_mail_source {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
- my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @k = $lei->url_folder_cache->keys($argv[-1] // undef, 1);
my @m = map { $match_cb->($_) } @k;
my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @k = $lms->folders($argv[-1], 1);
+ @k = $lms->folders($argv[-1] // undef, 1);
@m = map { $match_cb->($_) } grep(m!\A[a-z]+://!, @k);
if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index eb842843b51d..0cb9f3dccd48 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -101,7 +101,7 @@ sub _complete_refresh_mail_sync {
my ($lei, @argv) = @_;
my $lms = $lei->lms or return ();
my $match_cb = $lei->complete_url_prepare(\@argv);
- my @k = $lms->folders($argv[-1], 1);
+ my @k = $lms->folders($argv[-1] // undef, 1);
my @m = map { $match_cb->($_) } @k;
@m ? @m : @k
}
^ permalink raw reply related [relevance 58%]
* [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only
@ 2021-09-27 23:53 71% Kyle Meyer
2021-09-28 1:05 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-27 23:53 UTC (permalink / raw)
To: meta
---
Documentation/lei-rediff.pod | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index e711de97..4d5e8168 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -40,7 +40,7 @@ patch email.
=item --drq[=COUNT]
-De-Re-Quote. De-quote the input and re-quotes (the output).
+De-Re-Quote. De-quotes the input and re-quotes (the output).
Removes COUNT levels of C<E<gt> > email reply prefixes and
re-adds them upon regenerating the diff.
@@ -63,7 +63,7 @@ This can be useful for feeding a hunk to L<git-apply(1)>
or L<patch(1)> while writing a reply or further processing
by another diff viewer.
-Unlike L</--drq>, it does NOT implies L</--quiet>.
+Unlike L</--drq>, it does NOT imply L</--quiet>.
=item --git-dir=DIR
base-commit: 010159a7b789a3a27c917a5378db92c8adf78a9e
--
2.33.0
^ permalink raw reply related [relevance 71%]
* Re: [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only
2021-09-27 23:53 71% [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only Kyle Meyer
@ 2021-09-28 1:05 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-28 1:05 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed as commit 3dbd6dc31dd7e34bea87dbb626f811e093b6860b
^ permalink raw reply [relevance 71%]
* [PATCH 1/9] doc: lei-security: some more updates
@ 2021-10-01 9:54 69% ` Eric Wong
2021-10-01 9:54 49% ` [PATCH 9/9] doc: lei-daemon: new manpage Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-01 9:54 UTC (permalink / raw)
To: meta
Virtual users will probably be used for read-write IMAP/JMAP
support. The potential for various kernel/hardware bugs and
attacks also needs to be highlighted.
---
Documentation/lei-security.pod | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/Documentation/lei-security.pod b/Documentation/lei-security.pod
index 02305b9055c2..8cbd89934568 100644
--- a/Documentation/lei-security.pod
+++ b/Documentation/lei-security.pod
@@ -18,6 +18,9 @@ permissions support.
It does not use POSIX ACLs, extended attributes, nor any other
security-related functions which require non-standard Perl modules.
+There is preliminary support for "virtual users", but it is
+incomplete and undocumented.
+
=head1 INTERNAL FILES
lei runs with a umask of 077 to prevent other users on the
@@ -93,7 +96,7 @@ lei uses L<git-credential(1)> to prompt users for IMAP and NNTP
usernames and passwords. These passwords are not encrypted in
memory and get transferred across processes via anonymous UNIX
sockets and pipes. They may be exposed via syscall tracing
-tools (e.g. L<strace(1)>).
+tools (e.g. L<strace(1)>), kernel and hardware bugs/attacks.
While credentials are not written to the filesystem by default,
it is possible for them to end up on disk if processes are
^ permalink raw reply related [relevance 69%]
* [PATCH 9/9] doc: lei-daemon: new manpage
2021-10-01 9:54 69% ` [PATCH 1/9] doc: lei-security: some more updates Eric Wong
@ 2021-10-01 9:54 49% ` Eric Wong
2021-10-02 0:19 71% ` Kyle Meyer
1 sibling, 1 reply; 200+ results
From: Eric Wong @ 2021-10-01 9:54 UTC (permalink / raw)
To: meta
In case users see "lei-daemon" in ps(1) or syslog and wonder.
---
Documentation/lei-daemon.pod | 61 ++++++++++++++++++++++++++++++
Documentation/lei.pod | 9 ++++-
Documentation/lei_design_notes.txt | 2 +-
MANIFEST | 1 +
Makefile.PL | 2 +-
5 files changed, 72 insertions(+), 3 deletions(-)
create mode 100644 Documentation/lei-daemon.pod
diff --git a/Documentation/lei-daemon.pod b/Documentation/lei-daemon.pod
new file mode 100644
index 000000000000..1205e974706a
--- /dev/null
+++ b/Documentation/lei-daemon.pod
@@ -0,0 +1,61 @@
+=head1 NAME
+
+lei-daemon - technical information for local email interface daemon
+
+=head1 DESCRIPTION
+
+This documentation is a high-level overview for developers and
+administrators interested in how lei works.
+
+lei-daemon is a background daemon which powers the L<lei(1)>
+command-line tool. It may support virtual users and read-write
+IMAP+JMAP APIs in the future. It is designed to optimize shell
+completion by avoiding module loading costs, monitor Maildirs
+(and in the near future, IMAP folders) for changes.
+
+=head2 worker processes
+
+Most commands cause lei-daemon to L<fork(2)> new worker
+processes to isolate and parallelize work. lei-daemon is
+significantly more aggressive than read-only
+L<public-inbox-daemon(8)> processes with regards to resource use
+since it's not designed to support C10K/C100K scenarios.
+
+=head2 file descriptor passing
+
+FD passing is used to reduce IPC costs for bulk I/O when
+importing large mboxes from stdin and dumping large mboxes
+to stdout.
+
+=head2 SOCK_SEQPACKET
+
+SOCK_SEQPACKET sockets are used for both communicating with
+L<lei(1)> and to internal workers. SOCK_SEQPACKET is guarantee
+reliability (unlike SOCK_DGRAM), allows easy load distribution,
+and saves developers the trouble of maintaining stream parsers.
+
+=head2 File monitoring
+
+Inotify or EVFILT_VNODE is used depending on the platform
+to monitor Maildirs of changes and track keyword changes.
+
+The listen socket at (default: C<$XDG_RUNTIME_DIR/lei/5.seq.sock>)
+is also monitored, and the daemon will automatically shutdown
+if it is unlinked.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-overview(7)>, L<lei-daemon-kill(1)>, L<lei-daemon-pid(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 3af9e2eeacf7..63d5ee6900a7 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -115,6 +115,13 @@ Other subcommands include
By default storage is located at C<$XDG_DATA_HOME/lei/store>. The
configuration for lei resides at C<$XDG_CONFIG_HOME/lei/config>.
+=head1 ERRORS
+
+Errors and dianostics for interactive commands are reported to
+stderr. Some errors for background tasks are emitted via
+L<syslog(3)> as L<lei-daemon(8)> for the top-level daemon,
+and C<lei/store> for the L<lei-store-format(5)> worker.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -130,4 +137,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-overview(7)>
+L<lei-overview(7)>, L<lei-daemon(8)>
diff --git a/Documentation/lei_design_notes.txt b/Documentation/lei_design_notes.txt
index f1d2ab6f2169..b5180374ad05 100644
--- a/Documentation/lei_design_notes.txt
+++ b/Documentation/lei_design_notes.txt
@@ -6,7 +6,7 @@ Daemon architecture
The use of a persistent daemon works around slow startup time of
Perl. This is especially important for built-in support for
-shell completion. It will eventually support inotify and EVFILT_VNODE
+shell completion. It attempts to support inotify and EVFILT_VNODE
background monitoring of Maildir keyword changes.
If lei were reimplemented in a language with faster startup
diff --git a/MANIFEST b/MANIFEST
index 3e942855127f..929f5f869c5b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -27,6 +27,7 @@ Documentation/lei-config.pod
Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
+Documentation/lei-daemon.pod
Documentation/lei-edit-search.pod
Documentation/lei-export-kw.pod
Documentation/lei-forget-external.pod
diff --git a/Makefile.PL b/Makefile.PL
index 00a558d12472..c41c408a5704 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -60,7 +60,7 @@ $v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
$v->{-m7} = [ qw(lei-overview lei-security
public-inbox-overview public-inbox-tuning
public-inbox-glossary) ];
-$v->{-m8} = [ qw(public-inbox-daemon) ];
+$v->{-m8} = [ qw(public-inbox-daemon lei-daemon) ];
my @sections = (1, 5, 7, 8);
$v->{check_80} = [];
$v->{manuals} = [];
^ permalink raw reply related [relevance 49%]
* Re: [PATCH 9/9] doc: lei-daemon: new manpage
2021-10-01 9:54 49% ` [PATCH 9/9] doc: lei-daemon: new manpage Eric Wong
@ 2021-10-02 0:19 71% ` Kyle Meyer
2021-10-02 8:08 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-10-02 0:19 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> In case users see "lei-daemon" in ps(1) or syslog and wonder.
Thanks, this is very helpful/informative.
> diff --git a/Documentation/lei-daemon.pod b/Documentation/lei-daemon.pod
[...]
> +=head2 SOCK_SEQPACKET
> +
> +SOCK_SEQPACKET sockets are used for both communicating with
> +L<lei(1)> and to internal workers. SOCK_SEQPACKET is guarantee
> +reliability (unlike SOCK_DGRAM), allows easy load distribution,
s/is guarantee/guarantees/ ?
> +and saves developers the trouble of maintaining stream parsers.
> +
> +=head2 File monitoring
Obviously unimportant, but this casing is inconsistent with some of the
titles above (e.g., "file descriptor passing").
> +Inotify or EVFILT_VNODE is used depending on the platform
> +to monitor Maildirs of changes and track keyword changes.
s/of changes/for changes/ ?
> +The listen socket at (default: C<$XDG_RUNTIME_DIR/lei/5.seq.sock>)
I stumbled parsing this because target of "at" is parenthesized. At
least in my head, it'd read fine dropping the "at".
> +is also monitored, and the daemon will automatically shutdown
> +if it is unlinked.
^ permalink raw reply [relevance 71%]
* Re: [PATCH 9/9] doc: lei-daemon: new manpage
2021-10-02 0:19 71% ` Kyle Meyer
@ 2021-10-02 8:08 71% ` Eric Wong
2021-10-04 3:16 71% ` Kyle Meyer
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-02 8:08 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed with your Helped-by:.
Btw, did you have time to work on some of the other manpages?
No worries if not, I might take a stab at them sometime in the
next few days. I often realize shortcomings while writing
documentation, and I still need to figure out why some messages
go missing or get duplicated...
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei inspect: integerize "bytes" and "lines" fields
2021-10-02 8:16 71% [PATCH 0/2] lei inspect stuffs Eric Wong
@ 2021-10-02 8:16 71% ` Eric Wong
2021-10-02 8:16 53% ` [PATCH 2/2] lei inspect: fix "mid:" prefix, expand to Xapian Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-02 8:16 UTC (permalink / raw)
To: meta
These are always numeric, but none of the Perl code cares;
but we want to prevent JSON from quoting them.
---
lib/PublicInbox/LeiInspect.pm | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2158b996a3a4..f18e31c5c8f4 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -13,6 +13,12 @@ use PublicInbox::Config;
use PublicInbox::MID qw(mids);
use PublicInbox::NetReader qw(imap_uri nntp_uri);
+sub _json_prep ($) {
+ my ($smsg) = @_;
+ $smsg->{$_} += 0 for qw(bytes lines); # integerize
+ +{ %$smsg } # unbless and scalarize
+}
+
sub inspect_blob ($$) {
my ($lei, $oidhex) = @_;
my $ent = {};
@@ -143,7 +149,7 @@ sub inspect_num ($$) {
}
if ($ibx && $ibx->over) {
my $smsg = $ibx->over->get_art($num);
- $ent->{smsg} = { %$smsg } if $smsg;
+ $ent->{smsg} = _json_prep($smsg) if $smsg;
}
defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent;
}
@@ -164,7 +170,7 @@ sub inspect_mid ($$) {
if ($ibx && $ibx->over) {
my ($id, $prev);
while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
- push @{$ent->{smsg}}, { %$smsg }
+ push @{$ent->{smsg}}, _json_prep($smsg);
}
}
$ent;
^ permalink raw reply related [relevance 71%]
* [PATCH 0/2] lei inspect stuffs
@ 2021-10-02 8:16 71% Eric Wong
2021-10-02 8:16 71% ` [PATCH 1/2] lei inspect: integerize "bytes" and "lines" fields Eric Wong
2021-10-02 8:16 53% ` [PATCH 2/2] lei inspect: fix "mid:" prefix, expand to Xapian Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-10-02 8:16 UTC (permalink / raw)
To: meta
Some obvious fixes while working extindex stuff..
Eric Wong (2):
lei inspect: integerize "bytes" and "lines" fields
lei inspect: fix "mid:" prefix, expand to Xapian
MANIFEST | 1 +
lib/PublicInbox/LeiInspect.pm | 69 +++++++++++++++++++++--------------
t/lei-inspect.t | 14 +++++++
3 files changed, 56 insertions(+), 28 deletions(-)
create mode 100644 t/lei-inspect.t
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei inspect: fix "mid:" prefix, expand to Xapian
2021-10-02 8:16 71% [PATCH 0/2] lei inspect stuffs Eric Wong
2021-10-02 8:16 71% ` [PATCH 1/2] lei inspect: integerize "bytes" and "lines" fields Eric Wong
@ 2021-10-02 8:16 53% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-02 8:16 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
This fixes inspect for uninitialized instances, and adds Xapian
("xdoc") output if available.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Message-ID: <20211001204943.l4yl6xvc45c5eapz@meerkat.local>
---
MANIFEST | 1 +
lib/PublicInbox/LeiInspect.pm | 59 ++++++++++++++++++++---------------
t/lei-inspect.t | 14 +++++++++
3 files changed, 48 insertions(+), 26 deletions(-)
create mode 100644 t/lei-inspect.t
diff --git a/MANIFEST b/MANIFEST
index 929f5f869c5b..74b28d2d54ab 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -450,6 +450,7 @@ t/lei-import-maildir.t
t/lei-import-nntp.t
t/lei-import.t
t/lei-index.t
+t/lei-inspect.t
t/lei-lcat.t
t/lei-mirror.psgi
t/lei-mirror.t
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index f18e31c5c8f4..590dfdabca56 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -78,22 +78,9 @@ sub inspect_sync_folder ($$) {
$ent
}
-sub inspect_docid ($$;$) {
- my ($lei, $docid, $ent) = @_;
- require PublicInbox::Search;
- $ent //= {};
- my $xdb;
- if ($xdb = delete $ent->{xdb}) { # from inspect_num
- } elsif (defined(my $dir = $lei->{opt}->{dir})) {
- no warnings 'once';
- $xdb = $PublicInbox::Search::X{Database}->new($dir);
- } else {
- $xdb = $lei->{lse}->xdb;
- }
- $xdb or return $lei->fail('no Xapian DB');
- my $doc = $xdb->get_document($docid); # raises
+sub _inspect_doc ($$) {
+ my ($ent, $doc) = @_;
my $data = $doc->get_data;
- $ent->{docid} = $docid;
$ent->{data_length} = length($data);
$ent->{description} = $doc->get_description;
$ent->{$_} = $doc->$_ for (qw(termlist_count values_count));
@@ -119,6 +106,24 @@ sub inspect_docid ($$;$) {
$ent;
}
+sub inspect_docid ($$;$) {
+ my ($lei, $docid, $ent) = @_;
+ require PublicInbox::Search;
+ $ent //= {};
+ my $xdb;
+ if ($xdb = delete $ent->{xdb}) { # from inspect_num
+ } elsif (defined(my $dir = $lei->{opt}->{dir})) {
+ no warnings 'once';
+ $xdb = $PublicInbox::Search::X{Database}->new($dir);
+ } elsif ($lei->{lse}) {
+ $xdb = $lei->{lse}->xdb;
+ }
+ $xdb or return $lei->fail('no Xapian DB');
+ my $doc = $xdb->get_document($docid); # raises
+ $ent->{docid} = $docid;
+ _inspect_doc($ent, $doc);
+}
+
sub dir2ibx ($$) {
my ($lei, $dir) = @_;
if (-f "$dir/ei.lock") {
@@ -138,11 +143,9 @@ sub inspect_num ($$) {
my $ent = { num => $num };
if (defined(my $dir = $lei->{opt}->{dir})) {
$ibx = dir2ibx($lei, $dir) or return;
- if ($ent->{xdb} = $ibx->xdb) {
- my $num2docid = $lei->{lse}->can('num2docid');
- $docid = $num2docid->($ibx, $num);
- }
- } else {
+ $ent->{xdb} = $ibx->xdb and # for inspect_docid
+ $docid = PublicInbox::LeiSearch::num2docid($ibx, $num);
+ } elsif ($lei->{lse}) {
$ibx = $lei->{lse};
$lei->{lse}->xdb; # set {nshard} for num2docid
$docid = $lei->{lse}->num2docid($num);
@@ -156,16 +159,12 @@ sub inspect_num ($$) {
sub inspect_mid ($$) {
my ($lei, $mid) = @_;
- my ($ibx, $over);
+ my $ibx;
my $ent = { mid => $mid };
if (defined(my $dir = $lei->{opt}->{dir})) {
- my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid');
- $ibx = dir2ibx($lei, $dir) or return;
- # $ent->{xdb} = $ibx->xdb //
- # return $lei->fail("no Xapian DB for $dir");
+ $ibx = dir2ibx($lei, $dir)
} else {
$ibx = $lei->{lse};
- $lei->{lse}->xdb; # set {nshard} for num2docid
}
if ($ibx && $ibx->over) {
my ($id, $prev);
@@ -173,6 +172,14 @@ sub inspect_mid ($$) {
push @{$ent->{smsg}}, _json_prep($smsg);
}
}
+ if ($ibx && $ibx->search) {
+ my $mset = $ibx->search->mset(qq{mid:"$mid"});
+ for (sort { $a->get_docid <=> $b->get_docid } $mset->items) {
+ my $tmp = { docid => $_->get_docid };
+ _inspect_doc($tmp, $_->get_document);
+ push @{$ent->{xdoc}}, $tmp;
+ }
+ }
$ent;
}
diff --git a/t/lei-inspect.t b/t/lei-inspect.t
new file mode 100644
index 000000000000..077d0d13e84d
--- /dev/null
+++ b/t/lei-inspect.t
@@ -0,0 +1,14 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+
+test_lei(sub {
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
+ lei_ok qw(inspect --dir), "$ro_home/t1", 'mid:testmessage@example.com';
+ my $ent = json_utf8->decode($lei_out);
+ is(ref($ent->{smsg}), 'ARRAY', 'smsg array');
+ is(ref($ent->{xdoc}), 'ARRAY', 'xdoc array');
+});
+
+done_testing;
^ permalink raw reply related [relevance 53%]
* [PATCH 0/4] lei mail-diff and other small things
@ 2021-10-02 11:18 71% Eric Wong
2021-10-02 11:18 38% ` [PATCH 2/4] lei mail-diff: diagnostic command to diff mail contents Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-02 11:18 UTC (permalink / raw)
To: meta
1/4 doesn't matter, atm;
2/4 is something I finally got around to doing :x
3/4 is long overdue, I think (and a result of 2/4)
Not sure why 4/4 wasn't done earlier, either, I guess
I never notice missing blobs.
Eric Wong (4):
extsearchidx: attach_config: set {ibx_map} value to $ibx
lei mail-diff: diagnostic command to diff mail contents
content_hash: normalize whitespace before hashing addresses
extsearchidx: emit diagnostics for missing blobs
MANIFEST | 1 +
lib/PublicInbox/ContentHash.pm | 7 +-
lib/PublicInbox/ExtSearchIdx.pm | 8 ++-
lib/PublicInbox/LEI.pm | 5 ++
lib/PublicInbox/LeiInput.pm | 6 ++
lib/PublicInbox/LeiMailDiff.pm | 111 ++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiRediff.pm | 63 +++++++++---------
7 files changed, 167 insertions(+), 34 deletions(-)
create mode 100644 lib/PublicInbox/LeiMailDiff.pm
^ permalink raw reply [relevance 71%]
* [PATCH 2/4] lei mail-diff: diagnostic command to diff mail contents
2021-10-02 11:18 71% [PATCH 0/4] lei mail-diff and other small things Eric Wong
@ 2021-10-02 11:18 38% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-02 11:18 UTC (permalink / raw)
To: meta
This is useful in finding the cause of deduplication bugs,
and possibly the cause of missing threads reported by
Konstantin in <20211001130527.z7eivotlgqbgetzz@meerkat.local>
usage:
u=https://yhbt.net/lore/all/87czop5j33.fsf@tynnyri.adurom.net/raw
lei mail-diff $u
---
MANIFEST | 1 +
lib/PublicInbox/ContentHash.pm | 6 +-
lib/PublicInbox/LEI.pm | 5 ++
lib/PublicInbox/LeiInput.pm | 6 ++
lib/PublicInbox/LeiMailDiff.pm | 111 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiRediff.pm | 63 ++++++++++---------
6 files changed, 160 insertions(+), 32 deletions(-)
create mode 100644 lib/PublicInbox/LeiMailDiff.pm
diff --git a/MANIFEST b/MANIFEST
index 74b28d2d54ab..22b7df9bb89d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -237,6 +237,7 @@ lib/PublicInbox/LeiLsMailSource.pm
lib/PublicInbox/LeiLsMailSync.pm
lib/PublicInbox/LeiLsSearch.pm
lib/PublicInbox/LeiLsWatch.pm
+lib/PublicInbox/LeiMailDiff.pm
lib/PublicInbox/LeiMailSync.pm
lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiNoteEvent.pm
diff --git a/lib/PublicInbox/ContentHash.pm b/lib/PublicInbox/ContentHash.pm
index cc4a54c922d0..f6ae9011c1bf 100644
--- a/lib/PublicInbox/ContentHash.pm
+++ b/lib/PublicInbox/ContentHash.pm
@@ -52,9 +52,9 @@ sub content_dig_i {
$dig->add($s);
}
-sub content_digest ($) {
- my ($eml) = @_;
- my $dig = Digest::SHA->new(256);
+sub content_digest ($;$) {
+ my ($eml, $dig) = @_;
+ $dig //= Digest::SHA->new(256);
# References: and In-Reply-To: get used interchangeably
# in some "duplicates" in LKML. We treat them the same
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index fd59235846ae..51b0e95e1728 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -203,6 +203,11 @@ our %CMD = ( # sorted in order of importance/use:
qw(git-dir=s@ cwd! verbose|v+ color:s no-color drq:1 dequote-only:1),
@diff_opt, @lxs_opt, @net_opt, @c_opt ],
+'mail-diff' => [ '--stdin|LOCATION...', 'diff the contents of emails',
+ 'stdin|', # /|\z/ must be first for lone dash
+ qw(verbose|v+ color:s no-color raw-header),
+ @diff_opt, @net_opt, @c_opt ],
+
'add-external' => [ 'LOCATION',
'add/set priority of a publicinbox|extindex for extra matches',
qw(boost=i mirror=s inbox-version=i epoch=s verbose|v+),
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 22bedba6e6c3..83479221000b 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -57,6 +57,12 @@ sub check_input_format ($;$) {
1;
}
+sub input_mbox_cb { # base MboxReader callback
+ my ($eml, $self) = @_;
+ $eml->header_set($_) for (qw(Status X-Status));
+ $self->input_eml_cb($eml);
+}
+
# import a single file handle of $name
# Subclass must define ->input_eml_cb and ->input_mbox_cb
sub input_fh {
diff --git a/lib/PublicInbox/LeiMailDiff.pm b/lib/PublicInbox/LeiMailDiff.pm
new file mode 100644
index 000000000000..a29ae2259f71
--- /dev/null
+++ b/lib/PublicInbox/LeiMailDiff.pm
@@ -0,0 +1,111 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# The "lei mail-diff" sub-command, diffs input contents against
+# the first message of input
+package PublicInbox::LeiMailDiff;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use File::Temp 0.19 (); # 0.19 for ->newdir
+use PublicInbox::Spawn qw(spawn which);
+use PublicInbox::MsgIter qw(msg_part_text);
+use File::Path qw(remove_tree);
+use PublicInbox::ContentHash qw(content_digest);
+require PublicInbox::LeiRediff;
+use Data::Dumper ();
+
+sub write_part { # Eml->each_part callback
+ my ($ary, $self) = @_;
+ my ($part, $depth, $idx) = @$ary;
+ if ($idx ne '1' || $self->{lei}->{opt}->{'raw-header'}) {
+ open my $fh, '>', "$self->{curdir}/$idx.hdr" or die "open: $!";
+ print $fh ${$part->{hdr}} or die "print $!";
+ close $fh or die "close $!";
+ }
+ my $ct = $part->content_type || 'text/plain';
+ my ($s, $err) = msg_part_text($part, $ct);
+ my $sfx = defined($s) ? 'txt' : 'bin';
+ open my $fh, '>', "$self->{curdir}/$idx.$sfx" or die "open: $!";
+ print $fh ($s // $part->body) or die "print $!";
+ close $fh or die "close $!";
+}
+
+sub dump_eml ($$$) {
+ my ($self, $dir, $eml) = @_;
+ local $self->{curdir} = $dir;
+ mkdir $dir or die "mkdir($dir): $!";
+ $eml->each_part(\&write_part, $self);
+
+ open my $fh, '>', "$dir/content_digest" or die "open: $!";
+ my $dig = PublicInbox::ContentDigestDbg->new($fh);
+ local $Data::Dumper::Useqq = 1;
+ local $Data::Dumper::Terse = 1;
+ content_digest($eml, $dig);
+ print $fh "\n", $dig->hexdigest, "\n" or die "print $!";
+ close $fh or die "close: $!";
+}
+
+sub prep_a ($$) {
+ my ($self, $eml) = @_;
+ $self->{tmp} = File::Temp->newdir('lei-mail-diff-XXXX', TMPDIR => 1);
+ dump_eml($self, "$self->{tmp}/a", $eml);
+}
+
+sub diff_a ($$) {
+ my ($self, $eml) = @_;
+ ++$self->{nr};
+ my $dir = "$self->{tmp}/N$self->{nr}";
+ dump_eml($self, $dir, $eml);
+ my $cmd = [ qw(git diff --no-index) ];
+ my $lei = $self->{lei};
+ PublicInbox::LeiRediff::_lei_diff_prepare($lei, $cmd);
+ push @$cmd, qw(-- a), "N$self->{nr}";
+ my $rdr = { -C => "$self->{tmp}" };
+ @$rdr{1, 2} = @$lei{1, 2};
+ my $pid = spawn($cmd, $lei->{env}, $rdr);
+ waitpid($pid, 0);
+ $lei->child_error($?) if $?; # for git diff --exit-code
+ File::Path::remove_tree($self->{curdir});
+}
+
+sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
+ my ($self, $eml) = @_;
+ $self->{tmp} ? diff_a($self, $eml) : prep_a($self, $eml);
+}
+
+sub lei_mail_diff {
+ my ($lei, @argv) = @_;
+ $lei->{opt}->{'in-format'} //= 'eml';
+ my $self = bless {}, __PACKAGE__;
+ $self->prepare_inputs($lei, \@argv) or return;
+ my $isatty = -t $lei->{1};
+ $lei->{opt}->{color} //= $isatty;
+ $lei->start_pager if $isatty;
+ my $ops = {};
+ $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $lei->wait_wq_events($op_c, $ops);
+}
+
+no warnings 'once';
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
+package PublicInbox::ContentDigestDbg;
+use strict;
+use v5.10.1;
+use Data::Dumper;
+
+sub new { bless { dig => Digest::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
+
+sub add {
+ $_[0]->{dig}->add($_[1]);
+ print { $_[0]->{fh} } Dumper($_[1]) or die "print $!";
+}
+
+sub hexdigest { $_[0]->{dig}->hexdigest; }
+
+1;
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 1e95e55ac1cc..decb721b7a4c 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -56,6 +56,34 @@ sub solve_1 ($$$) {
$self->{blob}->{$oid_want}; # full OID
}
+sub _lei_diff_prepare ($$) {
+ my ($lei, $cmd) = @_;
+ my $opt = $lei->{opt};
+ push @$cmd, '--'.($opt->{color} && !$opt->{'no-color'} ? '' : 'no-').
+ 'color';
+ for my $o (@PublicInbox::LEI::diff_opt) {
+ my $c = '';
+ # remove single char short option
+ $o =~ s/\|([a-z0-9])\b//i and $c = $1;
+ if ($o =~ s/=[is]@\z//) {
+ my $v = $opt->{$o} or next;
+ push @$cmd, map { $c ? "-$c$_" : "--$o=$_" } @$v;
+ } elsif ($o =~ s/=[is]\z//) {
+ my $v = $opt->{$o} // next;
+ push @$cmd, $c ? "-$c$v" : "--$o=$v";
+ } elsif ($o =~ s/:[is]\z//) {
+ my $v = $opt->{$o} // next;
+ push @$cmd, $c ? "-$c$v" :
+ ($v eq '' ? "--$o" : "--$o=$v");
+ } elsif ($o =~ s/!\z//) {
+ my $v = $opt->{$o} // next;
+ push @$cmd, $v ? "--$o" : "--no-$o";
+ } elsif ($opt->{$o}) {
+ push @$cmd, $c ? "-$c" : "--$o";
+ }
+ }
+}
+
sub diff_ctxq ($$) {
my ($self, $ctxq) = @_;
return unless $ctxq;
@@ -103,35 +131,12 @@ EOM
waitpid($pid, 0);
die "fast-import failed: \$?=$?" if $?;
- my @cmd = qw(diff);
- my $opt = $lei->{opt};
- push @cmd, '--'.($opt->{color} && !$opt->{'no-color'} ? '' : 'no-').
- 'color';
- for my $o (@PublicInbox::LEI::diff_opt) {
- my $c = '';
- # remove single char short option
- $o =~ s/\|([a-z0-9])\b//i and $c = $1;
- if ($o =~ s/=[is]@\z//) {
- my $v = $opt->{$o} or next;
- push @cmd, map { $c ? "-$c$_" : "--$o=$_" } @$v;
- } elsif ($o =~ s/=[is]\z//) {
- my $v = $opt->{$o} // next;
- push @cmd, $c ? "-$c$v" : "--$o=$v";
- } elsif ($o =~ s/:[is]\z//) {
- my $v = $opt->{$o} // next;
- push @cmd, $c ? "-$c$v" :
- ($v eq '' ? "--$o" : "--$o=$v");
- } elsif ($o =~ s/!\z//) {
- my $v = $opt->{$o} // next;
- push @cmd, $v ? "--$o" : "--no-$o";
- } elsif ($opt->{$o}) {
- push @cmd, $c ? "-$c" : "--$o";
- }
- }
- $lei->qerr("# git @cmd");
- push @cmd, qw(A B);
- unshift @cmd, 'git', "--git-dir=$rw->{git_dir}";
- $pid = spawn(\@cmd, $lei->{env}, { 2 => $lei->{2}, 1 => $lei->{1} });
+ my $cmd = [ 'diff' ];
+ _lei_diff_prepare($lei, $cmd);
+ $lei->qerr("# git @$cmd");
+ push @$cmd, qw(A B);
+ unshift @$cmd, 'git', "--git-dir=$rw->{git_dir}";
+ $pid = spawn($cmd, $lei->{env}, { 2 => $lei->{2}, 1 => $lei->{1} });
waitpid($pid, 0);
$lei->child_error($?) if $?; # for git diff --exit-code
undef;
^ permalink raw reply related [relevance 38%]
* Re: [PATCH 9/9] doc: lei-daemon: new manpage
2021-10-02 8:08 71% ` Eric Wong
@ 2021-10-04 3:16 71% ` Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-10-04 3:16 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> Btw, did you have time to work on some of the other manpages?
> No worries if not, I might take a stab at them sometime in the
> next few days. [...]
Sorry, I haven't worked on manpages for the new commands. (Actually I
still need to catch up on what the new commands are). I suppose at this
point you might prefer to handle it yourself for a quicker turnaround,
but I think I could find time this week.
^ permalink raw reply [relevance 71%]
* [PATCH 7/8] lei/store: keep ".err-XXXX" in stderr tmpfile
@ 2021-10-10 14:25 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-10 14:25 UTC (permalink / raw)
To: meta
This is slighly more meaningful since the file is already
in ~/.local/share/lei/store, so "lei_store" was redundant
(and the "XXXX" are random characters replaced by File::Temp)
---
lib/PublicInbox/LeiStore.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 52a1456f..613d1d31 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -512,7 +512,7 @@ sub xchg_stderr {
return unless -e $dir;
my $old = delete $self->{-tmp_err};
my $pfx = POSIX::strftime('%Y%m%d%H%M%S', gmtime(time));
- my $err = File::Temp->new(TEMPLATE => "$pfx.$$.lei_storeXXXX",
+ my $err = File::Temp->new(TEMPLATE => "$pfx.$$.err-XXXX",
SUFFIX => '.err', DIR => $dir);
open STDERR, '>>', $err->filename or die "dup2: $!";
STDERR->autoflush(1); # shared with shard subprocesses
^ permalink raw reply related [relevance 71%]
* [PATCH] doc: lei-refresh-mail-sync: drop repeated word
@ 2021-10-11 4:53 71% Kyle Meyer
2021-10-11 5:09 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-10-11 4:53 UTC (permalink / raw)
To: meta
---
Spotted while (slowly :/) studying these commands and working on
adding missing lei manpages (add-watch, ls-watch, rm-watch,
ls-mail-source, forget-mail-sync, inspect, and mail-diff).
Documentation/lei-refresh-mail-sync.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/lei-refresh-mail-sync.pod b/Documentation/lei-refresh-mail-sync.pod
index aefd516f..92ca9044 100644
--- a/Documentation/lei-refresh-mail-sync.pod
+++ b/Documentation/lei-refresh-mail-sync.pod
@@ -10,7 +10,7 @@ lei refresh-mail-sync MFOLDER [MFOLDER...]
=head1 DESCRIPTION
-C<lei refresh-mail-sync> is intended to to keep old messages
+C<lei refresh-mail-sync> is intended to keep old messages
indexed with L<lei-index(1)> retrievable if Maildir flags change
a filename. It will prune invalid entries for messages which no
longer exist in a Maildir.
base-commit: dea4495677bbbdbce959551de34ef8f70fe2bc7c
--
2.33.0
^ permalink raw reply related [relevance 71%]
* Re: [PATCH] doc: lei-refresh-mail-sync: drop repeated word
2021-10-11 4:53 71% [PATCH] doc: lei-refresh-mail-sync: drop repeated word Kyle Meyer
@ 2021-10-11 5:09 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-11 5:09 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed as c9567f1e142931cf4c5f092ad1ec5904f7c5bdc1
^ permalink raw reply [relevance 71%]
* [PATCH 2/5] lei/store: use remove_doc to save some LoC
@ 2021-10-12 22:44 71% ` Eric Wong
2021-10-12 22:45 62% ` [PATCH 5/5] lei up --all: show output for warnings Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-12 22:44 UTC (permalink / raw)
To: meta
---
lib/PublicInbox/LeiStore.pm | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 613d1d31f581..bf41dcf53094 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -281,8 +281,7 @@ sub remove_docids ($;@) {
my ($self, @docids) = @_;
my $eidx = eidx_init($self);
for my $docid (@docids) {
- $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
- $eidx->{oidx}->delete_by_num($docid);
+ $eidx->remove_doc($docid);
$eidx->{oidx}->{dbh}->do(<<EOF, undef, $docid);
DELETE FROM xref3 WHERE docid = ?
EOF
^ permalink raw reply related [relevance 71%]
* [PATCH 5/5] lei up --all: show output for warnings
2021-10-12 22:44 71% ` [PATCH 2/5] lei/store: use remove_doc to save some LoC Eric Wong
@ 2021-10-12 22:45 62% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-12 22:45 UTC (permalink / raw)
To: meta
This helps users make sense of which saved searches some
warnings were coming from.
Since I often create and discard externals, some warnings
from saved searches were confusing to me without output context:
"`$FOO' is unknown"
"$FOO not indexed by Xapian"
---
lib/PublicInbox/LEI.pm | 12 ++++++++----
lib/PublicInbox/LeiUp.pm | 7 +++++++
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 51b0e95e1728..183cb545fe55 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -522,7 +522,7 @@ sub sigint_reap {
sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
$self->{failed}++;
- err($self, $buf) if defined $buf;
+ warn($buf, "\n") if defined $buf;
$self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
x_it($self, ($exit_code // 1) << 8);
undef;
@@ -542,7 +542,7 @@ sub puts ($;@) { out(shift, map { "$_\n" } @_) }
sub child_error { # passes non-fatal curl exit codes to user
my ($self, $child_error, $msg) = @_; # child_error is $?
$child_error ||= 1 << 8;
- $self->err($msg) if $msg;
+ warn($msg, "\n") if defined $msg;
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('child_error', $child_error);
} elsif ($self->{sock}) { # to lei(1) client
@@ -588,8 +588,12 @@ sub _lei_atfork_child {
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
undef $errors_log;
$quit = \&CORE::exit;
- $self->{-eml_noisy} or # only "lei import" sets this atm
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
+ if (!$self->{-eml_noisy}) { # only "lei import" sets this atm
+ my $cb = $SIG{__WARN__} // \&CORE::warn;
+ $SIG{__WARN__} = sub {
+ $cb->(@_) unless PublicInbox::Eml::warn_ignore(@_)
+ };
+ }
$current_lei = $persist ? undef : $self; # for SIG{__WARN__}
}
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 3e1ca21e29e7..3011300dd836 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -159,6 +159,13 @@ sub event_step { # runs via PublicInbox::DS::requeue
delete $l->{opt}->{all};
$l->qerr("# updating $self->{out}");
$l->{up_op_p} = $self->{op_p}; # ($l => $lei => script/lei)
+ my $cb = $SIG{__WARN__} // \&CORE::warn;
+ my $o = " (output: $self->{out})";
+ local $SIG{__WARN__} = sub {
+ my @m = @_;
+ push(@m, $o) if !@m || $m[-1] !~ s/\n\z/$o\n/;
+ $cb->(@m);
+ };
eval { $l->dispatch('up', $self->{out}) };
$lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
^ permalink raw reply related [relevance 62%]
* [PATCH 2/5] lei: use standard warn() in more places
@ 2021-10-13 7:00 28% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-13 7:00 UTC (permalink / raw)
To: meta
warn() is easier to augment with context information, and
frankly unavoidable in the presence of 3rd-party libraries
we don't control.
---
lib/PublicInbox/Fetch.pm | 2 +-
lib/PublicInbox/LEI.pm | 18 +++++++++++++-----
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiForgetExternal.pm | 5 ++---
lib/PublicInbox/LeiImport.pm | 7 +++----
lib/PublicInbox/LeiImportKw.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 4 ++--
lib/PublicInbox/LeiMailSync.pm | 4 ++--
lib/PublicInbox/LeiMirror.pm | 8 ++++----
lib/PublicInbox/LeiP2q.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiViewText.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 25 +++++++++++++++----------
15 files changed, 49 insertions(+), 38 deletions(-)
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 5ada1f49e4dc..e5756fb664e4 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -52,7 +52,7 @@ sub do_manifest ($$$) {
$m0 = eval {
PublicInbox::LeiMirror::decode_manifest($fh, $mf, $mf)
};
- $lei->err($@) if $@;
+ warn($@) if $@;
}
my ($bn) = ($fn =~ m!/([^/]+)\z!);
my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, qw(-R -o), $bn);
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 183cb545fe55..bd8a6bef632b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -471,6 +471,7 @@ sub _drop_wq {
# pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
sub x_it ($$) {
my ($self, $code) = @_;
+ local $current_lei = $self;
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
@@ -504,6 +505,7 @@ sub qfin { # show message on finalization (LeiFinmsg)
sub fail_handler ($;$$) {
my ($lei, $code, $io) = @_;
+ local $current_lei = $lei;
close($io) if $io; # needed to avoid warnings on SIGPIPE
_drop_wq($lei);
x_it($lei, $code // (1 << 8));
@@ -521,6 +523,7 @@ sub sigint_reap {
sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
+ local $current_lei = $self;
$self->{failed}++;
warn($buf, "\n") if defined $buf;
$self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
@@ -541,6 +544,7 @@ sub puts ($;@) { out(shift, map { "$_\n" } @_) }
sub child_error { # passes non-fatal curl exit codes to user
my ($self, $child_error, $msg) = @_; # child_error is $?
+ local $current_lei = $self;
$child_error ||= 1 << 8;
warn($msg, "\n") if defined $msg;
if ($self->{pkt_op_p}) { # to top lei-daemon
@@ -1019,7 +1023,7 @@ sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
$cmd = [ Text::ParseWords::shellwords($cmd) ];
send($sock, exec_buf($cmd, {}), MSG_EOR) if $sock;
} else {
- err($self, "W: unsupported --alert=$op"); # non-fatal
+ warn("W: unsupported --alert=$op\n"); # non-fatal
}
}
}
@@ -1068,7 +1072,7 @@ sub start_pager {
# display a message for user before spawning full-screen $VISUAL
sub pgr_err {
my ($self, @msg) = @_;
- return $self->err(@msg) unless $self->{sock} && -t $self->{2};
+ return warn(@msg) unless $self->{sock} && -t $self->{2};
start_pager($self, { LESS => 'RX' }); # no 'F' so we prompt
print { $self->{2} } @msg;
$self->{2}->autoflush(1);
@@ -1118,6 +1122,7 @@ sub accept_dispatch { # Listener {post_accept} callback
sub dclose {
my ($self) = @_;
+ local $current_lei = $self;
delete $self->{-progress};
_drop_wq($self) if $self->{failed};
$self->close if $self->{-event_init_done}; # PublicInbox::DS::close
@@ -1369,6 +1374,7 @@ sub DESTROY {
sub wq_done_wait { # dwaitpid callback
my ($arg, $pid) = @_;
my ($wq, $lei) = @$arg;
+ local $current_lei = $lei;
my $err_type = $lei->{-err_type};
$? and $lei->child_error($?,
$err_type ? "$err_type errors during $lei->{cmd}" : ());
@@ -1383,6 +1389,7 @@ sub fchdir {
sub wq_eof { # EOF callback for main daemon
my ($lei) = @_;
+ local $current_lei = $lei;
my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
$wq1->wq_wait_old($wq1->can('_wq_done_wait') // \&wq_done_wait, $lei);
}
@@ -1423,7 +1430,7 @@ sub refresh_watches {
$seen{$url} = undef;
my $state = $cfg->get_1("watch.$url.state");
if (!watch_state_ok($state)) {
- $lei->err("watch.$url.state=$state not supported");
+ warn("watch.$url.state=$state not supported\n");
next;
}
if ($url =~ /\Amaildir:(.+)/i) {
@@ -1492,6 +1499,7 @@ sub lms {
sub sto_done_request {
my ($lei, $sock) = @_;
+ local $current_lei = $lei;
eval {
if ($sock //= $lei->{sock}) { # issue, async wait
$lei->{sto}->wq_io_do('done', [ $sock ]);
@@ -1499,14 +1507,14 @@ sub sto_done_request {
my $wait = $lei->{sto}->wq_do('done');
}
};
- $lei->err($@) if $@;
+ warn($@) if $@;
}
sub cfg_dump ($$) {
my ($lei, $f) = @_;
my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
return $ret if !$@;
- $lei->err($@);
+ warn($@);
undef;
}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index b94f67a029ff..a3ddbbcec211 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -40,7 +40,7 @@ sub solver_user_cb { # called by solver when done
# don't try to support all the git-show(1) options for non-blob,
# this is just a convenience:
$type ne 'blob' and
- $lei->err("# $oid is a $type of $size bytes in:\n#\t$gd");
+ warn "# $oid is a $type of $size bytes in:\n#\t$gd\n";
my $cmd = [ 'git', "--git-dir=$gd", 'show', $oid ];
my $rdr = { 1 => $lei->{1}, 2 => $lei->{2} };
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 6550c242b1a2..9e98edc391a0 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -41,7 +41,7 @@ sub process_inputs { # via wq_do
delete $lei->{1};
delete $self->{wcb}; # commit
my $nr = delete($lei->{-nr_write}) // 0;
- $lei->err("# converted $nr messages") if $lei->{opt}->{verbose};
+ $lei->qerr("# converted $nr messages");
}
sub lei_convert { # the main "lei convert" method
diff --git a/lib/PublicInbox/LeiForgetExternal.pm b/lib/PublicInbox/LeiForgetExternal.pm
index 7a4bbcf80a3a..07f0ac803c88 100644
--- a/lib/PublicInbox/LeiForgetExternal.pm
+++ b/lib/PublicInbox/LeiForgetExternal.pm
@@ -20,10 +20,9 @@ sub lei_forget_external {
if ($? == 0) {
$lei->qerr("# $l forgotten ");
} elsif (($? >> 8) == 5) {
- $lei->err("# $l not found");
+ warn("# $l not found\n");
} else {
- $lei->err("# --unset $key error");
- return $lei->x_it($?);
+ $lei->child_error($?, "# --unset $key error");
}
}
}
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 69d63ab6b397..2f8fd6c64f69 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -38,7 +38,7 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
my $lse = $self->{lse} //= $self->{lei}->{sto}->search;
my $lms = $self->{-lms_ro} //= $self->{lei}->lms; # may be 0 or undef
my @oidbin = $lms ? $lms->name_oidbin($folder, $bn) : ();
- @oidbin > 1 and $self->{lei}->err("W: $folder/*/$$bn not unique:\n",
+ @oidbin > 1 and warn("W: $folder/*/$$bn not unique:\n",
map { "\t".unpack('H*', $_)."\n" } @oidbin);
my %seen;
my @docids = sort { $a <=> $b } grep { !$seen{$_}++ }
@@ -100,9 +100,8 @@ sub do_import_index ($$@) {
my $nproc = $self->detect_nproc;
$j = $nproc if $j > $nproc;
}
- if ($lei->{opt}->{'new-only'} && (!$net || !$net->{imap_order})) {
- $lei->err('# --new-only is only for IMAP');
- }
+ ($lei->{opt}->{'new-only'} && (!$net || !$net->{imap_order})) and
+ warn "# --new-only is only for IMAP\n";
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$lei->{-eml_noisy} = 1;
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index c35c5c266c4b..8359f3386668 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -36,7 +36,7 @@ sub ck_update_kw { # via wq_io_do
my ($self, $url, $uid, $kw) = @_;
my @oidbin = $self->{-lms_ro}->num_oidbin($url, $uid);
my $uid_url = "$url/;UID=$uid";
- @oidbin > 1 and $self->{lei}->err("W: $uid_url not unique:\n",
+ @oidbin > 1 and warn("W: $uid_url not unique:\n",
map { "\t".unpack('H*', $_)."\n" } @oidbin);
my %seen;
my @docids = sort { $a <=> $b } grep { !$seen{$_}++ }
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 83479221000b..6a90e7e1e756 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -343,8 +343,8 @@ $input is `eml', not --in-format=$in_fmt
--mail-sync specified but no inputs support it
# non-fatal if some inputs support support sync
- $lei->err("# --mail-sync will only be used for @{$sync->{ok}}");
- $lei->err("# --mail-sync is not supported for: @{$sync->{no}}");
+ warn("# --mail-sync will only be used for @{$sync->{ok}}\n");
+ warn("# --mail-sync is not supported for: @{$sync->{no}}\n");
}
if ($net) {
$net->{-can_die} = 1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index f7e37ad9ca80..f2f1e3ed2658 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -466,7 +466,7 @@ sub arg2folder {
# using `$res' instead of `$orig'
EOM
} else {
- $lei->err($res) if defined $res;
+ warn($res, "\n") if defined $res;
push @no, $orig;
}
} elsif (m!\A(?:nntps?|s?news)://!i) {
@@ -478,7 +478,7 @@ EOM
# using `$res' instead of `$orig'
EOM
} else {
- $lei->err($res) if defined $res;
+ warn($res, "\n") if defined $res;
push @no, $orig;
}
} else {
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 4be8f70ae401..ec41bec6f16b 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -20,7 +20,7 @@ sub _wq_done_wait { # dwaitpid callback (via wq_eof)
if ($?) {
$lei->child_error($?);
} elsif (!unlink($f)) {
- $lei->err("unlink($f): $!") unless $!{ENOENT};
+ warn("unlink($f): $!\n") unless $!{ENOENT};
} else {
if ($lei->{cmd} ne 'public-inbox-clone') {
$lei->lazy_cb('add-external', '_finish_'
@@ -120,7 +120,7 @@ sub _try_config {
-d $dst or die "mkpath($dst): $!\n";
}
my $err = _get_txt($self, qw(_/text/config/raw inbox.config.example));
- return $self->{lei}->err($err) if $err;
+ return warn($err, "\n") if $err;
my $f = "$self->{dst}/inbox.config.example";
chmod((stat($f))[2] & 0444, $f) or die "chmod(a-w, $f): $!";
my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2});
@@ -151,7 +151,7 @@ sub index_cloned_inbox {
my ($self, $iv) = @_;
my $lei = $self->{lei};
my $err = _get_txt($self, qw(description description));
- $lei->err($err) if $err; # non fatal
+ warn($err, "\n") if $err; # non fatal
eval { set_description($self) };
warn $@ if $@;
@@ -380,7 +380,7 @@ sub try_manifest {
my ($path_pfx, $v1_path, @v2_epochs) = deduce_epochs($m, $path);
if (@v2_epochs) {
# It may be possible to have v1 + v2 in parallel someday:
- $lei->err(<<EOM) if defined $v1_path;
+ warn(<<EOM) if defined $v1_path;
# `$v1_path' appears to be a v1 inbox while v2 epochs exist:
# @v2_epochs
# ignoring $v1_path (use --inbox-version=1 to force v1 instead)
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index c0c4563d6fbb..5c2ce0a13dfc 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -135,7 +135,7 @@ sub do_p2q { # via wq_do
if ($lei->{opt}->{debug}) {
my $json = ref(PublicInbox::Config->json)->new;
$json->utf8->canonical->pretty;
- $lei->err($json->encode($lei->{qterms}));
+ print { $lei->{2} } $json->encode($lei->{qterms});
}
my (@q, %seen);
for my $pfx (@want) {
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index decb721b7a4c..740cbcee334f 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -30,7 +30,7 @@ sub rediff_user_cb { # called by solver when done
# don't try to support all the git-show(1) options for non-blob,
# this is just a convenience:
- $type ne 'blob' and return $lei->err(<<EOF);
+ $type ne 'blob' and return warn(<<EOF);
# $oid is a $type of $size bytes in:
# $git->{git_dir} (wanted: $oid_want)
EOF
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 346aa6a40132..7782aa9dbfa1 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -75,7 +75,7 @@ sub smsg_eml {
if (my $bref = $self->{lei}->ale->git->cat_file($smsg->{blob})) {
return PublicInbox::Eml->new($bref);
}
- $self->{lei}->err("E: $self->{uri} $smsg->{blob} gone <$smsg->{mid}>");
+ warn("E: $self->{uri} $smsg->{blob} gone <$smsg->{mid}>\n");
undef;
}
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index c469d1ea5b1c..2dad3b780fa1 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -77,7 +77,7 @@ sub new {
my $cfg = PublicInbox::Config::config_fh_parse($r, "\0", "\n");
waitpid($pid, 0);
if ($?) {
- $lei->err("# git-config failed, no color (non-fatal)");
+ warn "# git-config failed, no color (non-fatal)\n";
return $self;
}
$self->{-colored} = \&my_colored;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index ae9f5881676d..fee1b859f9c6 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -16,6 +16,7 @@ use PublicInbox::Spawn qw(popen_rd spawn which);
use PublicInbox::MID qw(mids);
use PublicInbox::Smsg;
use PublicInbox::Eml;
+use PublicInbox::LEI;
use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
use PublicInbox::ContentHash qw(git_sha);
use POSIX qw(strftime);
@@ -392,11 +393,11 @@ sub query_remote_mboxrd {
$err = '';
if (-s $cerr) {
seek($cerr, 0, SEEK_SET) or
- $lei->err("seek($cmd stderr): $!");
+ warn "seek($cmd stderr): $!";
$err = do { local $/; <$cerr> } //
- "read($cmd stderr): $!";
+ warn "read($cmd stderr): $!";
truncate($cerr, 0) or
- $lei->err("truncate($cmd stderr): $!");
+ warn "truncate($cmd stderr): $!";
}
next if (($? >> 8) == 22 && $err =~ /\b404\b/);
$uri->query_form(q => $qstr);
@@ -416,6 +417,7 @@ sub xsearch_done_wait { # dwaitpid callback
sub query_done { # EOF callback for main daemon
my ($lei) = @_;
+ local $PublicInbox::LEI::current_lei = $lei;
my $l2m = delete $lei->{l2m};
$l2m->wq_wait_old(\&xsearch_done_wait, $lei) if $l2m;
if (my $lxs = delete $lei->{lxs}) {
@@ -462,6 +464,7 @@ Error closing $lei->{ovv}->{dst}: $!
sub do_post_augment {
my ($lei) = @_;
+ local $PublicInbox::LEI::current_lei = $lei;
my $l2m = $lei->{l2m} or return; # client disconnected
$lei->fchdir or return;
my $err;
@@ -497,9 +500,10 @@ sub concurrency {
$nl + $nr;
}
-sub start_query ($;$) { # always runs in main (lei-daemon) process
- my ($self, $l2m) = @_;
- if ($self->{opt_threads} || ($l2m && !$self->{opt_sort})) {
+sub start_query ($$) { # always runs in main (lei-daemon) process
+ my ($self, $lei) = @_;
+ local $PublicInbox::LEI::current_lei = $lei;
+ if ($self->{opt_threads} || ($lei->{l2m} && !$self->{opt_sort})) {
for my $ibxish (locals($self)) {
$self->wq_io_do('query_one_mset', [], $ibxish);
}
@@ -521,9 +525,10 @@ sub start_query ($;$) { # always runs in main (lei-daemon) process
}
sub incr_start_query { # called whenever an l2m shard starts do_post_auth
- my ($self, $l2m) = @_;
+ my ($self, $lei) = @_;
+ my $l2m = $lei->{l2m};
return if ++$self->{nr_start_query} != $l2m->{-wq_nr_workers};
- start_query($self, $l2m);
+ start_query($self, $lei);
}
sub ipc_atfork_child {
@@ -545,7 +550,7 @@ sub do_query {
'l2m_progress' => [ \&l2m_progress, $lei ],
'x_it' => [ $lei ],
'child_error' => [ $lei ],
- 'incr_start_query' => [ $self, $l2m ],
+ 'incr_start_query' => [ $self, $lei ],
};
$lei->{auth}->op_merge($ops, $l2m) if $l2m && $lei->{auth};
my $end = $lei->pkt_op_pair;
@@ -586,7 +591,7 @@ sub do_query {
if ($l2m) {
$l2m->net_merge_all_done($lei) unless $lei->{auth};
} else {
- start_query($self);
+ start_query($self, $lei);
}
$lei->event_step_init; # wait for shutdowns
$lei->wait_wq_events($op_c, $ops);
^ permalink raw reply related [relevance 28%]
* [PATCH 5/7] t/lei-mirror: avoid reading ~/.public-inbox/config in test
@ 2021-10-13 10:16 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-13 10:16 UTC (permalink / raw)
To: meta
Oops, we shouldn't attempt to read a users' actual HOME when
running -index, since mine has a bunch of invalid entries in
there.
---
t/lei-mirror.t | 1 +
1 file changed, 1 insertion(+)
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index b449e0b4..294eb654 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -167,6 +167,7 @@ SKIP: {
my $after = [ glob("$d/t1/*") ];
is_deeply($before, $after, 'no new files created');
+ local $ENV{HOME} = $tmpdir;
ok(run_script([qw(-index -Lbasic), "$d/t1"]), 'index v1');
ok(run_script([qw(-index -Lbasic), "$d/t2"]), 'index v2');
my $f = "$d/t1/public-inbox/msgmap.sqlite3";
^ permalink raw reply related [relevance 71%]
* [PATCH] lei inspect: account for non-extindex inboxes
@ 2021-10-14 3:12 56% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-14 3:12 UTC (permalink / raw)
To: meta
Inbox->xdb does not exist, but this code path was apparently
never tested :x I noticed this on basic v2 inbox, but it could
happen with any v1/v2 inbox. Move ->num2docid into Search
so it's less awkward to use.
---
lib/PublicInbox/LeiInspect.pm | 6 ++++--
lib/PublicInbox/LeiSearch.pm | 14 +++-----------
lib/PublicInbox/Search.pm | 8 ++++++++
t/lei-mirror.t | 6 ++++++
4 files changed, 21 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 590dfdab..5ba96056 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -143,8 +143,10 @@ sub inspect_num ($$) {
my $ent = { num => $num };
if (defined(my $dir = $lei->{opt}->{dir})) {
$ibx = dir2ibx($lei, $dir) or return;
- $ent->{xdb} = $ibx->xdb and # for inspect_docid
- $docid = PublicInbox::LeiSearch::num2docid($ibx, $num);
+ if (my $srch = $ibx->search) {
+ $ent->{xdb} = $srch->xdb and
+ $docid = $srch->num2docid($num);
+ }
} elsif ($lei->{lse}) {
$ibx = $lei->{lse};
$lei->{lse}->xdb; # set {nshard} for num2docid
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 4e048e9a..3e046b21 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -11,18 +11,10 @@ use PublicInbox::ContentHash qw(content_digest content_hash);
use PublicInbox::MID qw(mids mids_for_index);
use Carp qw(croak);
-# get combined docid from over.num:
-# (not generic Xapian, only works with our sharding scheme)
-sub num2docid ($$) {
- my ($self, $num) = @_;
- my $nshard = $self->{nshard};
- ($num - 1) * $nshard + $num % $nshard + 1;
-}
-
sub _msg_kw { # retry_reopen callback
my ($self, $num) = @_;
my $xdb = $self->xdb; # set {nshard} for num2docid;
- xap_terms('K', $xdb, num2docid($self, $num));
+ xap_terms('K', $xdb, $self->num2docid($num));
}
sub msg_keywords { # array or hashref
@@ -35,7 +27,7 @@ sub _oid_kw { # retry_reopen callback
my $xdb = $self->xdb; # set {nshard};
my %kw;
for my $num (@$nums) { # there should only be one...
- my $doc = $xdb->get_document(num2docid($self, $num));
+ my $doc = $xdb->get_document($self->num2docid($num));
my $x = xap_terms('K', $doc);
%kw = (%kw, %$x);
}
@@ -56,7 +48,7 @@ sub _xsmsg_vmd { # retry_reopen
$kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged});
my @num = $self->over->blob_exists($smsg->{blob});
for my $num (@num) { # there should only be one...
- $doc = $xdb->get_document(num2docid($self, $num));
+ $doc = $xdb->get_document($self->num2docid($num));
$x = xap_terms('K', $doc);
%kw = (%kw, %$x);
if ($want_label) { # JSON/JMAP only
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f0e7ed0c..d89bf545 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -570,4 +570,12 @@ sub xap_terms ($$;@) {
wantarray ? sort(keys(%ret)) : \%ret;
}
+# get combined docid from over.num:
+# (not generic Xapian, only works with our sharding scheme)
+sub num2docid ($$) {
+ my ($self, $num) = @_;
+ my $nshard = $self->{nshard};
+ ($num - 1) * $nshard + $num % $nshard + 1;
+}
+
1;
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 294eb654..646ff2b1 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -176,6 +176,12 @@ SKIP: {
$f = "$d/t2/msgmap.sqlite3";
$ca = PublicInbox::Msgmap->new_file($f)->created_at;
is($ca, $created{v2}, 'clone + index v1 synced ->created_at');
+ test_lei(sub {
+ lei_ok qw(inspect num:1 --dir), "$d/t1";
+ ok(ref(json_utf8->decode($lei_out)), 'inspect num: on v1');
+ lei_ok qw(inspect num:1 --dir), "$d/t2";
+ ok(ref(json_utf8->decode($lei_out)), 'inspect num: on v2');
+ });
}
ok($td->kill, 'killed -httpd');
^ permalink raw reply related [relevance 56%]
* [PATCH 2/3] lei add-external --mirror: respect client umask
@ 2021-10-14 4:32 65% ` Eric Wong
2021-10-14 4:32 61% ` [PATCH 3/3] lei: give workers their own process group Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-14 4:32 UTC (permalink / raw)
To: meta
While lei is intended for non-public mail and runs umask(077)
by default, externals are one area which can safely defer to
the user's umask.
Instead of sending it unconditionally with every command, only
have lei-daemon request it when necessary.
---
lib/PublicInbox/LEI.pm | 11 +++++++++++
lib/PublicInbox/LeiMirror.pm | 2 ++
script/lei | 2 ++
3 files changed, 15 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bd8a6bef632b..635cd0c5508a 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1518,4 +1518,15 @@ sub cfg_dump ($$) {
undef;
}
+sub request_umask {
+ my ($lei) = @_;
+ my $s = $lei->{sock} // return;
+ send($s, 'umask', MSG_EOR) // die "send: $!";
+ vec(my $rvec = '', fileno($s), 1) = 1;
+ select($rvec, undef, undef, 2) or die 'timeout waiting for umask';
+ recv($s, my $v, 5, 0) // die "recv: $!";
+ (my $u, $lei->{client_umask}) = unpack('AV', $v);
+ $u eq 'u' or warn "E: recv $v has no umask";
+}
+
1;
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 1369c00c57fd..fb73d8631670 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -424,6 +424,7 @@ sub start_clone_url {
sub do_mirror { # via wq_io_do
my ($self) = @_;
my $lei = $self->{lei};
+ umask($lei->{client_umask}) if defined $lei->{client_umask};
eval {
my $iv = $lei->{opt}->{'inbox-version'};
if (defined $iv) {
@@ -448,6 +449,7 @@ sub start {
require PublicInbox::Inbox;
require PublicInbox::Admin;
require PublicInbox::InboxWritable;
+ $lei->request_umask;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_mirror', []);
diff --git a/script/lei b/script/lei
index bc43779821e7..8f6e8aacb86b 100755
--- a/script/lei
+++ b/script/lei
@@ -122,6 +122,8 @@ while (1) {
$exec_cmd->(\@fds, split(/\0/, $1));
} elsif ($buf eq '-WINCH') {
kill($buf, @parent); # for MUA
+ } elsif ($buf eq 'umask') {
+ send($sock, 'u'.pack('V', umask), MSG_EOR) or die "send: $!"
} elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
$x_it_code ||= $1 + 0;
last;
^ permalink raw reply related [relevance 65%]
* [PATCH 3/3] lei: give workers their own process group
2021-10-14 4:32 65% ` [PATCH 2/3] lei add-external --mirror: respect client umask Eric Wong
@ 2021-10-14 4:32 61% ` Eric Wong
2021-10-14 13:16 70% ` [PATCH 0/7] lei: more process handling fixes Eric Wong
1 sibling, 1 reply; 200+ results
From: Eric Wong @ 2021-10-14 4:32 UTC (permalink / raw)
To: meta
This lets users Ctrl-Z from their terminal to pause an entire
git-clone process hierarchy.
---
lib/PublicInbox/LEI.pm | 10 ++++++----
lib/PublicInbox/LeiMirror.pm | 3 +--
lib/PublicInbox/LeiXSearch.pm | 2 +-
3 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 635cd0c5508a..145af7e2cb59 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -459,9 +459,9 @@ my @WQ_KEYS = qw(lxs l2m ikw pmd wq1 lne); # internal workers
sub _drop_wq {
my ($self) = @_;
for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) {
- if ($wq->wq_kill) {
+ if ($wq->wq_kill('-TERM')) {
$wq->wq_close(0, undef, $self);
- } elsif ($wq->wq_kill_old) {
+ } elsif ($wq->wq_kill_old('-TERM')) {
$wq->wq_wait_old(undef, $self);
}
$wq->DESTROY;
@@ -575,6 +575,7 @@ sub _lei_atfork_child {
} else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
STDERR->autoflush(1);
+ POSIX::setpgid(0, $$) // die "setpgid(0, $$): $!";
}
close($_) for (grep(defined, delete @$self{qw(3 old_1 au_done)}));
if (my $op_c = delete $self->{pkt_op_c}) {
@@ -1147,9 +1148,10 @@ sub event_step {
if ($buf eq '') {
_drop_wq($self); # EOF, client disconnected
dclose($self);
- } elsif ($buf =~ /\A(STOP|CONT)\z/) {
+ } elsif ($buf =~ /\A(?:STOP|CONT)\z/) {
+ my $sig = "-$buf";
for my $wq (grep(defined, @$self{@WQ_KEYS})) {
- $wq->wq_kill($buf) or $wq->wq_kill_old($buf);
+ $wq->wq_kill($sig) or $wq->wq_kill_old($sig);
}
} else {
die "unrecognized client signal: $buf";
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index fb73d8631670..f1bc82e27205 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -9,7 +9,7 @@ use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
use IO::Compress::Gzip qw(gzip $GzipError);
-use PublicInbox::Spawn qw(popen_rd spawn run_die);
+use PublicInbox::Spawn qw(popen_rd spawn);
use File::Temp ();
use Fcntl qw(SEEK_SET O_CREAT O_EXCL O_WRONLY);
use Carp qw(croak);
@@ -192,7 +192,6 @@ sub index_cloned_inbox {
sub run_reap {
my ($lei, $cmd, $opt) = @_;
$lei->qerr("# @$cmd");
- $opt->{pgid} = 0 if $lei->{sock};
my $pid = spawn($cmd, undef, $opt);
my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
waitpid($pid, 0) == $pid or die "waitpid @$cmd: $!";
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index ee9216feeb23..668d0b6e5df3 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -469,7 +469,7 @@ sub do_post_augment {
$err = $@;
if ($err) {
if (my $lxs = delete $lei->{lxs}) {
- $lxs->wq_kill;
+ $lxs->wq_kill('-TERM');
$lxs->wq_close(0, undef, $lei);
}
$lei->fail("$err");
^ permalink raw reply related [relevance 61%]
* Re: [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors
@ 2021-10-14 5:31 71% ` Eric Wong
2021-10-15 9:52 54% ` [PATCH] lei q: avoid kw lookup failure on remote mboxrd Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-14 5:31 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> Eric Wong <e@80x24.org> wrote:
> > 2/3 is probably a fix for a long-standing problem, 3/3 was
> > noticed while working on it. If 2/3 doesn't fix it, then maybe
> > 1/3 will help us narrow it down.
>
> s/is probably/was almost certainly/ \o/
No. :<
It still happens when there's overlap between various search
sources, but I think I know how to fix it...
^ permalink raw reply [relevance 71%]
* [PATCH] lei: -d (--dir) and -O (only) shortcuts
@ 2021-10-14 9:54 60% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-14 9:54 UTC (permalink / raw)
To: meta
`-d' seems like a non-brainer for --dir with inspect.
I find myself using `--only' a bit, too, and `-O' seems like
a reasonable shortcut for it.
---
Documentation/lei-q.pod | 2 ++
lib/PublicInbox/LEI.pm | 9 +++++----
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index e1e3666d797a..574c12eb9d3a 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -164,6 +164,8 @@ multiple times.
=item --only=LOCATION
+=item -O LOCATION
+
Use only the specified external for search. This option may be given
multiple times, in which case the search uses only the specified set.
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 145af7e2cb59..9620e2642213 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -148,7 +148,7 @@ sub index_opt {
my @c_opt = qw(c=s@ C=s@ quiet|q);
my @net_opt = (qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt());
-my @lxs_opt = qw(remote! local! external! include|I=s@ exclude=s@ only=s@
+my @lxs_opt = qw(remote! local! external! include|I=s@ exclude=s@ only|O=s@
import-remote!);
# we don't support -C as an alias for --find-copies since it's already
@@ -284,7 +284,7 @@ our %CMD = ( # sorted in order of importance/use:
}, qw(config-file|system|global|file|f=s), # for conflict detection
qw(edit|e c=s@ C=s@), pass_through('git config') ],
'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
- qw(stdin| pretty ascii dir=s), @c_opt ],
+ qw(stdin| pretty ascii dir|d=s), @c_opt ],
'init' => [ '[DIRNAME]', sub {
"initialize storage, default: ".store_path($_[0]);
@@ -337,7 +337,8 @@ my %OPTDESC = (
'path-a|a=s' => 'pre-image pathname associated with OID',
'path-b|b=s' => 'post-image pathname associated with OID',
'git-dir=s@' => 'additional git repository to scan',
-'dir=s inspect' => 'specify a inboxdir, extindex topdir or Xapian shard',
+'dir|d=s inspect' =>
+ 'specify a inboxdir, extindex topdir or Xapian shard',
'proxy=s' => [ 'PROTO://HOST[:PORT]', # shared with curl(1)
"proxy for (e.g. `socks5h://0:9050')" ],
'torsocks=s' => ['VAL|auto|no|yes',
@@ -395,7 +396,7 @@ my %OPTDESC = (
'exclude specified external(s) from search' ],
'include|I=s@ q' => [ 'LOCATION',
'include specified external(s) in search' ],
-'only=s@ q' => [ 'LOCATION',
+'only|O=s@ q' => [ 'LOCATION',
'only use specified external(s) for search' ],
'jobs=s q' => [ '[SEARCH_JOBS][,WRITER_JOBS]',
'control number of search and writer jobs' ],
^ permalink raw reply related [relevance 60%]
* [PATCH 1/7] lei: use send() perlop for signals
2021-10-14 13:16 70% ` [PATCH 0/7] lei: more process handling fixes Eric Wong
@ 2021-10-14 13:16 71% ` Eric Wong
2021-10-14 13:16 34% ` [PATCH 5/7] lei: TSTP affects all curl and related subprocesses Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-14 13:16 UTC (permalink / raw)
To: meta
This may save us a small bit of startup time since there's
fewer args and opcodes should be smaller.
---
script/lei | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/script/lei b/script/lei
index 8f6e8aacb86b..5cad19d77603 100755
--- a/script/lei
+++ b/script/lei
@@ -107,8 +107,8 @@ my $buf = join("\0", scalar(@ARGV), @ARGV);
while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
$buf .= "\0\0";
$send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR) or die "sendmsg: $!";
-$SIG{TSTP} = sub { $send_cmd->($sock, [], 'STOP', MSG_EOR); kill 'STOP', $$ };
-$SIG{CONT} = sub { $send_cmd->($sock, [], 'CONT', MSG_EOR) };
+$SIG{TSTP} = sub { send($sock, 'STOP', MSG_EOR); kill 'STOP', $$ };
+$SIG{CONT} = sub { send($sock, 'CONT', MSG_EOR) };
my $x_it_code = 0;
while (1) {
^ permalink raw reply related [relevance 71%]
* [PATCH 0/7] lei: more process handling fixes
2021-10-14 4:32 61% ` [PATCH 3/3] lei: give workers their own process group Eric Wong
@ 2021-10-14 13:16 70% ` Eric Wong
2021-10-14 13:16 71% ` [PATCH 1/7] lei: use send() perlop for signals Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-10-14 13:16 UTC (permalink / raw)
To: meta
"lei up --all" SIGTSTP/CONT/INT handling was totally
broken and now fixed. And we put cat-file processes
into their own pgrp, so it avoids scary errors when
hitting Ctrl-C on -extindex, too.
Automated testing of interactive stuff is tricky, so
it's not being done, currently :<
Eric Wong (7):
lei: use send() perlop for signals
git: async_err shows retried requests properly
git: ->fail invokes current callback
git: cat-file --batch are their own pgrp
lei: TSTP affects all curl and related subprocesses
lei up: actually rely on DESTROY for --all
lei up --all: send signals to workers, receive errors
MANIFEST | 1 +
lib/PublicInbox/AutoReap.pm | 34 +++++++++++++++++++++++++++++++++
lib/PublicInbox/Git.pm | 36 +++++++++++++++++++----------------
lib/PublicInbox/LEI.pm | 14 +++++++-------
lib/PublicInbox/LeiInput.pm | 7 ++++---
lib/PublicInbox/LeiMirror.pm | 8 +++-----
lib/PublicInbox/LeiRemote.pm | 13 +++++--------
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 22 ++++++++++++++++++---
lib/PublicInbox/LeiXSearch.pm | 20 +++++++++----------
lib/PublicInbox/TestCommon.pm | 36 +++--------------------------------
script/lei | 4 ++--
12 files changed, 108 insertions(+), 89 deletions(-)
create mode 100644 lib/PublicInbox/AutoReap.pm
^ permalink raw reply [relevance 70%]
* [PATCH 6/7] lei up: actually rely on DESTROY for --alllll
2021-10-14 13:16 70% ` [PATCH 0/7] lei: more process handling fixes Eric Wong
2021-10-14 13:16 71% ` [PATCH 1/7] lei: use send() perlop for signals Eric Wong
2021-10-14 13:16 34% ` [PATCH 5/7] lei: TSTP affects all curl and related subprocesses Eric Wong
@ 2021-10-14 13:16 67% ` Eric Wong
2021-10-14 13:16 62% ` [PATCH 7/7] lei up --all: send signals to workers, receive errors Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-14 13:16 UTC (permalink / raw)
To: meta
We need to use DESTROY here to ensure we wait for workers, too;
not just the initial dispatch.
Fixes: cafbd77b3c82167d ("lei up: avoid excessively parallel --all")
---
lib/PublicInbox/LeiUp.pm | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 3011300dd836..719736e8597e 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -63,6 +63,7 @@ sub redispatch_all ($$) {
$op_c->{ops} = { '' => [ $lei->can('dclose'), $lei ] };
my @first_batch = splice(@$upq, 0, $j); # initial parallelism
$lei->{-upq} = $upq;
+ $lei->{daemon_pid} = $$;
$lei->event_step_init; # wait for client disconnects
for my $out (@first_batch) {
PublicInbox::DS::requeue(
@@ -158,18 +159,22 @@ sub event_step { # runs via PublicInbox::DS::requeue
$l->{opt} = { %{$l->{opt}} }; # deep copy
delete $l->{opt}->{all};
$l->qerr("# updating $self->{out}");
- $l->{up_op_p} = $self->{op_p}; # ($l => $lei => script/lei)
+ my $o = " (output: $self->{out})"; # add to all warnings
my $cb = $SIG{__WARN__} // \&CORE::warn;
- my $o = " (output: $self->{out})";
local $SIG{__WARN__} = sub {
my @m = @_;
push(@m, $o) if !@m || $m[-1] !~ s/\n\z/$o\n/;
$cb->(@m);
};
+ $l->{-up1} = $self;
eval { $l->dispatch('up', $self->{out}) };
$lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
+}
- # onto the next:
+sub DESTROY {
+ my ($self) = @_;
+ my $lei = $self->{lei}; # the original, from lei_up
+ return if $lei->{daemon_pid} != $$;
my $out = shift(@{$lei->{-upq}}) or return;
PublicInbox::DS::requeue(nxt($lei, $out, $self->{op_p}));
}
^ permalink raw reply related [relevance 67%]
* [PATCH 7/7] lei up --all: send signals to workers, receive errors
2021-10-14 13:16 70% ` [PATCH 0/7] lei: more process handling fixes Eric Wong
` (2 preceding siblings ...)
2021-10-14 13:16 67% ` [PATCH 6/7] lei up: actually rely on DESTROY for --alllll Eric Wong
@ 2021-10-14 13:16 62% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-14 13:16 UTC (permalink / raw)
To: meta
The redispatch mechanism wasn't routing signals and messages
between redispatched workers and script/lei properly. We now
rely on PktOp to do bidirectional message forwarding and
carefully avoiding circular references by using PktOp.
---
lib/PublicInbox/LEI.pm | 7 ++++++-
lib/PublicInbox/LeiUp.pm | 13 ++++++++++++-
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d0905562f616..b6338377328f 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -573,6 +573,7 @@ sub _lei_atfork_child {
POSIX::setpgid(0, $$) // die "setpgid(0, $$): $!";
}
close($_) for (grep(defined, delete @$self{qw(3 old_1 au_done)}));
+ delete $self->{-socks};
if (my $op_c = delete $self->{pkt_op_c}) {
close(delete $op_c->{sock});
}
@@ -1144,7 +1145,9 @@ sub event_step {
if ($buf eq '') {
_drop_wq($self); # EOF, client disconnected
dclose($self);
- } elsif ($buf =~ /\A(?:STOP|CONT)\z/) {
+ $buf = 'TERM';
+ }
+ if ($buf =~ /\A(?:STOP|CONT|TERM)\z/) {
my $sig = "-$buf";
for my $wq (grep(defined, @$self{@WQ_KEYS})) {
$wq->wq_kill($sig) or $wq->wq_kill_old($sig);
@@ -1152,6 +1155,8 @@ sub event_step {
} else {
die "unrecognized client signal: $buf";
}
+ my $s = $self->{-socks} // []; # lei up --all
+ @$s = grep { send($_, $buf, MSG_EOR) } @$s;
};
if (my $err = $@) {
eval { $self->fail($err) };
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 719736e8597e..df65cb9b8474 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -166,7 +166,15 @@ sub event_step { # runs via PublicInbox::DS::requeue
push(@m, $o) if !@m || $m[-1] !~ s/\n\z/$o\n/;
$cb->(@m);
};
- $l->{-up1} = $self;
+ $l->{-up1} = $self; # for LeiUp1->DESTROY
+ delete @$l{qw(-socks -event_init_done)};
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ $self->{unref_on_destroy} = $op_c->{sock}; # to cleanup $lei->{-socks}
+ $lei->pkt_ops($op_c->{ops} //= {}); # errors from $l -> script/lei
+ push @{$lei->{-socks}}, $op_c->{sock}; # script/lei signals to $l
+ $l->{sock} = $op_p->{op_p}; # receive signals from op_c->{sock}
+ $op_c = $op_p = undef;
+
eval { $l->dispatch('up', $self->{out}) };
$lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
}
@@ -175,6 +183,9 @@ sub DESTROY {
my ($self) = @_;
my $lei = $self->{lei}; # the original, from lei_up
return if $lei->{daemon_pid} != $$;
+ my $sock = delete $self->{unref_on_destroy};
+ my $s = $lei->{-socks} // [];
+ @$s = grep { $_ != $sock } @$s;
my $out = shift(@{$lei->{-upq}}) or return;
PublicInbox::DS::requeue(nxt($lei, $out, $self->{op_p}));
}
^ permalink raw reply related [relevance 62%]
* [PATCH 5/7] lei: TSTP affects all curl and related subprocesses
2021-10-14 13:16 70% ` [PATCH 0/7] lei: more process handling fixes Eric Wong
2021-10-14 13:16 71% ` [PATCH 1/7] lei: use send() perlop for signals Eric Wong
@ 2021-10-14 13:16 34% ` Eric Wong
2021-10-14 13:16 67% ` [PATCH 6/7] lei up: actually rely on DESTROY for --alllll Eric Wong
2021-10-14 13:16 62% ` [PATCH 7/7] lei up --all: send signals to workers, receive errors Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-14 13:16 UTC (permalink / raw)
To: meta
By relying more on pgroups for remaining remaining processes,
this lets us pause all curl+tail subprocesses with a single
kill(2) to avoid cluttering stderr.
We won't bother pausing the pigz/gzip/bzip2/xz compressor
process not cat-file processes, though, since those don't write
to the terminal and they idle soon after the workers react to
SIGSTOP.
AutoReap is hoisted out from TestCommon.pm. CLONE_SKIP
is gone since we won't be using Perl threads any time
soon (they're discouraged by the maintainers of Perl).
---
MANIFEST | 1 +
lib/PublicInbox/AutoReap.pm | 34 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LEI.pm | 7 +------
lib/PublicInbox/LeiInput.pm | 7 ++++---
lib/PublicInbox/LeiMirror.pm | 8 +++-----
lib/PublicInbox/LeiRemote.pm | 13 +++++--------
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 20 +++++++++----------
lib/PublicInbox/TestCommon.pm | 36 +++--------------------------------
9 files changed, 61 insertions(+), 67 deletions(-)
create mode 100644 lib/PublicInbox/AutoReap.pm
diff --git a/MANIFEST b/MANIFEST
index 122ceda0a761..b89513d5afb5 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -147,6 +147,7 @@ lib/PublicInbox/AddressPP.pm
lib/PublicInbox/Admin.pm
lib/PublicInbox/AdminEdit.pm
lib/PublicInbox/AltId.pm
+lib/PublicInbox/AutoReap.pm
lib/PublicInbox/Cgit.pm
lib/PublicInbox/CmdIPC4.pm
lib/PublicInbox/CompressNoop.pm
diff --git a/lib/PublicInbox/AutoReap.pm b/lib/PublicInbox/AutoReap.pm
new file mode 100644
index 000000000000..23ecce772186
--- /dev/null
+++ b/lib/PublicInbox/AutoReap.pm
@@ -0,0 +1,34 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# automatically kill + reap children when this goes out-of-scope
+package PublicInbox::AutoReap;
+use v5.10.1;
+use strict;
+
+sub new {
+ my (undef, $pid, $cb) = @_;
+ bless { pid => $pid, cb => $cb, owner => $$ }, __PACKAGE__
+}
+
+sub kill {
+ my ($self, $sig) = @_;
+ CORE::kill($sig // 'TERM', $self->{pid});
+}
+
+sub join {
+ my ($self, $sig) = @_;
+ my $pid = delete $self->{pid} or return;
+ $self->{cb}->() if defined $self->{cb};
+ CORE::kill($sig, $pid) if defined $sig;
+ my $ret = waitpid($pid, 0) // die "waitpid($pid): $!";
+ $ret == $pid or die "BUG: waitpid($pid) != $ret";
+}
+
+sub DESTROY {
+ my ($self) = @_;
+ return if $self->{owner} != $$;
+ $self->join('TERM');
+}
+
+1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9620e2642213..d0905562f616 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -516,12 +516,6 @@ sub sigpipe_handler { # handles SIGPIPE from @WQ_KEYS workers
fail_handler($_[0], 13, delete $_[0]->{1});
}
-# PublicInbox::OnDestroy callback for SIGINT to take out the entire pgid
-sub sigint_reap {
- my ($pgid) = @_;
- dwaitpid($pgid) if kill('-INT', $pgid);
-}
-
sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
local $current_lei = $self;
@@ -600,6 +594,7 @@ sub _lei_atfork_child {
$cb->(@_) unless PublicInbox::Eml::warn_ignore(@_)
};
}
+ $SIG{TERM} = sub { exit(128 + 15) };
$current_lei = $persist ? undef : $self; # for SIG{__WARN__}
}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 6a90e7e1e756..dd40d83840c5 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -8,6 +8,7 @@ use v5.10.1;
use PublicInbox::DS;
use PublicInbox::Spawn qw(which popen_rd);
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::AutoReap;
# JMAP RFC 8621 4.1.1
# https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -102,13 +103,13 @@ sub handle_http_input ($$@) {
push @$curl, '-s', @$curl_opt;
my $cmd = $curl->for_uri($lei, $uri);
$lei->qerr("# $cmd");
- my $rdr = { 2 => $lei->{2}, pgid => 0 };
- my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+ my ($fh, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
+ my $ar = PublicInbox::AutoReap->new($pid);
grep(/\A--compressed\z/, @$curl) or
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
eval { $self->input_fh('mboxrd', $fh, $url, @args) };
my $err = $@;
- waitpid($pid, 0);
+ $ar->join;
$? || $err and
$lei->child_error($?, "@$cmd failed".$err ? " $err" : '');
}
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index f1bc82e27205..a75c99c4987f 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -7,6 +7,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
+use PublicInbox::AutoReap;
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
use IO::Compress::Gzip qw(gzip $GzipError);
use PublicInbox::Spawn qw(popen_rd spawn);
@@ -192,10 +193,8 @@ sub index_cloned_inbox {
sub run_reap {
my ($lei, $cmd, $opt) = @_;
$lei->qerr("# @$cmd");
- my $pid = spawn($cmd, undef, $opt);
- my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
- waitpid($pid, 0) == $pid or die "waitpid @$cmd: $!";
- @$reap = (); # cancel reap
+ my $ar = PublicInbox::AutoReap->new(spawn($cmd, undef, $opt));
+ $ar->join;
my $ret = $?;
$? = 0; # don't let it influence normal exit
$ret;
@@ -459,7 +458,6 @@ sub start {
sub ipc_atfork_child {
my ($self) = @_;
$self->{lei}->_lei_atfork_child;
- $SIG{TERM} = sub { exit(128 + 15) }; # trigger OnDestroy $reap
$self->SUPER::ipc_atfork_child;
}
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 7782aa9dbfa1..54750062fd5f 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -9,10 +9,10 @@ package PublicInbox::LeiRemote;
use v5.10.1;
use strict;
use IO::Uncompress::Gunzip;
-use PublicInbox::OnDestroy;
use PublicInbox::MboxReader;
use PublicInbox::Spawn qw(popen_rd);
use PublicInbox::LeiCurl;
+use PublicInbox::AutoReap;
use PublicInbox::ContentHash qw(git_sha);
sub new {
@@ -47,17 +47,14 @@ sub mset {
$uri->query_form(q => $qstr, x => 'm', r => 1); # r=1: relevance
my $cmd = $curl->for_uri($self->{lei}, $uri);
$self->{lei}->qerr("# $cmd");
- my $rdr = { 2 => $lei->{2}, pgid => 0 };
- my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
- my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
+ my ($fh, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
+ my $ar = PublicInbox::AutoReap->new($pid);
$self->{smsg} = [];
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
PublicInbox::MboxReader->mboxrd($fh, \&_each_mboxrd_eml, $self);
- my $err = waitpid($pid, 0) == $pid ? undef
- : "BUG: waitpid($cmd): $!";
- @$reap = (); # cancel OnDestroy
my $wait = $self->{lei}->{sto}->wq_do('done');
- die $err if $err;
+ $ar->join;
+ $lei->child_error($?) if $?;
$self; # we are the mset (and $ibx, and $self)
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 5a220ba39735..9c748deaed16 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -157,7 +157,7 @@ sub _post_augment_mbox { # open a compressor process from top-level process
my $zsfx = $self->{zsfx} or return;
my $cmd = PublicInbox::MboxReader::zsfx2cmd($zsfx, undef, $lei);
my ($r, $w) = @{delete $lei->{zpipe}};
- my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
+ my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2}, pgid => 0 };
my $pid = spawn($cmd, undef, $rdr);
my $pp = gensym;
my $dup = bless { "pid.$pid" => $cmd }, ref($lei);
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 668d0b6e5df3..fba168613d96 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -15,6 +15,7 @@ use PublicInbox::Search qw(xap_terms);
use PublicInbox::Spawn qw(popen_rd spawn which);
use PublicInbox::MID qw(mids);
use PublicInbox::Smsg;
+use PublicInbox::AutoReap;
use PublicInbox::Eml;
use PublicInbox::LEI;
use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
@@ -346,18 +347,17 @@ sub query_remote_mboxrd {
my @qform = (x => 'm');
push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
- my ($reap_tail, $reap_curl);
+ my $reap_tail;
my $cerr = File::Temp->new(TEMPLATE => 'curl.err-XXXX', TMPDIR => 1);
fcntl($cerr, F_SETFL, O_APPEND|O_RDWR) or warn "set O_APPEND: $!";
- my $rdr = { 2 => $cerr, pgid => 0 };
- my $sigint_reap = $lei->can('sigint_reap');
+ my $rdr = { 2 => $cerr };
if ($verbose) {
# spawn a process to force line-buffering, otherwise curl
# will write 1 character at-a-time and parallel outputs
# mmmaaayyy llloookkk llliiikkkeee ttthhhiiisss
- my $o = { 1 => $lei->{2}, 2 => $lei->{2}, pgid => 0 };
+ my $o = { 1 => $lei->{2}, 2 => $lei->{2} };
my $pid = spawn(['tail', '-f', $cerr->filename], undef, $o);
- $reap_tail = PublicInbox::OnDestroy->new($sigint_reap, $pid);
+ $reap_tail = PublicInbox::AutoReap->new($pid);
}
my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
push @$curl, '-s', '-d', '';
@@ -372,16 +372,13 @@ sub query_remote_mboxrd {
my $cmd = $curl->for_uri($lei, $uri);
$lei->qerr("# $cmd");
my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
- $reap_curl = PublicInbox::OnDestroy->new($sigint_reap, $pid);
+ my $reap_curl = PublicInbox::AutoReap->new($pid);
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml, $self,
$lei, $each_smsg);
- my $err = waitpid($pid, 0) == $pid ? undef
- : "BUG: waitpid($cmd): $!";
- @$reap_curl = (); # cancel OnDestroy
- die $err if $err;
my $nr = $lei->{-nr_remote_eml};
my $wait = $lei->{sto}->wq_do('done') if $nr && $lei->{sto};
+ $reap_curl->join;
if ($? == 0) {
# don't update if no results, maybe MTA is down
$key && $nr and
@@ -389,7 +386,7 @@ sub query_remote_mboxrd {
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
}
- $err = '';
+ my $err;
if (-s $cerr) {
seek($cerr, 0, SEEK_SET) //
warn "seek($cmd stderr): $!";
@@ -397,6 +394,7 @@ sub query_remote_mboxrd {
warn "read($cmd stderr): $!";
truncate($cerr, 0) // warn "truncate($cmd stderr): $!";
}
+ $err //= '';
next if (($? >> 8) == 22 && $err =~ /\b404\b/);
$uri->query_form(q => $qstr);
$lei->child_error($?, "E: <$uri> $err");
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 57f1db952e49..835779996d56 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -6,6 +6,7 @@ package PublicInbox::TestCommon;
use strict;
use parent qw(Exporter);
use v5.10.1;
+use PublicInbox::AutoReap;
use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD :seek);
use POSIX qw(dup2);
use IO::Socket::INET;
@@ -429,7 +430,7 @@ sub tail_f (@) {
require PublicInbox::Spawn;
my $pid = PublicInbox::Spawn::spawn($cmd, undef, { 1 => 2 });
wait_for_tail($pid, scalar @_);
- PublicInboxTestProcess->new($pid, \&wait_for_tail);
+ PublicInbox::AutoReap->new($pid, \&wait_for_tail);
}
sub start_script {
@@ -492,7 +493,7 @@ sub start_script {
die "FAIL: ",join(' ', $key, @argv), ": $!\n";
}
}
- my $td = PublicInboxTestProcess->new($pid);
+ my $td = PublicInbox::AutoReap->new($pid);
$td->{-extra} = $tail;
$td;
}
@@ -742,37 +743,6 @@ sub test_httpd ($$;$) {
};
-package PublicInboxTestProcess;
-use strict;
-
-# prevent new threads from inheriting these objects
-sub CLONE_SKIP { 1 }
-
-sub new {
- my ($cls, $pid, $cb) = @_;
- bless { pid => $pid, cb => $cb, owner => $$ }, $cls;
-}
-
-sub kill {
- my ($self, $sig) = @_;
- CORE::kill($sig // 'TERM', $self->{pid});
-}
-
-sub join {
- my ($self, $sig) = @_;
- my $pid = delete $self->{pid} or return;
- $self->{cb}->() if defined $self->{cb};
- CORE::kill($sig, $pid) if defined $sig;
- my $ret = waitpid($pid, 0) // die "waitpid($pid): $!";
- $ret == $pid or die "waitpid($pid) != $ret";
-}
-
-sub DESTROY {
- my ($self) = @_;
- return if $self->{owner} != $$;
- $self->join('TERM');
-}
-
package PublicInbox::TestCommon::InboxWakeup;
use strict;
sub on_inbox_unlock { ${$_[0]}->($_[1]) }
^ permalink raw reply related [relevance 34%]
* [PATCH] lei q: avoid kw lookup failure on remote mboxrd
2021-10-14 5:31 71% ` Eric Wong
@ 2021-10-15 9:52 54% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-15 9:52 UTC (permalink / raw)
To: meta
> > > 2/3 is probably a fix for a long-standing problem, 3/3 was
> > > noticed while working on it. If 2/3 doesn't fix it, then maybe
> > > 1/3 will help us narrow it down.
> >
> > s/is probably/was almost certainly/ \o/
>
> No. :<
>
> It still happens when there's overlap between various search
> sources, but I think I know how to fix it...
Ok, higher certainty for now :P
-----8<-----
Subject: [PATCH] lei q: avoid kw lookup failure on remote mboxrd
When importing several sources in parallel via http(s) mboxrd,
we need to be able to get keywords of uncommitted documents
directly from shard workers. Otherwise, Xapian DocNotFound
errors happen because the read-only LeiSearch won't see
documents from uncomitted transactions. Keep in mind that it's
possible the keywords can be changed on-the-fly even for
uncommitted documents because of inotify watches from LeiNoteEvent.
---
lib/PublicInbox/LeiStore.pm | 28 +++++++++++++++++++++++-----
lib/PublicInbox/LeiXSearch.pm | 8 +++-----
lib/PublicInbox/SearchIdx.pm | 6 ++++++
t/lei_store.t | 3 ++-
4 files changed, 34 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index bf41dcf5..c45380d1 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -328,6 +328,20 @@ sub _add_vmd ($$$$) {
sto_export_kw($self, $docid, $vmd);
}
+sub _docids_and_maybe_kw ($$) {
+ my ($self, $docids) = @_;
+ return $docids unless wantarray;
+ my $kw = {};
+ for my $num (@$docids) { # likely only 1, unless ContentHash changes
+ # can't use ->search->msg_keywords on uncommitted docs
+ my $idx = $self->{priv_eidx}->idx_shard($num);
+ my $tmp = eval { $idx->ipc_do('get_terms', 'K', $num) };
+ if ($@) { warn "#$num get_terms: $@" }
+ else { @$kw{keys %$tmp} = values(%$tmp) };
+ }
+ ($docids, [ sort keys %$kw ]);
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
@@ -339,7 +353,11 @@ sub add_eml {
if ($vmd && $vmd->{sync_info}) {
set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
}
- $im_mark or return; # duplicate blob returns undef
+ unless ($im_mark) { # duplicate blob returns undef
+ return unless wantarray;
+ my @docids = $oidx->blob_exists($smsg->{blob});
+ return _docids_and_maybe_kw $self, \@docids;
+ }
local $self->{current_info} = $smsg->{blob};
my $vivify_xvmd = delete($smsg->{-vivify_xvmd}) // []; # exact matches
@@ -373,7 +391,7 @@ sub add_eml {
}
_add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
- $vivify_xvmd;
+ _docids_and_maybe_kw $self, $vivify_xvmd;
} elsif (my @docids = _docids_for($self, $eml)) {
# fuzzy match from within lei/store
for my $docid (@docids) {
@@ -383,8 +401,8 @@ sub add_eml {
$idx->ipc_do('add_eidx_info', $docid, '.', $eml);
_add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
- \@docids;
- } else { # totally new message
+ _docids_and_maybe_kw $self, \@docids;
+ } else { # totally new message, no keywords
delete $smsg->{-oidx}; # for IPC-friendliness
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
$oidx->add_overview($eml, $smsg);
@@ -392,7 +410,7 @@ sub add_eml {
my $idx = $eidx->idx_shard($smsg->{num});
$idx->index_eml($eml, $smsg);
_add_vmd($self, $idx, $smsg->{num}, $vmd) if $vmd;
- $smsg;
+ wantarray ? ($smsg, []) : $smsg;
}
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index fba16861..3ec75528 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -282,11 +282,9 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
my $xoids = $lei->{ale}->xoids_for($eml, 1);
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- my $res = $self->{import_sto}->wq_do('add_eml', $eml);
- if (ref($res) eq ref($smsg)) { # totally new message
- $smsg = $res;
- $smsg->{kw} = []; # short-circuit xsmsg_vmd
- }
+ my ($res, $kw) = $self->{import_sto}->wq_do('add_eml', $eml);
+ $smsg = $res if ref($res) eq ref($smsg); # totally new message
+ $smsg->{kw} = $kw; # short-circuit xsmsg_vmd
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: $lei->git_oid($eml)->hexdigest;
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 928152ec..585f28f5 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -517,6 +517,12 @@ sub add_eidx_info {
$self->{xdb}->replace_document($docid, $doc);
}
+sub get_terms {
+ my ($self, $pfx, $docid) = @_;
+ begin_txn_lazy($self);
+ xap_terms($pfx, $self->{xdb}, $docid);
+}
+
sub remove_eidx_info {
my ($self, $docid, $eidx_key, $eml) = @_;
begin_txn_lazy($self);
diff --git a/t/lei_store.t b/t/lei_store.t
index c31e27a2..40ad7800 100644
--- a/t/lei_store.t
+++ b/t/lei_store.t
@@ -138,7 +138,8 @@ Subject: timezone-dependent test
WHAT IS TIME ANYMORE?
EOM
- ok($sto->add_eml($eml), 'recently received message');
+ my $smsg = $sto->add_eml($eml);
+ ok($smsg && $smsg->{blob}, 'recently received message');
$sto->done;
local $ENV{TZ} = 'GMT+5';
my $lse = $sto->search;
^ permalink raw reply related [relevance 54%]
* [PATCH 1/3] lei forget-search: fix for symlink-ed paths
2021-10-15 13:30 71% [PATCH 0/3] lei bugfixes and simplifications Eric Wong
@ 2021-10-15 13:30 71% ` Eric Wong
2021-10-15 13:30 33% ` [PATCH 2/3] lei + ipc: simplify process reaping Eric Wong
2021-10-15 13:30 67% ` [PATCH 3/3] lei note-event: fix explicit flush reliability Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-15 13:30 UTC (permalink / raw)
To: meta
If lei up and edit-search work on something, so should forget-search.
---
lib/PublicInbox/LeiForgetSearch.pm | 2 +-
t/lei-q-save.t | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiForgetSearch.pm b/lib/PublicInbox/LeiForgetSearch.pm
index b5fe5fb1..717fa5e9 100644
--- a/lib/PublicInbox/LeiForgetSearch.pm
+++ b/lib/PublicInbox/LeiForgetSearch.pm
@@ -12,7 +12,7 @@ use SelectSaver;
sub lei_forget_search {
my ($lei, $out) = @_;
- my $d = PublicInbox::LeiSavedSearch::lss_dir_for($lei, \$out);
+ my $d = PublicInbox::LeiSavedSearch::lss_dir_for($lei, \$out, 1);
if (-e $d) {
my $save;
my $opt = { safe => 1 };
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 5940018c..05d5d9f4 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -173,6 +173,7 @@ test_lei(sub {
symlink($o, "$home/ln -s") or
skip "symlinks not supported in $home?: $!", 1;
lei_ok('up', "$home/ln -s");
+ lei_ok('forget-search', "$home/ln -s");
};
my $v2 = "$home/v2"; # v2: as an output destination
^ permalink raw reply related [relevance 71%]
* [PATCH 0/3] lei bugfixes and simplifications
@ 2021-10-15 13:30 71% Eric Wong
2021-10-15 13:30 71% ` [PATCH 1/3] lei forget-search: fix for symlink-ed paths Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-10-15 13:30 UTC (permalink / raw)
To: meta
I think I finally tracked down the cause of very sporadic
test failures in 3/3.
Eric Wong (3):
lei forget-search: fix for symlink-ed paths
lei + ipc: simplify process reaping
lei note-event: fix explicit flush reliability
lib/PublicInbox/IPC.pm | 46 +++++++++++-------------------
lib/PublicInbox/LEI.pm | 17 ++++-------
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiForgetSearch.pm | 2 +-
lib/PublicInbox/LeiImportKw.pm | 1 -
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiInspect.pm | 2 +-
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 2 +-
lib/PublicInbox/LeiNoteEvent.pm | 14 ++++-----
lib/PublicInbox/LeiP2q.pm | 2 +-
lib/PublicInbox/LeiPmdir.pm | 1 -
lib/PublicInbox/LeiStore.pm | 1 +
lib/PublicInbox/LeiToMail.pm | 10 +++++--
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 16 ++++++-----
t/lei-q-save.t | 1 +
18 files changed, 57 insertions(+), 68 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei note-event: fix explicit flush reliability
2021-10-15 13:30 71% [PATCH 0/3] lei bugfixes and simplifications Eric Wong
2021-10-15 13:30 71% ` [PATCH 1/3] lei forget-search: fix for symlink-ed paths Eric Wong
2021-10-15 13:30 33% ` [PATCH 2/3] lei + ipc: simplify process reaping Eric Wong
@ 2021-10-15 13:30 67% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-15 13:30 UTC (permalink / raw)
To: meta
We need to send the socket over to lei/store and wait for the
kernel to drop the socket refcount down to zero before
script/lei can exit.
This is not a new bug and only caused very sporadic test
failures. I only noticed it while simplifying IPC stuff.
---
lib/PublicInbox/LeiNoteEvent.pm | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index ba4dfd49..1749c98f 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -11,10 +11,11 @@ use PublicInbox::DS;
our $to_flush; # { cfgpath => $lei }
-sub flush_lei ($) {
- my ($lei) = @_;
- my $lne = delete $lei->{cfg}->{-lei_note_event};
- $lne->wq_close if $lne; # runs _lei_wq_eof;
+sub flush_lei ($;$) {
+ my ($lei, $manual) = @_;
+ my $lne = delete $lei->{cfg}->{-lei_note_event} // return;
+ $lne->{lei_sock} = $lei->{sock} if $manual;
+ $lne->wq_close; # runs _lei_wq_eof;
}
# we batch up writes and flush every 5s (matching Linux default
@@ -67,7 +68,7 @@ sub lei_note_event {
die "BUG: unexpected: @rest" if @rest;
my $cfg = $lei->_lei_cfg or return; # gone (race)
my $sto = $lei->_lei_store or return; # gone
- return flush_lei($lei) if $folder eq 'done'; # special case
+ return flush_lei($lei, 1) if $folder eq 'done'; # special case
my $lms = $lei->lms or return;
$lms->lms_write_prepare if $new_cur eq ''; # for ->clear_src below
$lei->{opt}->{quiet} = 1;
@@ -111,8 +112,8 @@ sub ipc_atfork_child {
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
- delete $lei->{lne} or return $lei->fail;
- $lei->sto_done_request;
+ my $lne = delete $lei->{lne} or return $lei->fail;
+ $lei->sto_done_request($lne->{lei_sock});
}
1;
^ permalink raw reply related [relevance 67%]
* [PATCH 2/3] lei + ipc: simplify process reaping
2021-10-15 13:30 71% [PATCH 0/3] lei bugfixes and simplifications Eric Wong
2021-10-15 13:30 71% ` [PATCH 1/3] lei forget-search: fix for symlink-ed paths Eric Wong
@ 2021-10-15 13:30 33% ` Eric Wong
2021-10-15 13:45 71% ` [SQUASH PATCH 4/3] lei q: ensure all workers die on Ctrl-C Eric Wong
2021-10-15 13:30 67% ` [PATCH 3/3] lei note-event: fix explicit flush reliability Eric Wong
2 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-15 13:30 UTC (permalink / raw)
To: meta
Simplify our APIs and force dwaitpid() to work in async mode for
all lei workers. This avoids having lingering zombies for
parallel searches if one worker finishes soon before another.
The old distinction between "old" and "new" workers was
needlessly complex, error-prone, and embarrasingly bad.
---
lib/PublicInbox/IPC.pm | 46 ++++++++++++---------------------
lib/PublicInbox/LEI.pm | 17 ++++--------
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiImportKw.pm | 1 -
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiInspect.pm | 2 +-
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 2 +-
lib/PublicInbox/LeiNoteEvent.pm | 5 ++--
lib/PublicInbox/LeiP2q.pm | 2 +-
lib/PublicInbox/LeiPmdir.pm | 1 -
lib/PublicInbox/LeiStore.pm | 1 +
lib/PublicInbox/LeiToMail.pm | 10 +++++--
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 16 +++++++-----
16 files changed, 50 insertions(+), 63 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 6c189b64..3e299448 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -134,16 +134,22 @@ sub ipc_worker_spawn {
sub ipc_worker_reap { # dwaitpid callback
my ($args, $pid) = @_;
+ my ($self, @uargs) = @$args;
+ delete $self->{-wq_workers}->{$pid};
+ return $self->{-reap_do}->($args, $pid) if $self->{-reap_do};
return if !$?;
- # TERM(15) is our default exit signal, PIPE(13) is likely w/ pager
my $s = $? & 127;
- warn "PID:$pid died with \$?=$?\n" if $s != 15 && $s != 13;
+ # TERM(15) is our default exit signal, PIPE(13) is likely w/ pager
+ warn "$self->{-wq_ident} PID:$pid died \$?=$?\n" if $s != 15 && $s != 13
}
-sub wq_wait_old {
- my ($self, $cb, @args) = @_;
- my $pids = delete $self->{"-wq_old_pids.$$"} or return;
- dwaitpid($_, $cb // \&ipc_worker_reap, [$self, @args]) for @$pids;
+sub wq_wait_async {
+ my ($self, $cb, @uargs) = @_;
+ local $PublicInbox::DS::in_loop = 1;
+ $self->{-reap_async} = 1;
+ $self->{-reap_do} = $cb;
+ my @pids = keys %{$self->{-wq_workers}};
+ dwaitpid($_, \&ipc_worker_reap, [ $self, @uargs ]) for @pids;
}
# for base class, override in sub classes
@@ -394,42 +400,24 @@ sub wq_workers_start {
}
sub wq_close {
- my ($self, $nohang, $cb, @args) = @_;
+ my ($self) = @_;
delete @$self{qw(-wq_s1 -wq_s2)} or return;
- my $ppid = delete $self->{-wq_ppid} or return;
- my $workers = delete $self->{-wq_workers} // die 'BUG: no wq_workers';
- return if $ppid != $$; # can't reap siblings or parents
- my @pids = map { $_ + 0 } keys %$workers;
- if ($nohang) {
- push @{$self->{"-wq_old_pids.$$"}}, @pids;
- } else {
- $cb //= \&ipc_worker_reap;
- unshift @args, $self;
- dwaitpid($_, $cb, \@args) for @pids;
- }
-}
-
-sub wq_kill_old {
- my ($self, $sig) = @_;
- my $pids = $self->{"-wq_old_pids.$$"} or return;
- kill($sig // 'TERM', @$pids);
+ return if $self->{-reap_async};
+ my @pids = keys %{$self->{-wq_workers}};
+ dwaitpid($_, \&ipc_worker_reap, [ $self ]) for @pids;
}
sub wq_kill {
my ($self, $sig) = @_;
- my $workers = $self->{-wq_workers} or return;
- kill($sig // 'TERM', keys %$workers);
+ kill($sig // 'TERM', keys %{$self->{-wq_workers}});
}
sub DESTROY {
my ($self) = @_;
my $ppid = $self->{-wq_ppid};
wq_kill($self) if $ppid && $ppid == $$;
- my $err = $?;
wq_close($self);
- wq_wait_old($self);
ipc_worker_stop($self);
- $? = $err if $err;
}
sub detect_nproc () {
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b6338377..4a1f1652 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -459,14 +459,7 @@ my @WQ_KEYS = qw(lxs l2m ikw pmd wq1 lne); # internal workers
sub _drop_wq {
my ($self) = @_;
- for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) {
- if ($wq->wq_kill('-TERM')) {
- $wq->wq_close(0, undef, $self);
- } elsif ($wq->wq_kill_old('-TERM')) {
- $wq->wq_wait_old(undef, $self);
- }
- $wq->DESTROY;
- }
+ for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) { $wq->DESTROY }
}
# pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
@@ -644,6 +637,7 @@ sub workers_start {
my $op_c = delete $lei->{pkt_op_c};
@$end = ();
$lei->event_step_init;
+ $wq->wq_wait_async($wq->can('_wq_done_wait') // \&wq_done_wait, $lei);
($op_c, $ops);
}
@@ -651,7 +645,7 @@ sub workers_start {
sub wait_wq_events {
my ($lei, $op_c, $ops) = @_;
for my $wq (grep(defined, @$lei{qw(ikw pmd)})) { # auxiliary WQs
- $wq->wq_close(1);
+ $wq->wq_close;
}
$op_c->{ops} = $ops;
}
@@ -1150,7 +1144,7 @@ sub event_step {
if ($buf =~ /\A(?:STOP|CONT|TERM)\z/) {
my $sig = "-$buf";
for my $wq (grep(defined, @$self{@WQ_KEYS})) {
- $wq->wq_kill($sig) or $wq->wq_kill_old($sig);
+ $wq->wq_kill($sig);
}
} else {
die "unrecognized client signal: $buf";
@@ -1393,8 +1387,7 @@ sub fchdir {
sub wq_eof { # EOF callback for main daemon
my ($lei) = @_;
local $current_lei = $lei;
- my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
- $wq1->wq_wait_old($wq1->can('_wq_done_wait') // \&wq_done_wait, $lei);
+ delete $lei->{wq1} // return $lei->fail; # already failed
}
sub watch_state_ok ($) {
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index b6a62d24..004b156c 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -166,7 +166,7 @@ sub lei_blob {
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_solve_blob', []);
- $self->wq_close(1);
+ $self->wq_close;
$lei->wait_wq_events($op_c, $ops);
}
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 9e98edc3..68fc7c0b 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -58,7 +58,7 @@ sub lei_convert { # the main "lei convert" method
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('process_inputs', []);
- $self->wq_close(1);
+ $self->wq_close;
$lei->wait_wq_events($op_c, $ops);
}
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 8359f338..54454511 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -50,7 +50,6 @@ sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $ikw = delete $lei->{ikw} or return $lei->fail;
$lei->sto_done_request($ikw->{lei_sock});
- $ikw->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index dd40d838..2621fc1f 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -402,7 +402,7 @@ sub input_only_atfork_child {
sub input_only_net_merge_all_done {
my ($self) = @_;
$self->wq_io_do('process_inputs');
- $self->wq_close(1);
+ $self->wq_close;
}
# like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 5ba96056..05b6e21d 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -242,7 +242,7 @@ sub inspect_start ($$) {
$lei->{wq1} = $self;
$lei->wait_wq_events($op_c, $ops);
$self->wq_do('inspect_argv');
- $self->wq_close(1);
+ $self->wq_close;
}
sub ins_add { # InputPipe->consume callback
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index aebf0184..0193e590 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -75,7 +75,7 @@ sub bg_worker ($$$) {
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
- $self->wq_close(1);
+ $self->wq_close;
$lei->wait_wq_events($op_c, $ops);
}
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index a75c99c4..e20d30b4 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -451,7 +451,7 @@ sub start {
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_mirror', []);
- $self->wq_close(1);
+ $self->wq_close;
$lei->wait_wq_events($op_c, $ops);
}
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 1b714dae..ba4dfd49 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -14,7 +14,7 @@ our $to_flush; # { cfgpath => $lei }
sub flush_lei ($) {
my ($lei) = @_;
my $lne = delete $lei->{cfg}->{-lei_note_event};
- $lne->wq_close(1, undef, $lei) if $lne; # runs _lei_wq_eof;
+ $lne->wq_close if $lne; # runs _lei_wq_eof;
}
# we batch up writes and flush every 5s (matching Linux default
@@ -111,9 +111,8 @@ sub ipc_atfork_child {
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
- my $lne = delete $lei->{lne} or return $lei->fail;
+ delete $lei->{lne} or return $lei->fail;
$lei->sto_done_request;
- $lne->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index 5c2ce0a1..08ec81c5 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -191,7 +191,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_p2q', []);
- $self->wq_close(1);
+ $self->wq_close;
$lei->wait_wq_events($op_c, $ops);
}
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index 2d3b9755..f9b68fc2 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -51,7 +51,6 @@ sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $pmd = delete $lei->{pmd} or return $lei->fail;
$lei->sto_done_request($pmd->{lei_sock});
- $pmd->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index c45380d1..82104570 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -587,6 +587,7 @@ sub write_prepare {
-err_wr => $w,
to_close => [ $r ],
});
+ $self->wq_wait_async; # outlives $lei
require PublicInbox::LeiStoreErr;
PublicInbox::LeiStoreErr->new($r, $lei);
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 9c748dea..76e103c7 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -637,6 +637,12 @@ sub _do_augment_mbox {
$dedupe->pause_dedupe if $dedupe;
}
+sub v2w_done_wait { # dwaitpid callback
+ my ($arg, $pid) = @_;
+ my ($v2w, $lei) = @$arg;
+ $lei->child_error($?, "error for $v2w->{ibx}->{inboxdir}") if $?;
+}
+
sub _pre_augment_v2 {
my ($self, $lei) = @_;
my $dir = $self->{dst};
@@ -659,8 +665,8 @@ sub _pre_augment_v2 {
PublicInbox::InboxWritable->new($ibx, @creat);
$ibx->init_inbox if @creat;
my $v2w = $ibx->importer;
- $v2w->{-wq_no_bcast} = 1;
$v2w->wq_workers_start("lei/v2w $dir", 1, $lei->oldset, {lei => $lei});
+ $v2w->wq_wait_async(\&v2w_done_wait, $lei);
$lei->{v2w} = $v2w;
return if !$lei->{opt}->{shared};
my $d = "$lei->{ale}->{git}->{git_dir}/objects";
@@ -811,7 +817,7 @@ sub net_merge_all_done {
$self->{dst}, \$self->{-au_noted});
}
$self->wq_broadcast('do_post_auth');
- $self->wq_close(1);
+ $self->wq_close;
}
1;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index df65cb9b..39604177 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -122,7 +122,7 @@ EOM
sub net_merge_all_done {
my ($self, $lei) = @_;
$lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
- $self->wq_close(1);
+ $self->wq_close;
eval { redispatch_all($self, $lei) };
warn "E: $@" if $@;
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 3ec75528..fd2c8a37 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -413,14 +413,14 @@ sub query_done { # EOF callback for main daemon
my ($lei) = @_;
local $PublicInbox::LEI::current_lei = $lei;
my $l2m = delete $lei->{l2m};
- $l2m->wq_wait_old(\&xsearch_done_wait, $lei) if $l2m;
- if (my $lxs = delete $lei->{lxs}) {
- $lxs->wq_wait_old(\&xsearch_done_wait, $lei);
- }
+ delete $lei->{lxs};
($lei->{opt}->{'mail-sync'} && !$lei->{sto}) and
warn "BUG: {sto} missing with --mail-sync";
$lei->sto_done_request if $lei->{sto};
- my $wait = $lei->{v2w} ? $lei->{v2w}->wq_do('done') : undef;
+ if (my $v2w = delete $lei->{v2w}) {
+ $v2w->wq_do('done');
+ $v2w->wq_close;
+ }
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
@@ -466,7 +466,7 @@ sub do_post_augment {
if ($err) {
if (my $lxs = delete $lei->{lxs}) {
$lxs->wq_kill('-TERM');
- $lxs->wq_close(0, undef, $lei);
+ $lxs->wq_close;
}
$lei->fail("$err");
}
@@ -514,7 +514,7 @@ sub start_query ($$) { # always runs in main (lei-daemon) process
if ($self->{-do_lcat}) {
$self->wq_io_do('lcat_dump', []);
}
- $self->wq_close(1); # lei_xsearch workers stop when done
+ $self->wq_close; # lei_xsearch workers stop when done
}
sub incr_start_query { # called whenever an l2m shard starts do_post_auth
@@ -569,12 +569,14 @@ sub do_query {
}
$l2m->wq_workers_start('lei2mail', undef,
$lei->oldset, { lei => $lei });
+ $l2m->wq_wait_async(\&xsearch_done_wait, $lei);
pipe($lei->{startq}, $lei->{au_done}) or die "pipe: $!";
fcntl($lei->{startq}, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
delete $l2m->{au_peers};
}
$self->wq_workers_start('lei_xsearch', undef,
$lei->oldset, { lei => $lei });
+ $self->wq_wait_async(\&xsearch_done_wait, $lei);
my $op_c = delete $lei->{pkt_op_c};
delete $lei->{pkt_op_p};
@$end = ();
^ permalink raw reply related [relevance 33%]
* [SQUASH PATCH 4/3] lei q: ensure all workers die on Ctrl-C
2021-10-15 13:30 33% ` [PATCH 2/3] lei + ipc: simplify process reaping Eric Wong
@ 2021-10-15 13:45 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-15 13:45 UTC (permalink / raw)
To: meta
We never handled v2:// writers properly before. And we still
need to kill the entire process group of each worker :x
Oops
---
lib/PublicInbox/LEI.pm | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 4a1f1652..83534878 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -455,11 +455,14 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m ikw pmd wq1 lne); # internal workers
+my @WQ_KEYS = qw(lxs l2m ikw pmd wq1 lne v2w); # internal workers
sub _drop_wq {
my ($self) = @_;
- for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) { $wq->DESTROY }
+ for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) {
+ $wq->wq_kill('-TERM');
+ $wq->DESTROY;
+ }
}
# pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
^ permalink raw reply related [relevance 71%]
* [PATCH] lei forget-search: support multiple args
@ 2021-10-15 14:02 64% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-15 14:02 UTC (permalink / raw)
To: meta
I've been testing a lot of searches which I don't want to keep
around, so make it easy to remove a bunch at once. We'll behave
like rm(1) and keep going in the face of failure.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiForgetSearch.pm | 30 ++++++++++++++++++------------
2 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 83534878..7dfd3398 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -225,7 +225,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-search' => [ '[PREFIX]', 'list saved search queries',
qw(format|f=s pretty l ascii z|0), @c_opt ],
-'forget-search' => [ 'OUTPUT', 'forget a saved search',
+'forget-search' => [ 'OUTPUT...', 'forget a saved search',
qw(verbose|v+), @c_opt ],
'edit-search' => [ 'OUTPUT', "edit saved search via `git config --edit'",
@c_opt ],
diff --git a/lib/PublicInbox/LeiForgetSearch.pm b/lib/PublicInbox/LeiForgetSearch.pm
index 717fa5e9..0db9c75b 100644
--- a/lib/PublicInbox/LeiForgetSearch.pm
+++ b/lib/PublicInbox/LeiForgetSearch.pm
@@ -11,20 +11,26 @@ use File::Path ();
use SelectSaver;
sub lei_forget_search {
- my ($lei, $out) = @_;
- my $d = PublicInbox::LeiSavedSearch::lss_dir_for($lei, \$out, 1);
- if (-e $d) {
- my $save;
- my $opt = { safe => 1 };
- if ($lei->{opt}->{verbose}) {
- $opt->{verbose} = 1;
- $save = SelectSaver->new($lei->{2});
+ my ($lei, @outs) = @_;
+ my @dirs; # paths in ~/.local/share/lei/saved-search/
+ my $cwd;
+ for my $o (@outs) {
+ my $d = PublicInbox::LeiSavedSearch::lss_dir_for($lei, \$o, 1);
+ if (-e $d) {
+ push @dirs, $d
+ } else { # keep going, like rm(1):
+ $cwd //= $lei->rel2abs('.');
+ warn "--save was not used with $o cwd=$cwd\n";
}
- File::Path::remove_tree($d, $opt);
- } else {
- $lei->fail("--save was not used with $out cwd=".
- $lei->rel2abs('.'));
}
+ my $save;
+ my $opt = { safe => 1 };
+ if ($lei->{opt}->{verbose}) {
+ $opt->{verbose} = 1;
+ $save = SelectSaver->new($lei->{2});
+ }
+ File::Path::remove_tree(@dirs, $opt);
+ $lei->fail if defined $cwd;
}
*_complete_forget_search = \&PublicInbox::LeiUp::_complete_up;
^ permalink raw reply related [relevance 64%]
* [PATCH] lei q: guard query_done against die()
@ 2021-10-15 15:52 61% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-15 15:52 UTC (permalink / raw)
To: meta
v2w->wq_do('done') may die on I/O errors, and likely other
places. Just guard the entire block with an eval and ->fail
as appropriate.
---
lib/PublicInbox/LeiXSearch.pm | 75 ++++++++++++++++++-----------------
1 file changed, 38 insertions(+), 37 deletions(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index fd2c8a37..4aa2a81c 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -412,47 +412,48 @@ sub xsearch_done_wait { # dwaitpid callback
sub query_done { # EOF callback for main daemon
my ($lei) = @_;
local $PublicInbox::LEI::current_lei = $lei;
- my $l2m = delete $lei->{l2m};
- delete $lei->{lxs};
- ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) and
- warn "BUG: {sto} missing with --mail-sync";
- $lei->sto_done_request if $lei->{sto};
- if (my $v2w = delete $lei->{v2w}) {
- $v2w->wq_do('done');
- $v2w->wq_close;
- }
- $lei->{ovv}->ovv_end($lei);
- my $start_mua;
- if ($l2m) { # close() calls LeiToMail reap_compress
- if (my $out = delete $lei->{old_1}) {
- if (my $mbout = $lei->{1}) {
- close($mbout) or return $lei->fail(<<"");
-Error closing $lei->{ovv}->{dst}: $!
+ eval {
+ my $l2m = delete $lei->{l2m};
+ delete $lei->{lxs};
+ ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) and
+ warn "BUG: {sto} missing with --mail-sync";
+ $lei->sto_done_request if $lei->{sto};
+ if (my $v2w = delete $lei->{v2w}) {
+ my $wait = $v2w->wq_do('done'); # may die
+ $v2w->wq_close;
+ }
+ $lei->{ovv}->ovv_end($lei);
+ if ($l2m) { # close() calls LeiToMail reap_compress
+ if (my $out = delete $lei->{old_1}) {
+ if (my $mbout = $lei->{1}) {
+ close($mbout) or die <<"";
+Error closing $lei->{ovv}->{dst}: \$!=$! \$?=$?
+ }
+ $lei->{1} = $out;
+ }
+ if ($l2m->lock_free) {
+ $l2m->poke_dst;
+ $lei->poke_mua;
+ } else { # mbox users
+ delete $l2m->{mbl}; # drop dotlock
}
- $lei->{1} = $out;
- }
- if ($l2m->lock_free) {
- $l2m->poke_dst;
- $lei->poke_mua;
- } else { # mbox users
- delete $l2m->{mbl}; # drop dotlock
- $start_mua = 1;
}
- }
- if ($lei->{-progress}) {
- my $tot = $lei->{-mset_total} // 0;
- my $nr = $lei->{-nr_write} // 0;
- if ($l2m) {
- my $m = "# $nr written to " .
- "$lei->{ovv}->{dst} ($tot matches)";
- $nr ? $lei->qfin($m) : $lei->qerr($m);
- } else {
- $lei->qerr("# $tot matches");
+ if ($lei->{-progress}) {
+ my $tot = $lei->{-mset_total} // 0;
+ my $nr = $lei->{-nr_write} // 0;
+ if ($l2m) {
+ my $m = "# $nr written to " .
+ "$lei->{ovv}->{dst} ($tot matches)";
+ $nr ? $lei->qfin($m) : $lei->qerr($m);
+ } else {
+ $lei->qerr("# $tot matches");
+ }
}
- }
- $lei->start_mua if $start_mua;
- $lei->dclose;
+ $lei->start_mua if $l2m && !$l2m->lock_free;
+ $lei->dclose;
+ };
+ $lei->fail($@) if $@;
}
sub do_post_augment {
^ permalink raw reply related [relevance 61%]
* [PATCH 05/12] lei: golf PATH2CFG cleanup
@ 2021-10-16 1:00 71% ` Eric Wong
2021-10-16 1:00 63% ` [PATCH 06/12] lei: always keep cwd fd {3} for ->fchdir Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-16 1:00 UTC (permalink / raw)
To: meta
More code means more bugs.
---
lib/PublicInbox/LEI.pm | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a526a91f8035..e7f37efaf0a3 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -850,9 +850,7 @@ sub _lei_cfg ($;$) {
}
if (scalar(keys %PATH2CFG) > 5) {
# FIXME: use inotify/EVFILT_VNODE to detect unlinked configs
- for my $k (keys %PATH2CFG) {
- delete($PATH2CFG{$k}) unless -f $k
- }
+ delete(@PATH2CFG{grep(!-f, keys %PATH2CFG)});
}
$self->{cfg} = $PATH2CFG{$f} = $cfg;
refresh_watches($self);
^ permalink raw reply related [relevance 71%]
* [PATCH 06/12] lei: always keep cwd fd {3} for ->fchdir
2021-10-16 1:00 71% ` [PATCH 05/12] lei: golf PATH2CFG cleanup Eric Wong
@ 2021-10-16 1:00 63% ` Eric Wong
2021-10-16 1:00 55% ` [PATCH 07/12] lei: more eval guards for die on failure Eric Wong
2021-10-16 1:01 59% ` [PATCH 09/12] lei_overview: die rather than lei->fail Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-16 1:00 UTC (permalink / raw)
To: meta
The extra FD shouldn't cause noticeable overhead in short-lived
workers, and it lets us simplify lei->rel2abs. Get rid of a
2-argument form of open() while we're at it, since it's been
considered for warning+deprecation by Perl for safety reasons.
---
lib/PublicInbox/LEI.pm | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e7f37efaf0a3..0cdcf4492885 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -77,19 +77,16 @@ sub rel2abs {
return $p;
}
my $pwd = $self->{env}->{PWD};
- my $cwd;
if (defined $pwd) {
- my $xcwd = $self->{3} //
- ($cwd = getcwd() // die "getcwd(PWD=$pwd): $!");
if (my @st_pwd = stat($pwd)) {
- my @st_cwd = stat($xcwd) or die "stat($xcwd): $!";
+ my @st_cwd = stat($self->{3}) or die "stat({3}): $!";
"@st_pwd[1,0]" eq "@st_cwd[1,0]" or
$self->{env}->{PWD} = $pwd = undef;
} else { # PWD was invalid
$self->{env}->{PWD} = $pwd = undef;
}
}
- $pwd //= $self->{env}->{PWD} = $cwd // getcwd() // die "getcwd: $!";
+ $pwd //= $self->{env}->{PWD} = getcwd() // die "getcwd: $!";
File::Spec->rel2abs($p, $pwd);
}
@@ -558,7 +555,8 @@ sub _lei_atfork_child {
my ($self, $persist) = @_;
# we need to explicitly close things which are on stack
if ($persist) {
- chdir '/' or die "chdir(/): $!";
+ open $self->{3}, '<', '/' or die "open(/) $!";
+ fchdir($self) or die;
close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
if (my $cfg = $self->{cfg}) {
delete @$cfg{qw(-lei_store -watches -lei_note_event)};
@@ -568,7 +566,7 @@ sub _lei_atfork_child {
STDERR->autoflush(1);
POSIX::setpgid(0, $$) // die "setpgid(0, $$): $!";
}
- close($_) for (grep(defined, delete @$self{qw(3 old_1 au_done)}));
+ close($_) for (grep(defined, delete @$self{qw(old_1 au_done)}));
delete $self->{-socks};
if (my $op_c = delete $self->{pkt_op_c}) {
close(delete $op_c->{sock});
@@ -1190,7 +1188,7 @@ sub cfg2lei ($) {
open($lei->{0}, '<&', \*STDIN) or die "dup 0: $!";
open($lei->{1}, '>>&', \*STDOUT) or die "dup 1: $!";
open($lei->{2}, '>>&', \*STDERR) or die "dup 2: $!";
- open($lei->{3}, '/') or die "open /: $!";
+ open($lei->{3}, '<', '/') or die "open /: $!";
my ($x, $y);
socketpair($x, $y, AF_UNIX, SOCK_SEQPACKET, 0) or die "socketpair: $!";
$lei->{sock} = $x;
^ permalink raw reply related [relevance 63%]
* [PATCH 09/12] lei_overview: die rather than lei->fail
` (2 preceding siblings ...)
2021-10-16 1:00 55% ` [PATCH 07/12] lei: more eval guards for die on failure Eric Wong
@ 2021-10-16 1:01 59% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-16 1:01 UTC (permalink / raw)
To: meta
This will make our code more flexible in case it gets used in
non-lei things.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 24 +++++++++++-------------
2 files changed, 12 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 511b2c1d03a7..876598f9530e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1108,7 +1108,7 @@ sub accept_dispatch { # Listener {post_accept} callback
my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
$self->{env} = \%env;
eval { dispatch($self, @argv) };
- send($sock, $@, MSG_EOR) if $@;
+ $self->fail($@) if $@;
}
sub dclose {
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 223db22244ec..1b9dc9701c95 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -37,16 +37,16 @@ sub ovv_out_lk_cancel ($) {
unlink($lock_path);
}
-sub detect_fmt ($$) {
- my ($lei, $dst) = @_;
+sub detect_fmt ($) {
+ my ($dst) = @_;
if ($dst =~ m!\A([:/]+://)!) {
- $lei->fail("$1 support not implemented, yet\n");
+ die "$1 support not implemented, yet\n";
} elsif (!-e $dst || -d _) {
'maildir'; # the default TODO: MH?
} elsif (-f _ || -p _) {
- $lei->fail("unable to determine mbox family of $dst\n");
+ die "unable to determine mbox family of $dst\n";
} else {
- $lei->fail("unable to determine format of $dst\n");
+ die "unable to determine format of $dst\n";
}
}
@@ -60,20 +60,19 @@ sub new {
my $fmt = $opt->{$ofmt_key};
$fmt = lc($fmt) if defined $fmt;
if ($dst =~ m!\A([a-z0-9\+]+)://!is) {
- defined($fmt) and return $lei->fail(<<"");
+ defined($fmt) and die <<"";
--$ofmt_key=$fmt invalid with URL $dst
$fmt = lc $1;
} elsif ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/
my $ofmt = lc $1;
$fmt //= $ofmt;
- return $lei->fail(<<"") if $fmt ne $ofmt;
+ die <<"" if $fmt ne $ofmt;
--$ofmt_key=$fmt and --output=$ofmt conflict
}
-
my $devfd = $lei->path_to_fd($dst) // return;
- $fmt //= $devfd >= 0 ? 'json' : (detect_fmt($lei, $dst) or return);
+ $fmt //= $devfd >= 0 ? 'json' : detect_fmt($dst);
if (index($dst, '://') < 0) { # not a URL, so assume path
$dst = $lei->canonpath_harder($dst);
@@ -90,7 +89,7 @@ sub new {
$opt->{pretty} //= $isatty;
if (!$isatty && -f _) {
my $fl = fcntl($lei->{$devfd}, F_GETFL, 0) //
- return $lei->fail("fcntl(stdout): $!");
+ die("fcntl(/dev/fd/$devfd): $!\n");
ovv_out_lk_init($self) unless ($fl & O_APPEND);
} else {
ovv_out_lk_init($self);
@@ -101,14 +100,13 @@ sub new {
if ($json) {
$lei->{dedupe} //= PublicInbox::LeiDedupe->new($lei);
} else {
- $lei->{l2m} = eval { PublicInbox::LeiToMail->new($lei) };
- return $lei->fail($@) if $@;
+ $lei->{l2m} = PublicInbox::LeiToMail->new($lei);
if ($opt->{mua} && $lei->{l2m}->lock_free) {
$lei->{early_mua} = 1;
$opt->{alert} //= [ ':WINCH,:bell' ] if -t $lei->{1};
}
}
- return $lei->fail('--shared is only for v2 inbox output') if
+ die("--shared is only for v2 inbox output\n") if
$self->{fmt} ne 'v2' && $lei->{opt}->{shared};
$self;
}
^ permalink raw reply related [relevance 59%]
* [PATCH 07/12] lei: more eval guards for die on failure
2021-10-16 1:00 71% ` [PATCH 05/12] lei: golf PATH2CFG cleanup Eric Wong
2021-10-16 1:00 63% ` [PATCH 06/12] lei: always keep cwd fd {3} for ->fchdir Eric Wong
@ 2021-10-16 1:00 55% ` Eric Wong
2021-10-16 1:01 59% ` [PATCH 09/12] lei_overview: die rather than lei->fail Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-16 1:00 UTC (permalink / raw)
To: meta
Relying on $lei->fail is unsustainable since there'll always
be parts of our code and dependencies which can trigger die()
and break the event loop.
---
lib/PublicInbox/LEI.pm | 6 +++---
lib/PublicInbox/LeiLcat.pm | 21 +++++++++------------
lib/PublicInbox/LeiQuery.pm | 24 +++++++++++-------------
lib/PublicInbox/LeiXSearch.pm | 9 +++++----
4 files changed, 28 insertions(+), 32 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0cdcf4492885..511b2c1d03a7 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -556,7 +556,7 @@ sub _lei_atfork_child {
# we need to explicitly close things which are on stack
if ($persist) {
open $self->{3}, '<', '/' or die "open(/) $!";
- fchdir($self) or die;
+ fchdir($self);
close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
if (my $cfg = $self->{cfg}) {
delete @$cfg{qw(-lei_store -watches -lei_note_event)};
@@ -779,7 +779,7 @@ sub lazy_cb ($$$) {
sub dispatch {
my ($self, $cmd, @argv) = @_;
- fchdir($self) or return;
+ fchdir($self);
local %ENV = %{$self->{env}};
local $current_lei = $self; # for __WARN__
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
@@ -1381,7 +1381,7 @@ sub wq_done_wait { # dwaitpid callback
sub fchdir {
my ($lei) = @_;
my $dh = $lei->{3} // die 'BUG: lei->{3} (CWD) gone';
- chdir($dh) || $lei->fail("fchdir: $!");
+ chdir($dh) || die "fchdir: $!";
}
sub wq_eof { # EOF callback for main daemon
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index d553b18733da..191f6f244857 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -124,18 +124,15 @@ could not extract Message-ID from $x
sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
my ($lei) = @_; # $_[1] = $rbuf
- if (defined($_[1])) {
- $_[1] eq '' and return eval {
- $lei->fchdir or return;
- my @argv = split(/\s+/, $lei->{mset_opt}->{qstr});
- $lei->{mset_opt}->{qstr} = extract_all($lei, @argv)
- or return;
- $lei->_start_query;
- };
- $lei->{mset_opt}->{qstr} .= $_[1];
- } else {
- $lei->fail("error reading stdin: $!");
- }
+ $_[1] // return $lei->fail("error reading stdin: $!");
+ return $lei->{mset_opt}->{qstr} .= $_[1] if $_[1] ne '';
+ eval {
+ $lei->fchdir;
+ my @argv = split(/\s+/, $lei->{mset_opt}->{qstr});
+ $lei->{mset_opt}->{qstr} = extract_all($lei, @argv) or return;
+ $lei->_start_query;
+ };
+ $lei->fail($@) if $@;
}
sub lei_lcat {
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index c65b00ca0986..ec2ece051492 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -55,19 +55,17 @@ sub _start_query { # used by "lei q" and "lei up"
}
sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
- my ($self) = @_; # $_[1] = $rbuf
- if (defined($_[1])) {
- $_[1] eq '' and return eval {
- $self->fchdir or return;
- $self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr};
- $self->{lse}->query_approxidate($self->{lse}->git,
- $self->{mset_opt}->{qstr});
- _start_query($self);
- };
- $self->{mset_opt}->{qstr} .= $_[1];
- } else {
- $self->fail("error reading stdin: $!");
- }
+ my ($lei) = @_; # $_[1] = $rbuf
+ $_[1] // $lei->fail("error reading stdin: $!");
+ return $lei->{mset_opt}->{qstr} .= $_[1] if $_[1] ne '';
+ eval {
+ $lei->fchdir;
+ $lei->{mset_opt}->{q_raw} = $lei->{mset_opt}->{qstr};
+ $lei->{lse}->query_approxidate($lei->{lse}->git,
+ $lei->{mset_opt}->{qstr});
+ _start_query($lei);
+ };
+ $lei->fail($@) if $@;
}
sub lxs_prepare {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 4aa2a81c0025..8ab84b15c00b 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -460,10 +460,11 @@ sub do_post_augment {
my ($lei) = @_;
local $PublicInbox::LEI::current_lei = $lei;
my $l2m = $lei->{l2m} or return; # client disconnected
- $lei->fchdir or return;
- my $err;
- eval { $l2m->post_augment($lei) };
- $err = $@;
+ eval {
+ $lei->fchdir;
+ $l2m->post_augment($lei);
+ };
+ my $err = $@;
if ($err) {
if (my $lxs = delete $lei->{lxs}) {
$lxs->wq_kill('-TERM');
^ permalink raw reply related [relevance 55%]
* [PATCH 1/2] doc: lei: restore alphabetical order to some listings
2021-10-16 5:39 70% [PATCH 0/2] doc: lei manpages for remaining commands Kyle Meyer
@ 2021-10-16 5:39 65% ` Kyle Meyer
2021-10-16 5:39 29% ` [PATCH 2/2] doc: lei: add manpages for remaining commands Kyle Meyer
1 sibling, 0 replies; 200+ results
From: Kyle Meyer @ 2021-10-16 5:39 UTC (permalink / raw)
To: meta
Most the lei-related entries in txt2pre and Makefile.PL are in
alphabetical order. Reorder the few that aren't.
While at it, reflow the Makefile.PL entries in preparation for the
entries that will be added in the next commit.
---
Documentation/txt2pre | 2 +-
Makefile.PL | 14 +++++++-------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 7bc31d23..bc94d404 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -33,8 +33,8 @@ for (qw[lei(1)
lei-q(1)
lei-rediff(1)
lei-rm(1)
- lei-store-format(5)
lei-security(7)
+ lei-store-format(5)
lei-tag(1)
lei-up(1)
public-inbox.cgi(1)
diff --git a/Makefile.PL b/Makefile.PL
index c41c408a..22a58bee 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -45,13 +45,13 @@ $v->{-m1} = [ map {
}
} @EXE_FILES,
qw(
- lei-add-external lei-blob lei-config lei-convert lei-edit-search
- lei-export-kw
- lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
- lei-import lei-index lei-init lei-lcat lei-ls-external lei-ls-label
- lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff
- lei-refresh-mail-sync
- lei-rm lei-tag
+ lei-add-external lei-blob lei-config lei-convert
+ lei-daemon-kill lei-daemon-pid lei-edit-search lei-export-kw
+ lei-forget-external lei-forget-search
+ lei-import lei-index lei-init lei-lcat
+ lei-ls-external lei-ls-label lei-ls-mail-sync
+ lei-ls-search lei-p2q lei-q
+ lei-rediff lei-refresh-mail-sync lei-rm lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
--
2.33.0
^ permalink raw reply related [relevance 65%]
* [PATCH 0/2] doc: lei manpages for remaining commands
@ 2021-10-16 5:39 70% Kyle Meyer
2021-10-16 5:39 65% ` [PATCH 1/2] doc: lei: restore alphabetical order to some listings Kyle Meyer
2021-10-16 5:39 29% ` [PATCH 2/2] doc: lei: add manpages for remaining commands Kyle Meyer
0 siblings, 2 replies; 200+ results
From: Kyle Meyer @ 2021-10-16 5:39 UTC (permalink / raw)
To: meta
This lei manpage update should take care of the remaining lei commands
(aside from -help and -sucks) that don't have manpages.
[1/2] doc: lei: restore alphabetical order to some listings
[2/2] doc: lei: add manpages for remaining commands
Documentation/lei-add-watch.pod | 33 ++++++++++++++
Documentation/lei-forget-mail-sync.pod | 31 +++++++++++++
Documentation/lei-inspect.pod | 57 ++++++++++++++++++++++++
Documentation/lei-ls-mail-source.pod | 58 +++++++++++++++++++++++++
Documentation/lei-ls-mail-sync.pod | 2 +-
Documentation/lei-ls-watch.pod | 29 +++++++++++++
Documentation/lei-mail-diff.pod | 33 ++++++++++++++
Documentation/lei-refresh-mail-sync.pod | 2 +-
Documentation/lei-rm-watch.pod | 30 +++++++++++++
Documentation/lei.pod | 14 ++++++
Documentation/txt2pre | 9 +++-
MANIFEST | 7 +++
Makefile.PL | 14 +++---
13 files changed, 309 insertions(+), 10 deletions(-)
create mode 100644 Documentation/lei-add-watch.pod
create mode 100644 Documentation/lei-forget-mail-sync.pod
create mode 100644 Documentation/lei-inspect.pod
create mode 100644 Documentation/lei-ls-mail-source.pod
create mode 100644 Documentation/lei-ls-watch.pod
create mode 100644 Documentation/lei-mail-diff.pod
create mode 100644 Documentation/lei-rm-watch.pod
base-commit: 299b40d252cf4d4db6fa29ad18cb78777f1f55fc
--
2.33.0
^ permalink raw reply [relevance 70%]
* [PATCH 2/2] doc: lei: add manpages for remaining commands
2021-10-16 5:39 70% [PATCH 0/2] doc: lei manpages for remaining commands Kyle Meyer
2021-10-16 5:39 65% ` [PATCH 1/2] doc: lei: restore alphabetical order to some listings Kyle Meyer
@ 2021-10-16 5:39 29% ` Kyle Meyer
2021-10-16 7:07 71% ` Eric Wong
1 sibling, 1 reply; 200+ results
From: Kyle Meyer @ 2021-10-16 5:39 UTC (permalink / raw)
To: meta
At this point all of the current lei commands, aside from -help and
-sucks, should be covered.
---
Documentation/lei-add-watch.pod | 33 ++++++++++++++
Documentation/lei-forget-mail-sync.pod | 31 +++++++++++++
Documentation/lei-inspect.pod | 57 ++++++++++++++++++++++++
Documentation/lei-ls-mail-source.pod | 58 +++++++++++++++++++++++++
Documentation/lei-ls-mail-sync.pod | 2 +-
Documentation/lei-ls-watch.pod | 29 +++++++++++++
Documentation/lei-mail-diff.pod | 33 ++++++++++++++
Documentation/lei-refresh-mail-sync.pod | 2 +-
Documentation/lei-rm-watch.pod | 30 +++++++++++++
Documentation/lei.pod | 14 ++++++
Documentation/txt2pre | 7 +++
MANIFEST | 7 +++
Makefile.PL | 12 ++---
13 files changed, 307 insertions(+), 8 deletions(-)
create mode 100644 Documentation/lei-add-watch.pod
create mode 100644 Documentation/lei-forget-mail-sync.pod
create mode 100644 Documentation/lei-inspect.pod
create mode 100644 Documentation/lei-ls-mail-source.pod
create mode 100644 Documentation/lei-ls-watch.pod
create mode 100644 Documentation/lei-mail-diff.pod
create mode 100644 Documentation/lei-rm-watch.pod
diff --git a/Documentation/lei-add-watch.pod b/Documentation/lei-add-watch.pod
new file mode 100644
index 00000000..60984618
--- /dev/null
+++ b/Documentation/lei-add-watch.pod
@@ -0,0 +1,33 @@
+=head1 NAME
+
+lei-add-watch - watch for new messages and flag changes
+
+=head1 SYNOPSIS
+
+lei add-watch [OPTIONS] LOCATION [LOCATION...]
+
+=head1 DESCRIPTION
+
+Tell lei to watch C<LOCATION> for new messages and flag changes.
+Currently only Maildir locations are supported.
+
+=for comment
+TODO: Document --state? Believe valid values are pause, import-ro,
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-ls-watch(1)>, L<lei-rm-watch(1)>
diff --git a/Documentation/lei-forget-mail-sync.pod b/Documentation/lei-forget-mail-sync.pod
new file mode 100644
index 00000000..e70b4d33
--- /dev/null
+++ b/Documentation/lei-forget-mail-sync.pod
@@ -0,0 +1,31 @@
+=head1 NAME
+
+lei-forget-mail-sync - forget sync information for a mail folder
+
+=head1 SYNOPSIS
+
+lei forget-mail-sync [OPTIONS] LOCATION [LOCATION...]
+
+=head1 DESCRIPTION
+
+Forget synchronization information for C<LOCATION>, an IMAP or Maildir
+folder. Note that this won't delete any messages stored in Git,
+leaving C<lei-index(1)> users with dangling references.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-ls-mail-sync(1)>, L<lei-index(1)>
diff --git a/Documentation/lei-inspect.pod b/Documentation/lei-inspect.pod
new file mode 100644
index 00000000..19dd8ab5
--- /dev/null
+++ b/Documentation/lei-inspect.pod
@@ -0,0 +1,57 @@
+=head1 NAME
+
+lei-inspect - general purpose inspector
+
+=head1 SYNOPSIS
+
+lei inspect [OPTIONS] ITEM [ITEM...]
+
+lei inspect [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+This is a diagnostic command that provides a general purpose inspector
+of various things, including blobs, message IDs, Xapian document IDs,
+and mail sync sources.
+
+=head1 OPTIONS
+
+=over
+
+=item -d DIR
+
+=item --dir=DIR
+
+An inboxdir, extindex topdir, or Xapian shard
+
+=item --pretty
+
+Pretty print output. If stdout is opened to a tty, C<--pretty> is
+enabled by default.
+
+=item -
+
+=item --stdin
+
+Read message from stdin. This is implicit if no arguments are given
+and stdin is a pipe or regular file.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-mail-diff(1)>
diff --git a/Documentation/lei-ls-mail-source.pod b/Documentation/lei-ls-mail-source.pod
new file mode 100644
index 00000000..926bbe2c
--- /dev/null
+++ b/Documentation/lei-ls-mail-source.pod
@@ -0,0 +1,58 @@
+=head1 NAME
+
+lei-ls-mail-source - list IMAP or NNTP mail source folders
+
+=head1 SYNOPSIS
+
+lei ls-mail-source [OPTIONS] URL
+
+=head1 DESCRIPTION
+
+List information about the IMAP or NNTP mail source at C<URL>.
+
+=head1 OPTIONS
+
+=over
+
+=item -z
+
+=item -0
+
+Use C<\0> (NUL) instead of newline (CR) to delimit lines.
+
+=item -l
+
+Format output as JSON and include more information.
+
+=item --pretty
+
+Pretty print JSON output. If stdout is opened to a tty, C<--pretty>
+is enabled by default.
+
+=item --ascii
+
+Escape non-ASCII characters.
+
+=item --url
+
+Show full URL of newsgroup or IMAP folder.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-import(1)>
diff --git a/Documentation/lei-ls-mail-sync.pod b/Documentation/lei-ls-mail-sync.pod
index 86aede40..883eeead 100644
--- a/Documentation/lei-ls-mail-sync.pod
+++ b/Documentation/lei-ls-mail-sync.pod
@@ -52,4 +52,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-q(1)>, L<lei-up(1)>
+L<lei-refresh-mail-sync(1)>, L<lei-export-kw(1)>
diff --git a/Documentation/lei-ls-watch.pod b/Documentation/lei-ls-watch.pod
new file mode 100644
index 00000000..b1681ee4
--- /dev/null
+++ b/Documentation/lei-ls-watch.pod
@@ -0,0 +1,29 @@
+=head1 NAME
+
+lei-ls-watch - list active watches
+
+=head1 SYNOPSIS
+
+lei ls-watch
+
+=head1 DESCRIPTION
+
+List locations that lei is configured to watch.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-add-watch(1)>, L<lei-rm-watch(1)>
diff --git a/Documentation/lei-mail-diff.pod b/Documentation/lei-mail-diff.pod
new file mode 100644
index 00000000..96e49a8b
--- /dev/null
+++ b/Documentation/lei-mail-diff.pod
@@ -0,0 +1,33 @@
+=head1 NAME
+
+lei-mail-diff - diff the contents of emails
+
+=head1 SYNOPSIS
+
+
+lei mail-diff [OPTIONS] LOCATION
+
+lei mail-diff [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+This is a diagnostic command that's useful for finding deduplication
+bugs.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-inspect(1)>
diff --git a/Documentation/lei-refresh-mail-sync.pod b/Documentation/lei-refresh-mail-sync.pod
index 92ca9044..65150ae3 100644
--- a/Documentation/lei-refresh-mail-sync.pod
+++ b/Documentation/lei-refresh-mail-sync.pod
@@ -54,4 +54,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-index(1)>, L<lei-export-kw(1)>
+L<lei-index(1)>, L<lei-export-kw(1)>, L<lei-ls-mail-sync(1)>
diff --git a/Documentation/lei-rm-watch.pod b/Documentation/lei-rm-watch.pod
new file mode 100644
index 00000000..711d7dc4
--- /dev/null
+++ b/Documentation/lei-rm-watch.pod
@@ -0,0 +1,30 @@
+=head1 NAME
+
+lei-rm-watch - stop watching locations
+
+=head1 SYNOPSIS
+
+lei rm-watch [OPTIONS] LOCATION [LOCATION...]
+
+=head1 DESCRIPTION
+
+Tell lei to stop watching C<LOCATION> for new messages and flag
+changes. Currently only Maildir locations are supported.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-add-watch(1)>, L<lei-ls-watch(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 63d5ee69..24a585da 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -96,6 +96,8 @@ Other subcommands include
=over
+=item * L<lei-add-watch(1)>
+
=item * L<lei-config(1)>
=item * L<lei-convert(1)>
@@ -104,10 +106,22 @@ Other subcommands include
=item * L<lei-daemon-pid(1)>
+=item * lei-forget-mail-sync(1)
+
+=item * L<lei-mail-diff(1)>
+
+=item * L<lei-inspect(1)>
+
=item * L<lei-ls-label(1)>
+=item * L<lei-ls-mail-source(1)>
+
=item * L<lei-ls-mail-sync(1)>
+=item * L<lei-ls-watch(1)>
+
+=item * L<lei-rm-watch(1)>
+
=back
=head1 FILES
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index bc94d404..fb07579a 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -12,6 +12,7 @@ use PublicInbox::Hval qw(ascii_html);
my %xurls;
for (qw[lei(1)
lei-add-external(1)
+ lei-add-watch(1)
lei-blob(1)
lei-config(1)
lei-convert(1)
@@ -19,20 +20,26 @@ for (qw[lei(1)
lei-daemon-pid(1)
lei-edit-search(1)
lei-forget-external(1)
+ lei-forget-mail-sync(1)
lei-forget-search(1)
lei-import(1)
lei-index(1)
lei-init(1)
+ lei-inspect(1)
lei-lcat(1)
lei-ls-external(1)
lei-ls-label(1)
+ lei-ls-mail-source(1)
lei-ls-mail-sync(1)
lei-ls-search(1)
+ lei-ls-watch(1)
+ lei-mail-diff(1)
lei-overview(7)
lei-p2q(1)
lei-q(1)
lei-rediff(1)
lei-rm(1)
+ lei-rm-watch(1)
lei-security(7)
lei-store-format(5)
lei-tag(1)
diff --git a/MANIFEST b/MANIFEST
index b89513d5..c4cc6e33 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -22,6 +22,7 @@ Documentation/flow.txt
Documentation/hosted.txt
Documentation/include.mk
Documentation/lei-add-external.pod
+Documentation/lei-add-watch.pod
Documentation/lei-blob.pod
Documentation/lei-config.pod
Documentation/lei-convert.pod
@@ -31,15 +32,20 @@ Documentation/lei-daemon.pod
Documentation/lei-edit-search.pod
Documentation/lei-export-kw.pod
Documentation/lei-forget-external.pod
+Documentation/lei-forget-mail-sync.pod
Documentation/lei-forget-search.pod
Documentation/lei-import.pod
Documentation/lei-index.pod
Documentation/lei-init.pod
+Documentation/lei-inspect.pod
Documentation/lei-lcat.pod
Documentation/lei-ls-external.pod
Documentation/lei-ls-label.pod
+Documentation/lei-ls-mail-source.pod
Documentation/lei-ls-mail-sync.pod
Documentation/lei-ls-search.pod
+Documentation/lei-ls-watch.pod
+Documentation/lei-mail-diff.pod
Documentation/lei-mail-formats.pod
Documentation/lei-overview.pod
Documentation/lei-p2q.pod
@@ -47,6 +53,7 @@ Documentation/lei-q.pod
Documentation/lei-rediff.pod
Documentation/lei-refresh-mail-sync.pod
Documentation/lei-rm.pod
+Documentation/lei-rm-watch.pod
Documentation/lei-security.pod
Documentation/lei-store-format.pod
Documentation/lei-tag.pod
diff --git a/Makefile.PL b/Makefile.PL
index 22a58bee..348a343d 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -45,13 +45,13 @@ $v->{-m1} = [ map {
}
} @EXE_FILES,
qw(
- lei-add-external lei-blob lei-config lei-convert
+ lei-add-external lei-add-watch lei-blob lei-config lei-convert
lei-daemon-kill lei-daemon-pid lei-edit-search lei-export-kw
- lei-forget-external lei-forget-search
- lei-import lei-index lei-init lei-lcat
- lei-ls-external lei-ls-label lei-ls-mail-sync
- lei-ls-search lei-p2q lei-q
- lei-rediff lei-refresh-mail-sync lei-rm lei-tag
+ lei-forget-external lei-forget-mail-sync lei-forget-search
+ lei-import lei-index lei-init lei-inspect lei-lcat
+ lei-ls-external lei-ls-label lei-ls-mail-source lei-ls-mail-sync
+ lei-ls-search lei-ls-watch lei-mail-diff lei-p2q lei-q
+ lei-rediff lei-refresh-mail-sync lei-rm lei-rm-watch lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
--
2.33.0
^ permalink raw reply related [relevance 29%]
* Re: [PATCH 2/2] doc: lei: add manpages for remaining commands
2021-10-16 5:39 29% ` [PATCH 2/2] doc: lei: add manpages for remaining commands Kyle Meyer
@ 2021-10-16 7:07 71% ` Eric Wong
2021-10-16 15:13 67% ` Kyle Meyer
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-16 7:07 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> At this point all of the current lei commands, aside from -help and
> -sucks, should be covered.
Thanks, pushed as commit 9d72cc3f876e3d2bd1ecb2fc0f33c43a9a72b933
> diff --git a/MANIFEST b/MANIFEST
> index b89513d5..c4cc6e33 100644
> --- a/MANIFEST
> +++ b/MANIFEST
> @@ -47,6 +53,7 @@ Documentation/lei-q.pod
> Documentation/lei-rediff.pod
> Documentation/lei-refresh-mail-sync.pod
> Documentation/lei-rm.pod
> +Documentation/lei-rm-watch.pod
> Documentation/lei-security.pod
> Documentation/lei-store-format.pod
> Documentation/lei-tag.pod
Curious, was that from `git ls-files >MANIFEST' ?
Because that puts lei-rm.pod after lei-rm-watch.pod on different
systems I tested. I don't see git using LC_COLLATE anywhere,
but locale(1) gives:
FreeBSD:
LANG=
LC_CTYPE="C"
LC_COLLATE="C"
LC_TIME="C"
LC_NUMERIC="C"
LC_MONETARY="C"
LC_MESSAGES="C"
LC_ALL=
Debian:
LANG=en_US.UTF-8
LANGUAGE=
LC_CTYPE="en_US.UTF-8"
LC_NUMERIC="en_US.UTF-8"
LC_TIME="en_US.UTF-8"
LC_COLLATE="en_US.UTF-8"
LC_MONETARY="en_US.UTF-8"
LC_MESSAGES="en_US.UTF-8"
LC_PAPER="en_US.UTF-8"
LC_NAME="en_US.UTF-8"
LC_ADDRESS="en_US.UTF-8"
LC_TELEPHONE="en_US.UTF-8"
LC_MEASUREMENT="en_US.UTF-8"
LC_IDENTIFICATION="en_US.UTF-8"
LC_ALL=
No big deal, I can just flip and push it; I just don't want a
reproducibility issue popping up.
^ permalink raw reply [relevance 71%]
* [PATCH] t/lei*: set EDITOR for dumb terminals
@ 2021-10-16 7:54 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-16 7:54 UTC (permalink / raw)
To: meta
Running tests over a non-interactive ssh session fails,
otherwise.
---
t/lei-q-remote-import.t | 2 +-
t/lei.t | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index fdf6a11e..92d8c9b6 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -99,7 +99,7 @@ EOF
lei_ok('up', "$ENV{HOME}/md");
is_deeply(\@f, [ glob("$ENV{HOME}/md/*/*") ],
'lei up remote dedupe works on maildir');
- my $edit_env = { VISUAL => 'cat' };
+ my $edit_env = { VISUAL => 'cat', EDITOR => 'cat' };
lei_ok([qw(edit-search), "$ENV{HOME}/md"], $edit_env);
like($lei_out, qr/^\Q[external "$url"]\E\n\s*lastresult = \d+/sm,
'lastresult set');
diff --git a/t/lei.t b/t/lei.t
index 53fc43fb..f7de1b71 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -101,7 +101,7 @@ my $test_config = sub {
lei_ok(qw(-c imap.debug=a -c imap.debug=b config --get-all imap.debug));
is($lei_out, "a\nb\n", '-c and --get-all work together');
- lei_ok([qw(config -e)], { VISUAL => 'cat' });
+ lei_ok([qw(config -e)], { VISUAL => 'cat', EDITOR => 'cat' });
is($lei_out, "[a]\n\tb = c\n", '--edit works');
};
^ permalink raw reply related [relevance 70%]
* [PATCH 0/4] lei: prioritize signals
@ 2021-10-16 9:29 71% Eric Wong
2021-10-16 9:29 57% ` [PATCH 4/4] lei sockets: favor level-triggered epoll for fairness Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-16 9:29 UTC (permalink / raw)
To: meta
We drop Edge-Triggered kevent/epoll in nearly all places
to give signalfd / EVFILT_SIGNAL priority.
Eric Wong (4):
wqworker: favor level-triggered epoll for fairness
pkt_op: favor level-triggered epoll for fairness
input_pipe: do not loop in ->event_step for fairness
lei sockets: favor level-triggered epoll for fairness
lib/PublicInbox/InputPipe.pm | 17 +++++-----
lib/PublicInbox/LEI.pm | 20 +++++-------
lib/PublicInbox/LeiSelfSocket.pm | 27 +++++++---------
lib/PublicInbox/PktOp.pm | 53 +++++++++++++++-----------------
lib/PublicInbox/WQWorker.pm | 18 +++++------
5 files changed, 60 insertions(+), 75 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 4/4] lei sockets: favor level-triggered epoll for fairness
2021-10-16 9:29 71% [PATCH 0/4] lei: prioritize signals Eric Wong
@ 2021-10-16 9:29 57% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-16 9:29 UTC (permalink / raw)
To: meta
Sigfd->event_step needs priority over script/lei clients,
LeiSelfSocket, and everything else.
---
lib/PublicInbox/LEI.pm | 20 ++++++++------------
lib/PublicInbox/LeiSelfSocket.pm | 27 +++++++++++----------------
2 files changed, 19 insertions(+), 28 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 876598f9530e..6b989b33647e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -12,13 +12,13 @@ use parent qw(PublicInbox::DS PublicInbox::LeiExternal
PublicInbox::LeiQuery);
use Getopt::Long ();
use Socket qw(AF_UNIX SOCK_SEQPACKET MSG_EOR pack_sockaddr_un);
-use Errno qw(EPIPE EAGAIN EINTR ECONNREFUSED ENOENT ECONNRESET);
+use Errno qw(EPIPE EAGAIN ECONNREFUSED ENOENT ECONNRESET);
use Cwd qw(getcwd);
use POSIX qw(strftime);
use IO::Handle ();
use Fcntl qw(SEEK_SET);
use PublicInbox::Config;
-use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use PublicInbox::Syscall qw(EPOLLIN);
use PublicInbox::DS qw(now dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
@@ -1125,16 +1125,12 @@ sub event_step {
local %ENV = %{$self->{env}};
local $current_lei = $self;
eval {
- my $buf;
- while (my @fds = $recv_cmd->($self->{sock}, $buf, 4096)) {
- if (scalar(@fds) == 1 && !defined($fds[0])) {
- return if $! == EAGAIN;
- next if $! == EINTR;
- last if $! == ECONNRESET;
- die "recvmsg: $!";
- }
- for (@fds) { open my $rfh, '+<&=', $_ }
+ my @fds = $recv_cmd->($self->{sock}, my $buf, 4096);
+ if (scalar(@fds) == 1 && !defined($fds[0])) {
+ return if $! == EAGAIN;
+ die "recvmsg: $!" if $! != ECONNRESET;
}
+ for (@fds) { open my $rfh, '+<&=', $_ }
if ($buf eq '') {
_drop_wq($self); # EOF, client disconnected
dclose($self);
@@ -1162,7 +1158,7 @@ sub event_step_init {
my $sock = $self->{sock} or return;
$self->{-event_init_done} //= do { # persist til $ops done
$sock->blocking(0);
- $self->SUPER::new($sock, EPOLLIN|EPOLLET);
+ $self->SUPER::new($sock, EPOLLIN);
$sock;
};
}
diff --git a/lib/PublicInbox/LeiSelfSocket.pm b/lib/PublicInbox/LeiSelfSocket.pm
index 3d847649e3b1..dd64b6cfd491 100644
--- a/lib/PublicInbox/LeiSelfSocket.pm
+++ b/lib/PublicInbox/LeiSelfSocket.pm
@@ -10,7 +10,7 @@ use v5.10.1;
use parent qw(PublicInbox::DS);
use Data::Dumper;
$Data::Dumper::Useqq = 1; # should've been the Perl default :P
-use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use PublicInbox::Syscall qw(EPOLLIN);
use PublicInbox::Spawn;
my $recv_cmd;
@@ -20,26 +20,21 @@ sub new {
$r->blocking(0);
no warnings 'once';
$recv_cmd = $PublicInbox::LEI::recv_cmd;
- $self->SUPER::new($r, EPOLLIN|EPOLLET);
+ $self->SUPER::new($r, EPOLLIN);
}
sub event_step {
my ($self) = @_;
- while (1) {
- my (@fds) = $recv_cmd->($self->{sock}, my $buf, 4096 * 33);
- if (scalar(@fds) == 1 && !defined($fds[0])) {
- return if $!{EAGAIN};
- next if $!{EINTR};
- die "recvmsg: $!";
- }
- # open so perl can auto-close them:
- for my $fd (@fds) {
- open(my $newfh, '+<&=', $fd) or die "open +<&=$fd: $!";
- }
- return $self->close if $buf eq '';
- warn Dumper({ 'unexpected self msg' => $buf, fds => \@fds });
- # TODO: figure out what to do with these messages...
+ my (@fds) = $recv_cmd->($self->{sock}, my $buf, 4096 * 33);
+ if (scalar(@fds) == 1 && !defined($fds[0])) {
+ return if $!{EAGAIN};
+ die "recvmsg: $!" unless $!{ECONNRESET};
+ } else { # just in case open so perl can auto-close them:
+ for (@fds) { open my $fh, '+<&=', $_ };
}
+ return $self->close if $buf eq '';
+ warn Dumper({ 'unexpected self msg' => $buf, fds => \@fds });
+ # TODO: figure out what to do with these messages...
}
1;
^ permalink raw reply related [relevance 57%]
* Re: [PATCH 2/2] doc: lei: add manpages for remaining commands
2021-10-16 7:07 71% ` Eric Wong
@ 2021-10-16 15:13 67% ` Kyle Meyer
2021-10-16 16:58 71% ` Kyle Meyer
2021-10-16 17:03 71% ` Eric Wong
0 siblings, 2 replies; 200+ results
From: Kyle Meyer @ 2021-10-16 15:13 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
>> @@ -47,6 +53,7 @@ Documentation/lei-q.pod
>> Documentation/lei-rediff.pod
>> Documentation/lei-refresh-mail-sync.pod
>> Documentation/lei-rm.pod
>> +Documentation/lei-rm-watch.pod
>> Documentation/lei-security.pod
>> Documentation/lei-store-format.pod
>> Documentation/lei-tag.pod
>
> Curious, was that from `git ls-files >MANIFEST' ?
No, it wasn't. I hadn't realized that the content of MANIFEST
corresponded directly to git-ls-files output, but poking around I see
there is even a check-manifest make target [*]. Doh.
So, I manually added this one, like the others I added. I think my past
changes to this file have ended up in the expected order because I tend
to run Emacs's sort-lines to check my manual placement, but I must not
have done it here because sort-lines agrees with ls-files.
[...]
> No big deal, I can just flip and push it; I just don't want a
> reproducibility issue popping up.
Thanks, sorry about the mix up.
[*] I did run `make check' before sending, and it looks like that also
checks MANIFEST _after_ running the test suite. That didn't help me
catch the MANIFEST sorting issue in this case because the test suite
is failing on my end.
I believe these failures are recent and was planning on looking into
them today, either sending a patch or just reporting, depending on
whether I could figure out a fix. Anyway, here they are:
t/psgi_multipart_not.t .......
ok 1 - use HTTP::Request::Common;
ok 2 - use Plack::Test;
ok 3 - use PublicInbox::WWW;
not ok 4 - /v2test/?q=%22ain't what it used to be%22&x=t
not ok 5 - /v2test/?q=%22ain't what it used to be%22&x=t warns
Failed 2/5 subtests
^ permalink raw reply [relevance 67%]
* Re: [PATCH 2/2] doc: lei: add manpages for remaining commands
2021-10-16 15:13 67% ` Kyle Meyer
@ 2021-10-16 16:58 71% ` Kyle Meyer
2021-10-16 17:03 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Kyle Meyer @ 2021-10-16 16:58 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Kyle Meyer writes:
> t/psgi_multipart_not.t .......
> ok 1 - use HTTP::Request::Common;
> ok 2 - use Plack::Test;
> ok 3 - use PublicInbox::WWW;
> not ok 4 - /v2test/?q=%22ain't what it used to be%22&x=t
> not ok 5 - /v2test/?q=%22ain't what it used to be%22&x=t warns
> Failed 2/5 subtests
Hmm, trying a few times on 9ee1798e (2021-10-16), I'm not having any
luck triggering it again. All tests are now passing for me.
^ permalink raw reply [relevance 71%]
* Re: [PATCH 2/2] doc: lei: add manpages for remaining commands
2021-10-16 15:13 67% ` Kyle Meyer
2021-10-16 16:58 71% ` Kyle Meyer
@ 2021-10-16 17:03 71% ` Eric Wong
2021-10-16 17:21 71% ` Kyle Meyer
1 sibling, 1 reply; 200+ results
From: Eric Wong @ 2021-10-16 17:03 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> Thanks, sorry about the mix up.
No worries, will push a regen.
> [*] I did run `make check' before sending, and it looks like that also
> checks MANIFEST _after_ running the test suite. That didn't help me
> catch the MANIFEST sorting issue in this case because the test suite
> is failing on my end.
>
> I believe these failures are recent and was planning on looking into
> them today, either sending a patch or just reporting, depending on
> whether I could figure out a fix. Anyway, here they are:
>
> t/psgi_multipart_not.t .......
> ok 1 - use HTTP::Request::Common;
> ok 2 - use Plack::Test;
> ok 3 - use PublicInbox::WWW;
> not ok 4 - /v2test/?q=%22ain't what it used to be%22&x=t
> not ok 5 - /v2test/?q=%22ain't what it used to be%22&x=t warns
> Failed 2/5 subtests
The failure is probably caused by
00d5dff2cce9d2c9 (eml: avoid Encode 2.87..3.12 leak, 2021-10-13)
but I can't reproduce it across CentOS 7.x, FreeBSD 11.x, nor
Debian 10 & 11.
Which versions of Encode and Perl are you using?
At least in Debian, libencode-perl is available as a separate package
but it's also part of libperl5.xx (possibly w/ a different version);
only the latter is required for us, but two packages offering
the same thing gets confusing :/
I use: perl -MEncode -E 'say $Encode::VERSION'
to determine which gets loaded.
^ permalink raw reply [relevance 71%]
* Re: [PATCH 2/2] doc: lei: add manpages for remaining commands
2021-10-16 17:03 71% ` Eric Wong
@ 2021-10-16 17:21 71% ` Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-10-16 17:21 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> The failure is probably caused by
> 00d5dff2cce9d2c9 (eml: avoid Encode 2.87..3.12 leak, 2021-10-13)
> but I can't reproduce it across CentOS 7.x, FreeBSD 11.x, nor
> Debian 10 & 11.
Hmm, yeah, as mentioned in my other reply, I'm now not having any luck
triggering this either.
> Which versions of Encode and Perl are you using?
Perl v5.32.1 from Debian 11 and...
> At least in Debian, libencode-perl is available as a separate package
> but it's also part of libperl5.xx (possibly w/ a different version);
> only the latter is required for us, but two packages offering
> the same thing gets confusing :/
>
> I use: perl -MEncode -E 'say $Encode::VERSION'
> to determine which gets loaded.
... it looks like I'm using the on that's ships with libperl5:
$ perl -MEncode -E 'say $Encode::VERSION'
3.06
$ apt-cache policy libencode-perl
libencode-perl:
Installed: (none)
Candidate: 3.08-1+deb11u1
Version table:
3.08-1+deb11u1 500
500 http://ftp.us.debian.org/debian bullseye/main amd64 Packages
500 http://security.debian.org/debian-security bullseye-security/main amd64 Packages
^ permalink raw reply [relevance 71%]
* [PATCH 04/11] lei up: propagate redispatch_all failure via exit code
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
2021-10-19 9:33 65% ` [PATCH 02/11] lei up: prefix `remote' and `local' with `o_' Eric Wong
2021-10-19 9:33 50% ` [PATCH 03/11] lei: use die for external and query handling Eric Wong
@ 2021-10-19 9:33 71% ` Eric Wong
2021-10-19 9:33 70% ` [PATCH 05/11] lei: conditionally add "\n" to error messages Eric Wong
` (5 subsequent siblings)
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
We can still continue with some local externals, maybe;
but the error needs to be propagated to the calling process
for scripting purposes.
---
lib/PublicInbox/LeiUp.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 4fd0290c7612..fcdd535dc118 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -124,7 +124,7 @@ sub net_merge_all_done {
$lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
$self->wq_close;
eval { redispatch_all($self, $lei) };
- warn "E: $@" if $@;
+ $lei->child_error(0, "E: $@") if $@;
}
sub _complete_up { # lei__complete hook
^ permalink raw reply related [relevance 71%]
* [PATCH 00/11] refining lei up+inspect
@ 2021-10-19 9:33 70% Eric Wong
2021-10-19 9:33 65% ` [PATCH 02/11] lei up: prefix `remote' and `local' with `o_' Eric Wong
` (8 more replies)
0 siblings, 9 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
"lei up" gains some flexibility in dealing with offline
situations. "inspect" is slightly nicer-to-use
11/11 is preparation for "lei fsck"...
Eric Wong (11):
test_common: lazy-require AutoReap
lei up: prefix `remote' and `local' with `o_'
lei: use die for external and query handling
lei up: propagate redispatch_all failure via exit code
lei: conditionally add "\n" to error messages
lei up: support --exclude=, --no-(external|remote|local)
lei: remove unused ->busy time arg
doc: lei: describe lei-daemon-kill and upgrades
lei inspect: add atfork hook
lei inspect: show ISO8601 {rt} and {dt}, too
lei_mail_sync: show non-matching SHA
Documentation/lei-daemon-kill.pod | 29 +++++++++++++++++--
Documentation/lei-overview.pod | 8 +++++-
Documentation/lei-up.pod | 28 ++++++++++++++----
lib/PublicInbox/LEI.pm | 15 +++++-----
lib/PublicInbox/LeiExternal.pm | 15 ++++++++--
lib/PublicInbox/LeiInspect.pm | 11 +++++++
lib/PublicInbox/LeiMailSync.pm | 9 ++++--
lib/PublicInbox/LeiOverview.pm | 6 ++--
lib/PublicInbox/LeiQuery.pm | 27 ++++++-----------
lib/PublicInbox/LeiSavedSearch.pm | 8 +++---
lib/PublicInbox/LeiUp.pm | 48 ++++++++++++++++++++++---------
lib/PublicInbox/LeiXSearch.pm | 7 +++--
lib/PublicInbox/TestCommon.pm | 3 +-
13 files changed, 149 insertions(+), 65 deletions(-)
^ permalink raw reply [relevance 70%]
* [PATCH 05/11] lei: conditionally add "\n" to error messages
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
` (2 preceding siblings ...)
2021-10-19 9:33 71% ` [PATCH 04/11] lei up: propagate redispatch_all failure via exit code Eric Wong
@ 2021-10-19 9:33 70% ` Eric Wong
2021-10-19 9:33 47% ` [PATCH 06/11] lei up: support --exclude=, --no-(external|remote|local) Eric Wong
` (4 subsequent siblings)
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
Some error messages already include "\n" (w/ file+line info),
so don't add another one. (`warn' will automatically add its
caller location unless there's a final "\n").
---
lib/PublicInbox/LEI.pm | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6b989b33647e..553379e404fc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -510,10 +510,10 @@ sub sigpipe_handler { # handles SIGPIPE from @WQ_KEYS workers
}
sub fail ($$;$) {
- my ($self, $buf, $exit_code) = @_;
+ my ($self, $msg, $exit_code) = @_;
local $current_lei = $self;
$self->{failed}++;
- warn($buf, "\n") if defined $buf;
+ warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
$self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
x_it($self, ($exit_code // 1) << 8);
undef;
@@ -534,7 +534,7 @@ sub child_error { # passes non-fatal curl exit codes to user
my ($self, $child_error, $msg) = @_; # child_error is $?
local $current_lei = $self;
$child_error ||= 1 << 8;
- warn($msg, "\n") if defined $msg;
+ warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('child_error', $child_error);
} elsif ($self->{sock}) { # to lei(1) client
^ permalink raw reply related [relevance 70%]
* [PATCH 02/11] lei up: prefix `remote' and `local' with `o_'
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
@ 2021-10-19 9:33 65% ` Eric Wong
2021-10-19 9:33 50% ` [PATCH 03/11] lei: use die for external and query handling Eric Wong
` (7 subsequent siblings)
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
This will help distinguish between mail outputs and external
public-inboxes.
---
lib/PublicInbox/LeiUp.pm | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 396041771ff9..c35b2e42d49f 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -46,7 +46,7 @@ sub up1 ($$) {
sub redispatch_all ($$) {
my ($self, $lei) = @_;
- my $upq = [ (@{$self->{local} // []}, @{$self->{remote} // []}) ];
+ my $upq = [ (@{$self->{o_local} // []}, @{$self->{o_remote} // []}) ];
return up1($lei, $upq->[0]) if @$upq == 1; # just one, may start MUA
# FIXME: this is also used per-query, see lei->_start_query
@@ -81,12 +81,12 @@ sub lei_up {
$lei->fail('--all and --mua= are incompatible');
@outs = PublicInbox::LeiSavedSearch::list($lei);
if ($all eq 'local') {
- $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ $self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
} elsif ($all eq 'remote') {
- $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
} elsif ($all eq '') {
- $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
- $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ $self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
} else {
$lei->fail("only --all=$all not understood");
}
@@ -94,16 +94,16 @@ sub lei_up {
scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
return up1($lei, $outs[0]);
} else {
- $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
- $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ $self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
$lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
- ((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
+ ((@{$self->{o_local} // []} + @{$self->{o_remote} // []}) > 1 &&
defined($opt->{mua})) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
- if ($self->{remote}) { # setup lei->{auth}
- $self->prepare_inputs($lei, $self->{remote}) or return;
+ if ($self->{o_remote}) { # setup lei->{auth}
+ $self->prepare_inputs($lei, $self->{o_remote}) or return;
}
if ($lei->{auth}) { # start auth worker
require PublicInbox::NetWriter;
^ permalink raw reply related [relevance 65%]
* [PATCH 03/11] lei: use die for external and query handling
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
2021-10-19 9:33 65% ` [PATCH 02/11] lei up: prefix `remote' and `local' with `o_' Eric Wong
@ 2021-10-19 9:33 50% ` Eric Wong
2021-10-19 9:33 71% ` [PATCH 04/11] lei up: propagate redispatch_all failure via exit code Eric Wong
` (6 subsequent siblings)
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
This allows "lei up" to continue processing unrelated externals
if on output fails.
---
lib/PublicInbox/LeiExternal.pm | 4 ++--
lib/PublicInbox/LeiQuery.pm | 12 +++++-------
lib/PublicInbox/LeiSavedSearch.pm | 8 ++++----
lib/PublicInbox/LeiUp.pm | 4 ++--
lib/PublicInbox/LeiXSearch.pm | 7 ++++---
5 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 701d1ad53adf..851715d7099c 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -101,9 +101,9 @@ sub get_externals {
return (ext_canonicalize($loc));
}
if (scalar(@m) == 0) {
- $self->fail("`$loc' is unknown");
+ die "`$loc' is unknown\n";
} else {
- $self->fail("`$loc' is ambiguous:\n", map { "\t$_\n" } @m);
+ die("`$loc' is ambiguous:\n", map { "\t$_\n" } @m, "\n");
}
();
}
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index ec2ece051492..56b82acc8b5b 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -18,17 +18,15 @@ sub _start_query { # used by "lei q" and "lei up"
PublicInbox::LeiOverview->new($self) or return;
my $opt = $self->{opt};
my ($xj, $mj) = split(/,/, $opt->{jobs} // '');
- if (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) {
- return $self->fail("`$xj' search jobs must be >= 1");
- }
+ (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) and
+ die "`$xj' search jobs must be >= 1\n";
my $lxs = $self->{lxs};
$xj ||= $lxs->concurrency($opt); # allow: "--jobs ,$WRITER_ONLY"
my $nproc = $lxs->detect_nproc || 1; # don't memoize, schedtool(1) exists
$xj = $nproc if $xj > $nproc;
$lxs->{-wq_nr_workers} = $xj;
- if (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) {
- return $self->fail("`$mj' writer jobs must be >= 1");
- }
+ (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) and
+ die "`$mj' writer jobs must be >= 1\n";
my $l2m = $self->{l2m};
# we use \1 (a ref) to distinguish between default vs. user-supplied
if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
@@ -112,7 +110,7 @@ sub lxs_prepare {
}
}
($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) :
- $self->fail('no local or remote inboxes to search');
+ die("no local or remote inboxes to search\n");
}
# the main "lei q SEARCH_TERMS" method
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 3e10f780ad02..b2f1ad1027d3 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -96,7 +96,7 @@ sub translate_dedupe ($$) {
my $dd = $lei->{opt}->{dedupe} // 'content';
return 1 if $dd eq 'content'; # the default
return $self->{"-dedupe_$dd"} = 1 if ($dd eq 'oid' || $dd eq 'mid');
- $lei->fail("--dedupe=$dd requires --no-save");
+ die("--dedupe=$dd requires --no-save\n");
}
sub up { # updating existing saved search via "lei up"
@@ -105,9 +105,9 @@ sub up { # updating existing saved search via "lei up"
my $self = bless { ale => $lei->ale }, $cls;
my $dir = $dst;
output2lssdir($self, $lei, \$dir, \$f) or
- return $lei->fail("--no-save was used with $dst cwd=".
- $lei->rel2abs('.'));
- $self->{-cfg} = $lei->cfg_dump($f) // return $lei->fail;
+ return die("--no-save was used with $dst cwd=".
+ $lei->rel2abs('.')."\n");
+ $self->{-cfg} = $lei->cfg_dump($f) // return $lei->child_error;
$self->{-ovf} = "$dir/over.sqlite3";
$self->{'-f'} = $f;
$self->{lock_path} = "$self->{-f}.flock";
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index c35b2e42d49f..4fd0290c7612 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -19,7 +19,7 @@ sub up1 ($$) {
my $f = $lss->{'-f'};
my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
my $q = $mset_opt->{q_raw} = $lss->{-cfg}->{'lei.q'} //
- return $lei->fail("lei.q unset in $f");
+ die("lei.q unset in $f (out=$out)\n");
my $lse = $lei->{lse} // die 'BUG: {lse} missing';
if (ref($q)) {
$mset_opt->{qstr} = $lse->query_argv_to_string($lse->git, $q);
@@ -36,7 +36,7 @@ sub up1 ($$) {
$lei->{opt}->{$k} //= $v;
}
my $o = $lei->{opt}->{output} // '';
- return $lei->fail("lei.q.output unset in $f (out=$out)") if $o eq '';
+ return die("lei.q.output unset in $f (out=$out)\n") if $o eq '';
$lss->translate_dedupe($lei) or return;
$lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
my $lxs = $lei->lxs_prepare or return;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 8ab84b15c00b..119070a289de 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -142,11 +142,11 @@ sub wait_startq ($) {
delete $lei->{opt}->{verbose};
delete $lei->{-progress};
} else {
- $lei->fail("$$ WTF `$do_augment_done'");
+ die "BUG: do_augment_done=`$do_augment_done'";
}
return;
}
- return $lei->fail("$$ wait_startq: $!") unless $!{EINTR};
+ die "wait_startq: $!" unless $!{EINTR};
}
}
@@ -473,7 +473,8 @@ sub do_post_augment {
$lei->fail("$err");
}
if (!$err && delete $lei->{early_mua}) { # non-augment case
- $lei->start_mua;
+ eval { $lei->start_mua };
+ $lei->fail($@) if $@;
}
close(delete $lei->{au_done}); # triggers wait_startq in lei_xsearch
}
^ permalink raw reply related [relevance 50%]
* [PATCH 06/11] lei up: support --exclude=, --no-(external|remote|local)
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
` (3 preceding siblings ...)
2021-10-19 9:33 70% ` [PATCH 05/11] lei: conditionally add "\n" to error messages Eric Wong
@ 2021-10-19 9:33 47% ` Eric Wong
2021-10-19 9:33 71% ` [PATCH 07/11] lei: remove unused ->busy time arg Eric Wong
` (3 subsequent siblings)
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
These can be used to temporarily disable using certain
externals in case of temporary network failure or mount point
unavailability.
---
Documentation/lei-up.pod | 28 +++++++++++++++++++++++-----
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiExternal.pm | 11 ++++++++++-
lib/PublicInbox/LeiQuery.pm | 15 ++++-----------
lib/PublicInbox/LeiUp.pm | 22 ++++++++++++++++++++++
5 files changed, 61 insertions(+), 19 deletions(-)
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index f06ee5eb62b7..8fba0953b1ef 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -37,11 +37,26 @@ e.g C<1.hour> or C<3.days>
Default: 2.days
-=back
+=item --no-external
-The following options, described in L<lei-q(1)>, are supported.
+=item --no-local
-=over
+=item --no-remote
+
+These disable the use of all externals, local externals, or
+remote externals respectively. They are useful during
+temporary network or mount-point outages.
+
+Unlike C<lei q>, these switches override the original C<lei q --only>
+options saved as C<lei.q.only>.
+
+The combination C<--all=remote --no-remote> is supported for
+offline use in case a user is updating an IMAP folder on localhost.
+
+=item --exclude=LOCATION
+
+As with L<lei-q(1)>, but may also exclude externals originally
+specified via C<lei q --only>.
=item --lock=METHOD
@@ -49,7 +64,10 @@ The following options, described in L<lei-q(1)>, are supported.
=item --mua=CMD
-This option is incompatible with C<--all>.
+C<--lock>, C<--alert>, and C<--mua> are all supported and
+documented in L<lei-q(1)>.
+
+C<--mua> is incompatible with C<--all>.
=back
@@ -62,7 +80,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
=head1 COPYRIGHT
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 553379e404fc..5b726f71382e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -178,8 +178,8 @@ our %CMD = ( # sorted in order of importance/use:
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT...|--all', 'update saved search',
- qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+
- remote-fudge-time=s all:s), @c_opt ],
+ qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ exclude=s@
+ remote-fudge-time=s all:s remote! local! external!), @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 851715d7099c..30bb1a4579c7 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -105,7 +105,16 @@ sub get_externals {
} else {
die("`$loc' is ambiguous:\n", map { "\t$_\n" } @m, "\n");
}
- ();
+}
+
+sub canonicalize_excludes {
+ my ($lei, $excludes) = @_;
+ my %x;
+ for my $loc (@$excludes) {
+ my @l = get_externals($lei, $loc, 1);
+ $x{$_} = 1 for @l;
+ }
+ \%x;
}
# returns an anonymous sub which returns an array of potential results
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 56b82acc8b5b..effc572f6aaf 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -95,18 +95,11 @@ sub lxs_prepare {
}
# --external is enabled by default, but allow --no-external
if ($opt->{external} //= 1) {
- my %x;
- for my $loc (@{$opt->{exclude} // []}) {
- my @l = $self->get_externals($loc, 1) or return;
- $x{$_} = 1 for @l;
- }
- my $ne = $self->externals_each(\&prep_ext, $lxs, \%x);
+ my $ex = $self->canonicalize_excludes($opt->{exclude});
+ $self->externals_each(\&prep_ext, $lxs, $ex);
$opt->{remote} //= !($lxs->locals - $opt->{'local'});
- if ($opt->{'local'}) {
- $lxs->{remotes} = \@iremotes if !$opt->{remote};
- } else {
- $lxs->{locals} = \@ilocals;
- }
+ $lxs->{locals} = \@ilocals if !$opt->{'local'};
+ $lxs->{remotes} = \@iremotes if !$opt->{remote};
}
}
($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) :
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index fcdd535dc118..dac0fc287885 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -15,6 +15,14 @@ my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
sub up1 ($$) {
my ($lei, $out) = @_;
+ # precedence note for CLI switches between lei q and up:
+ # `lei q --only' > `lei q --no-(remote|local|external)'
+ # `lei up --no-(remote|local|external)' > `lei.q.only' in saved search
+ my %no = map {
+ my $v = $lei->{opt}->{$_}; # set by CLI
+ (defined($v) && !$v) ? ($_ => 1) : ();
+ } qw(remote local external);
+ my $cli_exclude = delete $lei->{opt}->{exclude};
my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
my $f = $lss->{'-f'};
my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
@@ -31,6 +39,20 @@ sub up1 ($$) {
my $v = $lss->{-cfg}->get_all("lei.q.$k") // next;
$lei->{opt}->{$k} //= $v;
}
+
+ # --no-(local|remote) CLI flags overrided saved `lei.q.only'
+ my $only = $lei->{opt}->{only};
+ @$only = map { $lei->get_externals($_) } @$only if $only;
+ if (scalar keys %no && $only) {
+ @$only = grep(!m!\Ahttps?://!i, @$only) if $no{remote};
+ @$only = grep(m!\Ahttps?://!i, @$only) if $no{'local'};
+ }
+ if ($cli_exclude) {
+ my $ex = $lei->canonicalize_excludes($cli_exclude);
+ @$only = grep { !$ex->{$_} } @$only if $only;
+ push @{$lei->{opt}->{exclude}}, @$cli_exclude;
+ }
+ delete $lei->{opt}->{only} if $no{external} || ($only && !@$only);
for my $k ($lss->BOOL_FIELDS, $lss->SINGLE_FIELDS) {
my $v = $lss->{-cfg}->get_1("lei.q.$k") // next;
$lei->{opt}->{$k} //= $v;
^ permalink raw reply related [relevance 47%]
* [PATCH 08/11] doc: lei: describe lei-daemon-kill and upgrades
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
` (5 preceding siblings ...)
2021-10-19 9:33 71% ` [PATCH 07/11] lei: remove unused ->busy time arg Eric Wong
@ 2021-10-19 9:33 80% ` Eric Wong
2021-10-19 9:33 71% ` [PATCH 09/11] lei inspect: add atfork hook Eric Wong
2021-10-19 9:33 64% ` [PATCH 10/11] lei inspect: show ISO8601 {rt} and {dt}, too Eric Wong
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
While we're at it, start dropping copyright years
since it seems acceptable to not have them:
https://www.linuxfoundation.org/blog/copyright-notices-in-open-source-software-projects/
Copyright years are also a noisy to update every year (maybe,
just maybe, we'll make it to 2022...)
---
Documentation/lei-daemon-kill.pod | 29 ++++++++++++++++++++++++++---
Documentation/lei-overview.pod | 8 +++++++-
2 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/Documentation/lei-daemon-kill.pod b/Documentation/lei-daemon-kill.pod
index 7fb0fb25ad0e..48c237b8d3a6 100644
--- a/Documentation/lei-daemon-kill.pod
+++ b/Documentation/lei-daemon-kill.pod
@@ -8,7 +8,30 @@ lei daemon-kill [-SIGNAL | -s SIGNAL | --signal SIGNAL]
=head1 DESCRIPTION
-Send a signal to the lei-daemon. C<SIGNAL> defaults to C<TERM>.
+Send a signal to the L<lei-daemon(8)>. C<SIGNAL> defaults to C<TERM>.
+
+This command should be run after updating the code of lei.
+
+=head1 SIGNALS
+
+=over 8
+
+=item SIGTERM
+
+Send a graceful termination signal. L<lei-daemon(8)> will exit
+when all currently running lei commands are done. The listen
+socket will be released as soon as the signal is processed
+so another L<lei-daemon(8)> process can take its place.
+
+=item SIGKILL
+
+Kills L<lei-daemon(8)> immediately. Some worker processes may
+remain running after a short while after this takes effect.
+
+=back
+
+=for comment
+SIGQUIT and SIGINT currently do what SIGTERM do, may change...
=head1 CONTACT
@@ -19,10 +42,10 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-daemon-pid(1)>
+L<lei-daemon-pid(1)>, L<lei-daemon(8)>
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index 40a7b0aadd04..bb2fe50f7cd9 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -131,6 +131,12 @@ C<contrib/completion/>. Contributions adding support for other
shells, as well as improvements to the existing Bash completion, are
welcome.
+=head1 UPGRADING
+
+Since lei runs as a daemon, L<lei-daemon-kill(1)> is required to kill
+the daemon so it can load new code. It will be restarted with the
+next invocation of any lei command.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -140,6 +146,6 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
^ permalink raw reply related [relevance 80%]
* [PATCH 07/11] lei: remove unused ->busy time arg
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
` (4 preceding siblings ...)
2021-10-19 9:33 47% ` [PATCH 06/11] lei up: support --exclude=, --no-(external|remote|local) Eric Wong
@ 2021-10-19 9:33 71% ` Eric Wong
2021-10-19 9:33 80% ` [PATCH 08/11] doc: lei: describe lei-daemon-kill and upgrades Eric Wong
` (2 subsequent siblings)
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
Our graceful shutdown doesn't time out clients.
---
lib/PublicInbox/LEI.pm | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5b726f71382e..c1f28f7b31bf 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -19,7 +19,7 @@ use IO::Handle ();
use Fcntl qw(SEEK_SET);
use PublicInbox::Config;
use PublicInbox::Syscall qw(EPOLLIN);
-use PublicInbox::DS qw(now dwaitpid);
+use PublicInbox::DS qw(dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
use PublicInbox::Eml;
@@ -1315,11 +1315,10 @@ sub lazy_start {
$quit->();
}
return 1 if defined($path);
- my $now = now();
my $n = 0;
for my $s (values %$dmap) {
$s->can('busy') or next;
- if ($s->busy($now)) {
+ if ($s->busy) {
++$n;
} else {
$s->close;
^ permalink raw reply related [relevance 71%]
* [PATCH 09/11] lei inspect: add atfork hook
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
` (6 preceding siblings ...)
2021-10-19 9:33 80% ` [PATCH 08/11] doc: lei: describe lei-daemon-kill and upgrades Eric Wong
@ 2021-10-19 9:33 71% ` Eric Wong
2021-10-19 9:33 64% ` [PATCH 10/11] lei inspect: show ISO8601 {rt} and {dt}, too Eric Wong
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
This is necessary for in case an inspect command is run
in a parallel with other commands.
---
lib/PublicInbox/LeiInspect.pm | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 05b6e21d298d..38ef3ad96df2 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -289,4 +289,10 @@ sub _complete_inspect {
# TODO: message-ids?, blobs? could get expensive...
}
+sub input_only_atfork_child {
+ my ($self) = @_;
+ $self->{lei}->_lei_atfork_child;
+ $self->SUPER::ipc_atfork_child;
+}
+
1;
^ permalink raw reply related [relevance 71%]
* [PATCH 10/11] lei inspect: show ISO8601 {rt} and {dt}, too
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
` (7 preceding siblings ...)
2021-10-19 9:33 71% ` [PATCH 09/11] lei inspect: add atfork hook Eric Wong
@ 2021-10-19 9:33 64% ` Eric Wong
8 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-19 9:33 UTC (permalink / raw)
To: meta
While inspect is intended for debugging, the Unix epoch in
seconds requires extra steps for human consumption; just
steal what we used for "lei q -f json" output.
---
lib/PublicInbox/LeiInspect.pm | 5 +++++
lib/PublicInbox/LeiOverview.pm | 6 +++---
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 38ef3ad96df2..5ea32ccb7e66 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -12,10 +12,15 @@ use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
use PublicInbox::MID qw(mids);
use PublicInbox::NetReader qw(imap_uri nntp_uri);
+use POSIX qw(strftime);
+use PublicInbox::LeiOverview;
+*iso8601 = \&PublicInbox::LeiOverview::iso8601;
sub _json_prep ($) {
my ($smsg) = @_;
$smsg->{$_} += 0 for qw(bytes lines); # integerize
+ $smsg->{dt} = iso8601($smsg->{ds}) if defined($smsg->{ds});
+ $smsg->{rt} = iso8601($smsg->{ts}) if defined($smsg->{ts});
+{ %$smsg } # unbless and scalarize
}
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 1b9dc9701c95..2d3db9f4ab92 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -21,7 +21,7 @@ use PublicInbox::LeiToMail;
# cf. https://en.wikipedia.org/wiki/JSON_streaming
my $JSONL = 'ldjson|ndjson|jsonl'; # 3 names for the same thing
-sub _iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) }
+sub iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) }
# we open this in the parent process before ->wq_io_do handoff
sub ovv_out_lk_init ($) {
@@ -139,8 +139,8 @@ sub _unbless_smsg {
# num/tid are nonsensical with multi-inbox search,
# lines/bytes are not generally useful
delete @$smsg{qw(num tid lines bytes)};
- $smsg->{rt} = _iso8601(delete $smsg->{ts}); # JMAP receivedAt
- $smsg->{dt} = _iso8601(delete $smsg->{ds}); # JMAP UTCDate
+ $smsg->{rt} = iso8601(delete $smsg->{ts}); # JMAP receivedAt
+ $smsg->{dt} = iso8601(delete $smsg->{ds}); # JMAP UTCDate
$smsg->{pct} = get_pct($mitem) if $mitem;
if (my $r = delete $smsg->{references}) {
$smsg->{refs} = [ map { $_ } ($r =~ m/$MID_EXTRACT/go) ];
^ permalink raw reply related [relevance 64%]
* [PATCH 02/15] t/lei-import-maildir: rename fix (SR -> RS)
2021-10-21 21:10 57% ` [PATCH 01/15] t/lei-{auto-watch,export-kw}: extra diagnostics on failure Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 03/15] t/lei-p2q: extra diagnostics Eric Wong
` (8 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
While it doesn't matter to us, the Maildir spec specifies
characters are to be sorted in alphabetical order.
---
t/lei-import-maildir.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index c81e7805fe7f..1e7eddd571d8 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -52,7 +52,7 @@ test_lei(sub {
my $r2 = json_utf8->decode($lei_out);
is_deeply($r2, $res, 'idempotent import')
or diag explain($imp_err, $res);
- rename("$md/cur/x:2,S", "$md/cur/x:2,SR") or BAIL_OUT "rename: $!";
+ rename("$md/cur/x:2,S", "$md/cur/x:2,RS") or BAIL_OUT "rename: $!";
lei_ok('import', "maildir:$md", \'import Maildir after +answered');
lei_ok(qw(q -d none s:boolean), \'lei q after +answered');
$res = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 71%]
* [PATCH 01/15] t/lei-{auto-watch,export-kw}: extra diagnostics on failure
@ 2021-10-21 21:10 57% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 02/15] t/lei-import-maildir: rename fix (SR -> RS) Eric Wong
` (9 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
Maybe these will help track down some failures and make
diagnosing bugs easier. "lei export-kw" should also become
optional, even, so allow disabling it easily in the test.
---
t/lei-auto-watch.t | 3 ++-
t/lei-export-kw.t | 39 ++++++++++++++++++++++++++++-----------
2 files changed, 30 insertions(+), 12 deletions(-)
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index e5e132eb3bfd..d5661ae5d618 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -41,7 +41,8 @@ test_lei(sub {
$ins = json_utf8->decode($lei_out);
$exp = { "maildir:$x" => [ map { basename($_) } glob("$x/*/*") ],
"maildir:$y" => [ map { basename($_) } glob("$y/*/*") ] };
- is_deeply($ins->{'mail-sync'}, $exp, 'mail_sync matches FS');
+ is_deeply($ins->{'mail-sync'}, $exp, 'mail_sync matches FS') or
+ diag explain($ins);
});
done_testing;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
index 1fe940bb6d89..55730e87c050 100644
--- a/t/lei-export-kw.t
+++ b/t/lei-export-kw.t
@@ -7,28 +7,45 @@ use File::Path qw(make_path);
require_mods(qw(lei -imapd Mail::IMAPClient));
my ($tmpdir, $for_destroy) = tmpdir;
my $expect = eml_load('t/data/0001.patch');
+my $do_export_kw = 1;
+my $wait_for = sub {
+ my ($f) = @_;
+ lei_ok qw(export-kw --all=local) if $do_export_kw;
+ my $x = $f;
+ $x =~ s!\Q$tmpdir\E/!\$TMPDIR/!;
+ for (0..10) {
+ last if -f $f;
+ diag "tick #$_ $x";
+ tick(0.1);
+ }
+ ok(-f $f, "$x exists") or xbail;
+};
+
test_lei({ tmpdir => $tmpdir }, sub {
my $home = $ENV{HOME};
my $md = "$home/md";
+ my $f;
make_path("$md/new", "$md/cur", "$md/tmp");
cp('t/data/0001.patch', "$md/new/y") or xbail "cp $md $!";
cp('t/data/message_embed.eml', "$md/cur/x:2,S") or xbail "cp $md $!";
- lei_ok qw(index -q), $md;
+ lei_ok qw(index), $md;
lei_ok qw(tag t/data/0001.patch +kw:seen);
- lei_ok qw(export-kw --all=local);
- ok(!-e "$md/new/y", 'original gone');
- is_deeply(eml_load("$md/cur/y:2,S"), $expect,
- "`seen' kw exported");
+ $wait_for->($f = "$md/cur/y:2,S");
+ ok(!-e "$md/new/y", 'original gone') or
+ diag explain([glob("$md/*/*")]);
+ is_deeply(eml_load($f), $expect, "`seen' kw exported");
lei_ok qw(tag t/data/0001.patch +kw:answered);
- lei_ok qw(export-kw --all=local);
- ok(!-e "$md/cur/y:2,S", 'seen-only file gone');
- is_deeply(eml_load("$md/cur/y:2,RS"), $expect, "`R' added");
+ $wait_for->($f = "$md/cur/y:2,RS");
+ ok(!-e "$md/cur/y:2,S", 'seen-only file gone') or
+ diag explain([glob("$md/*/*")]);
+ is_deeply(eml_load($f), $expect, "`R' added");
lei_ok qw(tag t/data/0001.patch -kw:answered -kw:seen);
- lei_ok qw(export-kw --mode=set --all=local);
- ok(!-e "$md/cur/y:2,RS", 'seen+answered file gone');
- is_deeply(eml_load("$md/cur/y:2,"), $expect, 'no keywords left');
+ $wait_for->($f = "$md/cur/y:2,");
+ ok(!-e "$md/cur/y:2,RS", 'seen+answered file gone') or
+ diag explain([glob("$md/*/*")]);
+ is_deeply(eml_load($f), $expect, 'no keywords left');
});
done_testing;
^ permalink raw reply related [relevance 57%]
* [PATCH 03/15] t/lei-p2q: extra diagnostics
2021-10-21 21:10 57% ` [PATCH 01/15] t/lei-{auto-watch,export-kw}: extra diagnostics on failure Eric Wong
2021-10-21 21:10 71% ` [PATCH 02/15] t/lei-import-maildir: rename fix (SR -> RS) Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 04/15] lei/store: check for any unexpected process death Eric Wong
` (7 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
I got one mysterious test failure here, once, and can't seem
to reproduce it...
---
t/lei-p2q.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index 495d81de78a5..bf40a43be466 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -7,7 +7,7 @@ require_mods(qw(json DBD::SQLite Search::Xapian));
test_lei(sub {
ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
- 'p2q fails on bogus arg');
+ 'p2q fails on bogus arg') or diag $lei_err;
like($lei_err, qr/format-patch.*failed/, 'notes format-patch failure');
lei_ok(qw(p2q -w dfpost t/data/0001.patch));
is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname') or diag $lei_err;
^ permalink raw reply related [relevance 71%]
* [PATCH 04/15] lei/store: check for any unexpected process death
` (2 preceding siblings ...)
2021-10-21 21:10 71% ` [PATCH 03/15] t/lei-p2q: extra diagnostics Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 05/15] lei note-event: drop unnecessary eval guard Eric Wong
` (6 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
The lei/store process should only exit from EOF on the
socket, so make sure we note any unintended signals
---
lib/PublicInbox/LeiStore.pm | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 821045701dfe..16e7d302dc2f 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -571,6 +571,12 @@ sub recv_and_run {
$self->SUPER::recv_and_run(@args);
}
+sub _sto_atexit { # dwaitpid callback
+ my ($args, $pid) = @_;
+ my $self = $args->[0];
+ warn "lei/store PID:$pid died \$?=$?\n" if $?;
+}
+
sub write_prepare {
my ($self, $lei) = @_;
$lei // die 'BUG: $lei not passed';
@@ -587,7 +593,7 @@ sub write_prepare {
-err_wr => $w,
to_close => [ $r ],
});
- $self->wq_wait_async; # outlives $lei
+ $self->wq_wait_async(\&_sto_atexit); # outlives $lei
require PublicInbox::LeiStoreErr;
PublicInbox::LeiStoreErr->new($r, $lei);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 05/15] lei note-event: drop unnecessary eval guard
` (3 preceding siblings ...)
2021-10-21 21:10 71% ` [PATCH 04/15] lei/store: check for any unexpected process death Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 06/15] lei note-event: wq_io_do => wq_do Eric Wong
` (5 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
We don't want to lose the failure message in case note-event
fails.
---
lib/PublicInbox/LeiNoteEvent.pm | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 1749c98f8312..0709754f6d2a 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -72,8 +72,7 @@ sub lei_note_event {
my $lms = $lei->lms or return;
$lms->lms_write_prepare if $new_cur eq ''; # for ->clear_src below
$lei->{opt}->{quiet} = 1;
- eval { $lms->arg2folder($lei, [ $folder ]) };
- return if $@;
+ $lms->arg2folder($lei, [ $folder ]);
my $state = $cfg->get_1("watch.$folder.state") // 'tag-rw';
return if $state eq 'pause';
return $lms->clear_src($folder, \$bn) if $new_cur eq '';
^ permalink raw reply related [relevance 71%]
* [PATCH 06/15] lei note-event: wq_io_do => wq_do
` (4 preceding siblings ...)
2021-10-21 21:10 71% ` [PATCH 05/15] lei note-event: drop unnecessary eval guard Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 69% ` [PATCH 07/15] lei_search: try harder to associate "lei index"-ed messages Eric Wong
` (4 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
No need to pass extra arrayref args, here.
---
lib/PublicInbox/LeiNoteEvent.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 0709754f6d2a..3472e73070d6 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -97,7 +97,7 @@ sub lei_note_event {
return if index($fl, 'T') >= 0;
my $kw = PublicInbox::MdirReader::flags2kw($fl);
my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] };
- $self->wq_io_do('maildir_event', [], $fn, $vmd, $state);
+ $self->wq_do('maildir_event', $fn, $vmd, $state);
} # else: TODO: imap
}
^ permalink raw reply related [relevance 71%]
* [PATCH 10/15] doc: lei-overview: add CAVEATS section
` (6 preceding siblings ...)
2021-10-21 21:10 69% ` [PATCH 07/15] lei_search: try harder to associate "lei index"-ed messages Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 11/15] lei note-event: clear_src on ENOENT Eric Wong
` (2 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
IMAP and NNTP client performance absolutely sucks compared to what
the read-only daemons are capable of...
---
Documentation/lei-overview.pod | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index bb2fe50f7cd9..99fd6ef72174 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -137,6 +137,11 @@ Since lei runs as a daemon, L<lei-daemon-kill(1)> is required to kill
the daemon so it can load new code. It will be restarted with the
next invocation of any lei command.
+=head1 CAVEATS
+
+IMAP and NNTP client performance is poor on high-latency connections.
+It will hopefully be fixed in 2022.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
^ permalink raw reply related [relevance 71%]
* [PATCH 11/15] lei note-event: clear_src on ENOENT
` (7 preceding siblings ...)
2021-10-21 21:10 71% ` [PATCH 10/15] doc: lei-overview: add CAVEATS section Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 13/15] lei: no Perl FileHandle for `undef' w/ ECONNRESET Eric Wong
2021-10-21 21:10 37% ` [PATCH 15/15] lei: use RENAME_NOREPLACE on Linux 3.15+ Eric Wong
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
When a file goes away, try to make sure we don't waste
time trying to access or store it.
---
lib/PublicInbox/LeiNoteEvent.pm | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 3472e73070d6..22d6ffac9feb 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -8,6 +8,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::DS;
+use Errno qw(ENOENT);
our $to_flush; # { cfgpath => $lei }
@@ -59,8 +60,11 @@ sub eml_event ($$$$) {
sub maildir_event { # via wq_io_do
my ($self, $fn, $vmd, $state) = @_;
- my $eml = PublicInbox::InboxWritable::eml_from_path($fn) // return;
- eml_event($self, $eml, $vmd, $state);
+ if (my $eml = PublicInbox::InboxWritable::eml_from_path($fn)) {
+ eml_event($self, $eml, $vmd, $state);
+ } elsif ($! == ENOENT) {
+ $self->{lms}->clear_src(@{$vmd->{sync_info}});
+ } # else: eml_from_path already warns
}
sub lei_note_event {
^ permalink raw reply related [relevance 71%]
* [PATCH 13/15] lei: no Perl FileHandle for `undef' w/ ECONNRESET
` (8 preceding siblings ...)
2021-10-21 21:10 71% ` [PATCH 11/15] lei note-event: clear_src on ENOENT Eric Wong
@ 2021-10-21 21:10 71% ` Eric Wong
2021-10-21 21:10 37% ` [PATCH 15/15] lei: use RENAME_NOREPLACE on Linux 3.15+ Eric Wong
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
Error reporting for recv_cmd4 methods is a bit wonky.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b68e526bf365..43baeeb3d51c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1129,6 +1129,7 @@ sub event_step {
if (scalar(@fds) == 1 && !defined($fds[0])) {
return if $! == EAGAIN;
die "recvmsg: $!" if $! != ECONNRESET;
+ @fds = (); # for open loop below:
}
for (@fds) { open my $rfh, '+<&=', $_ }
if ($buf eq '') {
^ permalink raw reply related [relevance 71%]
* [PATCH 07/15] lei_search: try harder to associate "lei index"-ed messages
` (5 preceding siblings ...)
2021-10-21 21:10 71% ` [PATCH 06/15] lei note-event: wq_io_do => wq_do Eric Wong
@ 2021-10-21 21:10 69% ` Eric Wong
2021-10-21 21:10 71% ` [PATCH 10/15] doc: lei-overview: add CAVEATS section Eric Wong
` (3 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
Allow checking for keyword changes if we have an known OID,
even if the blob isn't currently reachable.
---
lib/PublicInbox/LeiSearch.pm | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 3e046b2146c6..1fb38da1d7aa 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -7,7 +7,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::ExtSearch); # PublicInbox::Search->reopen
use PublicInbox::Search qw(xap_terms);
-use PublicInbox::ContentHash qw(content_digest content_hash);
+use PublicInbox::ContentHash qw(content_digest content_hash git_sha);
use PublicInbox::MID qw(mids mids_for_index);
use Carp qw(croak);
@@ -118,6 +118,13 @@ sub xoids_for {
}
}
$git->async_wait_all;
+
+ # it could be an 'lei index'-ed file that just got renamed
+ if (scalar(keys %$xoids) < ($min // 1) && defined($self->{topdir})) {
+ my $hex = git_sha(1, $eml)->hexdigest;
+ my @n = $overs[0]->blob_exists($hex);
+ for (@n) { $xoids->{$hex} //= $_ }
+ }
scalar(keys %$xoids) ? $xoids : undef;
}
@@ -129,6 +136,10 @@ sub kw_changed {
my $xoids = xoids_for($self, $eml) // return;
$docids //= [];
@$docids = sort { $a <=> $b } values %$xoids;
+ if (!@$docids && $self->over) {
+ my $bin = git_sha(1, $eml)->digest;
+ @$docids = $self->over->oidbin_exists($bin);
+ }
}
for my $id (@$docids) {
$cur_kw = eval { msg_keywords($self, $id) } and last;
^ permalink raw reply related [relevance 69%]
* [PATCH 15/15] lei: use RENAME_NOREPLACE on Linux 3.15+
` (9 preceding siblings ...)
2021-10-21 21:10 71% ` [PATCH 13/15] lei: no Perl FileHandle for `undef' w/ ECONNRESET Eric Wong
@ 2021-10-21 21:10 37% ` Eric Wong
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-21 21:10 UTC (permalink / raw)
To: meta
One syscall is better than two for atomicity in Maildirs. This
means there's no window where another process can see both the
old and new file at the same time (link && unlink), nor a window
where we might inadvertantly clobber an existing file if we were
to do `stat && rename'.
---
MANIFEST | 1 +
devel/syscall-list | 8 +++++-
lib/PublicInbox/LeiExportKw.pm | 19 +++++--------
lib/PublicInbox/LeiStore.pm | 8 +++---
lib/PublicInbox/LeiToMail.pm | 7 +++--
lib/PublicInbox/Syscall.pm | 49 +++++++++++++++++++++++++++++++---
t/rename_noreplace.t | 26 ++++++++++++++++++
7 files changed, 92 insertions(+), 26 deletions(-)
create mode 100644 t/rename_noreplace.t
diff --git a/MANIFEST b/MANIFEST
index af1522d71bd1..9fd979ef02fb 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -528,6 +528,7 @@ t/psgi_v2.t
t/purge.t
t/qspawn.t
t/reindex-time-range.t
+t/rename_noreplace.t
t/replace.t
t/reply.t
t/run.perl
diff --git a/devel/syscall-list b/devel/syscall-list
index b33401d98ce4..3d55df1fc1d7 100755
--- a/devel/syscall-list
+++ b/devel/syscall-list
@@ -1,4 +1,4 @@
-# Copyright 2021 all contributors <meta@public-inbox.org>
+# Copyright all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <http://www.gnu.org/licenses/agpl-3.0.txt>
# Dump syscall numbers under Linux and any other kernel which
# promises stable syscall numbers. This is to maintain
@@ -9,7 +9,10 @@
eval 'exec perl -S $0 ${1+"$@"}' # no shebang
if 0; # running under some shell
use strict;
+use v5.10.1;
use File::Temp 0.19;
+use POSIX qw(uname);
+say '$machine='.(POSIX::uname())[-1];
my $cc = $ENV{CC} // 'cc';
my @cflags = split(/\s+/, $ENV{CFLAGS} // '-Wall');
my $str = do { local $/; <DATA> };
@@ -43,6 +46,9 @@ int main(void)
D(SYS_inotify_add_watch);
D(SYS_inotify_rm_watch);
D(SYS_prctl);
+#ifdef SYS_renameat2
+ D(SYS_renameat2);
+#endif
#endif /* Linux, any other OSes with stable syscalls? */
printf("size_t=%zu off_t=%zu\n", sizeof(size_t), sizeof(off_t));
return 0;
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 0b65c2762633..ceeef7f21d54 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -7,6 +7,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use Errno qw(EEXIST ENOENT);
+use PublicInbox::Syscall qw(rename_noreplace);
sub export_kw_md { # LeiMailSync->each_src callback
my ($oidbin, $id, $self, $mdir) = @_;
@@ -30,30 +31,22 @@ sub export_kw_md { # LeiMailSync->each_src callback
my $lei = $self->{lei};
for my $d (@try) {
my $src = "$mdir/$d/$$id";
-
- # we use link(2) + unlink(2) since rename(2) may
- # inadvertently clobber if the "uniquefilename" part wasn't
- # actually unique.
- if (link($src, $dst)) { # success
- # unlink(2) may ENOENT from parallel invocation,
- # ignore it, but not other serious errors
- if (!unlink($src) and $! != ENOENT) {
- $lei->child_error(1, "E: unlink($src): $!");
- }
+ if (rename_noreplace($src, $dst)) { # success
$self->{lms}->mv_src("maildir:$mdir",
$oidbin, $id, $bn);
- return; # success anyways if link(2) worked
+ return; # success
} elsif ($! == EEXIST) { # lost race with lei/store?
return;
} elsif ($! != ENOENT) {
- $lei->child_error(1, "E: link($src -> $dst): $!");
+ $lei->child_error(1,
+ "E: rename_noreplace($src -> $dst): $!");
} # else loop @try
}
my $e = $!;
# both tries failed
my $oidhex = unpack('H*', $oidbin);
my $src = "$mdir/{".join(',', @try)."}/$$id";
- $lei->child_error(1, "link($src -> $dst) ($oidhex): $e");
+ $lei->child_error(1, "rename_noreplace($src -> $dst) ($oidhex): $e");
for (@try) { return if -e "$mdir/$_/$$id" }
$self->{lms}->clear_src("maildir:$mdir", $id);
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 16e7d302dc2f..f1316229bb32 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -32,6 +32,7 @@ use POSIX ();
use IO::Handle (); # ->autoflush
use Sys::Syslog qw(syslog openlog);
use Errno qw(EEXIST ENOENT);
+use PublicInbox::Syscall qw(rename_noreplace);
sub new {
my (undef, $dir, $opt) = @_;
@@ -185,10 +186,7 @@ sub export1_kw_md ($$$$$) {
my $dst = "$mdir/cur/$bn";
for my $d (@try) {
my $src = "$mdir/$d/$orig";
- if (link($src, $dst)) {
- if (!unlink($src) and $! != ENOENT) {
- syslog('warning', "unlink($src): $!");
- }
+ if (rename_noreplace($src, $dst)) {
# TODO: verify oidbin?
$self->{lms}->mv_src("maildir:$mdir",
$oidbin, \$orig, $bn);
@@ -196,7 +194,7 @@ sub export1_kw_md ($$$$$) {
} elsif ($! == EEXIST) { # lost race with "lei export-kw"?
return;
} elsif ($! != ENOENT) {
- syslog('warning', "link($src -> $dst): $!");
+ syslog('warning', "rename_noreplace($src -> $dst): $!");
}
}
for (@try) { return if -e "$mdir/$_/$orig" };
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ca4e92de48b7..d33d27aec006 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -12,6 +12,7 @@ use PublicInbox::Spawn qw(spawn);
use Symbol qw(gensym);
use IO::Handle; # ->autoflush
use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
+use PublicInbox::Syscall qw(rename_noreplace);
my %kw2char = ( # Maildir characters
draft => 'D',
@@ -262,10 +263,8 @@ sub _buf2maildir ($$$$) {
$rand = '';
do {
$base = $rand.$common.':2,'.kw2suffix($kw);
- } while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} &&
- ($rand = _rand.','));
- die "link($tmp, $dst$base): $!" unless $ok;
- unlink($tmp) or warn "W: failed to unlink $tmp: $!\n";
+ } while (!($ok = rename_noreplace($tmp, $dst.$base)) &&
+ $!{EEXIST} && ($rand = _rand.','));
\$base;
} else {
my $err = "Error writing $smsg->{blob} to $dst: $!\n";
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index 7ab4291119ea..c00385b94db8 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -13,8 +13,9 @@
# License or the Artistic License, as specified in the Perl README file.
package PublicInbox::Syscall;
use strict;
+use v5.10.1;
use parent qw(Exporter);
-use POSIX qw(ENOSYS O_NONBLOCK);
+use POSIX qw(ENOENT EEXIST ENOSYS O_NONBLOCK);
use Config;
# $VERSION = '0.25'; # Sys::Syscall version
@@ -22,7 +23,7 @@ our @EXPORT_OK = qw(epoll_ctl epoll_create epoll_wait
EPOLLIN EPOLLOUT EPOLLET
EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
EPOLLONESHOT EPOLLEXCLUSIVE
- signalfd);
+ signalfd rename_noreplace);
our %EXPORT_TAGS = (epoll => [qw(epoll_ctl epoll_create epoll_wait
EPOLLIN EPOLLOUT
EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
@@ -64,13 +65,16 @@ our (
$SYS_epoll_ctl,
$SYS_epoll_wait,
$SYS_signalfd4,
+ $SYS_renameat2,
);
my $SFD_CLOEXEC = 02000000; # Perl does not expose O_CLOEXEC
our $no_deprecated = 0;
if ($^O eq "linux") {
- my $machine = (POSIX::uname())[-1];
+ my (undef, undef, $release, undef, $machine) = POSIX::uname();
+ my ($maj, $min) = ($release =~ /\A([0-9]+)\.([0-9]+)/);
+ $SYS_renameat2 = 0 if "$maj.$min" < 3.15;
# whether the machine requires 64-bit numbers to be on 8-byte
# boundaries.
my $u64_mod_8 = 0;
@@ -91,22 +95,26 @@ if ($^O eq "linux") {
$SYS_epoll_ctl = 255;
$SYS_epoll_wait = 256;
$SYS_signalfd4 = 327;
+ $SYS_renameat2 //= 353;
} elsif ($machine eq "x86_64") {
$SYS_epoll_create = 213;
$SYS_epoll_ctl = 233;
$SYS_epoll_wait = 232;
$SYS_signalfd4 = 289;
+ $SYS_renameat2 //= 316;
} elsif ($machine eq 'x32') {
$SYS_epoll_create = 1073742037;
$SYS_epoll_ctl = 1073742057;
$SYS_epoll_wait = 1073742056;
$SYS_signalfd4 = 1073742113;
+ $SYS_renameat2 //= 0x40000000 + 316;
} elsif ($machine eq 'sparc64') {
$SYS_epoll_create = 193;
$SYS_epoll_ctl = 194;
$SYS_epoll_wait = 195;
$u64_mod_8 = 1;
$SYS_signalfd4 = 317;
+ $SYS_renameat2 //= 345;
$SFD_CLOEXEC = 020000000;
} elsif ($machine =~ m/^parisc/) {
$SYS_epoll_create = 224;
@@ -120,18 +128,21 @@ if ($^O eq "linux") {
$SYS_epoll_wait = 238;
$u64_mod_8 = 1;
$SYS_signalfd4 = 313;
+ $SYS_renameat2 //= 357;
} elsif ($machine eq "ppc") {
$SYS_epoll_create = 236;
$SYS_epoll_ctl = 237;
$SYS_epoll_wait = 238;
$u64_mod_8 = 1;
$SYS_signalfd4 = 313;
+ $SYS_renameat2 //= 357;
} elsif ($machine =~ m/^s390/) {
$SYS_epoll_create = 249;
$SYS_epoll_ctl = 250;
$SYS_epoll_wait = 251;
$u64_mod_8 = 1;
$SYS_signalfd4 = 322;
+ $SYS_renameat2 //= 347;
} elsif ($machine eq "ia64") {
$SYS_epoll_create = 1243;
$SYS_epoll_ctl = 1244;
@@ -153,6 +164,7 @@ if ($^O eq "linux") {
$u64_mod_8 = 1;
$no_deprecated = 1;
$SYS_signalfd4 = 74;
+ $SYS_renameat2 //= 276;
} elsif ($machine =~ m/arm(v\d+)?.*l/) {
# ARM OABI
$SYS_epoll_create = 250;
@@ -160,18 +172,21 @@ if ($^O eq "linux") {
$SYS_epoll_wait = 252;
$u64_mod_8 = 1;
$SYS_signalfd4 = 355;
+ $SYS_renameat2 //= 382;
} elsif ($machine =~ m/^mips64/) {
$SYS_epoll_create = 5207;
$SYS_epoll_ctl = 5208;
$SYS_epoll_wait = 5209;
$u64_mod_8 = 1;
$SYS_signalfd4 = 5283;
+ $SYS_renameat2 //= 5311;
} elsif ($machine =~ m/^mips/) {
$SYS_epoll_create = 4248;
$SYS_epoll_ctl = 4249;
$SYS_epoll_wait = 4250;
$u64_mod_8 = 1;
$SYS_signalfd4 = 4324;
+ $SYS_renameat2 //= 4351;
} else {
# as a last resort, try using the *.ph files which may not
# exist or may be wrong
@@ -280,6 +295,34 @@ sub signalfd ($$) {
}
}
+sub _rename_noreplace_racy ($$) {
+ my ($old, $new) = @_;
+ if (link($old, $new)) {
+ warn "unlink $old: $!\n" if !unlink($old) && $! != ENOENT;
+ 1
+ } else {
+ undef;
+ }
+}
+
+# TODO: support FD args?
+sub rename_noreplace ($$) {
+ my ($old, $new) = @_;
+ if ($SYS_renameat2) { # RENAME_NOREPLACE = 1, AT_FDCWD = -100
+ my $ret = syscall($SYS_renameat2, -100, $old, -100, $new, 1);
+ if ($ret == 0) {
+ 1; # like rename() perlop
+ } elsif ($! == ENOSYS) {
+ undef $SYS_renameat2;
+ _rename_noreplace_racy($old, $new);
+ } else {
+ undef
+ }
+ } else {
+ _rename_noreplace_racy($old, $new);
+ }
+}
+
1;
=head1 WARRANTY
diff --git a/t/rename_noreplace.t b/t/rename_noreplace.t
new file mode 100644
index 000000000000..bd1c4e9236a7
--- /dev/null
+++ b/t/rename_noreplace.t
@@ -0,0 +1,26 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use v5.10.1;
+use PublicInbox::TestCommon;
+use_ok 'PublicInbox::Syscall', 'rename_noreplace';
+my ($tmpdir, $for_destroy) = tmpdir;
+
+open my $fh, '>', "$tmpdir/a" or xbail $!;
+my @sa = stat($fh);
+is(rename_noreplace("$tmpdir/a", "$tmpdir/b"), 1, 'rename_noreplace');
+my @sb = stat("$tmpdir/b");
+ok(scalar(@sb), 'new file exists');
+ok(!-e "$tmpdir/a", 'original gone');
+is("@sa[0,1]", "@sb[0,1]", 'same st_dev + st_ino');
+
+is(rename_noreplace("$tmpdir/a", "$tmpdir/c"), undef, 'undef on ENOENT');
+ok($!{ENOENT}, 'ENOENT set when missing');
+
+open $fh, '>', "$tmpdir/a" or xbail $!;
+is(rename_noreplace("$tmpdir/a", "$tmpdir/b"), undef, 'undef on EEXIST');
+ok($!{EEXIST}, 'EEXIST set when missing');
+is_deeply([stat("$tmpdir/b")], \@sb, 'target unchanged on EEXIST');
+
+done_testing;
^ permalink raw reply related [relevance 37%]
* [PATCH 0/3] lei: some cleanup stuff
@ 2021-10-22 8:22 71% Eric Wong
2021-10-22 8:22 60% ` [PATCH 1/3] lei export-kw: don't recreate deleted IMAP folders Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-10-22 8:22 UTC (permalink / raw)
To: meta
Sometimes users delete stuff, and ordering shouldn't matter.
"lei fsck" should be easier to deal with if users have more
cleanup mechanisms available to them.
Eric Wong (3):
lei export-kw: don't recreate deleted IMAP folders
lei export-kw: completion returns all Maildir+IMAP
lei forget-search: support --prune=<local|remote>
Documentation/lei-forget-search.pod | 13 ++++++++
lib/PublicInbox/LEI.pm | 4 +--
lib/PublicInbox/LeiExportKw.pm | 13 +++++---
lib/PublicInbox/LeiForgetSearch.pm | 52 +++++++++++++++++++++++++++--
lib/PublicInbox/LeiUp.pm | 26 +++++++++------
lib/PublicInbox/NetReader.pm | 1 +
t/lei-q-save.t | 7 ++++
xt/net_writer-imap.t | 15 +++++++--
8 files changed, 110 insertions(+), 21 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei export-kw: completion returns all Maildir+IMAP
2021-10-22 8:22 71% [PATCH 0/3] lei: some cleanup stuff Eric Wong
2021-10-22 8:22 60% ` [PATCH 1/3] lei export-kw: don't recreate deleted IMAP folders Eric Wong
@ 2021-10-22 8:22 71% ` Eric Wong
2021-10-22 8:22 46% ` [PATCH 3/3] lei forget-search: support --prune=<local|remote> Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-22 8:22 UTC (permalink / raw)
To: meta
It's theoretically possible an AUTH=ANONYMOUS login could be
writable and allowed to store flags for various people (e.g.
within a private network).
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 12c8f4067e9e..5be9e51ff311 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -128,7 +128,7 @@ sub _complete_export_kw {
my $lms = $lei->lms or return ();
my $match_cb = $lei->complete_url_prepare(\@argv);
# filter-out read-only sources:
- my @k = grep(!m!(?://;AUTH=ANONYMOUS\@|\A(?:nntps?|s?news)://)!,
+ my @k = grep(m!(?:maildir|imaps?):!,
$lms->folders($argv[-1] // undef, 1));
my @m = map { $match_cb->($_) } @k;
@m ? @m : @k;
^ permalink raw reply related [relevance 71%]
* [PATCH 1/3] lei export-kw: don't recreate deleted IMAP folders
2021-10-22 8:22 71% [PATCH 0/3] lei: some cleanup stuff Eric Wong
@ 2021-10-22 8:22 60% ` Eric Wong
2021-10-22 8:22 71% ` [PATCH 2/3] lei export-kw: completion returns all Maildir+IMAP Eric Wong
2021-10-22 8:22 46% ` [PATCH 3/3] lei forget-search: support --prune=<local|remote> Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-22 8:22 UTC (permalink / raw)
To: meta
In case an IMAP folder is deleted, just set an error and
ignore it rather than creating an empty folder which we
attempt to export keywords to for non-existent messages.
---
lib/PublicInbox/LeiExportKw.pm | 11 ++++++++---
lib/PublicInbox/NetReader.pm | 1 +
xt/net_writer-imap.t | 15 +++++++++++++--
3 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index ceeef7f21d54..12c8f4067e9e 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -67,9 +67,13 @@ sub input_path_url {
$self->{lms}->each_src($input, \&export_kw_md, $self, $mdir);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
- my $mic = $self->{nwr}->mic_for_folder($uri);
- $self->{lms}->each_src($$uri, \&export_kw_imap, $self, $mic);
- $mic->expunge;
+ if (my $mic = $self->{nwr}->mic_for_folder($uri)) {
+ $self->{lms}->each_src($$uri, \&export_kw_imap,
+ $self, $mic);
+ $mic->expunge;
+ } else {
+ $self->{lei}->child_error(0, "$input unavailable: $@");
+ }
} else { die "BUG: $input not supported" }
}
@@ -108,6 +112,7 @@ EOM
$self->{nwr} = bless $net, 'PublicInbox::NetWriter';
$self->{imap_mod_kw} = $net->can($self->{-merge_kw} ?
'imap_add_kw' : 'imap_set_kw');
+ $self->{nwr}->{-skip_creat} = 1;
}
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 4da19ab969b5..032b4fda557c 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -359,6 +359,7 @@ sub imap_common_init ($;$) {
mic_for($self, $uri, $mic_common, $lei) //
die "Unable to continue\n";
next unless $self->isa('PublicInbox::NetWriter');
+ next if $self->{-skip_creat};
my $dst = $orig_uri->mailbox // next;
next if $mic->exists($dst); # already exists
$mic->create($dst) or die "CREATE $dst failed <$orig_uri>: $@";
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index cb2ea61ff8e3..afa4bcc3e881 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -83,8 +83,11 @@ my $mics = do {
};
my $mic = (values %$mics)[0];
my $cleanup = PublicInbox::OnDestroy->new($$, sub {
- my $mic = $nwr->mic_get($uri);
- $mic->delete($folder) or fail "delete $folder <$folder_uri>: $@";
+ if (defined($folder)) {
+ my $mic = $nwr->mic_get($uri);
+ $mic->delete($folder) or
+ fail "delete $folder <$folder_uri>: $@";
+ }
if ($tmpdir && -f "$tmpdir/.gitconfig") {
local $ENV{HOME} = $tmpdir;
system(qw(git credential-cache exit));
@@ -250,6 +253,14 @@ EOM
lei_ok qw(q m:testmessage --no-external -o), $folder_url;
lei_ok qw(up), $folder_url;
lei_ok qw(up --all=remote);
+ $mic = $nwr->mic_get($uri);
+ $mic->delete($folder) or fail "delete $folder <$folder_uri>: $@";
+ $mic->expunge;
+ undef $mic;
+ undef $folder;
+ ok(!lei(qw(export-kw), $folder_url),
+ 'export-kw fails w/ non-existent folder');
+
});
undef $cleanup; # remove temporary folder
^ permalink raw reply related [relevance 60%]
* [PATCH 3/3] lei forget-search: support --prune=<local|remote>
2021-10-22 8:22 71% [PATCH 0/3] lei: some cleanup stuff Eric Wong
2021-10-22 8:22 60% ` [PATCH 1/3] lei export-kw: don't recreate deleted IMAP folders Eric Wong
2021-10-22 8:22 71% ` [PATCH 2/3] lei export-kw: completion returns all Maildir+IMAP Eric Wong
@ 2021-10-22 8:22 46% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-22 8:22 UTC (permalink / raw)
To: meta
Instead of:
lei forget-search $OUTPUT && rm -r $OUTPUT
we'll also allow a user to do:
rm -r $OUTPUT && lei forget-search --prune
This gives users flexibility to choose whatever flow
is most natural to them.
---
Documentation/lei-forget-search.pod | 13 ++++++++
lib/PublicInbox/LEI.pm | 4 +--
lib/PublicInbox/LeiForgetSearch.pm | 52 +++++++++++++++++++++++++++--
lib/PublicInbox/LeiUp.pm | 26 +++++++++------
t/lei-q-save.t | 7 ++++
5 files changed, 87 insertions(+), 15 deletions(-)
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
index f3f043f93252..3a7f493cee8e 100644
--- a/Documentation/lei-forget-search.pod
+++ b/Documentation/lei-forget-search.pod
@@ -10,6 +10,19 @@ lei forget-search [OPTIONS] OUTPUT
Forget a saved search at C<OUTPUT>.
+=head1 OPTIONS
+
+=over
+
+=item --prune[=<local|remote>]
+
+C<--prune> will forget saved searches if the C<OUTPUT> no longer
+exists. C<--all=local> only prunes local mailboxes,
+C<--all=remote> only prunes remote mailboxes (currently
+C<imap://> and C<imaps://>).
+
+=back
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 43baeeb3d51c..cb1e5433cc2d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -222,8 +222,8 @@ our %CMD = ( # sorted in order of importance/use:
'ls-search' => [ '[PREFIX]', 'list saved search queries',
qw(format|f=s pretty l ascii z|0), @c_opt ],
-'forget-search' => [ 'OUTPUT...', 'forget a saved search',
- qw(verbose|v+), @c_opt ],
+'forget-search' => [ 'OUTPUT...|--prune', 'forget a saved search',
+ qw(verbose|v+ prune:s), @c_opt ],
'edit-search' => [ 'OUTPUT', "edit saved search via `git config --edit'",
@c_opt ],
'rm' => [ '--stdin|LOCATION...',
diff --git a/lib/PublicInbox/LeiForgetSearch.pm b/lib/PublicInbox/LeiForgetSearch.pm
index 0db9c75b8be3..f353fe52a958 100644
--- a/lib/PublicInbox/LeiForgetSearch.pm
+++ b/lib/PublicInbox/LeiForgetSearch.pm
@@ -5,12 +5,12 @@
package PublicInbox::LeiForgetSearch;
use strict;
use v5.10.1;
+use parent qw(PublicInbox::LeiUp);
use PublicInbox::LeiSavedSearch;
-use PublicInbox::LeiUp;
use File::Path ();
use SelectSaver;
-sub lei_forget_search {
+sub do_forget_search {
my ($lei, @outs) = @_;
my @dirs; # paths in ~/.local/share/lei/saved-search/
my $cwd;
@@ -30,9 +30,55 @@ sub lei_forget_search {
$save = SelectSaver->new($lei->{2});
}
File::Path::remove_tree(@dirs, $opt);
- $lei->fail if defined $cwd;
+ $lei->child_error(0) if defined $cwd;
+}
+
+sub lei_forget_search {
+ my ($lei, @outs) = @_;
+ my $prune = $lei->{opt}->{prune};
+ $prune // return do_forget_search($lei, @outs);
+ return $lei->fail("--prune and @outs incompatible") if @outs;
+ my @tmp = PublicInbox::LeiSavedSearch::list($lei);
+ my $self = bless { -mail_sync => 1 }, __PACKAGE__;
+ $self->filter_lss($lei, $prune) // return
+ $lei->fail("only --prune=$prune not understood");
+ if ($self->{o_remote}) { # setup lei->{auth}
+ $self->prepare_inputs($lei, $self->{o_remote}) or return;
+ }
+ my $ops = {};
+ $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
+ $lei->{wq1} = $self;
+ net_merge_all_done($self) unless $lei->{auth};
+ $lei->wait_wq_events($op_c, $ops);
+}
+
+sub do_prune {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ for my $o (@{$self->{o_local} // []}) {
+ next if -e $o;
+ $lei->qerr("# pruning $o");
+ eval { do_forget_search($lei, $o) };
+ $lei->child_error(0, "E: $@") if $@;
+ }
+ for my $o (@{$self->{o_remote} // []}) {
+ my $uri = PublicInbox::URIimap->new($o);
+ next if $lei->{net}->mic_for_folder($uri);
+ $lei->qerr("# pruning $uri");
+ eval { do_forget_search($lei, $o) };
+ $lei->child_error(0, "E: $@") if $@;
+ }
+}
+
+# called in top-level lei-daemon when LeiAuth is done
+sub net_merge_all_done {
+ my ($self) = @_;
+ $self->wq_do('do_prune');
+ $self->wq_close;
}
+*_wq_done_wait = \&PublicInbox::LEI::wq_done_wait;
*_complete_forget_search = \&PublicInbox::LeiUp::_complete_up;
1;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index dac0fc287885..79639d5e62a4 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -93,6 +93,21 @@ sub redispatch_all ($$) {
}
}
+sub filter_lss {
+ my ($self, $lei, $all) = @_;
+ my @outs = PublicInbox::LeiSavedSearch::list($lei);
+ if ($all eq 'local') {
+ $self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq 'remote') {
+ $self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq '') {
+ $self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } else {
+ undef;
+ }
+}
+
sub lei_up {
my ($lei, @outs) = @_;
my $opt = $lei->{opt};
@@ -101,17 +116,8 @@ sub lei_up {
return $lei->fail("--all and @outs incompatible") if @outs;
defined($opt->{mua}) and return
$lei->fail('--all and --mua= are incompatible');
- @outs = PublicInbox::LeiSavedSearch::list($lei);
- if ($all eq 'local') {
- $self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
- } elsif ($all eq 'remote') {
- $self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
- } elsif ($all eq '') {
- $self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
- $self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
- } else {
+ filter_lss($self, $lei, $all) // return
$lei->fail("only --all=$all not understood");
- }
} elsif ($lei->{lse}) { # redispatched
scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
return up1($lei, $outs[0]);
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 05d5d9f4436c..cd35461ce1b7 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -4,6 +4,7 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use PublicInbox::Smsg;
use List::Util qw(sum);
+use File::Path qw(remove_tree);
my $doc1 = eml_load('t/plack-qp.eml');
$doc1->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 5)}));
@@ -233,5 +234,11 @@ test_lei(sub {
and xbail "-ipe $lss[0]: $?";
lei_ok qw(ls-search);
is($lei_err, '', 'no errors w/ fixed config');
+
+ like($lei_out, qr!\Q$home/after\E!, "`after' in ls-search");
+ remove_tree("$home/after");
+ lei_ok qw(forget-search --prune);
+ lei_ok qw(ls-search);
+ unlike($lei_out, qr!\Q$home/after\E!, "`after' pruned");
});
done_testing;
^ permalink raw reply related [relevance 46%]
* [PATCH] doc: lei-forget-search: fix option name in --prune description
@ 2021-10-23 0:22 71% Kyle Meyer
2021-10-23 2:54 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-10-23 0:22 UTC (permalink / raw)
To: meta
Fixes: 6f8e16a266b30819 ("lei forget-search: support --prune=<local|remote>")
---
Documentation/lei-forget-search.pod | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
index 3a7f493c..adbe7638 100644
--- a/Documentation/lei-forget-search.pod
+++ b/Documentation/lei-forget-search.pod
@@ -17,8 +17,8 @@ Forget a saved search at C<OUTPUT>.
=item --prune[=<local|remote>]
C<--prune> will forget saved searches if the C<OUTPUT> no longer
-exists. C<--all=local> only prunes local mailboxes,
-C<--all=remote> only prunes remote mailboxes (currently
+exists. C<--prune=local> only prunes local mailboxes,
+C<--prune=remote> only prunes remote mailboxes (currently
C<imap://> and C<imaps://>).
=back
base-commit: 6f8e16a266b30819ff74c40bc532f8c3f4a9f4b7
--
2.33.1
^ permalink raw reply related [relevance 71%]
* Re: [PATCH] doc: lei-forget-search: fix option name in --prune description
2021-10-23 0:22 71% [PATCH] doc: lei-forget-search: fix option name in --prune description Kyle Meyer
@ 2021-10-23 2:54 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-23 2:54 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed as 0e22868f2c5ad03193ae077d17fdad64b51fdac2
^ permalink raw reply [relevance 71%]
* [PATCH 1/7] lei: always pass $lei to LeiAuth->op_merge
@ 2021-10-24 0:20 75% ` Eric Wong
2021-10-24 0:20 61% ` [PATCH 2/7] lei export-kw: skip read-only IMAP folders Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-24 0:20 UTC (permalink / raw)
To: meta
This will make future developments easier.
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
lib/PublicInbox/LeiForgetSearch.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 2 +-
lib/PublicInbox/LeiMailDiff.pm | 2 +-
lib/PublicInbox/LeiRefreshMailSync.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 2 +-
8 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 5be9e51f..756d0e9c 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -115,7 +115,7 @@ EOM
$self->{nwr}->{-skip_creat} = 1;
}
my $ops = {};
- $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $lei->{auth}->op_merge($ops, $self, $lei) if $lei->{auth};
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiForgetSearch.pm b/lib/PublicInbox/LeiForgetSearch.pm
index f353fe52..dfeb0293 100644
--- a/lib/PublicInbox/LeiForgetSearch.pm
+++ b/lib/PublicInbox/LeiForgetSearch.pm
@@ -46,7 +46,7 @@ sub lei_forget_search {
$self->prepare_inputs($lei, $self->{o_remote}) or return;
}
my $ops = {};
- $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $lei->{auth}->op_merge($ops, $self, $lei) if $lei->{auth};
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 2f8fd6c6..d8f39fdf 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -103,7 +103,7 @@ sub do_import_index ($$@) {
($lei->{opt}->{'new-only'} && (!$net || !$net->{imap_order})) and
warn "# --new-only is only for IMAP\n";
my $ops = {};
- $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $lei->{auth}->op_merge($ops, $self, $lei) if $lei->{auth};
$lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 7c3c0cda..5eb7032d 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -96,7 +96,7 @@ sub lei_ls_mail_source {
}
$lei->start_pager if $isatty;
my $ops = {};
- $lei->{auth}->op_merge($ops, $self);
+ $lei->{auth}->op_merge($ops, $self, $lei);
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiMailDiff.pm b/lib/PublicInbox/LeiMailDiff.pm
index a29ae225..4f3a4608 100644
--- a/lib/PublicInbox/LeiMailDiff.pm
+++ b/lib/PublicInbox/LeiMailDiff.pm
@@ -83,7 +83,7 @@ sub lei_mail_diff {
$lei->{opt}->{color} //= $isatty;
$lei->start_pager if $isatty;
my $ops = {};
- $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $lei->{auth}->op_merge($ops, $self, $lei) if $lei->{auth};
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 0cb9f3dc..f516f572 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -82,7 +82,7 @@ EOM
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $ops = {};
- $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $lei->{auth}->op_merge($ops, $self, $lei) if $lei->{auth};
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 817b87f8..77654d1a 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -49,7 +49,7 @@ sub lei_tag { # the "lei tag" method
grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or
return $lei->fail('no keywords or labels specified');
my $ops = {};
- $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $lei->{auth}->op_merge($ops, $self, $lei) if $lei->{auth};
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 119070a2..acc36897 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -548,7 +548,7 @@ sub do_query {
'child_error' => [ $lei ],
'incr_start_query' => [ $self, $lei ],
};
- $lei->{auth}->op_merge($ops, $l2m) if $l2m && $lei->{auth};
+ $lei->{auth}->op_merge($ops, $l2m, $lei) if $l2m && $lei->{auth};
my $end = $lei->pkt_op_pair;
$lei->{1}->autoflush(1);
$lei->start_pager if delete $lei->{need_pager};
^ permalink raw reply related [relevance 75%]
* [PATCH 2/7] lei export-kw: skip read-only IMAP folders
2021-10-24 0:20 75% ` [PATCH 1/7] lei: always pass $lei to LeiAuth->op_merge Eric Wong
@ 2021-10-24 0:20 61% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-24 0:20 UTC (permalink / raw)
To: meta
Since we want to store IMAP flags asynchronously and not wait
for results, we can't check for IMAP errors this way and end up
wasting bandwidth on public-inbox-imapd. Now, we just check
PERMANENTFLAGS up front to ensure a folder can handle IMAP flag
storage before proceeding.
---
lib/PublicInbox/LeiExportKw.pm | 12 +++++++++++-
lib/PublicInbox/NetWriter.pm | 9 +++++++++
t/lei-export-kw.t | 2 +-
t/lei-import-imap.t | 3 +++
4 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 756d0e9c..0ecfb782 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -67,7 +67,17 @@ sub input_path_url {
$self->{lms}->each_src($input, \&export_kw_md, $self, $mdir);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
- if (my $mic = $self->{nwr}->mic_for_folder($uri)) {
+ my $mic = $self->{nwr}->mic_for_folder($uri);
+ if ($mic && !$self->{nwr}->can_store_flags($mic)) {
+ my $m = "$input does not support PERMANENTFLAGS";
+ if (defined $self->{lei}->{opt}->{all}) {
+ $self->{lei}->qerr("# $m");
+ } else { # set error code if user explicitly requested
+ $self->{lei}->child_error(0, "E: $m");
+ }
+ return;
+ }
+ if ($mic) {
$self->{lms}->each_src($$uri, \&export_kw_imap,
$self, $mic);
$mic->expunge;
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 629a752a..4a1f34f6 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -56,4 +56,13 @@ sub imap_set_kw {
$mic; # caller must ->expunge
}
+sub can_store_flags {
+ my ($self, $mic) = @_;
+ for ($mic->Results) {
+ /^\* OK \[PERMANENTFLAGS \(([^\)]*)\)\].*/ and
+ return $self->can('perm_fl_ok')->($1);
+ }
+ undef;
+}
+
1;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
index 55730e87..88b2a80b 100644
--- a/t/lei-export-kw.t
+++ b/t/lei-export-kw.t
@@ -4,7 +4,7 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Copy qw(cp);
use File::Path qw(make_path);
-require_mods(qw(lei -imapd Mail::IMAPClient));
+require_mods(qw(lei)); # see lei-import-imap.t for IMAP tests
my ($tmpdir, $for_destroy) = tmpdir;
my $expect = eml_load('t/data/0001.patch');
my $do_export_kw = 1;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 315567b3..3b6cb299 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -110,6 +110,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
is(scalar(@$out), 2, 'got JSON') or diag explain($out);
lei_ok qw(lcat), $url_orig;
is($lei_out, $orig, 'lcat w/o UID works');
+
+ ok(!lei(qw(export-kw), $url_orig), 'export-kw fails on read-only IMAP');
+ like($lei_err, qr/does not support/, 'error noted in failure');
});
done_testing;
^ permalink raw reply related [relevance 61%]
* [PATCH] lei p2q: document --uri, add examples
@ 2021-10-25 19:31 64% Eric Wong
2021-10-25 23:39 71% ` Kyle Meyer
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-25 19:31 UTC (permalink / raw)
To: meta
This is useful for users lacking in local storage. Also,
referencing lei-add-external(1) seems to make less sense
than referencing lei-q(1).
We'll also stop dropping years from the copyright statement
to reduce future churn.
---
Documentation/lei-p2q.pod | 17 ++++++++++++++---
lib/PublicInbox/LEI.pm | 1 +
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/Documentation/lei-p2q.pod b/Documentation/lei-p2q.pod
index 1068ff0beb27..2e0b1ab676ed 100644
--- a/Documentation/lei-p2q.pod
+++ b/Documentation/lei-p2q.pod
@@ -57,6 +57,10 @@ Dump output that shows the information collected for every prefix.
This information can be useful for seeing how a patch is processed,
but the format should not be considered stable.
+=item --uri
+
+URI escape output for interacting with HTTP(S) public-inbox instances.
+
=item -q
=item --quiet
@@ -65,6 +69,14 @@ Suppress feedback messages.
=back
+=head1 EXAMPLES
+
+ # to search for all threads which touch a given thread:
+ lei p2q $COMMIT_OID | lei q -t -o /tmp/results
+
+ # to view results on a remote HTTP(S) public-inbox instance
+ $BROWSER https://example.com/pub-inbox/?q=$(lei p2q --uri $COMMIT_OID)
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -74,11 +86,10 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
-
=head1 SEE ALSO
-L<lei-add-external(1)>
+L<lei-q(1)>
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d24b8e2d6da2..e35339c16d8d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -355,6 +355,7 @@ my %OPTDESC = (
'want|w=s@' => [ 'PREFIX|dfpost|dfn', # common ones in help...
'search prefixes to extract (default: dfpost7)' ],
+'uri p2q' => [ 'URI escape output' ],
'alert=s@' => ['CMD,:WINCH,:bell,<any command>',
'run command(s) or perform ops when done writing to output ' .
^ permalink raw reply related [relevance 64%]
* Re: [PATCH] lei p2q: document --uri, add examples
2021-10-25 19:31 64% [PATCH] lei p2q: document --uri, add examples Eric Wong
@ 2021-10-25 23:39 71% ` Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-10-25 23:39 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> This is useful for users lacking in local storage. Also,
> referencing lei-add-external(1) seems to make less sense
> than referencing lei-q(1).
True, not sure why I went with lei-add-external there (probably forgot
to update that section after copying over an existing lei manpage as a
template).
> We'll also stop dropping years from the copyright statement
> to reduce future churn.
s/stop/start/ ?
^ permalink raw reply [relevance 71%]
* [PATCH 4/9] lei q: enable expensive Xapian flags
2021-10-26 10:35 69% [PATCH 0/9] lei p2q: more capable than originally thought Eric Wong
2021-10-26 10:35 67% ` [PATCH 2/9] doc: lei-store-format: mail sync section, update IPC Eric Wong
@ 2021-10-26 10:35 71% ` Eric Wong
2021-10-26 10:35 71% ` [PATCH 5/9] lei inspect: fix atfork hook Eric Wong
` (3 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:35 UTC (permalink / raw)
To: meta
FLAG_PURE_NOT is too expensive for public-facing WWW use, but
lei isn't public-facing. We'll also unconditionally enable
phrase search on old "chert" DBs since lei doesn't need to
worry about fairness across 10K users.
---
lib/PublicInbox/LeiSearch.pm | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 1fb38da1d7aa..d0ca13f0aa11 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -175,6 +175,8 @@ sub all_terms {
sub qparse_new {
my ($self) = @_;
my $qp = $self->SUPER::qparse_new; # PublicInbox::Search
+ $self->{qp_flags} |= PublicInbox::Search::FLAG_PHRASE() |
+ PublicInbox::Search::FLAG_PURE_NOT();
$qp->add_boolean_prefix('kw', 'K');
$qp->add_boolean_prefix('L', 'L');
$qp
^ permalink raw reply related [relevance 71%]
* [PATCH 5/9] lei inspect: fix atfork hook
2021-10-26 10:35 69% [PATCH 0/9] lei p2q: more capable than originally thought Eric Wong
2021-10-26 10:35 67% ` [PATCH 2/9] doc: lei-store-format: mail sync section, update IPC Eric Wong
2021-10-26 10:35 71% ` [PATCH 4/9] lei q: enable expensive Xapian flags Eric Wong
@ 2021-10-26 10:35 71% ` Eric Wong
2021-10-26 10:35 66% ` [PATCH 6/9] lei: add net getopt spec to various commands Eric Wong
` (2 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:35 UTC (permalink / raw)
To: meta
The misnamed sub wasn't firing, but was unlikely to be
noticeable given the short lifetime of the process.
Fixes: 1f887bd51d92b0d4 ("lei inspect: add atfork hook")
---
lib/PublicInbox/LeiInspect.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 5ea32ccb7e66..d7775d4b6162 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -294,7 +294,7 @@ sub _complete_inspect {
# TODO: message-ids?, blobs? could get expensive...
}
-sub input_only_atfork_child {
+sub ipc_atfork_child {
my ($self) = @_;
$self->{lei}->_lei_atfork_child;
$self->SUPER::ipc_atfork_child;
^ permalink raw reply related [relevance 71%]
* [PATCH 0/9] lei p2q: more capable than originally thought
@ 2021-10-26 10:35 69% Eric Wong
2021-10-26 10:35 67% ` [PATCH 2/9] doc: lei-store-format: mail sync section, update IPC Eric Wong
` (5 more replies)
0 siblings, 6 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:35 UTC (permalink / raw)
To: meta
I started writing documentation and managed to cobble up a p2q
example in patch 7/9 to find unapplied patches when combined
with "git log". 7/9 makes p2q use less memory, now.
Fixed a bunch of small bugs along the way and killed some
redundant code, too.
Eric Wong (9):
doc: tuning: additional notes for many inboxes
doc: lei-store-format: mail sync section, update IPC
eml: keep body if no headers are found
lei q: enable expensive Xapian flags
lei inspect: fix atfork hook
lei: add net getopt spec to various commands
lei p2q: use LeiInput for multi-patch series
lei rm|tag: drop redundant mbox+net callbacks
input_pipe: account for undefined {sock}
Documentation/lei-p2q.pod | 6 ++
Documentation/lei-store-format.pod | 14 +++-
Documentation/public-inbox-tuning.pod | 11 ++-
lib/PublicInbox/Eml.pm | 7 +-
lib/PublicInbox/InputPipe.pm | 2 +-
lib/PublicInbox/LEI.pm | 11 +--
lib/PublicInbox/LeiInput.pm | 30 +++++--
lib/PublicInbox/LeiInspect.pm | 2 +-
lib/PublicInbox/LeiP2q.pm | 115 +++++++++++++-------------
lib/PublicInbox/LeiRm.pm | 10 ---
lib/PublicInbox/LeiSearch.pm | 2 +
lib/PublicInbox/LeiTag.pm | 11 ---
t/eml.t | 11 +++
t/mbox_reader.t | 6 +-
14 files changed, 134 insertions(+), 104 deletions(-)
^ permalink raw reply [relevance 69%]
* [PATCH 8/9] lei rm|tag: drop redundant mbox+net callbacks
2021-10-26 10:35 69% [PATCH 0/9] lei p2q: more capable than originally thought Eric Wong
` (4 preceding siblings ...)
2021-10-26 10:35 35% ` [PATCH 7/9] lei p2q: use LeiInput for multi-patch series Eric Wong
@ 2021-10-26 10:35 68% ` Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:35 UTC (permalink / raw)
To: meta
These are supplied by the base LeiInput class
---
lib/PublicInbox/LeiRm.pm | 10 ----------
lib/PublicInbox/LeiTag.pm | 11 -----------
2 files changed, 21 deletions(-)
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 97b1c5c15dc4..524c178e3b47 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -13,16 +13,6 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
$self->{lei}->{sto}->wq_do('remove_eml', $eml);
}
-sub input_mbox_cb { # MboxReader callback
- my ($eml, $self) = @_;
- input_eml_cb($self, $eml);
-}
-
-sub input_net_cb { # callback for ->imap_each, ->nntp_each
- my (undef, undef, $kw, $eml, $self) = @_; # @_[0,1]: url + uid ignored
- input_eml_cb($self, $eml);
-}
-
sub input_maildir_cb {
my (undef, $kw, $eml, $self) = @_; # $_[0] $filename ignored
input_eml_cb($self, $eml);
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 77654d1a2f22..d64a9f86e05a 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -19,23 +19,12 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
}
}
-sub input_mbox_cb {
- my ($eml, $self) = @_;
- $eml->header_set($_) for (qw(X-Status Status));
- input_eml_cb($self, $eml);
-}
-
sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
my ($self, $f) = @_;
my $eml = eml_from_path($f) or return;
input_eml_cb($self, $eml);
}
-sub input_net_cb { # imap_each, nntp_each cb
- my ($url, $uid, $kw, $eml, $self) = @_;
- input_eml_cb($self, $eml);
-}
-
sub lei_tag { # the "lei tag" method
my ($lei, @argv) = @_;
my $sto = $lei->_lei_store(1);
^ permalink raw reply related [relevance 68%]
* [PATCH 2/9] doc: lei-store-format: mail sync section, update IPC
2021-10-26 10:35 69% [PATCH 0/9] lei p2q: more capable than originally thought Eric Wong
@ 2021-10-26 10:35 67% ` Eric Wong
2021-10-26 10:35 71% ` [PATCH 4/9] lei q: enable expensive Xapian flags Eric Wong
` (4 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:35 UTC (permalink / raw)
To: meta
mail_sync.sqlite3 needs to be documented, and brings the IPC
section up-to-date while we're in the area.
---
Documentation/lei-store-format.pod | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/Documentation/lei-store-format.pod b/Documentation/lei-store-format.pod
index 71aa72cb96e3..625c60f436a8 100644
--- a/Documentation/lei-store-format.pod
+++ b/Documentation/lei-store-format.pod
@@ -30,7 +30,6 @@ prevent them from being accidentally treated as a v2 inbox.
$SHARD - Integer starting with 0 based on parallelism
~/.local/share/lei/store
- - ipc.lock # lock file for internal lei IPC
- local/$EPOCH.git # normal bare git repositories
- mail_sync.sqlite3 # sync state IMAP, Maildir, NNTP
@@ -66,11 +65,18 @@ stored in Xapian indices, volatile metadata is associated with
the Xapian document, thus it is shared across different blobs of
the "same" message.
+=head2 mail_sync.sqlite3
+
+This SQLite database maintained for bidirectional mapping of
+git blobs to IMAP UIDs, Maildir file names, and NNTP article numbers.
+
+It is also used for retrieving messages from Maildirs indexed by
+L<lei-index(1)>.
+
=head1 IPC
-When L<lei(1)> is run in daemon mode, L<flock(2)> is used on
-C<ipc.lock> is used to serialize writes to C<lei/store> across
-multiple internal lei workers while minimizing commits.
+L<lei-daemon(8)> communicates with the C<lei/store> process using
+L<unix(7)> C<SOCK_SEQPACKET> sockets.
=head1 CAVEATS
^ permalink raw reply related [relevance 67%]
* [PATCH 6/9] lei: add net getopt spec to various commands
2021-10-26 10:35 69% [PATCH 0/9] lei p2q: more capable than originally thought Eric Wong
` (2 preceding siblings ...)
2021-10-26 10:35 71% ` [PATCH 5/9] lei inspect: fix atfork hook Eric Wong
@ 2021-10-26 10:35 66% ` Eric Wong
2021-10-26 10:35 35% ` [PATCH 7/9] lei p2q: use LeiInput for multi-patch series Eric Wong
2021-10-26 10:35 68% ` [PATCH 8/9] lei rm|tag: drop redundant mbox+net callbacks Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:35 UTC (permalink / raw)
To: meta
All of these commands should support --proxy, at least, if not
other curl options.
---
lib/PublicInbox/LEI.pm | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 32d4b9f3b427..93a7b426de3c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -179,7 +179,7 @@ our %CMD = ( # sorted in order of importance/use:
'up' => [ 'OUTPUT...|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ exclude=s@
- remote-fudge-time=s all:s remote! local! external!), @c_opt ],
+ remote-fudge-time=s all:s remote! local! external!), @net_opt, @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
@@ -215,7 +215,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
- qw(z|0 ascii l pretty url), @c_opt ],
+ qw(z|0 ascii l pretty url), @net_opt, @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
@@ -265,7 +265,8 @@ our %CMD = ( # sorted in order of importance/use:
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
'refresh-mail-sync' => [ 'LOCATION...|--all',
- 'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
+ 'prune dangling sync data for a mail folder', 'all:s',
+ @net_opt, @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
qw(all:s mode=s), @net_opt, @c_opt ],
^ permalink raw reply related [relevance 66%]
* [PATCH 7/9] lei p2q: use LeiInput for multi-patch series
2021-10-26 10:35 69% [PATCH 0/9] lei p2q: more capable than originally thought Eric Wong
` (3 preceding siblings ...)
2021-10-26 10:35 66% ` [PATCH 6/9] lei: add net getopt spec to various commands Eric Wong
@ 2021-10-26 10:35 35% ` Eric Wong
2021-10-26 10:35 68% ` [PATCH 8/9] lei rm|tag: drop redundant mbox+net callbacks Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:35 UTC (permalink / raw)
To: meta
The LeiInput backend now allows p2q to work like any other
command which reads .eml, .patch, mbox*, Maildir, IMAP, and NNTP
input. Running "git format-patch --stdout -1 $COMMIT" remains
supported.
This is intended to allow lower memory use while parsing
"git log --pretty=mboxrd -p" output. Previously, the entire
output of "git log" would be slurped into memory at once.
The intended use is to allow easy(-ish :P) searching for
unapplied patches as documented in the new example in the
manpage.
---
Documentation/lei-p2q.pod | 6 ++
lib/PublicInbox/LEI.pm | 4 +-
lib/PublicInbox/LeiInput.pm | 30 ++++++++--
lib/PublicInbox/LeiP2q.pm | 115 ++++++++++++++++++------------------
4 files changed, 89 insertions(+), 66 deletions(-)
diff --git a/Documentation/lei-p2q.pod b/Documentation/lei-p2q.pod
index 2e0b1ab676ed..4bc5d25f8ef0 100644
--- a/Documentation/lei-p2q.pod
+++ b/Documentation/lei-p2q.pod
@@ -77,6 +77,12 @@ Suppress feedback messages.
# to view results on a remote HTTP(S) public-inbox instance
$BROWSER https://example.com/pub-inbox/?q=$(lei p2q --uri $COMMIT_OID)
+ # to view unapplied patches for a given $FILE from the past year:
+ echo \( rt:last.year.. AND dfn:$FILE \) AND NOT \( \
+ $(git log -p --pretty=mboxrd --since=last.year $FILE |
+ lei p2q -F mboxrd )
+ \) | lei q -o /tmp/unapplied
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 93a7b426de3c..f9d24f29c87d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -274,9 +274,9 @@ our %CMD = ( # sorted in order of importance/use:
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
@net_opt, @c_opt ],
-'p2q' => [ 'FILE|COMMIT_OID|--stdin',
+'p2q' => [ 'LOCATION_OR_COMMIT...|--stdin',
"use a patch to generate a query for `lei q --stdin'",
- qw(stdin| want|w=s@ uri debug), @c_opt ],
+ qw(stdin| in-format|F=s want|w=s@ uri debug), @net_opt, @c_opt ],
'config' => [ '[...]', sub {
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 2621fc1f9d05..540681e3ff6b 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -64,6 +64,11 @@ sub input_mbox_cb { # base MboxReader callback
$self->input_eml_cb($eml);
}
+sub input_net_cb { # imap_each, nntp_each cb
+ my ($url, $uid, $kw, $eml, $self) = @_;
+ $self->input_eml_cb($eml);
+}
+
# import a single file handle of $name
# Subclass must define ->input_eml_cb and ->input_mbox_cb
sub input_fh {
@@ -108,10 +113,10 @@ sub handle_http_input ($$@) {
grep(/\A--compressed\z/, @$curl) or
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
eval { $self->input_fh('mboxrd', $fh, $url, @args) };
- my $err = $@;
+ my @err = ($@ ? $@ : ());
$ar->join;
- $? || $err and
- $lei->child_error($?, "@$cmd failed".$err ? " $err" : '');
+ push(@err, "\$?=$?") if $?;
+ $lei->child_error($?, "@$cmd failed: @err") if @err;
}
sub input_path_url {
@@ -184,7 +189,17 @@ EOM
$self, @args);
}
} elsif ($self->{missing_ok} && !-e $input) { # don't ->fail
- $self->folder_missing("$ifmt:$input");
+ if ($lei->{cmd} eq 'p2q') {
+ my $fp = [ qw(git format-patch --stdout -1), $input ];
+ my $rdr = { 2 => $lei->{2} };
+ my $fh = popen_rd($fp, undef, $rdr);
+ eval { $self->input_fh('eml', $fh, $input, @args) };
+ my @err = ($@ ? $@ : ());
+ close($fh) or push @err, "\$?=$?";
+ $lei->child_error($?, "@$fp failed: @err") if @err;
+ } else {
+ $self->folder_missing("$ifmt:$input");
+ }
} else {
$lei->fail("$ifmt_pfx$input unsupported (TODO)");
}
@@ -330,9 +345,12 @@ $input is `eml', not --in-format=$in_fmt
}
push @md, $input;
} elsif ($self->{missing_ok} && !-e $input) {
- # for lei rm-watch
- $may_sync and $input = 'maildir:'.
+ if ($lei->{cmd} eq 'p2q') {
+ # will run "git format-patch"
+ } elsif ($may_sync) { # for lei rm-watch
+ $input = 'maildir:'.
$lei->abs_path($input);
+ }
} else {
return $lei->fail("Unable to handle $input")
}
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index 08ec81c5295e..09ec0a079bb9 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -1,16 +1,16 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# front-end for the "lei patch-to-query" sub-command
package PublicInbox::LeiP2q;
use strict;
use v5.10.1;
-use parent qw(PublicInbox::IPC);
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::Eml;
use PublicInbox::Smsg;
use PublicInbox::MsgIter qw(msg_part_text);
use PublicInbox::Git qw(git_unquote);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::OnDestroy;
use URI::Escape qw(uri_escape_utf8);
my $FN = qr!((?:"?[^/\n]+/[^\r\n]+)|/dev/null)!;
@@ -28,8 +28,16 @@ sub xphrase ($) {
} ($s =~ m!(\w[\|=><,\./:\\\@\-\w\s]+)!g);
}
+sub add_qterm ($$@) {
+ my ($self, $p, @v) = @_;
+ for (@v) {
+ $self->{qseen}->{"$p\0$_"} //=
+ push(@{$self->{qterms}->{$p}}, $_);
+ }
+}
+
sub extract_terms { # eml->each_part callback
- my ($p, $lei) = @_;
+ my ($p, $self) = @_;
my $part = $p->[0]; # ignore $depth and @idx;
my $ct = $part->content_type || 'text/plain';
my ($s, undef) = msg_part_text($part, $ct);
@@ -38,7 +46,7 @@ sub extract_terms { # eml->each_part callback
# TODO: b: nq: q:
for (split(/\n/, $s)) {
if ($in_diff && s/^ //) { # diff context
- push @{$lei->{qterms}->{dfctx}}, xphrase($_);
+ add_qterm($self, 'dfctx', xphrase($_));
} elsif (/^-- $/) { # email signature begins
$in_diff = undef;
} elsif (m!^diff --git $FN $FN!) {
@@ -46,21 +54,21 @@ sub extract_terms { # eml->each_part callback
$in_diff = 1;
} elsif (/^index ([a-f0-9]+)\.\.([a-f0-9]+)\b/) {
my ($oa, $ob) = ($1, $2);
- push @{$lei->{qterms}->{dfpre}}, $oa;
- push @{$lei->{qterms}->{dfpost}}, $ob;
+ add_qterm($self, 'dfpre', $oa);
+ add_qterm($self, 'dfpost', $ob);
# who uses dfblob?
} elsif (m!^(?:---|\+{3}) ($FN)!) {
next if $1 eq '/dev/null';
my $fn = (split(m!/!, git_unquote($1.''), 2))[1];
- push @{$lei->{qterms}->{dfn}}, xphrase($fn);
+ add_qterm($self, 'dfn', xphrase($fn));
} elsif ($in_diff && s/^\+//) { # diff added
- push @{$lei->{qterms}->{dfb}}, xphrase($_);
+ add_qterm($self, 'dfb', xphrase($_));
} elsif ($in_diff && s/^-//) { # diff removed
- push @{$lei->{qterms}->{dfa}}, xphrase($_);
+ add_qterm($self, 'dfa', xphrase($_));
} elsif (/^@@ (?:\S+) (?:\S+) @@\s*$/) {
# traditional diff w/o -p
} elsif (/^@@ (?:\S+) (?:\S+) @@\s*(\S+.*)/) {
- push @{$lei->{qterms}->{dfhh}}, xphrase($1);
+ add_qterm($self, 'dfhh', xphrase($1));
} elsif (/^(?:dis)similarity index/ ||
/^(?:old|new) mode/ ||
/^(?:deleted|new) file mode/ ||
@@ -92,53 +100,43 @@ my %pfx2smsg = (
rt => [ qw(ts) ], # ditto...
);
-sub do_p2q { # via wq_do
- my ($self) = @_;
- my $lei = $self->{lei};
- my $want = $lei->{opt}->{want} // [ qw(dfpost7) ];
- my @want = split(/[, ]+/, "@$want");
- for (@want) {
- /\A(?:(d|dt|rt):)?([0-9]+)(\.(?:day|weeks)s?)?\z/ or next;
- my ($pfx, $n, $unit) = ($1, $2, $3);
- $n *= 86400 * ($unit =~ /week/i ? 7 : 1);
- $_ = [ $pfx, $n ];
- }
- my $smsg = bless {}, 'PublicInbox::Smsg';
- my $in = $self->{0};
- my @cmd;
- unless ($in) {
- my $input = $self->{input};
- my $devfd = $lei->path_to_fd($input) // return;
- if ($devfd >= 0) {
- $in = $lei->{$devfd};
- } elsif (-e $input) {
- open($in, '<', $input) or
- return $lei->fail("open < $input: $!");
- } else {
- @cmd = (qw(git format-patch --stdout -1), $input);
- $in = popen_rd(\@cmd, undef, { 2 => $lei->{2} });
+sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
+ my ($self, $eml) = @_;
+ my $diff_want = $self->{diff_want} // do {
+ my $want = $self->{lei}->{opt}->{want} // [ qw(dfpost7) ];
+ my @want = split(/[, ]+/, "@$want");
+ for (@want) {
+ /\A(?:(d|dt|rt):)?([0-9]+)(\.(?:day|weeks)s?)?\z/
+ or next;
+ my ($pfx, $n, $unit) = ($1, $2, $3);
+ $n *= 86400 * ($unit =~ /week/i ? 7 : 1);
+ $_ = [ $pfx, $n ];
}
+ $self->{want_order} = \@want;
+ $self->{diff_want} = +{ map { $_ => 1 } @want };
};
- my $str = do { local $/; <$in> };
- @cmd && !close($in) and return $lei->fail("E: @cmd failed: $?");
- my $eml = PublicInbox::Eml->new(\$str);
- $lei->{diff_want} = +{ map { $_ => 1 } @want };
+ my $smsg = bless {}, 'PublicInbox::Smsg';
$smsg->populate($eml);
while (my ($pfx, $fields) = each %pfx2smsg) {
- next unless $lei->{diff_want}->{$pfx};
+ next unless $diff_want->{$pfx};
for my $f (@$fields) {
my $v = $smsg->{$f} // next;
- push @{$lei->{qterms}->{$pfx}}, xphrase($v);
+ add_qterm($self, $pfx, xphrase($v));
}
}
- $eml->each_part(\&extract_terms, $lei, 1);
+ $eml->each_part(\&extract_terms, $self, 1);
+}
+
+sub emit_query {
+ my ($self) = @_;
+ my $lei = $self->{lei};
if ($lei->{opt}->{debug}) {
my $json = ref(PublicInbox::Config->json)->new;
$json->utf8->canonical->pretty;
- print { $lei->{2} } $json->encode($lei->{qterms});
+ print { $lei->{2} } $json->encode($self->{qterms});
}
my (@q, %seen);
- for my $pfx (@want) {
+ for my $pfx (@{$self->{want_order}}) {
if (ref($pfx) eq 'ARRAY') {
my ($p, $t_range) = @$pfx; # TODO
@@ -148,7 +146,7 @@ sub do_p2q { # via wq_do
} else {
my $plusminus = ($pfx =~ s/\A([\+\-])//) ? $1 : '';
my $end = ($pfx =~ s/([0-9\*]+)\z//) ? $1 : '';
- my $x = delete($lei->{qterms}->{$pfx}) or next;
+ my $x = delete($self->{qterms}->{$pfx}) or next;
my $star = $end =~ tr/*//d ? '*' : '';
my $min_len = ($end || 0) + 0;
@@ -181,24 +179,25 @@ sub do_p2q { # via wq_do
}
sub lei_p2q { # the "lei patch-to-query" entry point
- my ($lei, $input) = @_;
- my $self = bless {}, __PACKAGE__;
- if ($lei->{opt}->{stdin}) {
- $self->{0} = delete $lei->{0}; # guard from _lei_atfork_child
- } else {
- $self->{input} = $input;
- }
- my ($op_c, $ops) = $lei->workers_start($self, 1);
+ my ($lei, @inputs) = @_;
+ $lei->{opt}->{'in-format'} //= 'eml' if $lei->{opt}->{stdin};
+ my $self = bless { missing_ok => 1 }, __PACKAGE__;
+ $self->prepare_inputs($lei, \@inputs) or return;
+ my $ops = {};
+ $lei->{auth}->op_merge($ops, $self, $lei) if $lei->{auth};
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
- $self->wq_io_do('do_p2q', []);
- $self->wq_close;
+ net_merge_all_done($self) unless $lei->{auth};
$lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
my ($self) = @_;
- $self->{lei}->_lei_atfork_child;
- $self->SUPER::ipc_atfork_child;
+ PublicInbox::LeiInput::input_only_atfork_child($self);
+ PublicInbox::OnDestroy->new($$, \&emit_query, $self);
}
+no warnings 'once';
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
1;
^ permalink raw reply related [relevance 35%]
* [PATCH] t/lei-watch: add diagnostics for failure
@ 2021-10-26 10:47 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 10:47 UTC (permalink / raw)
To: meta
I just got a difficult-to-reproduce failure, here; so there's
still some issues with the up-to-dateness of the inotify watcher.
---
t/lei-watch.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-watch.t b/t/lei-watch.t
index df887a03dc77..9158571c60cc 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -58,7 +58,7 @@ test_lei(sub {
lei_ok qw(q mid:testmessage@example.com -o), $md2, '-I', "$ro_home/t1";
my @f2 = glob("$md2/*/*");
is(scalar(@f2), 1, 'got one result');
- like($f2[0], qr/S\z/, 'seen set from rename');
+ like($f2[0], qr/S\z/, 'seen set from rename') or diag explain(\@f2);
my $e2 = eml_load($f2[0]);
my $e1 = eml_load("$f[0]S");
is_deeply($e2, $e1, 'results match');
^ permalink raw reply related [relevance 71%]
* [PATCH] lei mail-diff: support more inputs, split newlines
@ 2021-10-26 21:18 64% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-26 21:18 UTC (permalink / raw)
To: meta
Support --in-format like the rest of LeiInput users, and don't
default to .eml if a per-input format was specified. In any
case, I saved a bunch of messages from mutt which uses mboxcl2.
We'll also split newlines for diff, since it's a pain to read
diffs with escaped "\n" characters in them.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiMailDiff.pm | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f9d24f29c87d..96f7c5e315a9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -202,7 +202,7 @@ our %CMD = ( # sorted in order of importance/use:
'mail-diff' => [ '--stdin|LOCATION...', 'diff the contents of emails',
'stdin|', # /|\z/ must be first for lone dash
- qw(verbose|v+ color:s no-color raw-header),
+ qw(verbose|v+ in-format|F=s color:s no-color raw-header),
@diff_opt, @net_opt, @c_opt ],
'add-external' => [ 'LOCATION',
diff --git a/lib/PublicInbox/LeiMailDiff.pm b/lib/PublicInbox/LeiMailDiff.pm
index 4f3a4608d8e0..b21a0c366fd6 100644
--- a/lib/PublicInbox/LeiMailDiff.pm
+++ b/lib/PublicInbox/LeiMailDiff.pm
@@ -76,7 +76,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
sub lei_mail_diff {
my ($lei, @argv) = @_;
- $lei->{opt}->{'in-format'} //= 'eml';
+ $lei->{opt}->{'in-format'} //= 'eml' if !grep(/\A[a-z0-9]+:/i, @argv);
my $self = bless {}, __PACKAGE__;
$self->prepare_inputs($lei, \@argv) or return;
my $isatty = -t $lei->{1};
@@ -94,7 +94,7 @@ sub lei_mail_diff {
no warnings 'once';
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
-package PublicInbox::ContentDigestDbg;
+package PublicInbox::ContentDigestDbg; # cf. PublicInbox::ContentDigest
use strict;
use v5.10.1;
use Data::Dumper;
@@ -103,7 +103,7 @@ sub new { bless { dig => Digest::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
sub add {
$_[0]->{dig}->add($_[1]);
- print { $_[0]->{fh} } Dumper($_[1]) or die "print $!";
+ print { $_[0]->{fh} } Dumper([split(/^/sm, $_[1])]) or die "print $!";
}
sub hexdigest { $_[0]->{dig}->hexdigest; }
^ permalink raw reply related [relevance 64%]
* [PATCH] lei q: fix remote import accounting
@ 2021-10-27 21:09 64% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-27 21:09 UTC (permalink / raw)
To: meta; +Cc: Thomas Weißschuh
We need to update the {-nr_remote_eml} counter regardless
of progress display being enabled since it's needed for
saved searches. We'll also split out the {-imported} flag
separately and only call LeiStore->done if a new message
was imported.
Note: this change is NOT expected to fix errors reported by
Thomas in <ebf92218-1470-4602-b534-6dae59639dc6@t-8ch.de>
Cc: Thomas Weißschuh <thomas@t-8ch.de>
---
lib/PublicInbox/LeiXSearch.pm | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index acc36897d4e3..2a037f2bd79b 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -283,20 +283,22 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
my ($res, $kw) = $self->{import_sto}->wq_do('add_eml', $eml);
- $smsg = $res if ref($res) eq ref($smsg); # totally new message
+ if (ref($res) eq ref($smsg)) { # totally new message
+ $smsg = $res;
+ $self->{-imported} = 1;
+ }
$smsg->{kw} = $kw; # short-circuit xsmsg_vmd
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: $lei->git_oid($eml)->hexdigest;
_smsg_fill($smsg, $eml);
wait_startq($lei);
+ my $nr = ++$lei->{-nr_remote_eml}; # needed for lss->cfg_set
if ($lei->{-progress}) {
- ++$lei->{-nr_remote_eml};
my $now = now();
my $next = $lei->{-next_progress} //= ($now + 1);
if ($now > $next) {
$lei->{-next_progress} = $now + 1;
- my $nr = $lei->{-nr_remote_eml};
mset_progress($lei, $lei->{-current_url}, $nr, '?');
}
}
@@ -374,13 +376,14 @@ sub query_remote_mboxrd {
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml, $self,
$lei, $each_smsg);
- my $nr = $lei->{-nr_remote_eml};
- my $wait = $lei->{sto}->wq_do('done') if $nr && $lei->{sto};
+ if ($self->{import_sto} && delete($self->{-imported})) {
+ my $wait = $self->{import_sto}->wq_do('done');
+ }
$reap_curl->join;
if ($? == 0) {
# don't update if no results, maybe MTA is down
- $key && $nr and
- $lei->{lss}->cfg_set($key, $start);
+ my $nr = $lei->{-nr_remote_eml};
+ $lei->{lss}->cfg_set($key, $start) if $key && $nr;
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
}
^ permalink raw reply related [relevance 64%]
* lei-q-remote-import failures [was: [PATCH] t/index-git-times: support non-master default branch]
@ 2021-10-27 21:15 71% ` Eric Wong
2021-10-27 21:24 39% ` Thomas Weißschuh
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-10-27 21:15 UTC (permalink / raw)
To: Thomas Weißschuh; +Cc: meta
Thomas Weißschuh <thomas@t-8ch.de> wrote:
> On 2021-10-26 05:28+0000, Eric Wong wrote:
> > Thomas Weißschuh <thomas@t-8ch.de> wrote:
> > > The only failures I still see are in t/lei-q-remote-import.t which is unrelated
> > > to the git branch and some more that fail only during parallel test execution.
> > >
> > > In t/lei-q-remote-import.t it seems the search results are not memoized to the
> > > local store. (See attachment)
> >
> > Thanks, I'll check out lei-q-remote-import.t separately.
> > That could be related to SMP or slow/fast storage.
> > Any details you can share about CPU core count, speeds
> > or storage speeds, or CPU scheduler, CONFIG_HZ?
>
> CPU:
>
> Model name: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz
> CPU family: 6
> Model: 140
> Thread(s) per core: 2
> Core(s) per socket: 4
> Socket(s): 1
> Stepping: 1
> CPU max MHz: 4700,0000
> CPU min MHz: 400,0000
>
> Storage: Samsung EVO 970 (with plenty of memory cache)
> CONFIG_HZ: 300
I'm puzzled by this failure; I'm not sure if I've
ever seen it. Does it fail when run standalone?
(prove -bvw t/lei-q-remote-import.t)
Maybe running under TEST_LEI_ERR_LOUD=1 or checking
syslog can reveal something.
Adding a sleep shouldn't be necessary, but maybe something
else is broken I'm not seeing...:
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 92d8c9b6058c..a2d643cd06c7 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -35,6 +35,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok(@cmd, '-I', $url);
is_deeply($slurp_emls->($o), $exp1, 'got results after remote search');
unlink $o or BAIL_OUT $!;
+ sleep 1;
lei_ok(@cmd);
ok(-f $o && -s _, 'output exists after import but is not empty') or
diag $lei_err;
^ permalink raw reply related [relevance 71%]
* Re: lei-q-remote-import failures [was: [PATCH] t/index-git-times: support non-master default branch]
2021-10-27 21:15 71% ` lei-q-remote-import failures [was: [PATCH] t/index-git-times: support non-master default branch] Eric Wong
@ 2021-10-27 21:24 39% ` Thomas Weißschuh
0 siblings, 0 replies; 200+ results
From: Thomas Weißschuh @ 2021-10-27 21:24 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
[-- Attachment #1: Type: text/plain, Size: 2131 bytes --]
On 2021-10-27 21:15+0000, Eric Wong wrote:
> Thomas Weißschuh <thomas@t-8ch.de> wrote:
> > On 2021-10-26 05:28+0000, Eric Wong wrote:
> > > Thomas Weißschuh <thomas@t-8ch.de> wrote:
> > > > The only failures I still see are in t/lei-q-remote-import.t which is unrelated
> > > > to the git branch and some more that fail only during parallel test execution.
> > > >
> > > > In t/lei-q-remote-import.t it seems the search results are not memoized to the
> > > > local store. (See attachment)
> > >
> > > Thanks, I'll check out lei-q-remote-import.t separately.
> > > That could be related to SMP or slow/fast storage.
> > > Any details you can share about CPU core count, speeds
> > > or storage speeds, or CPU scheduler, CONFIG_HZ?
> >
> > CPU:
> >
> > Model name: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz
> > CPU family: 6
> > Model: 140
> > Thread(s) per core: 2
> > Core(s) per socket: 4
> > Socket(s): 1
> > Stepping: 1
> > CPU max MHz: 4700,0000
> > CPU min MHz: 400,0000
> >
> > Storage: Samsung EVO 970 (with plenty of memory cache)
> > CONFIG_HZ: 300
>
> I'm puzzled by this failure; I'm not sure if I've
> ever seen it. Does it fail when run standalone?
> (prove -bvw t/lei-q-remote-import.t)
It also does. The protocol is attached.
> Maybe running under TEST_LEI_ERR_LOUD=1 or checking
> syslog can reveal something.
>
> Adding a sleep shouldn't be necessary, but maybe something
> else is broken I'm not seeing...:
It didn't help, neither did larger sleeps.
> diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
> index 92d8c9b6058c..a2d643cd06c7 100644
> --- a/t/lei-q-remote-import.t
> +++ b/t/lei-q-remote-import.t
> @@ -35,6 +35,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
> lei_ok(@cmd, '-I', $url);
> is_deeply($slurp_emls->($o), $exp1, 'got results after remote search');
> unlink $o or BAIL_OUT $!;
> + sleep 1;
> lei_ok(@cmd);
> ok(-f $o && -s _, 'output exists after import but is not empty') or
> diag $lei_err;
I'll see if I can debug it in the coming days.
[-- Attachment #2: output-lei-q-remote-import.t --]
[-- Type: text/troff, Size: 6315 bytes --]
$ git checkout origin/master
Note: switching to 'origin/master'.
HEAD is now at ded9dad2 test_common: key test inboxes to init.defaultBranch
$ git clean -fdx t/data-gen/
...
$ TEST_LEI_ERR_LOUD=1 prove -bvw t/lei-q-remote-import.t
# # /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (0 matches)
ok 1 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:qp@example.com
ok 2 - output exists but is empty
# lei_err=# /usr/bin/curl -Sf -s -d '' http://[::1]:46299/t2/?x=m&q=m%3Aqp%40example.com
# # /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # http://[::1]:46299/t2/ 1/1
# # 1 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (1 matches)
ok 3 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:qp@example.com -I http://[::1]:46299/t2/
ok 4 - got results after remote search
ok 5 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:qp@example.com
not ok 6 - output exists after import but is not empty
# lei_err=# /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (0 matches)
# Failed test 'output exists after import but is not empty'
# at t/lei-q-remote-import.t line 39.
# # /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (0 matches)
not ok 7 - got results w/o remote search
# Failed test 'got results w/o remote search'
# at t/lei-q-remote-import.t line 41.
# Structures begin differing at:
# $got->[0] = Does not exist
# $expected->[0] = PublicInbox::Eml=HASH(0x565282aeff78)
ok 8 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:199707281508.AAA24167@hoyogw.example -I http://[::1]:46299/t2/ --no-import-remote
# lei_err=# /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # /usr/bin/curl -Sf -s -d '' http://[::1]:46299/t2/?x=m&q=m%3A199707281508.AAA24167%40hoyogw.example
# # http://[::1]:46299/t2/ 1/1
# # 1 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (1 matches)
ok 9 - got another after remote search
ok 10 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:199707281508.AAA24167@hoyogw.example
ok 11 - --no-import-remote did not memoize
# lei_err=# /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (0 matches)
ok 12 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:qp@example.com --lock=none
not ok 13 - --lock=none respected
# lei_err=# /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (0 matches)
# Failed test '--lock=none respected'
# at t/lei-q-remote-import.t line 55.
# # /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (0 matches)
# lei_err=dotlock timeout /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd.lock
ok 14 - dotlock fails
ok 15 - timeout noted
ok 16 - nothing output on lock failure
ok 17 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:qp@example.com --lock=dotlock,timeout=0.000001 (succeeds after lock removal)
# lei_err=# /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (0 matches)
ok 18 - lei add-external -q /home/t-8ch/Projekte/public-inbox.org/t/data-gen/lei-q-remote-import.local-external-master
ok 19 - lei q -q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd --only http://[::1]:46299/t2/ m:testmessage@example.com
ok 20 - no warnings or errors
ok 21 - got result from remote external
ok 22 - got expected result
ok 23 - lei q --no-external -o mboxrd:/dev/stdout m:testmessage@example.com
ok 24 - message not imported when in local external
ok 25 - lei q -o mboxrd:/tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd m:testmessage@example.com
# lei_err=# /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/.local/share/lei/store 0/0
# # /home/t-8ch/Projekte/public-inbox.org/t/data-gen/lei-q-remote-import.local-external-master 1/1
# # 1 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/o.mboxrd (1 matches)
ok 26 - got expected result after clobber
ok 27 - lei q -o mboxrd:/dev/stdout m:never-before-seen@example.com
ok 28 - --import-before imported totally unseen message
ok 29 - lei q --save z:0.. -o /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/md --only http://[::1]:46299/t2/
# lei_err=# /usr/bin/curl -Sf -s -d '' http://[::1]:46299/t2/?x=m&q=z%3A0..
# # http://[::1]:46299/t2/ 15/15
# # 15 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/md/ (15 matches)
ok 30 - lei up /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/md
ok 31 - lei up remote dedupe works on maildir
# lei_err=# http://[::1]:46299/t2/ limiting to 2021-10-25 23:18 +0200 and newer
# # /usr/bin/curl -Sf -s -d '' http://[::1]:46299/t2/?x=m&q=(z%3A0..)+AND+dt%3A20211025211849..
# # http://[::1]:46299/t2/ 11/11
# # 0 written to /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/md/ (11 matches)
ok 32 - lei edit-search /tmp/pi-lei-q-remote-import-5668-gmA0/lei-daemon/md
ok 33 - lastresult set
ok 34 - lei daemon-pid (daemon-pid after t/lei-q-remote-import.t:106)
ok 35 - daemon running after t/lei-q-remote-import.t:106
ok 36 - lei daemon-kill (daemon-kill after t/lei-q-remote-import.t:106)
ok 37 - t/lei-q-remote-import.t:106 daemon stopped
ok 38 - t/lei-q-remote-import.t:106 daemon XDG_RUNTIME_DIR/lei/errors.log empty
1..38
# Looks like you failed 3 tests of 38.
Dubious, test returned 3 (wstat 768, 0x300)
Failed 3/38 subtests
Test Summary Report
-------------------
t/lei-q-remote-import.t (Wstat: 768 Tests: 38 Failed: 3)
Failed tests: 6-7, 13
Non-zero exit status: 3
Files=1, Tests=38, 1 wallclock secs ( 0.01 usr 0.01 sys + 0.32 cusr 0.08 csys = 0.42 CPU)
Result: FAIL
^ permalink raw reply [relevance 39%]
Results 801-1000 of ~1329 next (newer) | prev (older) | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-08-14 0:29 [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-24 20:14 ` [PATCH 0/3] lei: hopefully^W " Eric Wong
2021-10-14 5:31 71% ` Eric Wong
2021-10-15 9:52 54% ` [PATCH] lei q: avoid kw lookup failure on remote mboxrd Eric Wong
2021-09-11 0:19 [PATCH 0/3] lei saved-search fixes Eric Wong
2021-09-11 0:19 53% ` [PATCH 1/3] lei: fix handling of broken lei.saved-search config files Eric Wong
2021-09-11 8:33 71% [PATCH 0/1] lei q -f reply ... perhaps my new favorite feature Eric Wong
2021-09-11 8:33 40% ` [PATCH 1/1] lei q|lcat: support "-f reply" output format Eric Wong
2021-09-12 8:42 [PATCH 0/3] test dependency fixes for minimal installs Eric Wong
2021-09-12 8:42 67% ` [PATCH 1/3] t/lei-*.t: guard setup_public_inboxes with test_lei Eric Wong
2021-09-12 8:42 65% ` [PATCH 2/3] lei sucks: allow it to work without SQLite Eric Wong
2021-09-13 20:53 [PATCH 0/6] a batch of misc fixes Eric Wong
2021-09-13 20:53 67% ` [PATCH 2/6] lei: stop_pager: restore stdout when done Eric Wong
2021-09-13 20:53 71% ` [PATCH 4/6] lei up: localize %ENV in redispatch Eric Wong
2021-09-13 20:53 62% ` [PATCH 6/6] lei up: fix --mua with single output Eric Wong
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
2021-09-14 2:39 71% ` [PATCH 2/5] lei: sto_done_request: add eval guard Eric Wong
2021-09-14 2:39 56% ` [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders Eric Wong
2021-09-14 20:42 71% RFC: lei-daemon and auto-up Konstantin Ryabitsev
2021-09-14 22:32 71% ` Eric Wong
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
2021-09-15 23:12 71% ` Eric Wong
2021-09-15 22:36 71% ` Kyle Meyer
2021-09-15 23:31 71% ` Eric Wong
2021-09-16 2:59 71% ` Eric Wong
2021-09-15 23:06 57% ` Eric Wong
2021-09-16 17:43 71% ` Konstantin Ryabitsev
2021-09-16 20:04 70% ` IMAP vs NNTP [was: make menuconfig interface for lei / grok-pull] Eric Wong
2021-09-19 21:21 71% ` make menuconfig interface for lei / grok-pull Eric Wong
2021-09-16 17:38 71% ` Konstantin Ryabitsev
2021-09-16 23:54 71% ` Luis Chamberlain
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
2021-09-16 23:53 90% ` Luis Chamberlain
2021-09-17 0:22 89% ` Eric Wong
2021-09-17 0:48 90% ` Luis Chamberlain
2021-09-16 2:19 [PATCH 0/3] IMAP and Tor .onion niceities Eric Wong
2021-09-16 2:19 71% ` [PATCH 2/3] lei ls-mail-source: sort IMAP folder names Eric Wong
2021-09-16 7:45 63% [PATCH] doc: lei-mail-formats: add "eml" and expand on git things Eric Wong
2021-09-16 9:41 71% [PATCH 0/3] lei refresh-mail-sync Eric Wong
2021-09-16 9:41 59% ` [PATCH 1/3] lei: git_oid: replace git_blob_id Eric Wong
2021-09-16 9:41 39% ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-16 11:04 66% [PATCH] lei: sqlite: do not forcibly enable WAL Eric Wong
2021-09-17 2:16 71% ` Eric Wong
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path Eric Wong
2021-09-17 1:56 61% ` [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders Eric Wong
2021-09-17 1:56 56% ` [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info Eric Wong
2021-09-17 12:12 71% [PATCH] script/lei: umask(077) before execve Eric Wong
2021-09-17 12:38 38% [PATCH] doc: add lei-security(7) manpage Eric Wong
2021-09-18 9:33 67% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
2021-09-18 9:33 51% ` [PATCH 1/9] lei: lock worker counts Eric Wong
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
2021-09-18 14:31 71% ` Kyle Meyer
2021-09-18 20:30 71% ` Eric Wong
2021-09-18 22:38 66% [PATCH] t/lei-refresh-mail-sync: improve test reliability Eric Wong
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
2021-09-19 12:50 58% ` [PATCH 04/16] lei: simplify sto_done_request Eric Wong
2021-09-19 12:50 68% ` [PATCH 07/16] lei: clamp internal worker processes to 4 Eric Wong
2021-09-19 12:50 71% ` [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP Eric Wong
2021-09-19 12:50 62% ` [PATCH 09/16] lei ls-mail-source: pretty JSON support Eric Wong
2021-09-19 12:50 36% ` [PATCH 14/16] lei config --edit: use controlling terminal Eric Wong
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
2021-09-19 16:14 71% ` Kyle Meyer
2021-09-19 20:00 65% ` Eric Wong
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
2021-09-21 7:41 42% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
2021-09-21 7:41 45% ` [PATCH 04/12] lei: simplify internal arg2folder usage Eric Wong
2021-09-21 7:41 61% ` [PATCH 05/12] lei lcat: use single queue for ordering Eric Wong
2021-09-21 7:41 71% ` [PATCH 06/12] doc: lei-security: section for WIP auth methods Eric Wong
2021-09-21 7:41 44% ` [PATCH 07/12] lei lcat: support NNTP URLs Eric Wong
2021-09-21 7:41 39% ` [PATCH 08/12] lei: various completion improvements Eric Wong
2021-09-21 7:41 63% ` [PATCH 09/12] lei q: show progress on >1s preparation phase Eric Wong
2021-09-21 7:41 61% ` [PATCH 11/12] lei q: update messages to reflect --save default Eric Wong
2021-09-21 7:41 39% ` [PATCH 12/12] lei q: improve --limit behavior and progress Eric Wong
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
2021-09-21 9:29 62% ` [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT Eric Wong
2021-09-21 9:29 71% ` [PATCH 3/3] lei: umask(077) before opening errors.log Eric Wong
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
2021-09-22 2:24 71% ` [PATCH 4/7] lei: dclose: do not close unnecessarily Eric Wong
2021-09-22 2:24 49% ` [PATCH 6/7] lei up: avoid excessively parallel --all Eric Wong
2021-09-22 2:24 58% ` [PATCH 7/7] lei: drop redundant WQ EOF callbacks Eric Wong
2021-09-23 10:36 71% [PATCH] lei: common --all[=remote|local] help message Eric Wong
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
2021-09-24 12:51 56% ` [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes Eric Wong
2021-09-24 12:51 68% ` [PATCH 3/4] doc: lei-index: remove --stdin, reword -F Eric Wong
2021-09-24 12:51 49% ` [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync Eric Wong
2021-09-25 5:49 71% [PATCH 0/3] lei: some robustness fixes Eric Wong
2021-09-25 5:49 71% ` [PATCH 1/3] lei up: show timezone offset with localtime Eric Wong
2021-09-25 5:49 71% ` [PATCH 2/3] lei: restore old sigmask before daemon exit Eric Wong
2021-09-25 6:17 55% [PATCH] lei: make pkt_op easier-to-use and understand Eric Wong
2021-09-25 7:08 71% [PATCH] doc: lei-rm: remove unnecessary -F values Eric Wong
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
2021-09-25 8:49 48% ` [PATCH 2/3] lei add-external: " Eric Wong
2021-09-25 8:49 58% ` [PATCH 3/3] lei ls-external: " Eric Wong
2021-09-26 0:02 62% [PATCH] lei note-event: ignore kw_changed exceptions Eric Wong
2021-09-26 1:42 71% [PATCH] lei -f reply: fix Cc: header combining Eric Wong
2021-09-26 5:38 71% [PATCH] lei: ensure refresh_watches isn't called from workers Eric Wong
2021-09-27 4:59 71% [PATCH 0/2] lei rediff: --drq and --dequote-only Eric Wong
2021-09-27 4:59 69% ` [PATCH 1/2] lei rediff: quiet warnings from Import and Eml Eric Wong
2021-09-27 4:59 42% ` [PATCH 2/2] lei rediff: add --drq and --dequote-only Eric Wong
2021-09-27 21:05 [PATCH 0/3] fixes for odd/old/missing dependencies Eric Wong
2021-09-27 21:05 63% ` [PATCH 2/3] t/lei-index: IMAP and NNTP dependencies are optional Eric Wong
2021-09-27 21:05 58% ` [PATCH 3/3] lei completion: workaround old Perl bug Eric Wong
2021-09-27 23:53 71% [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only Kyle Meyer
2021-09-28 1:05 71% ` Eric Wong
2021-10-01 9:54 [PATCH 0/9] daemon-related things Eric Wong
2021-10-01 9:54 69% ` [PATCH 1/9] doc: lei-security: some more updates Eric Wong
2021-10-01 9:54 49% ` [PATCH 9/9] doc: lei-daemon: new manpage Eric Wong
2021-10-02 0:19 71% ` Kyle Meyer
2021-10-02 8:08 71% ` Eric Wong
2021-10-04 3:16 71% ` Kyle Meyer
2021-10-02 8:16 71% [PATCH 0/2] lei inspect stuffs Eric Wong
2021-10-02 8:16 71% ` [PATCH 1/2] lei inspect: integerize "bytes" and "lines" fields Eric Wong
2021-10-02 8:16 53% ` [PATCH 2/2] lei inspect: fix "mid:" prefix, expand to Xapian Eric Wong
2021-10-02 11:18 71% [PATCH 0/4] lei mail-diff and other small things Eric Wong
2021-10-02 11:18 38% ` [PATCH 2/4] lei mail-diff: diagnostic command to diff mail contents Eric Wong
2021-10-10 14:25 [PATCH 0/8] extindex and then some Eric Wong
2021-10-10 14:25 71% ` [PATCH 7/8] lei/store: keep ".err-XXXX" in stderr tmpfile Eric Wong
2021-10-11 4:53 71% [PATCH] doc: lei-refresh-mail-sync: drop repeated word Kyle Meyer
2021-10-11 5:09 71% ` Eric Wong
2021-10-12 22:44 [PATCH 0/5] fix extindex reindex harder Eric Wong
2021-10-12 22:44 71% ` [PATCH 2/5] lei/store: use remove_doc to save some LoC Eric Wong
2021-10-12 22:45 62% ` [PATCH 5/5] lei up --all: show output for warnings Eric Wong
2021-10-13 7:00 [PATCH 0/5] warnings and such Eric Wong
2021-10-13 7:00 28% ` [PATCH 2/5] lei: use standard warn() in more places Eric Wong
2021-10-13 10:16 [PATCH 0/7] workaround Encode leak, several test fixes Eric Wong
2021-10-13 10:16 71% ` [PATCH 5/7] t/lei-mirror: avoid reading ~/.public-inbox/config in test Eric Wong
2021-10-14 3:12 56% [PATCH] lei inspect: account for non-extindex inboxes Eric Wong
2021-10-14 4:32 [PATCH 0/3] clone+fetch stuff Eric Wong
2021-10-14 4:32 65% ` [PATCH 2/3] lei add-external --mirror: respect client umask Eric Wong
2021-10-14 4:32 61% ` [PATCH 3/3] lei: give workers their own process group Eric Wong
2021-10-14 13:16 70% ` [PATCH 0/7] lei: more process handling fixes Eric Wong
2021-10-14 13:16 71% ` [PATCH 1/7] lei: use send() perlop for signals Eric Wong
2021-10-14 13:16 34% ` [PATCH 5/7] lei: TSTP affects all curl and related subprocesses Eric Wong
2021-10-14 13:16 67% ` [PATCH 6/7] lei up: actually rely on DESTROY for --alllll Eric Wong
2021-10-14 13:16 62% ` [PATCH 7/7] lei up --all: send signals to workers, receive errors Eric Wong
2021-10-14 9:54 60% [PATCH] lei: -d (--dir) and -O (only) shortcuts Eric Wong
2021-10-15 13:30 71% [PATCH 0/3] lei bugfixes and simplifications Eric Wong
2021-10-15 13:30 71% ` [PATCH 1/3] lei forget-search: fix for symlink-ed paths Eric Wong
2021-10-15 13:30 33% ` [PATCH 2/3] lei + ipc: simplify process reaping Eric Wong
2021-10-15 13:45 71% ` [SQUASH PATCH 4/3] lei q: ensure all workers die on Ctrl-C Eric Wong
2021-10-15 13:30 67% ` [PATCH 3/3] lei note-event: fix explicit flush reliability Eric Wong
2021-10-15 14:02 64% [PATCH] lei forget-search: support multiple args Eric Wong
2021-10-15 15:52 61% [PATCH] lei q: guard query_done against die() Eric Wong
2021-10-16 1:00 [PATCH 00/16] some yak-shaving and annoyance fixes Eric Wong
2021-10-16 1:00 71% ` [PATCH 05/12] lei: golf PATH2CFG cleanup Eric Wong
2021-10-16 1:00 63% ` [PATCH 06/12] lei: always keep cwd fd {3} for ->fchdir Eric Wong
2021-10-16 1:00 55% ` [PATCH 07/12] lei: more eval guards for die on failure Eric Wong
2021-10-16 1:01 59% ` [PATCH 09/12] lei_overview: die rather than lei->fail Eric Wong
2021-10-16 5:39 70% [PATCH 0/2] doc: lei manpages for remaining commands Kyle Meyer
2021-10-16 5:39 65% ` [PATCH 1/2] doc: lei: restore alphabetical order to some listings Kyle Meyer
2021-10-16 5:39 29% ` [PATCH 2/2] doc: lei: add manpages for remaining commands Kyle Meyer
2021-10-16 7:07 71% ` Eric Wong
2021-10-16 15:13 67% ` Kyle Meyer
2021-10-16 16:58 71% ` Kyle Meyer
2021-10-16 17:03 71% ` Eric Wong
2021-10-16 17:21 71% ` Kyle Meyer
2021-10-16 7:54 70% [PATCH] t/lei*: set EDITOR for dumb terminals Eric Wong
2021-10-16 9:29 71% [PATCH 0/4] lei: prioritize signals Eric Wong
2021-10-16 9:29 57% ` [PATCH 4/4] lei sockets: favor level-triggered epoll for fairness Eric Wong
2021-10-19 9:33 70% [PATCH 00/11] refining lei up+inspect Eric Wong
2021-10-19 9:33 65% ` [PATCH 02/11] lei up: prefix `remote' and `local' with `o_' Eric Wong
2021-10-19 9:33 50% ` [PATCH 03/11] lei: use die for external and query handling Eric Wong
2021-10-19 9:33 71% ` [PATCH 04/11] lei up: propagate redispatch_all failure via exit code Eric Wong
2021-10-19 9:33 70% ` [PATCH 05/11] lei: conditionally add "\n" to error messages Eric Wong
2021-10-19 9:33 47% ` [PATCH 06/11] lei up: support --exclude=, --no-(external|remote|local) Eric Wong
2021-10-19 9:33 71% ` [PATCH 07/11] lei: remove unused ->busy time arg Eric Wong
2021-10-19 9:33 80% ` [PATCH 08/11] doc: lei: describe lei-daemon-kill and upgrades Eric Wong
2021-10-19 9:33 71% ` [PATCH 09/11] lei inspect: add atfork hook Eric Wong
2021-10-19 9:33 64% ` [PATCH 10/11] lei inspect: show ISO8601 {rt} and {dt}, too Eric Wong
2021-10-21 21:10 [PATCH 00/15] use RENAME_NOREPLACE on Linux 3.15+ Eric Wong
2021-10-21 21:10 57% ` [PATCH 01/15] t/lei-{auto-watch,export-kw}: extra diagnostics on failure Eric Wong
2021-10-21 21:10 71% ` [PATCH 02/15] t/lei-import-maildir: rename fix (SR -> RS) Eric Wong
2021-10-21 21:10 71% ` [PATCH 03/15] t/lei-p2q: extra diagnostics Eric Wong
2021-10-21 21:10 71% ` [PATCH 04/15] lei/store: check for any unexpected process death Eric Wong
2021-10-21 21:10 71% ` [PATCH 05/15] lei note-event: drop unnecessary eval guard Eric Wong
2021-10-21 21:10 71% ` [PATCH 06/15] lei note-event: wq_io_do => wq_do Eric Wong
2021-10-21 21:10 69% ` [PATCH 07/15] lei_search: try harder to associate "lei index"-ed messages Eric Wong
2021-10-21 21:10 71% ` [PATCH 10/15] doc: lei-overview: add CAVEATS section Eric Wong
2021-10-21 21:10 71% ` [PATCH 11/15] lei note-event: clear_src on ENOENT Eric Wong
2021-10-21 21:10 71% ` [PATCH 13/15] lei: no Perl FileHandle for `undef' w/ ECONNRESET Eric Wong
2021-10-21 21:10 37% ` [PATCH 15/15] lei: use RENAME_NOREPLACE on Linux 3.15+ Eric Wong
2021-10-22 8:22 71% [PATCH 0/3] lei: some cleanup stuff Eric Wong
2021-10-22 8:22 60% ` [PATCH 1/3] lei export-kw: don't recreate deleted IMAP folders Eric Wong
2021-10-22 8:22 71% ` [PATCH 2/3] lei export-kw: completion returns all Maildir+IMAP Eric Wong
2021-10-22 8:22 46% ` [PATCH 3/3] lei forget-search: support --prune=<local|remote> Eric Wong
2021-10-23 0:22 71% [PATCH] doc: lei-forget-search: fix option name in --prune description Kyle Meyer
2021-10-23 2:54 71% ` Eric Wong
2021-10-24 0:20 [PATCH 0/7] misc tweaks and fixes Eric Wong
2021-10-24 0:20 75% ` [PATCH 1/7] lei: always pass $lei to LeiAuth->op_merge Eric Wong
2021-10-24 0:20 61% ` [PATCH 2/7] lei export-kw: skip read-only IMAP folders Eric Wong
2021-10-25 19:31 64% [PATCH] lei p2q: document --uri, add examples Eric Wong
2021-10-25 23:39 71% ` Kyle Meyer
2021-10-25 22:24 [PATCH] t/index-git-times: support non-master default branch Thomas Weißschuh
2021-10-25 22:58 ` Eric Wong
2021-10-26 5:05 ` Thomas Weißschuh
2021-10-26 5:28 ` Eric Wong
2021-10-26 18:43 ` Thomas Weißschuh
2021-10-27 21:15 71% ` lei-q-remote-import failures [was: [PATCH] t/index-git-times: support non-master default branch] Eric Wong
2021-10-27 21:24 39% ` Thomas Weißschuh
2021-10-26 10:35 69% [PATCH 0/9] lei p2q: more capable than originally thought Eric Wong
2021-10-26 10:35 67% ` [PATCH 2/9] doc: lei-store-format: mail sync section, update IPC Eric Wong
2021-10-26 10:35 71% ` [PATCH 4/9] lei q: enable expensive Xapian flags Eric Wong
2021-10-26 10:35 71% ` [PATCH 5/9] lei inspect: fix atfork hook Eric Wong
2021-10-26 10:35 66% ` [PATCH 6/9] lei: add net getopt spec to various commands Eric Wong
2021-10-26 10:35 35% ` [PATCH 7/9] lei p2q: use LeiInput for multi-patch series Eric Wong
2021-10-26 10:35 68% ` [PATCH 8/9] lei rm|tag: drop redundant mbox+net callbacks Eric Wong
2021-10-26 10:47 71% [PATCH] t/lei-watch: add diagnostics for failure Eric Wong
2021-10-26 21:18 64% [PATCH] lei mail-diff: support more inputs, split newlines Eric Wong
2021-10-27 21:09 64% [PATCH] lei q: fix remote import accounting Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).