* [PATCH 2/2] lei inspect: fix "mid:" prefix, expand to Xapian
@ 2021-10-02 8:16 53% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-02 8:16 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
This fixes inspect for uninitialized instances, and adds Xapian
("xdoc") output if available.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Message-ID: <20211001204943.l4yl6xvc45c5eapz@meerkat.local>
---
MANIFEST | 1 +
lib/PublicInbox/LeiInspect.pm | 59 ++++++++++++++++++++---------------
t/lei-inspect.t | 14 +++++++++
3 files changed, 48 insertions(+), 26 deletions(-)
create mode 100644 t/lei-inspect.t
diff --git a/MANIFEST b/MANIFEST
index 929f5f869c5b..74b28d2d54ab 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -450,6 +450,7 @@ t/lei-import-maildir.t
t/lei-import-nntp.t
t/lei-import.t
t/lei-index.t
+t/lei-inspect.t
t/lei-lcat.t
t/lei-mirror.psgi
t/lei-mirror.t
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index f18e31c5c8f4..590dfdabca56 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -78,22 +78,9 @@ sub inspect_sync_folder ($$) {
$ent
}
-sub inspect_docid ($$;$) {
- my ($lei, $docid, $ent) = @_;
- require PublicInbox::Search;
- $ent //= {};
- my $xdb;
- if ($xdb = delete $ent->{xdb}) { # from inspect_num
- } elsif (defined(my $dir = $lei->{opt}->{dir})) {
- no warnings 'once';
- $xdb = $PublicInbox::Search::X{Database}->new($dir);
- } else {
- $xdb = $lei->{lse}->xdb;
- }
- $xdb or return $lei->fail('no Xapian DB');
- my $doc = $xdb->get_document($docid); # raises
+sub _inspect_doc ($$) {
+ my ($ent, $doc) = @_;
my $data = $doc->get_data;
- $ent->{docid} = $docid;
$ent->{data_length} = length($data);
$ent->{description} = $doc->get_description;
$ent->{$_} = $doc->$_ for (qw(termlist_count values_count));
@@ -119,6 +106,24 @@ sub inspect_docid ($$;$) {
$ent;
}
+sub inspect_docid ($$;$) {
+ my ($lei, $docid, $ent) = @_;
+ require PublicInbox::Search;
+ $ent //= {};
+ my $xdb;
+ if ($xdb = delete $ent->{xdb}) { # from inspect_num
+ } elsif (defined(my $dir = $lei->{opt}->{dir})) {
+ no warnings 'once';
+ $xdb = $PublicInbox::Search::X{Database}->new($dir);
+ } elsif ($lei->{lse}) {
+ $xdb = $lei->{lse}->xdb;
+ }
+ $xdb or return $lei->fail('no Xapian DB');
+ my $doc = $xdb->get_document($docid); # raises
+ $ent->{docid} = $docid;
+ _inspect_doc($ent, $doc);
+}
+
sub dir2ibx ($$) {
my ($lei, $dir) = @_;
if (-f "$dir/ei.lock") {
@@ -138,11 +143,9 @@ sub inspect_num ($$) {
my $ent = { num => $num };
if (defined(my $dir = $lei->{opt}->{dir})) {
$ibx = dir2ibx($lei, $dir) or return;
- if ($ent->{xdb} = $ibx->xdb) {
- my $num2docid = $lei->{lse}->can('num2docid');
- $docid = $num2docid->($ibx, $num);
- }
- } else {
+ $ent->{xdb} = $ibx->xdb and # for inspect_docid
+ $docid = PublicInbox::LeiSearch::num2docid($ibx, $num);
+ } elsif ($lei->{lse}) {
$ibx = $lei->{lse};
$lei->{lse}->xdb; # set {nshard} for num2docid
$docid = $lei->{lse}->num2docid($num);
@@ -156,16 +159,12 @@ sub inspect_num ($$) {
sub inspect_mid ($$) {
my ($lei, $mid) = @_;
- my ($ibx, $over);
+ my $ibx;
my $ent = { mid => $mid };
if (defined(my $dir = $lei->{opt}->{dir})) {
- my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid');
- $ibx = dir2ibx($lei, $dir) or return;
- # $ent->{xdb} = $ibx->xdb //
- # return $lei->fail("no Xapian DB for $dir");
+ $ibx = dir2ibx($lei, $dir)
} else {
$ibx = $lei->{lse};
- $lei->{lse}->xdb; # set {nshard} for num2docid
}
if ($ibx && $ibx->over) {
my ($id, $prev);
@@ -173,6 +172,14 @@ sub inspect_mid ($$) {
push @{$ent->{smsg}}, _json_prep($smsg);
}
}
+ if ($ibx && $ibx->search) {
+ my $mset = $ibx->search->mset(qq{mid:"$mid"});
+ for (sort { $a->get_docid <=> $b->get_docid } $mset->items) {
+ my $tmp = { docid => $_->get_docid };
+ _inspect_doc($tmp, $_->get_document);
+ push @{$ent->{xdoc}}, $tmp;
+ }
+ }
$ent;
}
diff --git a/t/lei-inspect.t b/t/lei-inspect.t
new file mode 100644
index 000000000000..077d0d13e84d
--- /dev/null
+++ b/t/lei-inspect.t
@@ -0,0 +1,14 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+
+test_lei(sub {
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
+ lei_ok qw(inspect --dir), "$ro_home/t1", 'mid:testmessage@example.com';
+ my $ent = json_utf8->decode($lei_out);
+ is(ref($ent->{smsg}), 'ARRAY', 'smsg array');
+ is(ref($ent->{xdoc}), 'ARRAY', 'xdoc array');
+});
+
+done_testing;
^ permalink raw reply related [relevance 53%]
* Re: [PATCH 9/9] doc: lei-daemon: new manpage
2021-10-02 0:19 71% ` Kyle Meyer
@ 2021-10-02 8:08 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-10-02 8:08 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed with your Helped-by:.
Btw, did you have time to work on some of the other manpages?
No worries if not, I might take a stab at them sometime in the
next few days. I often realize shortcomings while writing
documentation, and I still need to figure out why some messages
go missing or get duplicated...
^ permalink raw reply [relevance 71%]
* Re: [PATCH 9/9] doc: lei-daemon: new manpage
2021-10-01 9:54 49% ` [PATCH 9/9] doc: lei-daemon: new manpage Eric Wong
@ 2021-10-02 0:19 71% ` Kyle Meyer
2021-10-02 8:08 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-10-02 0:19 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> In case users see "lei-daemon" in ps(1) or syslog and wonder.
Thanks, this is very helpful/informative.
> diff --git a/Documentation/lei-daemon.pod b/Documentation/lei-daemon.pod
[...]
> +=head2 SOCK_SEQPACKET
> +
> +SOCK_SEQPACKET sockets are used for both communicating with
> +L<lei(1)> and to internal workers. SOCK_SEQPACKET is guarantee
> +reliability (unlike SOCK_DGRAM), allows easy load distribution,
s/is guarantee/guarantees/ ?
> +and saves developers the trouble of maintaining stream parsers.
> +
> +=head2 File monitoring
Obviously unimportant, but this casing is inconsistent with some of the
titles above (e.g., "file descriptor passing").
> +Inotify or EVFILT_VNODE is used depending on the platform
> +to monitor Maildirs of changes and track keyword changes.
s/of changes/for changes/ ?
> +The listen socket at (default: C<$XDG_RUNTIME_DIR/lei/5.seq.sock>)
I stumbled parsing this because target of "at" is parenthesized. At
least in my head, it'd read fine dropping the "at".
> +is also monitored, and the daemon will automatically shutdown
> +if it is unlinked.
^ permalink raw reply [relevance 71%]
* [PATCH 9/9] doc: lei-daemon: new manpage
2021-10-01 9:54 69% ` [PATCH 1/9] doc: lei-security: some more updates Eric Wong
@ 2021-10-01 9:54 49% ` Eric Wong
2021-10-02 0:19 71% ` Kyle Meyer
1 sibling, 1 reply; 200+ results
From: Eric Wong @ 2021-10-01 9:54 UTC (permalink / raw)
To: meta
In case users see "lei-daemon" in ps(1) or syslog and wonder.
---
Documentation/lei-daemon.pod | 61 ++++++++++++++++++++++++++++++
Documentation/lei.pod | 9 ++++-
Documentation/lei_design_notes.txt | 2 +-
MANIFEST | 1 +
Makefile.PL | 2 +-
5 files changed, 72 insertions(+), 3 deletions(-)
create mode 100644 Documentation/lei-daemon.pod
diff --git a/Documentation/lei-daemon.pod b/Documentation/lei-daemon.pod
new file mode 100644
index 000000000000..1205e974706a
--- /dev/null
+++ b/Documentation/lei-daemon.pod
@@ -0,0 +1,61 @@
+=head1 NAME
+
+lei-daemon - technical information for local email interface daemon
+
+=head1 DESCRIPTION
+
+This documentation is a high-level overview for developers and
+administrators interested in how lei works.
+
+lei-daemon is a background daemon which powers the L<lei(1)>
+command-line tool. It may support virtual users and read-write
+IMAP+JMAP APIs in the future. It is designed to optimize shell
+completion by avoiding module loading costs, monitor Maildirs
+(and in the near future, IMAP folders) for changes.
+
+=head2 worker processes
+
+Most commands cause lei-daemon to L<fork(2)> new worker
+processes to isolate and parallelize work. lei-daemon is
+significantly more aggressive than read-only
+L<public-inbox-daemon(8)> processes with regards to resource use
+since it's not designed to support C10K/C100K scenarios.
+
+=head2 file descriptor passing
+
+FD passing is used to reduce IPC costs for bulk I/O when
+importing large mboxes from stdin and dumping large mboxes
+to stdout.
+
+=head2 SOCK_SEQPACKET
+
+SOCK_SEQPACKET sockets are used for both communicating with
+L<lei(1)> and to internal workers. SOCK_SEQPACKET is guarantee
+reliability (unlike SOCK_DGRAM), allows easy load distribution,
+and saves developers the trouble of maintaining stream parsers.
+
+=head2 File monitoring
+
+Inotify or EVFILT_VNODE is used depending on the platform
+to monitor Maildirs of changes and track keyword changes.
+
+The listen socket at (default: C<$XDG_RUNTIME_DIR/lei/5.seq.sock>)
+is also monitored, and the daemon will automatically shutdown
+if it is unlinked.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-overview(7)>, L<lei-daemon-kill(1)>, L<lei-daemon-pid(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 3af9e2eeacf7..63d5ee6900a7 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -115,6 +115,13 @@ Other subcommands include
By default storage is located at C<$XDG_DATA_HOME/lei/store>. The
configuration for lei resides at C<$XDG_CONFIG_HOME/lei/config>.
+=head1 ERRORS
+
+Errors and dianostics for interactive commands are reported to
+stderr. Some errors for background tasks are emitted via
+L<syslog(3)> as L<lei-daemon(8)> for the top-level daemon,
+and C<lei/store> for the L<lei-store-format(5)> worker.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -130,4 +137,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-overview(7)>
+L<lei-overview(7)>, L<lei-daemon(8)>
diff --git a/Documentation/lei_design_notes.txt b/Documentation/lei_design_notes.txt
index f1d2ab6f2169..b5180374ad05 100644
--- a/Documentation/lei_design_notes.txt
+++ b/Documentation/lei_design_notes.txt
@@ -6,7 +6,7 @@ Daemon architecture
The use of a persistent daemon works around slow startup time of
Perl. This is especially important for built-in support for
-shell completion. It will eventually support inotify and EVFILT_VNODE
+shell completion. It attempts to support inotify and EVFILT_VNODE
background monitoring of Maildir keyword changes.
If lei were reimplemented in a language with faster startup
diff --git a/MANIFEST b/MANIFEST
index 3e942855127f..929f5f869c5b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -27,6 +27,7 @@ Documentation/lei-config.pod
Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
+Documentation/lei-daemon.pod
Documentation/lei-edit-search.pod
Documentation/lei-export-kw.pod
Documentation/lei-forget-external.pod
diff --git a/Makefile.PL b/Makefile.PL
index 00a558d12472..c41c408a5704 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -60,7 +60,7 @@ $v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
$v->{-m7} = [ qw(lei-overview lei-security
public-inbox-overview public-inbox-tuning
public-inbox-glossary) ];
-$v->{-m8} = [ qw(public-inbox-daemon) ];
+$v->{-m8} = [ qw(public-inbox-daemon lei-daemon) ];
my @sections = (1, 5, 7, 8);
$v->{check_80} = [];
$v->{manuals} = [];
^ permalink raw reply related [relevance 49%]
* [PATCH 1/9] doc: lei-security: some more updates
@ 2021-10-01 9:54 69% ` Eric Wong
2021-10-01 9:54 49% ` [PATCH 9/9] doc: lei-daemon: new manpage Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-10-01 9:54 UTC (permalink / raw)
To: meta
Virtual users will probably be used for read-write IMAP/JMAP
support. The potential for various kernel/hardware bugs and
attacks also needs to be highlighted.
---
Documentation/lei-security.pod | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/Documentation/lei-security.pod b/Documentation/lei-security.pod
index 02305b9055c2..8cbd89934568 100644
--- a/Documentation/lei-security.pod
+++ b/Documentation/lei-security.pod
@@ -18,6 +18,9 @@ permissions support.
It does not use POSIX ACLs, extended attributes, nor any other
security-related functions which require non-standard Perl modules.
+There is preliminary support for "virtual users", but it is
+incomplete and undocumented.
+
=head1 INTERNAL FILES
lei runs with a umask of 077 to prevent other users on the
@@ -93,7 +96,7 @@ lei uses L<git-credential(1)> to prompt users for IMAP and NNTP
usernames and passwords. These passwords are not encrypted in
memory and get transferred across processes via anonymous UNIX
sockets and pipes. They may be exposed via syscall tracing
-tools (e.g. L<strace(1)>).
+tools (e.g. L<strace(1)>), kernel and hardware bugs/attacks.
While credentials are not written to the filesystem by default,
it is possible for them to end up on disk if processes are
^ permalink raw reply related [relevance 69%]
* Re: [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only
2021-09-27 23:53 71% [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only Kyle Meyer
@ 2021-09-28 1:05 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-28 1:05 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed as commit 3dbd6dc31dd7e34bea87dbb626f811e093b6860b
^ permalink raw reply [relevance 71%]
* [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only
@ 2021-09-27 23:53 71% Kyle Meyer
2021-09-28 1:05 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-27 23:53 UTC (permalink / raw)
To: meta
---
Documentation/lei-rediff.pod | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index e711de97..4d5e8168 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -40,7 +40,7 @@ patch email.
=item --drq[=COUNT]
-De-Re-Quote. De-quote the input and re-quotes (the output).
+De-Re-Quote. De-quotes the input and re-quotes (the output).
Removes COUNT levels of C<E<gt> > email reply prefixes and
re-adds them upon regenerating the diff.
@@ -63,7 +63,7 @@ This can be useful for feeding a hunk to L<git-apply(1)>
or L<patch(1)> while writing a reply or further processing
by another diff viewer.
-Unlike L</--drq>, it does NOT implies L</--quiet>.
+Unlike L</--drq>, it does NOT imply L</--quiet>.
=item --git-dir=DIR
base-commit: 010159a7b789a3a27c917a5378db92c8adf78a9e
--
2.33.0
^ permalink raw reply related [relevance 71%]
* [PATCH 3/3] lei completion: workaround old Perl bug
2021-09-27 21:05 63% ` [PATCH 2/3] t/lei-index: IMAP and NNTP dependencies are optional Eric Wong
@ 2021-09-27 21:05 58% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 21:05 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
While `$argv[-1]' is `undef' on an empty @argv, using `$argv[-1]'
as a subroutine argument would fail incorrectly with:
Modification of non-creatable array value attempted, subscript -1 at ...
...even though we'd never attempt to modify @_ itself in the
subroutines being called. Work around the bug (tested on
5.16.3) by passing `undef' explicitly when `$argv[-1]' is
already `undef'.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210927124056.kj5okiefvs4ztk27@meerkat.local/
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 4 ++--
lib/PublicInbox/LeiLsMailSource.pm | 4 ++--
lib/PublicInbox/LeiRefreshMailSync.pm | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index cea9beeb5694..0b65c2762633 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -131,7 +131,7 @@ sub _complete_export_kw {
my $match_cb = $lei->complete_url_prepare(\@argv);
# filter-out read-only sources:
my @k = grep(!m!(?://;AUTH=ANONYMOUS\@|\A(?:nntps?|s?news)://)!,
- $lms->folders($argv[-1], 1));
+ $lms->folders($argv[-1] // undef, 1));
my @m = map { $match_cb->($_) } @k;
@m ? @m : @k;
}
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 397292d4c787..69d63ab6b397 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -122,11 +122,11 @@ sub lei_import { # the main "lei import" method
sub _complete_import {
my ($lei, @argv) = @_;
my ($re, $cur, $match_cb) = $lei->complete_url_prepare(\@argv);
- my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @k = $lei->url_folder_cache->keys($argv[-1] // undef, 1);
my @m = map { $match_cb->($_) } @k;
my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @k = $lms->folders($argv[-1], 1);
+ @k = $lms->folders($argv[-1] // undef, 1);
@m = map { $match_cb->($_) } @k;
if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 1db15d5742b5..7c3c0cda18d6 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -107,11 +107,11 @@ sub lei_ls_mail_source {
sub _complete_ls_mail_source {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
- my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @k = $lei->url_folder_cache->keys($argv[-1] // undef, 1);
my @m = map { $match_cb->($_) } @k;
my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @k = $lms->folders($argv[-1], 1);
+ @k = $lms->folders($argv[-1] // undef, 1);
@m = map { $match_cb->($_) } grep(m!\A[a-z]+://!, @k);
if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index eb842843b51d..0cb9f3dccd48 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -101,7 +101,7 @@ sub _complete_refresh_mail_sync {
my ($lei, @argv) = @_;
my $lms = $lei->lms or return ();
my $match_cb = $lei->complete_url_prepare(\@argv);
- my @k = $lms->folders($argv[-1], 1);
+ my @k = $lms->folders($argv[-1] // undef, 1);
my @m = map { $match_cb->($_) } @k;
@m ? @m : @k
}
^ permalink raw reply related [relevance 58%]
* [PATCH 2/3] t/lei-index: IMAP and NNTP dependencies are optional
@ 2021-09-27 21:05 63% ` Eric Wong
2021-09-27 21:05 58% ` [PATCH 3/3] lei completion: workaround old Perl bug Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 21:05 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
"lei index" support for IMAP and NNTP is incomplete, so there's
no point in requiring them.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210927124056.kj5okiefvs4ztk27@meerkat.local/
---
t/lei-index.t | 35 +++++++++++++++++++++++------------
1 file changed, 23 insertions(+), 12 deletions(-)
diff --git a/t/lei-index.t b/t/lei-index.t
index eeda5196f411..aab8f7e63f06 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -3,21 +3,30 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Spec;
-require_mods(qw(lei -nntpd));
+require_mods(qw(lei));
my ($ro_home, $cfg_path) = setup_public_inboxes;
my ($tmpdir, $for_destroy) = tmpdir;
my $env = { PI_CONFIG => $cfg_path };
+my $srv = {};
-my $sock = tcp_server;
-my $cmd = [ '-nntpd', '-W0', "--stdout=$tmpdir/n1", "--stderr=$tmpdir/n2" ];
-my $nntpd = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-nntpd $?");
-my $nntp_host_port = tcp_host_port($sock);
+SKIP: {
+ require_mods(qw(-nntpd Net::NNTP), 1);
+ my $rdr = { 3 => tcp_server };
+ $srv->{nntpd} = start_script(
+ [qw(-nntpd -W0), "--stdout=$tmpdir/n1", "--stderr=$tmpdir/n2"],
+ $env, $rdr) or xbail "nntpd: $?";
+ $srv->{nntp_host_port} = tcp_host_port($rdr->{3});
+}
+
+SKIP: {
+ require_mods(qw(-imapd Mail::IMAPClient), 1);
+ my $rdr = { 3 => tcp_server };
+ $srv->{imapd} = start_script(
+ [qw(-imapd -W0), "--stdout=$tmpdir/i1", "--stderr=$tmpdir/i2"],
+ $env, $rdr) or xbail("-imapd $?");
+ $srv->{imap_host_port} = tcp_host_port($rdr->{3});
+}
-$sock = tcp_server;
-$cmd = [ '-imapd', '-W0', "--stdout=$tmpdir/i1", "--stderr=$tmpdir/i2" ];
-my $imapd = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-imapd $?");
-my $imap_host_port = tcp_host_port($sock);
-undef $sock;
for ('', qw(cur new)) {
mkdir "$tmpdir/md/$_" or xbail "mkdir: $!";
mkdir "$tmpdir/md1/$_" or xbail "mkdir: $!";
@@ -77,8 +86,10 @@ test_lei({ tmpdir => $tmpdir }, sub {
is_deeply(json_utf8->decode($lei_out)->[0]->{'kw'},
['seen'], 'keyword set');
- lei_ok('index', "nntp://$nntp_host_port/t.v2");
- lei_ok('index', "imap://$imap_host_port/t.v2.0");
+ $srv->{nntpd} and
+ lei_ok('index', "nntp://$srv->{nntp_host_port}/t.v2");
+ $srv->{imapd} and
+ lei_ok('index', "imap://$srv->{imap_host_port}/t.v2.0");
is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
lei_ok qw(q m:multipart-html-sucks@11);
^ permalink raw reply related [relevance 63%]
* [PATCH 0/2] lei rediff: --drq and --dequote-only
@ 2021-09-27 4:59 71% Eric Wong
2021-09-27 4:59 69% ` [PATCH 1/2] lei rediff: quiet warnings from Import and Eml Eric Wong
2021-09-27 4:59 42% ` [PATCH 2/2] lei rediff: add --drq and --dequote-only Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-27 4:59 UTC (permalink / raw)
To: meta
Things to make lei-rediff more useful inside a text editor
while writing replies to patch emails.
Eric Wong (2):
lei rediff: quiet warnings from Import and Eml
lei rediff: add --drq and --dequote-only
Documentation/lei-rediff.pod | 27 +++++++++++++++++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 58 ++++++++++++++++++++++++++++++++++--
t/solver_git.t | 41 +++++++++++++++++++++++--
4 files changed, 123 insertions(+), 5 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei rediff: quiet warnings from Import and Eml
2021-09-27 4:59 71% [PATCH 0/2] lei rediff: --drq and --dequote-only Eric Wong
@ 2021-09-27 4:59 69% ` Eric Wong
2021-09-27 4:59 42% ` [PATCH 2/2] lei rediff: add --drq and --dequote-only Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 4:59 UTC (permalink / raw)
To: meta
lei rediff is expected to see partial patch fragments and such,
so silence warnings when something isn't exactly a valid email
message.
---
lib/PublicInbox/LeiRediff.pm | 11 +++++++++--
t/solver_git.t | 1 +
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index ea9b2a643f9a..f6960560f1a2 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -193,8 +193,15 @@ sub extract_oids { # Eml each_part callback
sub input_eml_cb { # callback for all emails
my ($self, $eml) = @_;
- $self->{tmp_sto}->add_eml($eml);
- $self->{tmp_sto}->done;
+ {
+ local $SIG{__WARN__} = sub {
+ return if "@_" =~ /^no email in From: .*? or Sender:/;
+ return if PublicInbox::Eml::warn_ignore(@_);
+ warn @_;
+ };
+ $self->{tmp_sto}->add_eml($eml);
+ $self->{tmp_sto}->done;
+ }
$eml->each_part(\&extract_oids, $self, 1);
}
diff --git a/t/solver_git.t b/t/solver_git.t
index d5f4c4a02758..3c0b7f6524c6 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -93,6 +93,7 @@ index 15ac20eb..771486c4
EOM
like($lei_out, qr/\Q$exp\E/,
'preserve mode, regen header + context from -U0 patch');
+ is($lei_err, '', 'no warnings from bare patch');
my $e = { GIT_DIR => "$ENV{HOME}/.local/share/lei/store/ALL.git" };
my @x = xqx([qw(git cat-file --batch-all-objects --batch-check)], $e);
is_deeply(\@x, [], 'no objects stored') or diag explain(\@x);
^ permalink raw reply related [relevance 69%]
* [PATCH 2/2] lei rediff: add --drq and --dequote-only
2021-09-27 4:59 71% [PATCH 0/2] lei rediff: --drq and --dequote-only Eric Wong
2021-09-27 4:59 69% ` [PATCH 1/2] lei rediff: quiet warnings from Import and Eml Eric Wong
@ 2021-09-27 4:59 42% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-27 4:59 UTC (permalink / raw)
To: meta
More switches which can be useful for users who pipe from text
editors. --drq can be helpful while writing patch review email
replies, and perhaps --dequote-only, too.
---
Documentation/lei-rediff.pod | 27 +++++++++++++++++++++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 47 ++++++++++++++++++++++++++++++++++++
t/solver_git.t | 40 ++++++++++++++++++++++++++++--
4 files changed, 113 insertions(+), 3 deletions(-)
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index f34e946ffe52..e711de97015c 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -38,6 +38,33 @@ to C<lei rediff> before piping it to L<git-am(1)>. The output
of C<lei rediff> is compatible with C<git am> if its input was a
patch email.
+=item --drq[=COUNT]
+
+De-Re-Quote. De-quote the input and re-quotes (the output).
+Removes COUNT levels of C<E<gt> > email reply prefixes and
+re-adds them upon regenerating the diff.
+
+This switch is intended as a convenience for running inside a
+pipe-capable text editor when writing replies to a patch email.
+Note: this may over-add C<E<gt> > prefixes if some input lines
+are missing C<E<gt> > prefixes.
+
+COUNT is 1 if unspecified; in other words, C<--drq=1> and
+C<--drq> are equivalent.
+
+It implies L</--quiet> unless L</--verbose> is specified
+since text editors tend to combine stderr with stdout.
+
+=item --dequote-only[=COUNT]
+
+Like L</--drq>, but does not re-add quote prefixes to the output.
+
+This can be useful for feeding a hunk to L<git-apply(1)>
+or L<patch(1)> while writing a reply or further processing
+by another diff viewer.
+
+Unlike L</--drq>, it does NOT implies L</--quiet>.
+
=item --git-dir=DIR
Specify an additional .git/ directory to scan. This option may be
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8faf74a29466..b8159cba2922 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -201,7 +201,7 @@ our %CMD = ( # sorted in order of importance/use:
'rediff' => [ '--stdin|LOCATION...',
'regenerate a diff with different options',
'stdin|', # /|\z/ must be first for lone dash
- qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
+ qw(git-dir=s@ cwd! verbose|v+ color:s no-color drq:1 dequote-only:1),
@diff_opt, @lxs_opt, @net_opt, @c_opt ],
'add-external' => [ 'LOCATION',
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index f6960560f1a2..1e95e55ac1cc 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -134,6 +134,27 @@ EOM
$pid = spawn(\@cmd, $lei->{env}, { 2 => $lei->{2}, 1 => $lei->{1} });
waitpid($pid, 0);
$lei->child_error($?) if $?; # for git diff --exit-code
+ undef;
+}
+
+sub wait_requote ($$$) { # OnDestroy callback
+ my ($lei, $pid, $old_1) = @_;
+ $lei->{1} = $old_1; # closes stdin of `perl -pE 's/^/> /'`
+ waitpid($pid, 0) == $pid or die "BUG(?) waitpid: \$!=$! \$?=$?";
+ $lei->child_error($?) if $?;
+}
+
+sub requote ($$) {
+ my ($lei, $pfx) = @_;
+ pipe(my($r, $w)) or die "pipe: $!";
+ my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
+ # $^X (perl) is overkill, but maybe there's a weird system w/o sed
+ my $pid = spawn([$^X, '-pE', "s/^/$pfx/"], $lei->{env}, $rdr);
+ my $old_1 = $lei->{1};
+ $w->autoflush(1);
+ binmode $w, ':utf8';
+ $lei->{1} = $w;
+ PublicInbox::OnDestroy->new(\&wait_requote, $lei, $pid, $old_1);
}
sub extract_oids { # Eml each_part callback
@@ -142,6 +163,10 @@ sub extract_oids { # Eml each_part callback
$self->{lei}->out($p->header_obj->as_string, "\n");
my ($s, undef) = msg_part_text($p, $p->content_type || 'text/plain');
defined $s or return;
+ my $rq;
+ if ($self->{dqre} && $s =~ s/$self->{dqre}//g) { # '> ' prefix(es)
+ $rq = requote($self->{lei}, $1) if $self->{lei}->{opt}->{drq};
+ }
my @top = split($PublicInbox::ViewDiff::EXTRACT_DIFFS, $s);
undef $s;
my $blobs = $self->{blobs}; # blobs to resolve
@@ -191,6 +216,19 @@ sub extract_oids { # Eml each_part callback
$ctxq = diff_ctxq($self, $ctxq);
}
+# ensure dequoted parts are available for rebuilding patches:
+sub dequote_add { # Eml each_part callback
+ my ($ary, $self) = @_;
+ my ($p, undef, $idx) = @$ary;
+ my ($s, undef) = msg_part_text($p, $p->content_type || 'text/plain');
+ defined $s or return;
+ if ($s =~ s/$self->{dqre}//g) { # remove '> ' prefix(es)
+ substr($s, 0, 0, "part-dequoted: $idx\n\n");
+ utf8::encode($s);
+ $self->{tmp_sto}->add_eml(PublicInbox::Eml->new(\$s));
+ }
+}
+
sub input_eml_cb { # callback for all emails
my ($self, $eml) = @_;
{
@@ -200,6 +238,7 @@ sub input_eml_cb { # callback for all emails
warn @_;
};
$self->{tmp_sto}->add_eml($eml);
+ $eml->each_part(\&dequote_add, $self) if $self->{dqre};
$self->{tmp_sto}->done;
}
$eml->each_part(\&extract_oids, $self, 1);
@@ -207,6 +246,10 @@ sub input_eml_cb { # callback for all emails
sub lei_rediff {
my ($lei, @inputs) = @_;
+ ($lei->{opt}->{drq} && $lei->{opt}->{'dequote-only'}) and return
+ $lei->fail('--drq and --dequote-only are mutually exclusive');
+ ($lei->{opt}->{drq} && !$lei->{opt}->{verbose}) and
+ $lei->{opt}->{quiet} //= 1;
$lei->_lei_store(1)->write_prepare($lei);
$lei->{opt}->{'in-format'} //= 'eml';
# maybe it's a non-email (code) blob from a coderepo
@@ -257,6 +300,10 @@ sub ipc_atfork_child {
} @{$self->{lei}->{opt}->{'git-dir'}} ];
$lei->{env}->{'psgi.errors'} = $lei->{2}; # ugh...
$lei->{env}->{TMPDIR} = $self->{rdtmp}->dirname;
+ if (my $nr = ($lei->{opt}->{drq} || $lei->{opt}->{'dequote-only'})) {
+ my $re = '\s*> ' x $nr;
+ $self->{dqre} = qr/^($re)/ms;
+ }
undef;
}
diff --git a/t/solver_git.t b/t/solver_git.t
index 3c0b7f6524c6..cf450e2459e5 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -80,12 +80,48 @@ test_lei({tmpdir => "$tmpdir/rediff"}, sub {
lei_ok(qw(rediff -q -U9 t/solve/0001-simple-mod.patch));
like($lei_out, qr!^\Q+++\E b/TODO\n@@ -103,9 \+103,11 @@!sm,
'got more context with -U9');
- lei_ok(qw(rediff -q -U9 t/solve/bare.patch));
+
+ my (undef, $re) = split(/\n\n/, $lei_out, 2);
+ $re =~ s/^/> /sgm;
+ substr($re, 0, 0, <<EOM);
+From: me\@example.com
+Subject: Re: awesome advice
+
+WEB DESIGN EXPERT wrote:
+EOM
+ lei_ok([qw(rediff --abbrev=40 -U16 --drq)], undef,
+ { 0 => \$re, %$lei_opt });
my $exp = <<'EOM';
+From: me@example.com
+Subject: Re: awesome advice
+
+EOM
+ like($lei_out, qr/\Q$exp\E/, '--drq preserved header');
+
+ # n.b. --drq can requote the attribution line ("So-and-so wrote:"),
+ # but it's probably not worth preventing...
+
+ $exp = <<'EOM';
+> ---
+> TODO | 2 ++
+> Ω | 5 --
+> 1 file changed, 2 insertions(+)
+>
+> diff --git a/TODO b/TODO
+> index 605013e4904baabecd4a0a55997aebd8e8477a8f..69df7d565d49fbaaeb0a067910f03dc22cd52bd0 100644
+> --- a/TODO
+> +++ b/TODO
+> @@ -96,16 +96,18 @@ all need to be considered for everything we introduce)
+EOM
+ $exp =~ s/^>$/> /sgm; # re-add trailing white space
+ like($lei_out, qr/\Q$exp\E/, '--drq diffstat + context');
+
+ lei_ok(qw(rediff -q --abbrev=40 -U9 t/solve/bare.patch));
+ $exp = <<'EOM';
diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
old mode 100644
new mode 100755
-index 15ac20eb..771486c4
+index 15ac20eb871bf47697377e58a27db23102a38fca..771486c425b315bae70fd8a82d62ab0331e0a827
--- a/script/public-inbox-extindex
+++ b/script/public-inbox-extindex
@@ -1,13 +1,12 @@
^ permalink raw reply related [relevance 42%]
* [PATCH] lei: ensure refresh_watches isn't called from workers
@ 2021-09-26 5:38 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-26 5:38 UTC (permalink / raw)
To: meta
Only the top-level lei-daemon will do inotify/kevent.
---
lib/PublicInbox/LEI.pm | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a337fb0d80c2..8faf74a29466 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -578,6 +578,7 @@ sub _lei_atfork_child {
close $listener if $listener;
undef $listener;
$dir_idle->force_close if $dir_idle;
+ undef $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
@@ -1413,6 +1414,7 @@ sub add_maildir_watch ($$) {
sub refresh_watches {
my ($lei) = @_;
+ $dir_idle or return;
my $cfg = _lei_cfg($lei) or return;
my $old = $cfg->{-watches};
my $watches = $cfg->{-watches} //= {};
^ permalink raw reply related [relevance 71%]
* [PATCH] lei -f reply: fix Cc: header combining
@ 2021-09-26 1:42 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-26 1:42 UTC (permalink / raw)
To: meta
When combining lines from To: and Cc: headers, ", " needs to be
used to separate them.
---
lib/PublicInbox/LeiViewText.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 34612711..1f002ccd 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -96,10 +96,11 @@ sub quote_hdr_buf ($$) {
for my $f (qw(To Cc)) {
for my $v ($eml->header_raw($f)) {
next if $v !~ /\S/;
- $cc .= $v;
+ $cc .= ", $v";
$to //= $v;
}
}
+ substr($cc, 0, 2, ''); # s/^, //;
PublicInbox::View::fold_addresses($to);
PublicInbox::View::fold_addresses($cc);
_xs($to);
^ permalink raw reply related [relevance 71%]
* [PATCH] lei note-event: ignore kw_changed exceptions
@ 2021-09-26 0:02 62% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-26 0:02 UTC (permalink / raw)
To: meta
The note-event worker may see changes before a Xapian shard
commit happens, meaning keyword lookups fail as a result.
Just emit the request to the lei/store worker since it's a
fairly cheap operation at this point.
We'll try harder to look for kw changes, too, since
deduplication changes may lead to multiple docids being
resolved for a single message.
---
lib/PublicInbox/LeiNoteEvent.pm | 4 +++-
lib/PublicInbox/LeiSearch.pm | 12 +++++++++---
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index d24294850fd2..2a7b52c13487 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -42,7 +42,9 @@ sub eml_event ($$$$) {
$sto->wq_do('index_eml_only', $eml, $vmd, $xoids);
} elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
my $docids = [];
- my $c = $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
+ my $c = eval {
+ $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
+ } // 1; # too new, assume changed since still to-be-committed.
if (scalar @$docids) { # already in lei/store
$sto->wq_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index a10e6e176775..568277a6f404 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -9,6 +9,7 @@ use parent qw(PublicInbox::ExtSearch); # PublicInbox::Search->reopen
use PublicInbox::Search qw(xap_terms);
use PublicInbox::ContentHash qw(content_digest content_hash);
use PublicInbox::MID qw(mids mids_for_index);
+use Carp qw(croak);
# get combined docid from over.num:
# (not generic Xapian, only works with our sharding scheme)
@@ -131,14 +132,19 @@ sub xoids_for {
# returns true if $eml is indexed by lei/store and keywords don't match
sub kw_changed {
my ($self, $eml, $new_kw_sorted, $docids) = @_;
+ my $cur_kw;
if ($eml) {
my $xoids = xoids_for($self, $eml) // return;
$docids //= [];
@$docids = sort { $a <=> $b } values %$xoids;
}
- my $cur_kw = eval { msg_keywords($self, $docids->[0]) };
- die "E: #$docids->[0] keyword lookup failure: $@\n" if $@;
-
+ for my $id (@$docids) {
+ $cur_kw = eval { msg_keywords($self, $id) } and last;
+ }
+ if (!defined($cur_kw) && $@) {
+ $docids = join(', num:', @$docids);
+ croak "E: num:$docids keyword lookup failure: $@";
+ }
# RFC 5550 sec 5.9 on the $Forwarded keyword states:
# "Once set, the flag SHOULD NOT be cleared"
if (exists($cur_kw->{forwarded}) &&
^ permalink raw reply related [relevance 62%]
* [PATCH 0/3] lei *-external: split off into separate files
@ 2021-09-25 8:49 71% Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
Some of the early parts of lei predated the current autoloading
using lazy_cb. None of these commands are expected to be
frequently used, so wasting memory on them is unnecessary.
Eric Wong (3):
lei forget-external: split into separate file
lei add-external: split into separate file
lei ls-external: split into separate file
MANIFEST | 3 +
lib/PublicInbox/LeiAddExternal.pm | 72 ++++++++++++++
lib/PublicInbox/LeiExternal.pm | 140 +--------------------------
lib/PublicInbox/LeiForgetExternal.pm | 46 +++++++++
lib/PublicInbox/LeiLsExternal.pm | 32 ++++++
lib/PublicInbox/LeiMirror.pm | 3 +-
lib/PublicInbox/LeiQuery.pm | 6 +-
7 files changed, 161 insertions(+), 141 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddExternal.pm
create mode 100644 lib/PublicInbox/LeiForgetExternal.pm
create mode 100644 lib/PublicInbox/LeiLsExternal.pm
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei ls-external: split into separate file
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
2021-09-25 8:49 48% ` [PATCH 2/3] lei add-external: " Eric Wong
@ 2021-09-25 8:49 58% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
This was written before we had auto-loading and rarely used.
---
MANIFEST | 1 +
lib/PublicInbox/LeiExternal.pm | 26 +-------------------------
lib/PublicInbox/LeiLsExternal.pm | 32 ++++++++++++++++++++++++++++++++
3 files changed, 34 insertions(+), 25 deletions(-)
create mode 100644 lib/PublicInbox/LeiLsExternal.pm
diff --git a/MANIFEST b/MANIFEST
index 8db9cdc791bb..3e942855127f 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -230,6 +230,7 @@ lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
lib/PublicInbox/LeiLcat.pm
+lib/PublicInbox/LeiLsExternal.pm
lib/PublicInbox/LeiLsLabel.pm
lib/PublicInbox/LeiLsMailSource.pm
lib/PublicInbox/LeiLsMailSync.pm
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 5a54cc19281e..701d1ad53adf 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -50,7 +50,7 @@ my %re_map = ( '*' => '[^/]*?', '?' => '[^/]',
'[' => '[', ']' => ']', ',' => ',' );
sub glob2re {
- my $re = $_[-1];
+ my $re = $_[-1]; # $_[0] may be $lei
my $p = '';
my $in_bracket = 0;
my $qm = 0;
@@ -108,30 +108,6 @@ sub get_externals {
();
}
-# TODO: does this need JSON output?
-sub lei_ls_external {
- my ($self, $filter) = @_;
- my $opt = $self->{opt};
- my $do_glob = !$opt->{globoff}; # glob by default
- my ($OFS, $ORS) = $opt->{z} ? ("\0", "\0\0") : (" ", "\n");
- $filter //= '*';
- my $re = $do_glob ? glob2re($filter) : undef;
- $re //= index($filter, '/') < 0 ?
- qr!/\Q$filter\E/?\z! : # exact basename match
- qr/\Q$filter\E/; # grep -F semantics
- my @ext = externals_each($self, my $boost = {});
- @ext = $opt->{'invert-match'} ? grep(!/$re/, @ext)
- : grep(/$re/, @ext);
- if ($opt->{'local'} && !$opt->{remote}) {
- @ext = grep(!m!\A[a-z\+]+://!, @ext);
- } elsif ($opt->{remote} && !$opt->{'local'}) {
- @ext = grep(m!\A[a-z\+]+://!, @ext);
- }
- for my $loc (@ext) {
- $self->out($loc, $OFS, 'boost=', $boost->{$loc}, $ORS);
- }
-}
-
# returns an anonymous sub which returns an array of potential results
sub complete_url_prepare {
my $argv = $_[-1]; # $_[0] may be $lei
diff --git a/lib/PublicInbox/LeiLsExternal.pm b/lib/PublicInbox/LeiLsExternal.pm
new file mode 100644
index 000000000000..dd2eb2e7d16e
--- /dev/null
+++ b/lib/PublicInbox/LeiLsExternal.pm
@@ -0,0 +1,32 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei ls-external" command
+package PublicInbox::LeiLsExternal;
+use strict;
+use v5.10.1;
+
+# TODO: does this need JSON output?
+sub lei_ls_external {
+ my ($lei, $filter) = @_;
+ my $do_glob = !$lei->{opt}->{globoff}; # glob by default
+ my ($OFS, $ORS) = $lei->{opt}->{z} ? ("\0", "\0\0") : (" ", "\n");
+ $filter //= '*';
+ my $re = $do_glob ? $lei->glob2re($filter) : undef;
+ $re //= index($filter, '/') < 0 ?
+ qr!/\Q$filter\E/?\z! : # exact basename match
+ qr/\Q$filter\E/; # grep -F semantics
+ my @ext = $lei->externals_each(my $boost = {});
+ @ext = $lei->{opt}->{'invert-match'} ? grep(!/$re/, @ext)
+ : grep(/$re/, @ext);
+ if ($lei->{opt}->{'local'} && !$lei->{opt}->{remote}) {
+ @ext = grep(!m!\A[a-z\+]+://!, @ext);
+ } elsif ($lei->{opt}->{remote} && !$lei->{opt}->{'local'}) {
+ @ext = grep(m!\A[a-z\+]+://!, @ext);
+ }
+ for my $loc (@ext) {
+ $lei->out($loc, $OFS, 'boost=', $boost->{$loc}, $ORS);
+ }
+}
+
+1;
^ permalink raw reply related [relevance 58%]
* [PATCH 1/3] lei forget-external: split into separate file
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
@ 2021-09-25 8:49 50% ` Eric Wong
2021-09-25 8:49 48% ` [PATCH 2/3] lei add-external: " Eric Wong
2021-09-25 8:49 58% ` [PATCH 3/3] lei ls-external: " Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
This was written before we had auto-loading, and forget-external
should be a rarely-used command that's not worth loading at
startup. Do some golfing while we're in the area, too.
---
MANIFEST | 1 +
lib/PublicInbox/LeiExternal.pm | 48 ++--------------------------
lib/PublicInbox/LeiForgetExternal.pm | 46 ++++++++++++++++++++++++++
lib/PublicInbox/LeiQuery.pm | 6 ++--
4 files changed, 52 insertions(+), 49 deletions(-)
create mode 100644 lib/PublicInbox/LeiForgetExternal.pm
diff --git a/MANIFEST b/MANIFEST
index 3595195a6996..a39afe24aa87 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -218,6 +218,7 @@ lib/PublicInbox/LeiEditSearch.pm
lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
lib/PublicInbox/LeiFinmsg.pm
+lib/PublicInbox/LeiForgetExternal.pm
lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index f8e610cacb21..35a7d68a17b5 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -31,7 +31,7 @@ sub externals_each {
}
sub ext_canonicalize {
- my ($location) = @_;
+ my $location = $_[-1]; # $_[0] may be $lei
if ($location !~ m!\Ahttps?://!) {
PublicInbox::Config::rel2abs_collapsed($location);
} else {
@@ -185,39 +185,9 @@ sub lei_add_external {
}
}
-sub lei_forget_external {
- my ($self, @locations) = @_;
- my $cfg = $self->_lei_cfg(1);
- my $quiet = $self->{opt}->{quiet};
- my %seen;
- for my $loc (@locations) {
- my (@unset, @not_found);
- for my $l ($loc, ext_canonicalize($loc)) {
- next if $seen{$l}++;
- my $key = "external.$l.boost";
- delete($cfg->{$key});
- $self->_config('--unset', $key);
- if ($? == 0) {
- push @unset, $l;
- } elsif (($? >> 8) == 5) {
- push @not_found, $l;
- } else {
- $self->err("# --unset $key error");
- return $self->x_it($?);
- }
- }
- if (@unset) {
- next if $quiet;
- $self->err("# $_ gone") for @unset;
- } elsif (@not_found) {
- $self->err("# $_ not found") for @not_found;
- } # else { already exited
- }
-}
-
# returns an anonymous sub which returns an array of potential results
sub complete_url_prepare {
- my $argv = $_[-1];
+ my $argv = $_[-1]; # $_[0] may be $lei
# Workaround bash word-splitting URLs to ['https', ':', '//' ...]
# Maybe there's a better way to go about this in
# contrib/completion/lei-completion.bash
@@ -253,20 +223,6 @@ sub complete_url_prepare {
wantarray ? ($re, $cur, $match_cb) : $match_cb;
}
-# shell completion helper called by lei__complete
-sub _complete_forget_external {
- my ($self, @argv) = @_;
- my $cfg = $self->_lei_cfg;
- my ($cur, $re, $match_cb) = complete_url_prepare(\@argv);
- # FIXME: bash completion off "http:" or "https:" when the last
- # character is a colon doesn't work properly even if we're
- # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
- # be a bash issue.
- map {
- $match_cb->(substr($_, length('external.')));
- } grep(/\Aexternal\.$re\Q$cur/, @{$cfg->{-section_order}});
-}
-
sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
my ($self, @argv) = @_;
my $cfg = $self->_lei_cfg;
diff --git a/lib/PublicInbox/LeiForgetExternal.pm b/lib/PublicInbox/LeiForgetExternal.pm
new file mode 100644
index 000000000000..7a4bbcf80a3a
--- /dev/null
+++ b/lib/PublicInbox/LeiForgetExternal.pm
@@ -0,0 +1,46 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei forget-external" command
+package PublicInbox::LeiForgetExternal;
+use strict;
+use v5.10.1;
+
+sub lei_forget_external {
+ my ($lei, @locations) = @_;
+ my $cfg = $lei->_lei_cfg or
+ return $lei->fail('no externals configured');
+ my %seen;
+ for my $loc (@locations) {
+ for my $l ($loc, $lei->ext_canonicalize($loc)) {
+ next if $seen{$l}++;
+ my $key = "external.$l.boost";
+ delete($cfg->{$key});
+ $lei->_config('--unset', $key);
+ if ($? == 0) {
+ $lei->qerr("# $l forgotten ");
+ } elsif (($? >> 8) == 5) {
+ $lei->err("# $l not found");
+ } else {
+ $lei->err("# --unset $key error");
+ return $lei->x_it($?);
+ }
+ }
+ }
+}
+
+# shell completion helper called by lei__complete
+sub _complete_forget_external {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg or return ();
+ my ($cur, $re, $match_cb) = $lei->complete_url_prepare(\@argv);
+ # FIXME: bash completion off "http:" or "https:" when the last
+ # character is a colon doesn't work properly even if we're
+ # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
+ # be a bash issue.
+ map {
+ $match_cb->(substr($_, length('external.')));
+ } grep(/\Aexternal\.$re\Q$cur/, @{$cfg->{-section_order}});
+}
+
+1;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index cb5ac8fb84a7..c65b00ca0986 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -158,11 +158,11 @@ no query allowed on command-line with --stdin
# shell completion helper called by lei__complete
sub _complete_q {
my ($self, @argv) = @_;
- my $ext = qr/\A(?:-I|(?:--(?:include|exclude|only)))\z/;
my @cur;
+ my $cb = $self->lazy_cb(qw(forget-external _complete_));
while (@argv) {
- if ($argv[-1] =~ $ext) {
- my @c = $self->_complete_forget_external(@cur);
+ if ($argv[-1] =~ /\A(?:-I|(?:--(?:include|exclude|only)))\z/) {
+ my @c = $cb->($self, @cur);
# try basename match:
if (scalar(@cur) == 1 && index($cur[0], '/') < 0) {
my $all = $self->externals_each;
^ permalink raw reply related [relevance 50%]
* [PATCH 2/3] lei add-external: split into separate file
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
@ 2021-09-25 8:49 48% ` Eric Wong
2021-09-25 8:49 58% ` [PATCH 3/3] lei ls-external: " Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 8:49 UTC (permalink / raw)
To: meta
Also was written before we had auto-loading and rarely used.
---
MANIFEST | 1 +
lib/PublicInbox/LeiAddExternal.pm | 72 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiExternal.pm | 66 ----------------------------
lib/PublicInbox/LeiMirror.pm | 3 +-
4 files changed, 75 insertions(+), 67 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddExternal.pm
diff --git a/MANIFEST b/MANIFEST
index a39afe24aa87..8db9cdc791bb 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -207,6 +207,7 @@ lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
lib/PublicInbox/LI2Wrap.pm
lib/PublicInbox/LeiALE.pm
+lib/PublicInbox/LeiAddExternal.pm
lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
lib/PublicInbox/LeiBlob.pm
diff --git a/lib/PublicInbox/LeiAddExternal.pm b/lib/PublicInbox/LeiAddExternal.pm
new file mode 100644
index 000000000000..5eef206c761c
--- /dev/null
+++ b/lib/PublicInbox/LeiAddExternal.pm
@@ -0,0 +1,72 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei add-external" command
+package PublicInbox::LeiAddExternal;
+use strict;
+use v5.10.1;
+
+sub _finish_add_external {
+ my ($lei, $location) = @_;
+ my $new_boost = $lei->{opt}->{boost} // 0;
+ my $key = "external.$location.boost";
+ my $cur_boost = $lei->_lei_cfg(1)->{$key};
+ return if defined($cur_boost) && $cur_boost == $new_boost; # idempotent
+ $lei->_config($key, $new_boost);
+}
+
+sub lei_add_external {
+ my ($lei, $location) = @_;
+ my $mirror = $lei->{opt}->{mirror} // do {
+ my @fail;
+ for my $sw ($lei->index_opt, $lei->curl_opt,
+ qw(no-torsocks torsocks inbox-version)) {
+ my ($f) = (split(/|/, $sw, 2))[0];
+ next unless defined $lei->{opt}->{$f};
+ $f = length($f) == 1 ? "-$f" : "--$f";
+ push @fail, $f;
+ }
+ if (scalar(@fail) == 1) {
+ return $lei->("@fail requires --mirror");
+ } elsif (@fail) {
+ my $last = pop @fail;
+ my $fail = join(', ', @fail);
+ return $lei->("@fail and $last require --mirror");
+ }
+ undef;
+ };
+ $location = $lei->ext_canonicalize($location);
+ if (defined($mirror) && -d $location) {
+ $lei->fail(<<""); # TODO: did you mean "update-external?"
+--mirror destination `$location' already exists
+
+ } elsif (-d $location) {
+ index($location, "\n") >= 0 and
+ return $lei->fail("`\\n' not allowed in `$location'");
+ }
+ if ($location !~ m!\Ahttps?://! && !-d $location) {
+ $mirror // return $lei->fail("$location not a directory");
+ index($location, "\n") >= 0 and
+ return $lei->fail("`\\n' not allowed in `$location'");
+ $mirror = $lei->ext_canonicalize($mirror);
+ require PublicInbox::LeiMirror;
+ PublicInbox::LeiMirror->start($lei, $mirror => $location);
+ } else {
+ _finish_add_external($lei, $location);
+ }
+}
+
+sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg or return ();
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ require URI;
+ map {
+ my $u = URI->new(substr($_, length('external.')));
+ my ($base) = ($u->path =~ m!((?:/?.*)?/)[^/]+/?\z!);
+ $u->path($base);
+ $match_cb->($u->as_string);
+ } grep(m!\Aexternal\.https?://!, @{$cfg->{-section_order}});
+}
+
+1;
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 35a7d68a17b5..5a54cc19281e 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -132,59 +132,6 @@ sub lei_ls_external {
}
}
-sub add_external_finish {
- my ($self, $location) = @_;
- my $cfg = $self->_lei_cfg(1);
- my $new_boost = $self->{opt}->{boost} // 0;
- my $key = "external.$location.boost";
- my $cur_boost = $cfg->{$key};
- return if defined($cur_boost) && $cur_boost == $new_boost; # idempotent
- $self->_config($key, $new_boost);
-}
-
-sub lei_add_external {
- my ($self, $location) = @_;
- my $opt = $self->{opt};
- my $mirror = $opt->{mirror} // do {
- my @fail;
- for my $sw ($self->index_opt, $self->curl_opt,
- qw(no-torsocks torsocks inbox-version)) {
- my ($f) = (split(/|/, $sw, 2))[0];
- next unless defined $opt->{$f};
- $f = length($f) == 1 ? "-$f" : "--$f";
- push @fail, $f;
- }
- if (scalar(@fail) == 1) {
- return $self->("@fail requires --mirror");
- } elsif (@fail) {
- my $last = pop @fail;
- my $fail = join(', ', @fail);
- return $self->("@fail and $last require --mirror");
- }
- undef;
- };
- my $new_boost = $opt->{boost} // 0;
- $location = ext_canonicalize($location);
- if (defined($mirror) && -d $location) {
- $self->fail(<<""); # TODO: did you mean "update-external?"
---mirror destination `$location' already exists
-
- } elsif (-d $location) {
- index($location, "\n") >= 0 and
- return $self->fail("`\\n' not allowed in `$location'");
- }
- if ($location !~ m!\Ahttps?://! && !-d $location) {
- $mirror // return $self->fail("$location not a directory");
- index($location, "\n") >= 0 and
- return $self->fail("`\\n' not allowed in `$location'");
- $mirror = ext_canonicalize($mirror);
- require PublicInbox::LeiMirror;
- PublicInbox::LeiMirror->start($self, $mirror => $location);
- } else {
- add_external_finish($self, $location);
- }
-}
-
# returns an anonymous sub which returns an array of potential results
sub complete_url_prepare {
my $argv = $_[-1]; # $_[0] may be $lei
@@ -223,17 +170,4 @@ sub complete_url_prepare {
wantarray ? ($re, $cur, $match_cb) : $match_cb;
}
-sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
- my ($self, @argv) = @_;
- my $cfg = $self->_lei_cfg;
- my $match_cb = complete_url_prepare(\@argv);
- require URI;
- map {
- my $u = URI->new(substr($_, length('external.')));
- my ($base) = ($u->path =~ m!((?:/?.*)?/)[^/]+/?\z!);
- $u->path($base);
- $match_cb->($u->as_string);
- } grep(m!\Aexternal\.https?://!, @{$cfg->{-section_order}});
-}
-
1;
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 1ab5e0d898d2..5cfa6fea851c 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -23,7 +23,8 @@ sub _wq_done_wait { # dwaitpid callback (via wq_eof)
$lei->err("unlink($f): $!") unless $!{ENOENT};
} else {
if ($lei->{cmd} ne 'public-inbox-clone') {
- $lei->add_external_finish($mrr->{dst});
+ $lei->lazy_cb('add-external', '_finish_'
+ )->($lei, $mrr->{dst});
}
$lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
}
^ permalink raw reply related [relevance 48%]
* [PATCH] doc: lei-rm: remove unnecessary -F values
@ 2021-09-25 7:08 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 7:08 UTC (permalink / raw)
To: meta
-F is really only useful for distinguishing between mbox
variants and single message/rfc822 files. URLs and
directory-based formats can be auto-detected easily enough.
---
Documentation/lei-rm.pod | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/Documentation/lei-rm.pod b/Documentation/lei-rm.pod
index f2a0c0f046e1..67bfb5510d63 100644
--- a/Documentation/lei-rm.pod
+++ b/Documentation/lei-rm.pod
@@ -34,10 +34,13 @@ the command-line.
=item --in-format=MAIL_FORMAT
-Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
-C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+Message input format: C<eml>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>
+when reading from stdin or using one of the mbox variants.
-Default: C<eml> when reading from stdin
+Not necessary when using an IMAP URL, NNTP URL or Maildir.
+
+Default: C<eml> when reading from stdin or if the file suffix
+ends in C<.patch> or C<.eml>.
=item --lock=METHOD
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: make pkt_op easier-to-use and understand
@ 2021-09-25 6:17 55% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 6:17 UTC (permalink / raw)
To: meta
Since switching to SOCK_SEQUENTIAL, we no longer have to use
fixed-width records to guarantee atomic reads. Thus we can
maintain more human-readable/searchable PktOp opcodes.
Furthermore, we can infer the subroutine name in many cases
to avoid repeating ourselves by specifying a command-name
twice (e.g. $ops->{CMD} => [ \&CMD, $obj ]; can now simply be
written as: $ops->{CMD} => [ $obj ] if CMD is a method of
$obj.
---
lib/PublicInbox/LEI.pm | 15 +++++++--------
lib/PublicInbox/LeiToMail.pm | 7 +++----
lib/PublicInbox/LeiXSearch.pm | 14 +++++++-------
lib/PublicInbox/PktOp.pm | 5 +++--
4 files changed, 20 insertions(+), 21 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3ff8a347af8b..a337fb0d80c2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -519,8 +519,7 @@ sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
$self->{failed}++;
err($self, $buf) if defined $buf;
- # calls fail_handler
- $self->{pkt_op_p}->pkt_do('!') if $self->{pkt_op_p};
+ $self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
x_it($self, ($exit_code // 1) << 8);
undef;
}
@@ -552,7 +551,7 @@ sub child_error { # passes non-fatal curl exit codes to user
sub note_sigpipe { # triggers sigpipe_handler
my ($self, $fd) = @_;
close(delete($self->{$fd})); # explicit close silences Perl warning
- $self->{pkt_op_p}->pkt_do('|') if $self->{pkt_op_p};
+ $self->{pkt_op_p}->pkt_do('sigpipe_handler') if $self->{pkt_op_p};
x_it($self, 13);
}
@@ -614,11 +613,11 @@ sub incr {
sub pkt_ops {
my ($lei, $ops) = @_;
- $ops->{'!'} = [ \&fail_handler, $lei ];
- $ops->{'|'} = [ \&sigpipe_handler, $lei ];
- $ops->{x_it} = [ \&x_it, $lei ];
- $ops->{child_error} = [ \&child_error, $lei ];
- $ops->{incr} = [ \&incr, $lei ];
+ $ops->{fail_handler} = [ $lei ];
+ $ops->{sigpipe_handler} = [ $lei ];
+ $ops->{x_it} = [ $lei ];
+ $ops->{child_error} = [ $lei ];
+ $ops->{incr} = [ $lei ];
$ops;
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 467b27bf275d..d42759cf71d4 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -714,16 +714,15 @@ sub do_post_auth {
} else { # Maildir
$self->{shard_info} = [ $mod, $shard ];
}
- $aug = '+'; # incr_post_augment
+ $aug = 'incr_post_augment';
} elsif ($self->{-wq_worker_nr} == 0) { # 1st worker do_augment
- $aug = '.'; # do_post_augment
+ $aug = 'do_post_augment';
}
if ($aug) {
local $0 = 'do_augment';
eval { do_augment($self, $lei) };
$lei->fail($@) if $@;
- $lei->{pkt_op_p}->pkt_do($aug) == 1 or
- die "do_post_augment trigger: $!";
+ $lei->{pkt_op_p}->pkt_do($aug) or die "pkt_do($aug): $!";
}
# done augmenting, connect the compressor pipe for each worker
if (my $zpipe = delete $lei->{zpipe}) {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 99f887ce730f..ae9f5881676d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -536,16 +536,16 @@ sub do_query {
my ($self, $lei) = @_;
my $l2m = $lei->{l2m};
my $ops = {
- '|' => [ $lei->can('sigpipe_handler'), $lei ],
- '!' => [ $lei->can('fail_handler'), $lei ],
- '.' => [ \&do_post_augment, $lei ],
- '+' => [ \&incr_post_augment, $lei ],
+ 'sigpipe_handler' => [ $lei ],
+ 'fail_handler' => [ $lei ],
+ 'do_post_augment' => [ \&do_post_augment, $lei ],
+ 'incr_post_augment' => [ \&incr_post_augment, $lei ],
'' => [ \&query_done, $lei ],
'mset_progress' => [ \&mset_progress, $lei ],
'l2m_progress' => [ \&l2m_progress, $lei ],
- 'x_it' => [ $lei->can('x_it'), $lei ],
- 'child_error' => [ $lei->can('child_error'), $lei ],
- 'incr_start_query' => [ \&incr_start_query, $self, $l2m ],
+ 'x_it' => [ $lei ],
+ 'child_error' => [ $lei ],
+ 'incr_start_query' => [ $self, $l2m ],
};
$lei->{auth}->op_merge($ops, $l2m) if $l2m && $lei->{auth};
my $end = $lei->pkt_op_pair;
diff --git a/lib/PublicInbox/PktOp.pm b/lib/PublicInbox/PktOp.pm
index 10942dd19b68..fd2569badd74 100644
--- a/lib/PublicInbox/PktOp.pm
+++ b/lib/PublicInbox/PktOp.pm
@@ -13,6 +13,7 @@ use Errno qw(EAGAIN EINTR);
use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
use Socket qw(AF_UNIX MSG_EOR SOCK_SEQPACKET);
use PublicInbox::IPC qw(ipc_freeze ipc_thaw);
+use Scalar::Util qw(blessed);
sub new {
my ($cls, $r) = @_;
@@ -57,8 +58,8 @@ sub event_step {
}
my $op = $self->{ops}->{$cmd //= $msg};
if ($op) {
- my ($sub, @args) = @$op;
- $sub->(@args, @pargs);
+ my ($obj, @args) = (@$op, @pargs);
+ blessed($obj) ? $obj->$cmd(@args) : $obj->(@args);
} elsif ($msg ne '') {
die "BUG: unknown message: `$cmd'";
}
^ permalink raw reply related [relevance 55%]
* [PATCH 1/3] lei up: show timezone offset with localtime
2021-09-25 5:49 71% [PATCH 0/3] lei: some robustness fixes Eric Wong
@ 2021-09-25 5:49 71% ` Eric Wong
2021-09-25 5:49 71% ` [PATCH 2/3] lei: restore old sigmask before daemon exit Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 5:49 UTC (permalink / raw)
To: meta
Sometimes a user (e.g. me) isn't really sure what timezone
they're in...
---
lib/PublicInbox/LeiXSearch.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 88540bc22367..99f887ce730f 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -324,7 +324,7 @@ sub fudge_qstr_time ($$$) {
}
$lr -= ($rft || (48 * 60 * 60));
$lei->qerr("# $uri limiting to ".
- strftime('%Y-%m-%d %k:%M', localtime($lr)). ' and newer');
+ strftime('%Y-%m-%d %k:%M %z', localtime($lr)). ' and newer');
# this should really be rt: (received-time), but no stable
# public-inbox releases support it, yet.
my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..';
^ permalink raw reply related [relevance 71%]
* [PATCH 0/3] lei: some robustness fixes
@ 2021-09-25 5:49 71% Eric Wong
2021-09-25 5:49 71% ` [PATCH 1/3] lei up: show timezone offset with localtime Eric Wong
2021-09-25 5:49 71% ` [PATCH 2/3] lei: restore old sigmask before daemon exit Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-25 5:49 UTC (permalink / raw)
To: meta
It out I can still fine localtime confusing if I'm constantly
switching timezones to fit wherever insomnia puts me.
I also noticed an odd failure triggered by augment_inprogress
which I can't reproduce. In any case, a progress message
shouldn't cause catastrophic failure.
Eric Wong (3):
lei up: show timezone offset with localtime
lei: restore old sigmask before daemon exit
lei2mail: augment_inprogress: guard against closed FDs
lib/PublicInbox/LEI.pm | 2 ++
lib/PublicInbox/LeiToMail.pm | 15 +++++++++------
lib/PublicInbox/LeiXSearch.pm | 2 +-
3 files changed, 12 insertions(+), 7 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei: restore old sigmask before daemon exit
2021-09-25 5:49 71% [PATCH 0/3] lei: some robustness fixes Eric Wong
2021-09-25 5:49 71% ` [PATCH 1/3] lei up: show timezone offset with localtime Eric Wong
@ 2021-09-25 5:49 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-25 5:49 UTC (permalink / raw)
To: meta
If the event loop fails, we want blocking waitpid (wait4) calls
to be interruptible with SIGTERM via "kill $PID" rather than
SIGKILL. Though a failing event loop is something we should
avoid...
---
lib/PublicInbox/LEI.pm | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9d5a5a46b398..3ff8a347af8b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1346,6 +1346,8 @@ sub lazy_start {
# $daemon pipe to `lei' closed, main loop begins:
eval { PublicInbox::DS->EventLoop };
warn "event loop error: $@\n" if $@;
+ # exit() may trigger waitpid via various DESTROY, ensure interruptible
+ PublicInbox::DS::sig_setmask($oldset);
dump_and_clear_log();
exit($exit_code // 0);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/4] doc: lei: some manpage updaets
@ 2021-09-24 12:51 71% Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Tring ot ut somthing mildly coharent. Insomnia :<
Eric Wong (4):
doc: lei blob+rediff+p2q: add notes about git directory
doc: lei-overview: implicit stdin, correct Inline::C notes
doc: lei-index: remove --stdin, reword -F
doc: lei: manpages for export-kw and refresh-mail-sync
Documentation/lei-blob.pod | 6 ++-
Documentation/lei-export-kw.pod | 52 ++++++++++++++++++++++
Documentation/lei-index.pod | 18 +++-----
Documentation/lei-overview.pod | 22 +++++-----
Documentation/lei-p2q.pod | 2 +-
Documentation/lei-rediff.pod | 23 +++++++++-
Documentation/lei-refresh-mail-sync.pod | 57 +++++++++++++++++++++++++
MANIFEST | 2 +
Makefile.PL | 5 ++-
9 files changed, 158 insertions(+), 29 deletions(-)
create mode 100644 Documentation/lei-export-kw.pod
create mode 100644 Documentation/lei-refresh-mail-sync.pod
^ permalink raw reply [relevance 71%]
* [PATCH 3/4] doc: lei-index: remove --stdin, reword -F
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
2021-09-24 12:51 56% ` [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes Eric Wong
@ 2021-09-24 12:51 68% ` Eric Wong
2021-09-24 12:51 49% ` [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
lei-index really only works for Maildir, at the moment.
---
Documentation/lei-index.pod | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
diff --git a/Documentation/lei-index.pod b/Documentation/lei-index.pod
index bd125bcceb63..9e72026351a1 100644
--- a/Documentation/lei-index.pod
+++ b/Documentation/lei-index.pod
@@ -6,8 +6,6 @@ lei-index - index messages without importing them into lei/store
lei index [OPTIONS] FOLDER
-lei index [OPTIONS] --stdin
-
=head1 DESCRIPTION
Similar to L<lei-import(1)>, but does not store a copy of
@@ -26,22 +24,16 @@ messages into C<lei/store>.
=over
-=item -
-
-=item --stdin
-
-Read input from standard input. This is the default if standard
-input is a pipe or regular file and there are no arguments on
-the command-line.
-
=item -F MAIL_FORMAT
=item --in-format=MAIL_FORMAT
-Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
-C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+There is currently no need for this option. It will support C<mh>,
+eventually. For now, the default (and only supported) format is
+C<maildir>. When IMAP and NNTP support are fleshed out, those
+formats will be inferred from their URLs.
-Default: C<eml> when reading from stdin
+Default: C<maildir>
=item -q
^ permalink raw reply related [relevance 68%]
* [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
@ 2021-09-24 12:51 56% ` Eric Wong
2021-09-24 12:51 68% ` [PATCH 3/4] doc: lei-index: remove --stdin, reword -F Eric Wong
2021-09-24 12:51 49% ` [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Implicit stdin based on standard input being a pipe or regular
file is here to stay, so save users the trouble of typing '-'
or '--stdin'.
Inline::C is required as of commit 1d6e1f9a6a66 (lei: require
Socket::MsgHdr or Inline::C, drop oneshot, 2021-05-26); but
Socket::MsgHdr still gives a noticeable improvement in bash
completion speed.
Also, spell-out "MESSAGE-ID" since "MID" is actually not a
common abbreviation ("MSGID" is used by RFC 3977 and several
other RFCs, I recall).
---
Documentation/lei-overview.pod | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index e80cb094cecc..40a7b0aadd04 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -33,7 +33,7 @@ Show message with the git blob OID of 59ec517f9. If a message with
that OID isn't found, check if the current git repository has the
blob, trying to reconstruct it from a message if needed.
-=item $ lei blob 59ec517f9 | lei tag - -F eml +kw:flagged +L:next
+=item $ lei blob 59ec517f9 | lei tag -F eml +kw:flagged +L:next
Set the "flagged" keyword and "next" label on the message with the
blob OID of 59ec517f9.
@@ -92,18 +92,18 @@ and written.
Search for all flagged messages that also have a "next" label.
-=item $ lei p2q HEAD | lei q --stdin -tt -o /tmp/mdir
+=item $ lei p2q HEAD | lei q -tt -o /tmp/mdir
Search for messages that have post-image git blob IDs that match those
of the current repository's HEAD commit, writing them to the Maildir
directory "mdir" and flagging the messages that were an exact match.
-=item $ git show -s HEAD | lei lcat -
+=item $ git show -s HEAD | lei lcat
Display a local message for the public-inbox link contained in a
commit message.
-=item $ lei q -f text m:MID | lei rediff -U5 -
+=item $ lei q -f text m:MESSAGE-ID | lei rediff -U5
Feed a message containing a diff to L<lei-rediff(1)> to regenerate its
diff with five context lines. Unless C<--git-dir> is specified, this
@@ -114,13 +114,15 @@ code repository.
=head1 PERFORMANCE NOTES
-L<Inline::C> is recommended for performance. To enable it, create
-C<~/.cache/public-inbox/inline-c/>.
+L<Inline::C> is required, lei runs as a background daemon to reduce
+startup costs and can provide real-time L<kqueue(2)>/L<inotify(7)>
+Maildir monitoring. L<IO::KQueue> (p5-IO-KQueue on FreeBSD) and
+L<Linux::Inotify2> (liblinux-inotify2-perl and perl-Linux-Inotify2 in
+.deb and .rpm-based distros, respectively) are recommended.
-If Socket::MsgHdr is installed (libsocket-msghdr-perl in Debian), the
-first invocation of lei starts a daemon, reducing the startup cost of
-for future invocations (which is particularly important for Bash
-completion).
+L<Socket::MsgHdr> is optional (libsocket-msghdr-perl in Debian),
+and further improves startup performance. Its effect is most felt
+when using shell completion.
=head1 BASH COMPLETION
^ permalink raw reply related [relevance 56%]
* [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
@ 2021-09-24 12:51 53% ` Eric Wong
2021-09-24 12:51 56% ` [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Try to clarify these commands are intended to be useful for
git-using (usually software) projects (and not the bare git
repos we use internally).
We'll also document some commonly useful git-diff switches
in the lei-rediff man page to highlight the usefulness
of the command.
---
Documentation/lei-blob.pod | 6 ++++--
Documentation/lei-p2q.pod | 2 +-
Documentation/lei-rediff.pod | 23 +++++++++++++++++++++--
3 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index 5b611d11ba1f..429d206e358e 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -10,8 +10,8 @@ lei blob [OPTIONS] OID
Display a git blob. The blob may correspond to a message from the
local store, an existing blob in the current repository, or a
-not-yet-created blob in the current repository that can be
-reconstructed from a message.
+not-yet-created blob in the current git project repository (if any)
+that can be reconstructed from a message.
=head1 OPTIONS
@@ -22,6 +22,8 @@ reconstructed from a message.
Specify an additional .git/ directory to scan. This option may be
given multiple times.
+Default: the output of C<git rev-parse --git-dir>
+
=item --no-cwd
Do not look in the git repository of the current working directory.
diff --git a/Documentation/lei-p2q.pod b/Documentation/lei-p2q.pod
index 44798ac3d2a0..1068ff0beb27 100644
--- a/Documentation/lei-p2q.pod
+++ b/Documentation/lei-p2q.pod
@@ -16,7 +16,7 @@ inbox.
The patch can be provided on stdin or as a file. Alternatively, when
an argument is given that does not point to an existing file, it is
-taken as a reference to a commit in the current repository, and
+taken as a reference to a commit in the current git repository, and
L<git-format-patch(1)> is used to generate the patch.
=head1 OPTIONS
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index c7db6c1e20b6..f34e946ffe52 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -15,10 +15,15 @@ the specified L<git-diff(1)> options. This is useful if you want to
change the display of the original patch (e.g., increasing context,
coloring moved lines differently, or using an external diff viewer).
+It relies on the contents of the .git directory of your current
+project working tree. In other words, it works anywhere
+L<git-am(1)> works. Otherwise, C<--git-dir=> may be specified
+any number of times to add repositories to build blob data from.
+
=head1 OPTIONS
-In addition to many L<git-diff(1)>, the following options are
-supported.
+In addition to many L<git-diff(1)> options (e.g. C<-W>, C<-w>,
+C<-U $LINES>) the following options are supported:
=over
@@ -27,15 +32,29 @@ supported.
Read message from stdin. This is implicit if no arguments are given
and stdin is a pipe or regular file.
+For users of text editors and pagers capable of piping its
+buffer to arbitrary commands, it is useful to pipe a patch email
+to C<lei rediff> before piping it to L<git-am(1)>. The output
+of C<lei rediff> is compatible with C<git am> if its input was a
+patch email.
+
=item --git-dir=DIR
Specify an additional .git/ directory to scan. This option may be
given multiple times.
+Default: the output of C<git rev-parse --git-dir>
+
=item --no-cwd
Do not look in the git repository of the current working directory.
+=item -q
+
+=item --quiet
+
+Suppress progress output.
+
=item -v
=item --verbose
^ permalink raw reply related [relevance 53%]
* [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
` (2 preceding siblings ...)
2021-09-24 12:51 68% ` [PATCH 3/4] doc: lei-index: remove --stdin, reword -F Eric Wong
@ 2021-09-24 12:51 49% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-24 12:51 UTC (permalink / raw)
To: meta
Something is better than nothing.
---
Documentation/lei-export-kw.pod | 52 ++++++++++++++++++++++
Documentation/lei-refresh-mail-sync.pod | 57 +++++++++++++++++++++++++
MANIFEST | 2 +
Makefile.PL | 5 ++-
4 files changed, 115 insertions(+), 1 deletion(-)
create mode 100644 Documentation/lei-export-kw.pod
create mode 100644 Documentation/lei-refresh-mail-sync.pod
diff --git a/Documentation/lei-export-kw.pod b/Documentation/lei-export-kw.pod
new file mode 100644
index 000000000000..cf482ca0d307
--- /dev/null
+++ b/Documentation/lei-export-kw.pod
@@ -0,0 +1,52 @@
+=head1 NAME
+
+lei-export-kw - export keywords (flags) to Maildir and IMAP folders
+
+=head1 SYNOPSIS
+
+lei export-kw --all=[<remote|local>]
+
+lei export-kw MFOLDER [MFOLDER...]
+
+=head1 DESCRIPTION
+
+C<lei export-kw> propagates keywords (e.g. C<seen>, C<answered>,
+C<flagged>, etc.) from lei/store to IMAP folders and/or Maildirs.
+
+It does not delete, write, nor modify messages themselves;
+it only sets metadata on Maildirs and IMAP folders.
+
+=head1 OPTIONS
+
+=over
+
+=item --all
+
+Export to all local Maildirs and remote IMAP folders
+
+=item --all=local
+
+Export all local Maildirs
+
+=item --all=remote
+
+Export all remote IMAP folders
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-refresh-mail-sync(1)>, L<lei-tag(1)>
diff --git a/Documentation/lei-refresh-mail-sync.pod b/Documentation/lei-refresh-mail-sync.pod
new file mode 100644
index 000000000000..aefd516fad93
--- /dev/null
+++ b/Documentation/lei-refresh-mail-sync.pod
@@ -0,0 +1,57 @@
+=head1 NAME
+
+lei-refresh-mail-sync - refresh sync info with Maildir, IMAP
+
+=head1 SYNOPSIS
+
+lei refresh-mail-sync --all[=<remote|local>]
+
+lei refresh-mail-sync MFOLDER [MFOLDER...]
+
+=head1 DESCRIPTION
+
+C<lei refresh-mail-sync> is intended to to keep old messages
+indexed with L<lei-index(1)> retrievable if Maildir flags change
+a filename. It will prune invalid entries for messages which no
+longer exist in a Maildir.
+
+It is also useful for ensuring L<lei-export-kw(1)> can propagate
+keyword (flag) changes to Maildirs and IMAP folders.
+
+It only needs read-only access to Maildirs and IMAP folders
+and will not attempt to write to them at all.
+
+=head1 OPTIONS
+
+=over
+
+=item --all
+
+Refresh all local Maildirs and remote IMAP folders
+
+=item --all=local
+
+Refresh all local Maildirs
+
+=item --all=remote
+
+Refresh all remote IMAP folders
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-index(1)>, L<lei-export-kw(1)>
diff --git a/MANIFEST b/MANIFEST
index 8c2e964b3dce..3595195a6996 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -28,6 +28,7 @@ Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
Documentation/lei-edit-search.pod
+Documentation/lei-export-kw.pod
Documentation/lei-forget-external.pod
Documentation/lei-forget-search.pod
Documentation/lei-import.pod
@@ -43,6 +44,7 @@ Documentation/lei-overview.pod
Documentation/lei-p2q.pod
Documentation/lei-q.pod
Documentation/lei-rediff.pod
+Documentation/lei-refresh-mail-sync.pod
Documentation/lei-rm.pod
Documentation/lei-security.pod
Documentation/lei-store-format.pod
diff --git a/Makefile.PL b/Makefile.PL
index f56445ae422d..00a558d12472 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -46,9 +46,12 @@ $v->{-m1} = [ map {
} @EXE_FILES,
qw(
lei-add-external lei-blob lei-config lei-convert lei-edit-search
+ lei-export-kw
lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
lei-import lei-index lei-init lei-lcat lei-ls-external lei-ls-label
- lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-rm lei-tag
+ lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff
+ lei-refresh-mail-sync
+ lei-rm lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
^ permalink raw reply related [relevance 49%]
* [PATCH] lei: common --all[=remote|local] help message
@ 2021-09-23 10:36 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-23 10:36 UTC (permalink / raw)
To: meta
It helps to be consistent and reduce the learning curve, here.
---
lib/PublicInbox/LEI.pm | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 1305dfb8..96f63805 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -420,7 +420,9 @@ my %OPTDESC = (
'remote' => 'limit operations to those requiring network access',
'remote!' => 'prevent operations requiring network access',
-'all:s up' => ['local|remote', 'update all remote or local saved searches' ],
+# up, refresh-mail-sync, export-kw
+'all:s' => ['TYPE|local|remote', 'all remote or local folders' ],
+
'remote-fudge-time=s' => [ 'INTERVAL',
'look for mail INTERVAL older than the last successful query' ],
^ permalink raw reply related [relevance 71%]
* [PATCH 3/7] script/lei: describe purpose of sleep loop
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
@ 2021-09-22 2:24 71% ` Eric Wong
2021-09-22 2:24 71% ` [PATCH 4/7] lei: dclose: do not close unnecessarily Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
It looks dumb, but I'm not about to take a runtime penalty to
use signalfd|EVFILT_SIGNAL, here, either.
---
script/lei | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/script/lei b/script/lei
index 399296ba..bc437798 100755
--- a/script/lei
+++ b/script/lei
@@ -137,6 +137,6 @@ while (1) {
$sigchld->();
if (my $sig = ($x_it_code & 127)) {
kill $sig, $$;
- sleep(1) while 1;
+ sleep(1) while 1; # no self-pipe/signalfd, here, so we loop
}
exit($x_it_code >> 8);
^ permalink raw reply related [relevance 71%]
* [PATCH 4/7] lei: dclose: do not close unnecessarily
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
@ 2021-09-22 2:24 71% ` Eric Wong
2021-09-22 2:24 49% ` [PATCH 6/7] lei up: avoid excessively parallel --all Eric Wong
2021-09-22 2:24 58% ` [PATCH 7/7] lei: drop redundant WQ EOF callbacks Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
The bit about reap_compress is no longer true since
LeiXSearch->query_done triggers it, instead. I only noticed
this while working on "lei up".
---
lib/PublicInbox/LEI.pm | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 29293e6c..a1cab55a 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1110,7 +1110,6 @@ sub dclose {
my ($self) = @_;
delete $self->{-progress};
_drop_wq($self) if $self->{failed};
- close(delete $self->{1}) if $self->{1}; # may reap_compress
$self->close if $self->{-event_init_done}; # PublicInbox::DS::close
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/7] lei bugfixes and other fixes
@ 2021-09-22 2:24 70% Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
How did I forget 2/7 for all these months? :<
6/7 took a while for me to figure out, I only noticed it
because of background processes running after MUA exit
(from "lei up --mua=... MFOLDER"), but I got 4/7 and 7/7
fixes out of it.
Eric Wong (7):
ipc: do not add "0" to $0 of solo workers
treewide: fix %SIG localization, harder
script/lei: describe purpose of sleep loop
lei: dclose: do not close unnecessarily
inbox: do not waste hash slot on httpbackend_limiter
lei up: avoid excessively parallel --all
lei: drop redundant WQ EOF callbacks
lib/PublicInbox/Admin.pm | 2 +-
lib/PublicInbox/ExtSearchIdx.pm | 4 +-
lib/PublicInbox/IPC.pm | 7 +--
lib/PublicInbox/Inbox.pm | 10 ++--
lib/PublicInbox/LEI.pm | 3 +-
lib/PublicInbox/LeiImportKw.pm | 8 +--
lib/PublicInbox/LeiMirror.pm | 10 +---
lib/PublicInbox/LeiNoteEvent.pm | 8 +--
lib/PublicInbox/LeiPmdir.pm | 8 +--
lib/PublicInbox/LeiUp.pm | 86 +++++++++++++++++++++------------
lib/PublicInbox/TestCommon.pm | 3 +-
lib/PublicInbox/Watch.pm | 4 +-
lib/PublicInbox/Xapcmd.pm | 8 +--
script/lei | 2 +-
script/public-inbox-clone | 2 +-
t/run.perl | 1 +
16 files changed, 86 insertions(+), 80 deletions(-)
^ permalink raw reply [relevance 70%]
* [PATCH 7/7] lei: drop redundant WQ EOF callbacks
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
` (2 preceding siblings ...)
2021-09-22 2:24 49% ` [PATCH 6/7] lei up: avoid excessively parallel --all Eric Wong
@ 2021-09-22 2:24 58% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
Redundant code is noise and therefore confusing :<
---
lib/PublicInbox/LeiImportKw.pm | 8 +-------
lib/PublicInbox/LeiMirror.pm | 10 ++--------
lib/PublicInbox/LeiNoteEvent.pm | 8 +-------
lib/PublicInbox/LeiPmdir.pm | 8 +-------
script/public-inbox-clone | 2 +-
5 files changed, 6 insertions(+), 30 deletions(-)
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 21c93515..c35c5c26 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -46,17 +46,11 @@ sub ck_update_kw { # via wq_io_do
$self->{sto}->wq_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
}
-sub ikw_done_wait {
- my ($arg, $pid) = @_;
- my ($self, $lei) = @$arg;
- $lei->can('wq_done_wait')->($arg, $pid);
-}
-
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $ikw = delete $lei->{ikw} or return $lei->fail;
$lei->sto_done_request($ikw->{lei_sock});
- $ikw->wq_wait_old(\&ikw_done_wait, $lei);
+ $ikw->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index d9c13f05..6bfa4b6f 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -11,7 +11,7 @@ use PublicInbox::Spawn qw(popen_rd spawn run_die);
use File::Temp ();
use Fcntl qw(SEEK_SET O_CREAT O_EXCL O_WRONLY);
-sub do_finish_mirror { # dwaitpid callback
+sub _wq_done_wait { # dwaitpid callback (via wq_eof)
my ($arg, $pid) = @_;
my ($mrr, $lei) = @$arg;
my $f = "$mrr->{dst}/mirror.done";
@@ -28,12 +28,6 @@ sub do_finish_mirror { # dwaitpid callback
$lei->dclose;
}
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $mrr = delete $lei->{wq1} or return $lei->fail;
- $mrr->wq_wait_old(\&do_finish_mirror, $lei);
-}
-
# for old installations without manifest.js.gz
sub try_scrape {
my ($self) = @_;
@@ -176,7 +170,7 @@ sub index_cloned_inbox {
PublicInbox::Admin::progress_prepare($opt, $lei->{2});
PublicInbox::Admin::index_inbox($ibx, undef, $opt);
}
- open my $x, '>', "$self->{dst}/mirror.done"; # for do_finish_mirror
+ open my $x, '>', "$self->{dst}/mirror.done"; # for _wq_done_wait
}
sub run_reap {
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 43d5ed0f..d2429485 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -107,17 +107,11 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
-sub lne_done_wait {
- my ($arg, $pid) = @_;
- my ($self, $lei) = @$arg;
- $lei->can('wq_done_wait')->($arg, $pid);
-}
-
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $lne = delete $lei->{lne} or return $lei->fail;
$lei->sto_done_request;
- $lne->wq_wait_old(\&lne_done_wait, $lei);
+ $lne->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index 23bccb4f..2d3b9755 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -47,17 +47,11 @@ sub mdir_iter { # via wq_io_do
$self->{ipt}->pmdir_cb($f, $fl, @args);
}
-sub pmd_done_wait {
- my ($arg, $pid) = @_;
- my ($self, $lei) = @$arg;
- $lei->can('wq_done_wait')->($arg, $pid);
-}
-
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $pmd = delete $lei->{pmd} or return $lei->fail;
$lei->sto_done_request($pmd->{lei_sock});
- $pmd->wq_wait_old(\&pmd_done_wait, $lei);
+ $pmd->wq_wait_old($lei->can('wq_done_wait'), $lei);
}
1;
diff --git a/script/public-inbox-clone b/script/public-inbox-clone
index 2b18969f..0efde1a8 100755
--- a/script/public-inbox-clone
+++ b/script/public-inbox-clone
@@ -54,5 +54,5 @@ my $mrr = bless {
dst => $dst,
}, 'PublicInbox::LeiMirror';
$mrr->do_mirror;
-$mrr->can('do_finish_mirror')->([$mrr, $lei], $$);
+$mrr->can('_wq_done_wait')->([$mrr, $lei], $$);
exit(($lei->{child_error} // 0) >> 8);
^ permalink raw reply related [relevance 58%]
* [PATCH 6/7] lei up: avoid excessively parallel --all
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
2021-09-22 2:24 71% ` [PATCH 4/7] lei: dclose: do not close unnecessarily Eric Wong
@ 2021-09-22 2:24 49% ` Eric Wong
2021-09-22 2:24 58% ` [PATCH 7/7] lei: drop redundant WQ EOF callbacks Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-22 2:24 UTC (permalink / raw)
To: meta
We shouldn't dispatch all outputs right away since they
can be expensive CPU-wise. Instead, rely on DESTROY to
trigger further redispatches.
This also fixes a circular reference bug for the single-output
case that could lead to a leftover script/lei after MUA exit.
I'm not sure how --jobs/-j should work when the actual xsearch
and lei2mail has it's own parallelism ("--jobs=$X,$M"), but
it's better than having thousands of subtasks running.
Fixes: b34a267efff7b831 ("lei up: fix --mua with single output")
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 86 +++++++++++++++++++++++++---------------
2 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a1cab55a..1305dfb8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1384,7 +1384,7 @@ sub fchdir {
sub wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
- $wq1->wq_wait_old(\&wq_done_wait, $lei);
+ $wq1->wq_wait_old($wq1->can('_wq_done_wait') // \&wq_done_wait, $lei);
}
sub watch_state_ok ($) {
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 89cf0112..377a720e 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -36,7 +36,7 @@ sub up1 ($$) {
$lei->{opt}->{$k} //= $v;
}
my $o = $lei->{opt}->{output} // '';
- return $lei->fail("lei.q.output unset in $f") if $o eq '';
+ return $lei->fail("lei.q.output unset in $f (out=$out)") if $o eq '';
$lss->translate_dedupe($lei) or return;
$lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
my $lxs = $lei->lxs_prepare or return;
@@ -44,39 +44,30 @@ sub up1 ($$) {
$lei->_start_query;
}
-sub up1_redispatch {
- my ($lei, $out, $op_p) = @_;
- my $l;
- if (defined($lei->{opt}->{mua})) { # single output
- $l = $lei;
- } else { # multiple outputs
- $l = bless { %$lei }, ref($lei);
- $l->{opt} = { %{$l->{opt}} }; # deep copy
- delete $l->{opt}->{all};
- delete $l->{sock}; # do not close
- # make close($l->{1}) happy in lei->dclose
- open my $fh, '>&', $l->{1} or
- return $l->child_error(0, "dup: $!");
- $l->{1} = $fh;
- $l->qerr("# updating $out");
- }
- $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
- eval { $l->dispatch('up', $out) };
- $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
-}
-
sub redispatch_all ($$) {
my ($self, $lei) = @_;
+ my $upq = [ (@{$self->{local} // []}, @{$self->{remote} // []}) ];
+ return up1($lei, $upq->[0]) if @$upq == 1; # just one, may start MUA
+
+ # FIXME: this is also used per-query, see lei->_start_query
+ my $j = $lei->{opt}->{jobs} || do {
+ my $n = $self->detect_nproc // 1;
+ $n > 4 ? 4 : $n;
+ };
+ $j = ($j =~ /\A([0-9]+)/) ? $1 + 0 : 1; # may be --jobs=$x,$m on CLI
# re-dispatch into our event loop w/o creating an extra fork-level
+ # $upq will be drained via DESTROY as each query finishes
$lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei);
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
+ # call lei->dclose when upq is done processing:
+ $op_c->{ops} = { '' => [ $lei->can('dclose'), $lei ] };
+ my @first_batch = splice(@$upq, 0, $j); # initial parallelism
+ $lei->{-upq} = $upq;
+ $lei->event_step_init; # wait for client disconnects
+ for my $out (@first_batch) {
+ PublicInbox::DS::requeue(
+ PublicInbox::LeiUp1::nxt($lei, $out, $op_p));
}
- $lei->event_step_init;
- $lei->pkt_ops($op_c->{ops} = { '' => [$lei->can('dclose'), $lei] });
}
sub lei_up {
@@ -98,7 +89,7 @@ sub lei_up {
} else {
$lei->fail("only --all=$all not understood");
}
- } elsif ($lei->{lse}) {
+ } elsif ($lei->{lse}) { # redispatched
scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
return up1($lei, $outs[0]);
} else {
@@ -131,16 +122,49 @@ sub net_merge_all_done {
my ($self, $lei) = @_;
$lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
$self->wq_close(1);
- redispatch_all($self, $lei);
+ eval { redispatch_all($self, $lei) };
+ warn "E: $@" if $@;
}
-sub _complete_up {
+sub _complete_up { # lei__complete hook
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
}
+sub _wq_done_wait { # dwaitpid callback
+ my ($arg, $pid) = @_;
+ my ($wq, $lei) = @$arg;
+ $lei->child_error($?, 'auth failure') if $?
+}
+
no warnings 'once';
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+package PublicInbox::LeiUp1; # for redispatch_all
+use strict;
+use v5.10.1;
+
+sub nxt ($$$) {
+ my ($lei, $out, $op_p) = @_;
+ bless { lei => $lei, out => $out, op_p => $op_p }, __PACKAGE__;
+}
+
+sub event_step { # runs via PublicInbox::DS::requeue
+ my ($self) = @_;
+ my $lei = $self->{lei}; # the original, from lei_up
+ my $l = bless { %$lei }, ref($lei); # per-output copy
+ delete($l->{sock}) or return; # client disconnected if {sock} is gone
+ $l->{opt} = { %{$l->{opt}} }; # deep copy
+ delete $l->{opt}->{all};
+ $l->qerr("# updating $self->{out}");
+ $l->{up_op_p} = $self->{op_p}; # ($l => $lei => script/lei)
+ eval { $l->dispatch('up', $self->{out}) };
+ $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
+
+ # onto the next:
+ my $out = shift(@{$lei->{-upq}}) or return;
+ PublicInbox::DS::requeue(nxt($lei, $out, $self->{op_p}));
+}
+
1;
^ permalink raw reply related [relevance 49%]
* [PATCH 0/3] lei: a few more annoyances fixed
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (9 preceding siblings ...)
2021-09-21 7:41 39% ` [PATCH 12/12] lei q: improve --limit behavior and progress Eric Wong
@ 2021-09-21 9:29 71% ` Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
` (2 more replies)
10 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
The more I use it, the more I find wrong...
Eric Wong (3):
t/lei-up: use '-q' to silence non-redirected test
script/lei: handle SIGTSTP and SIGCONT
lei: umask(077) before opening errors.log
lib/PublicInbox/LEI.pm | 20 +++++++++++++-------
script/lei | 9 ++++-----
t/lei-up.t | 2 +-
3 files changed, 18 insertions(+), 13 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
@ 2021-09-21 9:29 71% ` Eric Wong
2021-09-21 9:29 62% ` [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT Eric Wong
2021-09-21 9:29 71% ` [PATCH 3/3] lei: umask(077) before opening errors.log Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
We could redirect, too, but just use -q since we don't care
for the output with run_mode => 0.
---
t/lei-up.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-up.t b/t/lei-up.t
index 8937cfb1..fc369156 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -35,7 +35,7 @@ test_lei(sub {
$uc = do { local $/; <$fh> };
is($uc, $exp, 'uncompressed both match');
- lei_ok [ 'up', "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
+ lei_ok [ qw(up -q), "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
undef, { run_mode => 0 };
ok(-f "$ENV{HOME}/c", '--mua works with single output');
});
^ permalink raw reply related [relevance 71%]
* [PATCH 3/3] lei: umask(077) before opening errors.log
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
2021-09-21 9:29 62% ` [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT Eric Wong
@ 2021-09-21 9:29 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
There's a chance some sensitive information (e.g. folder names)
can end up in errors.log, though $XDG_RUNTIME_DIR or
/tmp/lei-$UID/ will have 0700 permissions, anyways.
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 2df1f326..29293e6c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1225,6 +1225,7 @@ sub lazy_start {
$errors_log = "$sock_dir/errors.log";
my $addr = pack_sockaddr_un($path);
my $lk = bless { lock_path => $errors_log }, 'PublicInbox::Lock';
+ umask(077) // die("umask(077): $!");
$lk->lock_acquire;
socket($listener, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
if ($errno == ECONNREFUSED || $errno == ENOENT) {
@@ -1236,7 +1237,6 @@ sub lazy_start {
$! = $errno; # allow interpolation to stringify in die
die "connect($path): $!";
}
- umask(077) // die("umask(077): $!");
bind($listener, $addr) or die "bind($path): $!";
$lk->lock_release;
undef $lk;
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
@ 2021-09-21 9:29 62% ` Eric Wong
2021-09-21 9:29 71% ` [PATCH 3/3] lei: umask(077) before opening errors.log Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 9:29 UTC (permalink / raw)
To: meta
Sometimes it's useful to pause an expensive query or
refresh-mail-sync to do something else. While lei-daemon and
lei/store can't be paused since they're shared across clients,
per-invocation WQ workers can be paused safely using the
unblockable SIGSTOP.
While we're at it, drop the ETOOMANYREFS hint since it
hasn't been a problem since we drastically reduced FD passing
early in development.
---
lib/PublicInbox/LEI.pm | 18 ++++++++++++------
script/lei | 9 ++++-----
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f94bfa45..2df1f326 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1118,23 +1118,28 @@ sub dclose {
sub event_step {
my ($self) = @_;
local %ENV = %{$self->{env}};
- my $sock = $self->{sock};
local $current_lei = $self;
eval {
- while (my @fds = $recv_cmd->($sock, my $buf, 4096)) {
+ my $buf;
+ while (my @fds = $recv_cmd->($self->{sock}, $buf, 4096)) {
if (scalar(@fds) == 1 && !defined($fds[0])) {
return if $! == EAGAIN;
next if $! == EINTR;
last if $! == ECONNRESET;
die "recvmsg: $!";
}
- for my $fd (@fds) {
- open my $rfh, '+<&=', $fd;
+ for (@fds) { open my $rfh, '+<&=', $_ }
+ }
+ if ($buf eq '') {
+ _drop_wq($self); # EOF, client disconnected
+ dclose($self);
+ } elsif ($buf =~ /\A(STOP|CONT)\z/) {
+ for my $wq (grep(defined, @$self{@WQ_KEYS})) {
+ $wq->wq_kill($buf) or $wq->wq_kill_old($buf);
}
+ } else {
die "unrecognized client signal: $buf";
}
- _drop_wq($self); # EOF, client disconnected
- dclose($self);
};
if (my $err = $@) {
eval { $self->fail($err) };
@@ -1146,6 +1151,7 @@ sub event_step_init {
my ($self) = @_;
my $sock = $self->{sock} or return;
$self->{-event_init_done} //= do { # persist til $ops done
+ $sock->blocking(0);
$self->SUPER::new($sock, EPOLLIN|EPOLLET);
$sock;
};
diff --git a/script/lei b/script/lei
index 591013e3..399296ba 100755
--- a/script/lei
+++ b/script/lei
@@ -106,11 +106,10 @@ open my $dh, '<', '.' or die "open(.) $!";
my $buf = join("\0", scalar(@ARGV), @ARGV);
while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
$buf .= "\0\0";
-my $n = $send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR);
-if (!$n) {
- die "sendmsg: $! (check RLIMIT_NOFILE)\n" if $!{ETOOMANYREFS};
- die "sendmsg: $!\n";
-}
+$send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR) or die "sendmsg: $!";
+$SIG{TSTP} = sub { $send_cmd->($sock, [], 'STOP', MSG_EOR); kill 'STOP', $$ };
+$SIG{CONT} = sub { $send_cmd->($sock, [], 'CONT', MSG_EOR) };
+
my $x_it_code = 0;
while (1) {
my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
^ permalink raw reply related [relevance 62%]
* [PATCH 11/12] lei q: update messages to reflect --save default
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (7 preceding siblings ...)
2021-09-21 7:41 63% ` [PATCH 09/12] lei q: show progress on >1s preparation phase Eric Wong
@ 2021-09-21 7:41 61% ` Eric Wong
2021-09-21 7:41 39% ` [PATCH 12/12] lei q: improve --limit behavior and progress Eric Wong
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
I wanted to try --dedupe=none for something, but it failed
since I forgot --no-save :x So hint users towards --no-save
if necessary.
---
lib/PublicInbox/LeiSavedSearch.pm | 4 ++--
t/lei-q-save.t | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 89f5c359..637456e4 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -94,7 +94,7 @@ sub translate_dedupe ($$$) {
$dd //= 'content';
return 1 if $dd eq 'content'; # the default
return $self->{"-dedupe_$dd"} = 1 if ($dd eq 'oid' || $dd eq 'mid');
- $lei->fail("--dedupe=$dd unsupported with --save");
+ $lei->fail("--dedupe=$dd requires --no-save");
}
sub up { # updating existing saved search via "lei up"
@@ -103,7 +103,7 @@ sub up { # updating existing saved search via "lei up"
my $self = bless { ale => $lei->ale }, $cls;
my $dir = $dst;
output2lssdir($self, $lei, \$dir, \$f) or
- return $lei->fail("--save was not used with $dst cwd=".
+ return $lei->fail("--no-save was used with $dst cwd=".
$lei->rel2abs('.'));
$self->{-cfg} = $lei->cfg_dump($f) // return $lei->fail;
$self->{-ovf} = "$dir/over.sqlite3";
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 9c17a011..5940018c 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -69,11 +69,11 @@ test_lei(sub {
ok(-s "$home/mbcl2" > $size, 'size increased after up');
ok(!lei(qw(up -q), $home), 'up fails on unknown dir');
- like($lei_err, qr/--save was not used/, 'error noted --save');
+ like($lei_err, qr/--no-save was used/, 'error noted --no-save');
lei_ok(qw(q --no-save d:last.week.. -q -o), "$home/no-save");
ok(!lei(qw(up -q), "$home/no-save"), 'up fails on --no-save');
- like($lei_err, qr/--save was not used/, 'error noted --save');
+ like($lei_err, qr/--no-save was used/, 'error noted --no-save');
lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
lei_ok qw(ls-search -z); my @z = split(/\0/, $lei_out);
@@ -131,7 +131,7 @@ test_lei(sub {
unlike($lei_out, qr/mbrd-aug/,
'forget-search completion cleared after forget');
ok(!lei('up', "$home/mbrd-aug"), 'lei up fails after forget');
- like($lei_err, qr/--save was not used/, 'error noted --save');
+ like($lei_err, qr/--no-save was used/, 'error noted --no-save');
# dedupe=mid
my $o = "$home/dd-mid";
^ permalink raw reply related [relevance 61%]
* [PATCH 12/12] lei q: improve --limit behavior and progress
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (8 preceding siblings ...)
2021-09-21 7:41 61% ` [PATCH 11/12] lei q: update messages to reflect --save default Eric Wong
@ 2021-09-21 7:41 39% ` Eric Wong
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Avoid slurping gigantic (e.g. 100000) result sets into a single
response if a giant limit is specified, and instead use 10000
as a window for the mset with a given offset. We'll also warn
and hint towards about the --limit= switch when the estimated
result set is larger than the default limit.
---
Documentation/lei-q.pod | 6 ++++--
lib/PublicInbox/LeiLcat.pm | 3 +--
lib/PublicInbox/LeiQuery.pm | 9 +++++++--
lib/PublicInbox/LeiSavedSearch.pm | 18 +++++++++++-------
lib/PublicInbox/LeiUp.pm | 25 +++++++++----------------
lib/PublicInbox/LeiXSearch.pm | 18 +++++++++++++++---
6 files changed, 47 insertions(+), 32 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 2823ced8..e1e3666d 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -10,7 +10,7 @@ lei q [OPTIONS] (--stdin|-)
=head1 DESCRIPTION
-Search for messages across the lei store and externals.
+Search for messages across the lei/store and externals.
=for comment
TODO: Give common prefixes, or at least a description/reference.
@@ -192,7 +192,9 @@ Default: fcntl,dotlock
=item -n NUMBER
-Limit the number of matches.
+Fuzzy limit the number of matches per-local external and lei/store.
+Messages added by the L<--threads> switch do not count towards this
+limit, and there is no limit on remote externals.
Default: 10000
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index c13e2153..d553b187 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -144,9 +144,8 @@ sub lei_lcat {
$lei->ale->refresh_externals($lxs, $lei);
$lei->_lei_store(1);
my $opt = $lei->{opt};
- my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
+ my %mset_opt;
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
- $mset_opt{limit} //= 10000;
$opt->{sort} //= 'relevance';
$mset_opt{relevance} = 1;
$lei->{mset_opt} = \%mset_opt;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index d5f132f1..cb5ac8fb 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -41,6 +41,12 @@ sub _start_query { # used by "lei q" and "lei up"
# descending docid order is cheapest, MUA controls sorting order
$self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads};
+
+ my $tot = $self->{mset_opt}->{total} //= $self->{opt}->{limit} // 10000;
+ $self->{mset_opt}->{limit} = $tot > 10000 ? 10000 : $tot;
+ $self->{mset_opt}->{offset} //= 0;
+ $self->{mset_opt}->{threads} //= $opt->{threads};
+
if ($self->{net}) {
require PublicInbox::LeiAuth;
$self->{auth} = PublicInbox::LeiAuth->new
@@ -118,9 +124,8 @@ sub lei_q {
my $lxs = lxs_prepare($self) or return;
$self->ale->refresh_externals($lxs, $self);
my $opt = $self->{opt};
- my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
+ my %mset_opt;
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
- $mset_opt{limit} //= 10000;
if (defined(my $sort = $opt->{'sort'})) {
if ($sort eq 'relevance') {
$mset_opt{relevance} = 1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 637456e4..3e10f780 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -29,6 +29,8 @@ sub BOOL_FIELDS () {
qw(external local remote import-remote import-before threads)
}
+sub SINGLE_FIELDS () { qw(limit dedupe output) }
+
sub lss_dir_for ($$;$) {
my ($lei, $dstref, $on_fs) = @_;
my $pfx;
@@ -89,9 +91,9 @@ sub list {
} @$out
}
-sub translate_dedupe ($$$) {
- my ($self, $lei, $dd) = @_;
- $dd //= 'content';
+sub translate_dedupe ($$) {
+ my ($self, $lei) = @_;
+ my $dd = $lei->{opt}->{dedupe} // 'content';
return 1 if $dd eq 'content'; # the default
return $self->{"-dedupe_$dd"} = 1 if ($dd eq 'oid' || $dd eq 'mid');
$lei->fail("--dedupe=$dd requires --no-save");
@@ -128,8 +130,7 @@ sub new { # new saved search "lei q --save"
File::Path::make_path($dir); # raises on error
$self->{-cfg} = {};
my $f = $self->{'-f'} = "$dir/lei.saved-search";
- my $dd = $lei->{opt}->{dedupe};
- translate_dedupe($self, $lei, $dd) or return;
+ translate_dedupe($self, $lei) or return;
open my $fh, '>', $f or return $lei->fail("open $f: $!");
my $sq_dst = PublicInbox::Config::squote_maybe($dst);
my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
@@ -139,15 +140,14 @@ sub new { # new saved search "lei q --save"
$q = "\tq = ".cquote_val($q);
}
$dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i;
+ $lei->{opt}->{output} = $dst;
print $fh <<EOM;
; to refresh with new results, run: lei up $sq_dst
; `maxuid' and `lastresult' lines are maintained by "lei up" for optimization
[lei]
$q
[lei "q"]
- output = $dst
EOM
- print $fh "\tdedupe = $dd\n" if $dd;
for my $k (ARRAY_FIELDS) {
my $ary = $lei->{opt}->{$k} // next;
for my $x (@$ary) {
@@ -158,6 +158,10 @@ EOM
my $val = $lei->{opt}->{$k} // next;
print $fh "\t$k = ".($val ? 1 : 0)."\n";
}
+ for my $k (SINGLE_FIELDS) {
+ my $val = $lei->{opt}->{$k} // next;
+ print $fh "\t$k = $val\n";
+ }
close($fh) or return $lei->fail("close $f: $!");
$self->{lock_path} = "$self->{-f}.flock";
$self->{-ovf} = "$dir/over.sqlite3";
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index abb05d46..89cf0112 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -18,7 +18,6 @@ sub up1 ($$) {
my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
my $f = $lss->{'-f'};
my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
- $mset_opt->{limit} = $lei->{opt}->{limit} // 10000;
my $q = $mset_opt->{q_raw} = $lss->{-cfg}->{'lei.q'} //
return $lei->fail("lei.q unset in $f");
my $lse = $lei->{lse} // die 'BUG: {lse} missing';
@@ -27,24 +26,18 @@ sub up1 ($$) {
} else {
$lse->query_approxidate($lse->git, $mset_opt->{qstr} = $q);
}
- my $o = $lei->{opt}->{output} = $lss->{-cfg}->{'lei.q.output'} //
- return $lei->fail("lei.q.output unset in $f");
- ref($o) and return $lei->fail("multiple values of lei.q.output in $f");
- if (defined(my $dd = $lss->{-cfg}->{'lei.q.dedupe'})) {
- $lss->translate_dedupe($lei, $dd) or return;
- $lei->{opt}->{dedupe} = $dd;
- }
- for my $k (qw(only include exclude)) {
+ # n.b. only a few CLI args are accepted for "up", so //= usually sets
+ for my $k ($lss->ARRAY_FIELDS) {
my $v = $lss->{-cfg}->get_all("lei.q.$k") // next;
- $lei->{opt}->{$k} = $v;
+ $lei->{opt}->{$k} //= $v;
}
- for my $k (qw(external local remote
- import-remote import-before threads)) {
- my $c = "lei.q.$k";
- my $v = $lss->{-cfg}->{$c} // next;
- ref($v) and return $lei->fail("multiple values of $c in $f");
- $lei->{opt}->{$k} = $v;
+ for my $k ($lss->BOOL_FIELDS, $lss->SINGLE_FIELDS) {
+ my $v = $lss->{-cfg}->get_1('lei.q', $k) // next;
+ $lei->{opt}->{$k} //= $v;
}
+ my $o = $lei->{opt}->{output} // '';
+ return $lei->fail("lei.q.output unset in $f") if $o eq '';
+ $lss->translate_dedupe($lei) or return;
$lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
my $lxs = $lei->lxs_prepare or return;
$lei->ale->refresh_externals($lxs, $lei);
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 2227c2ac..584ffde9 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -110,10 +110,20 @@ sub recent {
sub over {}
+sub _check_mset_limit ($$$) {
+ my ($lei, $desc, $mset) = @_;
+ return if defined($lei->{opt}->{limit}); # user requested limit
+ my $est = $mset->get_matches_estimated;
+ my $tot = $lei->{mset_opt}->{total};
+ $est > $tot and $lei->qerr(<<"");
+# $desc estimated matches ($est) exceeds default --limit=$tot
+
+}
+
sub _mset_more ($$) {
my ($mset, $mo) = @_;
my $size = $mset->size;
- $size >= $mo->{limit} && (($mo->{offset} += $size) < $mo->{limit});
+ $size >= $mo->{limit} && (($mo->{offset} += $size) < $mo->{total});
}
# $startq will EOF when do_augment is done augmenting and allow
@@ -182,7 +192,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $first_ids;
do {
$mset = $srch->mset($mo->{qstr}, $mo);
- mset_progress($lei, $dir, $mset->size,
+ mset_progress($lei, $dir, $mo->{offset} + $mset->size,
$mset->get_matches_estimated);
wait_startq($lei); # wait for keyword updates
my $ids = $srch->mset_to_artnums($mset, $mo);
@@ -222,6 +232,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
}
}
} while (_mset_more($mset, $mo));
+ _check_mset_limit($lei, $dir, $mset);
if ($lss && scalar(@$first_ids)) {
undef $stop_at;
my $max = $first_ids->[0];
@@ -244,7 +255,7 @@ sub query_combined_mset { # non-parallel for non-"--threads" users
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
do {
$mset = $self->mset($mo->{qstr}, $mo);
- mset_progress($lei, 'xsearch', $mset->size,
+ mset_progress($lei, 'xsearch', $mo->{offset} + $mset->size,
$mset->get_matches_estimated);
wait_startq($lei); # wait for keyword updates
for my $mitem ($mset->items) {
@@ -252,6 +263,7 @@ sub query_combined_mset { # non-parallel for non-"--threads" users
$each_smsg->($smsg, $mitem);
}
} while (_mset_more($mset, $mo));
+ _check_mset_limit($lei, 'xsearch', $mset);
undef $each_smsg; # may commit
$lei->{ovv}->ovv_atexit_child($lei);
}
^ permalink raw reply related [relevance 39%]
* [PATCH 06/12] doc: lei-security: section for WIP auth methods
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (3 preceding siblings ...)
2021-09-21 7:41 61% ` [PATCH 05/12] lei lcat: use single queue for ordering Eric Wong
@ 2021-09-21 7:41 71% ` Eric Wong
2021-09-21 7:41 44% ` [PATCH 07/12] lei lcat: support NNTP URLs Eric Wong
` (5 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Lots of stuff out there that becomes a pain to setup
configuration for and test...
---
Documentation/lei-security.pod | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/Documentation/lei-security.pod b/Documentation/lei-security.pod
index 4b712c2d..02305b90 100644
--- a/Documentation/lei-security.pod
+++ b/Documentation/lei-security.pod
@@ -99,6 +99,14 @@ While credentials are not written to the filesystem by default,
it is possible for them to end up on disk if processes are
swapped out. Use of an encrypted swap partition is recommended.
+=head1 AUTHENTICATION METHODS
+
+LOGIN (username + password) is known to work over IMAP(S),
+as does AUTH=ANONYMOUS (which is used by L<public-inbox-imapd(1)>
+as part of our test suite). AUTHINFO may work for NNTP, but
+is untested. Testers will be needed for other authentication
+methods.
+
=head1 DENIAL-OF-SERVICE VECTORS
lei uses the same MIME parsing library as L<public-inbox-mda(1)>
^ permalink raw reply related [relevance 71%]
* [PATCH 00/12] lei: fix various annoyances
@ 2021-09-21 7:41 68% Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
` (10 more replies)
0 siblings, 11 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Eric Wong (12):
lei inspect: convert to WQ worker
lei inspect: support NNTP URLs
lei_mail_sync: account for non-unique cases
lei: simplify internal arg2folder usage
lei lcat: use single queue for ordering
doc: lei-security: section for WIP auth methods
lei lcat: support NNTP URLs
lei: various completion improvements
lei q: show progress on >1s preparation phase
search: drop reopen retry message
lei q: update messages to reflect --save default
lei q: improve --limit behavior and progress
Documentation/lei-q.pod | 6 +-
Documentation/lei-security.pod | 8 ++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 12 +--
lib/PublicInbox/LeiExternal.pm | 5 ++
lib/PublicInbox/LeiForgetMailSync.pm | 9 +--
lib/PublicInbox/LeiImport.pm | 20 +++--
lib/PublicInbox/LeiImportKw.pm | 12 ++-
lib/PublicInbox/LeiInspect.pm | 103 +++++++++++++++++---------
lib/PublicInbox/LeiLcat.pm | 91 +++++++++++++----------
lib/PublicInbox/LeiLsMailSource.pm | 11 +--
lib/PublicInbox/LeiMailSync.pm | 102 +++++++++++++++++--------
lib/PublicInbox/LeiNoteEvent.pm | 5 +-
lib/PublicInbox/LeiQuery.pm | 9 ++-
lib/PublicInbox/LeiRefreshMailSync.pm | 18 +++--
lib/PublicInbox/LeiSavedSearch.pm | 22 +++---
lib/PublicInbox/LeiTag.pm | 14 ++--
lib/PublicInbox/LeiToMail.pm | 23 +++++-
lib/PublicInbox/LeiUp.pm | 25 +++----
lib/PublicInbox/LeiXSearch.pm | 36 ++++++---
lib/PublicInbox/Search.pm | 1 -
lib/PublicInbox/SharedKV.pm | 19 +++--
lib/PublicInbox/TestCommon.pm | 11 ++-
t/lei-import-nntp.t | 44 +++++++++++
t/lei-q-save.t | 6 +-
t/lei-watch.t | 2 +-
26 files changed, 414 insertions(+), 202 deletions(-)
^ permalink raw reply [relevance 68%]
* [PATCH 01/12] lei inspect: convert to WQ worker
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
@ 2021-09-21 7:41 64% ` Eric Wong
2021-09-21 7:41 42% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
` (9 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Xapian and SQLite access can be slow when a DB is large and/or
on high-latency storage.
---
lib/PublicInbox/LeiInspect.pm | 31 +++++++++++++++++++++----------
1 file changed, 21 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index f06cea61..48da826b 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -8,6 +8,7 @@
package PublicInbox::LeiInspect;
use strict;
use v5.10.1;
+use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
use PublicInbox::MID qw(mids);
@@ -184,9 +185,11 @@ sub inspect1 ($$$) {
1;
}
-sub _inspect_argv ($$) {
- my ($lei, $argv) = @_;
+sub inspect_argv { # via wq_do
+ my ($self) = @_;
+ my ($lei, $argv) = delete @$self{qw(lei argv)};
my $multi = scalar(@$argv) > 1;
+ $lei->{1}->autoflush(0);
$lei->out('[') if $multi;
while (defined(my $x = shift @$argv)) {
inspect1($lei, $x, scalar(@$argv)) or return;
@@ -194,6 +197,16 @@ sub _inspect_argv ($$) {
$lei->out(']') if $multi;
}
+sub inspect_start ($$) {
+ my ($lei, $argv) = @_;
+ my $self = bless { lei => $lei, argv => $argv }, __PACKAGE__;
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
+ $lei->{wq1} = $self;
+ $lei->wait_wq_events($op_c, $ops);
+ $self->wq_do('inspect_argv');
+ $self->wq_close(1);
+}
+
sub ins_add { # InputPipe->consume callback
my ($lei) = @_; # $_[1] = $rbuf
if (defined $_[1]) {
@@ -201,7 +214,7 @@ sub ins_add { # InputPipe->consume callback
my $str = delete $lei->{istr};
$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
my $eml = PublicInbox::Eml->new(\$str);
- _inspect_argv($lei, [
+ inspect_start($lei, [
'blob:'.$lei->git_oid($eml)->hexdigest,
map { "mid:$_" } @{mids($eml)} ]);
};
@@ -218,20 +231,18 @@ sub lei_inspect {
my $sto = $lei->_lei_store;
$sto ? $sto->search : undef;
} : undef;
- if ($lei->{opt}->{pretty} || -t $lei->{1}) {
- $lei->{json}->pretty(1)->indent(2);
- }
- $lei->start_pager if -t $lei->{1};
- $lei->{1}->autoflush(0);
+ my $isatty = -t $lei->{1};
+ $lei->{json}->pretty(1)->indent(2) if $lei->{opt}->{pretty} || $isatty;
+ $lei->start_pager if $isatty;
if ($lei->{opt}->{stdin}) {
return $lei->fail(<<'') if @argv;
no args allowed on command-line with --stdin
require PublicInbox::InputPipe;
PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
- return;
+ } else {
+ inspect_start($lei, \@argv);
}
- _inspect_argv($lei, \@argv);
}
sub _complete_inspect {
^ permalink raw reply related [relevance 64%]
* [PATCH 09/12] lei q: show progress on >1s preparation phase
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (6 preceding siblings ...)
2021-09-21 7:41 39% ` [PATCH 08/12] lei: various completion improvements Eric Wong
@ 2021-09-21 7:41 63% ` Eric Wong
2021-09-21 7:41 61% ` [PATCH 11/12] lei q: update messages to reflect --save default Eric Wong
` (2 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Overwriting existing destinations safe (but slow) by default,
so show a progress message noting what we're doing while
a user waits.
---
lib/PublicInbox/LeiToMail.pm | 23 ++++++++++++++++++++++-
lib/PublicInbox/LeiXSearch.pm | 2 +-
t/lei-watch.t | 2 +-
3 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index a419b83f..ed609081 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -689,6 +689,8 @@ sub do_augment { # slow, runs in wq worker
# fast (spawn compressor or mkdir), runs in same process as pre_augment
sub post_augment {
my ($self, $lei, @args) = @_;
+ $self->{-au_noted}++ and $lei->qerr("# writing to $self->{dst} ...");
+
my $wait = $lei->{opt}->{'import-before'} ?
$lei->{sto}->wq_do('checkpoint', 1) : 0;
# _post_augment_mbox
@@ -784,9 +786,28 @@ sub wq_atexit_child {
$lei->{pkt_op_p}->pkt_do('l2m_progress', $nr);
}
+# runs on a 1s timer in lei-daemon
+sub augment_inprogress {
+ my ($err, $opt, $dst, $au_noted) = @_;
+ $$au_noted++ and return;
+ print $err '# '.($opt->{'import-before'} ?
+ "importing non-external contents of $dst" : (
+ ($opt->{dedupe} // 'content') ne 'none') ?
+ "scanning old contents of $dst for dedupe" :
+ "removing old contents of $dst")." ...\n";
+}
+
# called in top-level lei-daemon when LeiAuth is done
sub net_merge_all_done {
- my ($self) = @_;
+ my ($self, $lei) = @_;
+ if ($PublicInbox::DS::in_loop &&
+ $self->can("_do_augment_$self->{base_type}") &&
+ !$lei->{opt}->{quiet}) {
+ $self->{-au_noted} = 0;
+ PublicInbox::DS::add_timer(1, \&augment_inprogress,
+ $lei->{2}, $lei->{opt},
+ $self->{dst}, \$self->{-au_noted});
+ }
$self->wq_broadcast('do_post_auth');
$self->wq_close(1);
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 3ce8f32d..2227c2ac 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -572,7 +572,7 @@ sub do_query {
$self->{opt_sort} = $lei->{opt}->{'sort'};
$self->{-do_lcat} = !!(delete $lei->{lcat_todo});
if ($l2m) {
- $l2m->net_merge_all_done unless $lei->{auth};
+ $l2m->net_merge_all_done($lei) unless $lei->{auth};
} else {
start_query($self);
}
diff --git a/t/lei-watch.t b/t/lei-watch.t
index a881fbb9..df887a03 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -91,7 +91,7 @@ test_lei(sub {
$ino_fdinfo or skip 'Linux/inotify-only removal removal', 1;
open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
my $cmp = [ <$fh> ];
- is_deeply($cmp, $ino_contents, 'inotify Maildir watches gone');
+ is_xdeeply($cmp, $ino_contents, 'inotify Maildir watches gone');
};
});
^ permalink raw reply related [relevance 63%]
* [PATCH 05/12] lei lcat: use single queue for ordering
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (2 preceding siblings ...)
2021-09-21 7:41 45% ` [PATCH 04/12] lei: simplify internal arg2folder usage Eric Wong
@ 2021-09-21 7:41 61% ` Eric Wong
2021-09-21 7:41 71% ` [PATCH 06/12] doc: lei-security: section for WIP auth methods Eric Wong
` (6 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
If lcat-ing multiple argument types (blobs vs folders),
maintain the original order of the arguments instead of
dumping all blobs before folder contents.
---
lib/PublicInbox/LeiLcat.pm | 13 ++++++-------
lib/PublicInbox/LeiXSearch.pm | 16 ++++++++--------
2 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index ccb1823d..1a4a988e 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -21,7 +21,7 @@ sub lcat_folder ($$$) {
} else {
for my $f (@$folders) {
my $fid = $lms->fid_for($f);
- push @{$lei->{lcat_fid}}, $fid;
+ push @{$lei->{lcat_todo}}, { fid => $fid };
}
}
}
@@ -31,10 +31,9 @@ sub lcat_imap_uri ($$) {
my $lms = $lei->lms or return;
# cf. LeiXsearch->lcat_dump
if (defined $uri->uid) {
- my @oidhex = $lms->imap_oidhex($lei, $uri);
- push @{$lei->{lcat_blob}}, @oidhex;
+ push @{$lei->{lcat_todo}}, $lms->imap_oidhex($lei, $uri);
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
- push @{$lei->{lcat_fid}}, $fid;
+ push @{$lei->{lcat_todo}}, { fid => $fid };
} else {
lcat_folder($lei, $lms, $$uri);
}
@@ -46,10 +45,10 @@ sub extract_1 ($$) {
my $u = $1;
require PublicInbox::URIimap;
lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
- '""'; # blank query, using {lcat_blob} or {lcat_fid}
+ '""'; # blank query, using {lcat_todo}
} elsif ($x =~ m!\b(maildir:.+)!i) {
lcat_folder($lei, undef, $1);
- '""'; # blank query, using {lcat_blob} or {lcat_fid}
+ '""'; # blank query, using {lcat_todo}
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
@@ -82,7 +81,7 @@ sub extract_1 ($$) {
} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
"mid:$1";
} elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
- push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+ push @{$lei->{lcat_todo}}, $1; # cf. LeiToMail->wq_atexit_child
'""'; # blank query
} else {
undef;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 756183a9..3ce8f32d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -570,7 +570,7 @@ sub do_query {
@$end = ();
$self->{opt_threads} = $lei->{opt}->{threads};
$self->{opt_sort} = $lei->{opt}->{'sort'};
- $self->{-do_lcat} = $lei->{lcat_blob} // $lei->{lcat_fid};
+ $self->{-do_lcat} = !!(delete $lei->{lcat_todo});
if ($l2m) {
$l2m->net_merge_all_done unless $lei->{auth};
} else {
@@ -646,13 +646,13 @@ sub lcat_dump { # via wq_io_do
$git->cat_async($smsg->{blob}, \&_lcat2smsg, $smsg);
};
}
- for my $oid (@{$lei->{lcat_blob} // []}) {
- $each_smsg->({ blob => $oid, pct => 100 });
- }
- if (my $fids = delete $lei->{lcat_fid}) {
- my $lms = $lei->{lse}->lms;
- for my $fid (@$fids) {
- $lms->each_src({fid => $fid}, \&_lcat_i, $each_smsg);
+ my $lms;
+ for my $ent (@{$lei->{lcat_todo}}) {
+ if (ref $ent eq 'HASH') { # { fid => $fid ,.. }
+ $lms //= $lei->{lse}->lms;
+ $lms->each_src($ent, \&_lcat_i, $each_smsg);
+ } else { # oidhex
+ $each_smsg->({ blob => $ent, pct => 100 });
}
}
$git->async_wait_all;
^ permalink raw reply related [relevance 61%]
* [PATCH 04/12] lei: simplify internal arg2folder usage
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
2021-09-21 7:41 42% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
@ 2021-09-21 7:41 45% ` Eric Wong
2021-09-21 7:41 61% ` [PATCH 05/12] lei lcat: use single queue for ordering Eric Wong
` (7 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
We can set opt->{quiet} for (internal) 'note-event' command
to quiet ->qerr, since we use ->qerr everywhere else. And
we'll just die() instead of setting a ->{fail} message, since
eval + die are more inline with the rest of our Perl code.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 4 +---
lib/PublicInbox/LeiForgetMailSync.pm | 4 +---
lib/PublicInbox/LeiInspect.pm | 17 +++++------------
lib/PublicInbox/LeiLcat.pm | 5 ++---
lib/PublicInbox/LeiMailSync.pm | 17 ++++++-----------
lib/PublicInbox/LeiNoteEvent.pm | 5 +++--
lib/PublicInbox/LeiRefreshMailSync.pm | 4 +---
8 files changed, 20 insertions(+), 38 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 148a5b1e..f94bfa45 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1194,7 +1194,7 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
$lei->dispatch('note-event',
"maildir:$mdir", $nc, $bn, $fn);
};
- warn "E note-event $f: $@\n" if $@;
+ warn "E: note-event $f: $@\n" if $@;
}
}
if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 8c5fbc13..d5533a2a 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -92,9 +92,7 @@ EOM
$lms->group2folders($lei, $all, \@folders) or return;
@folders = grep(/\A(?:maildir|imaps?):/i, @folders);
} else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ $lms->arg2folder($lei, \@folders); # may die
}
$lms->lms_pause;
my $self = bless { lse => $sto->search, lms => $lms }, __PACKAGE__;
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 701f48d2..d85616cc 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -16,9 +16,7 @@ sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
my $lms = $lei->lms or return;
$lms->lms_write_prepare;
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ $lms->arg2folder($lei, \@folders); # may die
$lms->forget_folders(@folders);
}
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 722ba5b2..8e128580 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -46,10 +46,9 @@ sub inspect_nntp_range {
my $ent = {};
my $ret = { "$uri" => $ent };
my $lms = $lei->lms or return $ret;
- my $err = $lms->arg2folder($lei, my $folders = [ $$uri ]);
- if ($err) {
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- }
+ my $folders = [ $$uri ];
+ eval { $lms->arg2folder($lei, $folders) };
+ $lei->qerr("# no folders match $$uri (non-fatal)") if $@;
$end //= $beg;
for my $art ($beg..$end) {
my @oidhex = map { unpack('H*', $_) }
@@ -65,14 +64,8 @@ sub inspect_sync_folder ($$) {
my $ent = {};
my $lms = $lei->lms or return $ent;
my $folders = [ $folder ];
- my $err = $lms->arg2folder($lei, $folders);
- if ($err) {
- if ($err->{fail}) {
- $lei->qerr("# no folders match $folder (non-fatal)");
- @$folders = ();
- }
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- }
+ eval { $lms->arg2folder($lei, $folders) };
+ $lei->qerr("# no folders match $folder (non-fatal)") if $@;
for my $f (@$folders) {
$ent->{$f} = $lms->location_stats($f); # may be undef
}
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 8f8e83bc..ccb1823d 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -15,9 +15,8 @@ sub lcat_folder ($$$) {
my ($lei, $lms, $folder) = @_;
$lms //= $lei->lms or return;
my $folders = [ $folder];
- my $err = $lms->arg2folder($lei, $folders);
- $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
- if ($err && $err->{fail}) {
+ eval { $lms->arg2folder($lei, $folders) };
+ if ($@) {
$lei->child_error(0, "# unknown folder: $folder");
} else {
for my $f (@$folders) {
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 3e725d30..f83c7de2 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -437,7 +437,7 @@ sub arg2folder {
my ($self, $lei, $folders) = @_;
my @all = $self->folders;
my %all = map { $_ => 1 } @all;
- my ($err, @no);
+ my @no;
for (@$folders) {
next if $all{$_}; # ok
if (m!\A(maildir|mh):(.+)!i) {
@@ -454,7 +454,7 @@ sub arg2folder {
my $res = match_imap_url($self, $orig, \@all);
if (ref $res) {
$_ = $$res;
- push(@{$err->{qerr}}, <<EOM);
+ $lei->qerr(<<EOM);
# using `$res' instead of `$orig'
EOM
} else {
@@ -466,7 +466,7 @@ EOM
my $res = match_nntp_url($self, $orig, \@all);
if (ref $res) {
$_ = $$res;
- push(@{$err->{qerr}}, <<EOM);
+ $lei->qerr(<<EOM);
# using `$res' instead of `$orig'
EOM
} else {
@@ -479,12 +479,11 @@ EOM
}
if (@no) {
my $no = join("\n\t", @no);
- $err->{fail} = <<EOF;
+ die <<EOF;
No sync information for: $no
Run `lei ls-mail-sync' to display valid choices
EOF
}
- $err;
}
sub forget_folders {
@@ -549,12 +548,8 @@ sub imap_oidhex {
my $mailbox_uri = $uid_uri->clone;
$mailbox_uri->uid(undef);
my $folders = [ $$mailbox_uri ];
- if (my $err = $self->arg2folder($lei, $folders)) {
- if ($err->{fail}) {
- $lei->qerr("# no sync information for $mailbox_uri");
- }
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- }
+ eval { $self->arg2folder($lei, $folders) };
+ $lei->qerr("# no sync information for $mailbox_uri") if $@;
map { unpack('H*',$_) } num_oidbin($self, $folders->[0], $uid_uri->uid)
}
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index a0591a09..43d5ed0f 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -68,8 +68,9 @@ sub lei_note_event {
return flush_lei($lei) if $folder eq 'done'; # special case
my $lms = $lei->lms or return;
$lms->lms_write_prepare if $new_cur eq ''; # for ->clear_src below
- my $err = $lms->arg2folder($lei, [ $folder ]);
- return if $err->{fail};
+ $lei->{opt}->{quiet} = 1;
+ eval { $lms->arg2folder($lei, [ $folder ]) };
+ return if $@;
my $state = $cfg->get_1("watch.$folder", 'state') // 'tag-rw';
return if $state eq 'pause';
return $lms->clear_src($folder, \$bn) if $new_cur eq '';
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 92673492..51e89b23 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -74,9 +74,7 @@ EOM
if (defined(my $all = $lei->{opt}->{all})) {
$lms->group2folders($lei, $all, \@folders) or return;
} else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ $lms->arg2folder($lei, \@folders); # may die
}
$lms->lms_pause; # must be done before fork
$sto->write_prepare($lei);
^ permalink raw reply related [relevance 45%]
* [PATCH 07/12] lei lcat: support NNTP URLs
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (4 preceding siblings ...)
2021-09-21 7:41 71% ` [PATCH 06/12] doc: lei-security: section for WIP auth methods Eric Wong
@ 2021-09-21 7:41 44% ` Eric Wong
2021-09-21 7:41 39% ` [PATCH 08/12] lei: various completion improvements Eric Wong
` (4 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
NNTP URLs are probably more prevalent in public message archives
than IMAP URLs.
---
lib/PublicInbox/LeiLcat.pm | 66 +++++++++++++++++++++-------------
lib/PublicInbox/LeiMailSync.pm | 14 +++++---
t/lei-import-nntp.t | 23 ++++++++++++
3 files changed, 74 insertions(+), 29 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 1a4a988e..0902c213 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -11,47 +11,64 @@ use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_folder ($$$) {
- my ($lei, $lms, $folder) = @_;
- $lms //= $lei->lms or return;
- my $folders = [ $folder];
+sub lcat_folder ($$;$$) {
+ my ($lei, $folder, $beg, $end) = @_;
+ my $lms = $lei->{-lms_ro} //= $lei->lms // return;
+ my $folders = [ $folder ];
eval { $lms->arg2folder($lei, $folders) };
- if ($@) {
- $lei->child_error(0, "# unknown folder: $folder");
- } else {
- for my $f (@$folders) {
- my $fid = $lms->fid_for($f);
- push @{$lei->{lcat_todo}}, { fid => $fid };
- }
+ return $lei->child_error(0, "# unknown folder: $folder") if $@;
+ my %range;
+ if (defined($beg)) { # NNTP article range
+ $range{min} = $beg;
+ $range{max} = $end // $beg;
+ }
+ for my $f (@$folders) {
+ my $fid = $lms->fid_for($f);
+ push @{$lei->{lcat_todo}}, { fid => $fid, %range };
}
}
sub lcat_imap_uri ($$) {
my ($lei, $uri) = @_;
- my $lms = $lei->lms or return;
- # cf. LeiXsearch->lcat_dump
+ # cf. LeiXSearch->lcat_dump
+ my $lms = $lei->{-lms_ro} //= $lei->lms // return;
if (defined $uri->uid) {
push @{$lei->{lcat_todo}}, $lms->imap_oidhex($lei, $uri);
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
push @{$lei->{lcat_todo}}, { fid => $fid };
} else {
- lcat_folder($lei, $lms, $$uri);
+ lcat_folder($lei, $$uri);
}
}
+sub lcat_nntp_uri ($$) {
+ my ($lei, $uri) = @_;
+ my $mid = $uri->message; # already unescaped by URI::news
+ return "mid:$mid" if defined($mid);
+ my $lms = $lei->{-lms_ro} //= $lei->lms // return;
+ my ($ng, $beg, $end) = $uri->group;
+ $uri->group($ng);
+ lcat_folder($lei, $$uri, $beg, $end);
+ '""';
+}
+
sub extract_1 ($$) {
my ($lei, $x) = @_;
- if ($x =~ m!\b(imaps?://[^>]+)!i) {
- my $u = $1;
- require PublicInbox::URIimap;
- lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
- '""'; # blank query, using {lcat_todo}
- } elsif ($x =~ m!\b(maildir:.+)!i) {
- lcat_folder($lei, undef, $1);
+ if ($x =~ m!\b(maildir:.+)!i) {
+ lcat_folder($lei, $1);
'""'; # blank query, using {lcat_todo}
- } elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
- my $u = $1;
+ } elsif ($x =~ m!\b(([a-z]+)://\S+)!i) {
+ my ($u, $scheme) = ($1, $2);
$u =~ s/[\>\]\)\,\.\;]+\z//;
+ if ($scheme =~ m!\A(imaps?)\z!i) {
+ require PublicInbox::URIimap;
+ lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
+ return '""'; # blank query, using {lcat_todo}
+ } elsif ($scheme =~ m!\A(?:nntps?|s?news)\z!i) {
+ require PublicInbox::URInntps;
+ $u = PublicInbox::URInntps->new($u);
+ return lcat_nntp_uri($lei, $u);
+ } # http, or something else:
require URI;
$u = URI->new($u);
my $p = $u->path;
@@ -93,7 +110,7 @@ sub extract_all {
my $strict = !$lei->{opt}->{stdin};
my @q;
for my $x (@argv) {
- if (my $term = extract_1($lei,$x)) {
+ if (my $term = extract_1($lei, $x)) {
push @q, $term;
} elsif ($strict) {
return $lei->fail(<<"");
@@ -101,6 +118,7 @@ could not extract Message-ID from $x
}
}
+ delete $lei->{-lms_ro};
@q ? join(' OR ', @q) : $lei->fail("no Message-ID in: @argv");
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index f83c7de2..522a5ebc 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -197,9 +197,12 @@ INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?)
sub each_src {
my ($self, $folder, $cb, @args) = @_;
my $dbh = $self->{dbh} //= dbh_new($self);
- my $fid;
+ my ($fid, @rng);
+ my $and_ge_le = '';
if (ref($folder) eq 'HASH') {
$fid = $folder->{fid} // die "BUG: no `fid'";
+ @rng = grep(defined, @$folder{qw(min max)});
+ $and_ge_le = 'AND uid >= ? AND uid <= ?' if @rng;
} else {
$fid = $self->{fmap}->{$folder} //=
fid_for($self, $folder) // return;
@@ -208,16 +211,17 @@ sub each_src {
# minimize implicit txn time to avoid blocking writers by
# batching SELECTs. This looks wonky but is necessary since
# $cb-> may access the DB on its own.
- my $ary = $dbh->selectall_arrayref(<<'', undef, $fid);
-SELECT _rowid_,oidbin,uid FROM blob2num WHERE fid = ?
+ my $ary = $dbh->selectall_arrayref(<<"", undef, $fid, @rng);
+SELECT _rowid_,oidbin,uid FROM blob2num WHERE fid = ? $and_ge_le
ORDER BY _rowid_ ASC LIMIT 1000
my $min = @$ary ? $ary->[-1]->[0] : undef;
while (defined $min) {
for my $row (@$ary) { $cb->($row->[1], $row->[2], @args) }
- $ary = $dbh->selectall_arrayref(<<'', undef, $fid, $min);
-SELECT _rowid_,oidbin,uid FROM blob2num WHERE fid = ? AND _rowid_ > ?
+ $ary = $dbh->selectall_arrayref(<<"", undef, $fid, @rng, $min);
+SELECT _rowid_,oidbin,uid FROM blob2num
+WHERE fid = ? $and_ge_le AND _rowid_ > ?
ORDER BY _rowid_ ASC LIMIT 1000
$min = @$ary ? $ary->[-1]->[0] : undef;
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index 0b080781..eb1ae312 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -25,6 +25,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON');
lei_ok('import', $url);
+ lei_ok "lcat", "nntp://$host_port/testmessage\@example.com";
+ my $local = $lei_out;
+ lei_ok "lcat", "nntp://example.com/testmessage\@example.com";
+ my $remote = $lei_out;
+ is($local, $remote, 'Message-ID used even from unknown host');
lei_ok(qw(q z:1..));
$out = json_utf8->decode($lei_out);
ok(scalar(@$out) > 1, 'got imported messages');
@@ -57,6 +62,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('inspect', "$url/$high");
my $x = json_utf8->decode($lei_out);
like($x->{$url}->{$high}, qr/\A[a-f0-9]{40,}\z/, 'inspect shows blob');
+ lei_ok qw(lcat -f json), "$url/$high";
+ my $lcat = json_utf8->decode($lei_out);
+ is($lcat->[1], undef, 'only one result for lcat');
+ is($lcat->[0]->{blob}, $x->{$url}->{$high},
+ 'lcat showed correct blob');
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article number not stored as folder');
@@ -78,6 +88,19 @@ test_lei({ tmpdir => $tmpdir }, sub {
is(scalar(grep(/\A[a-f0-9]{40,}\z/, values %{$x->{$url}})),
$end - $low + 1, 'all values are git blobs');
+ lei_ok qw(lcat -f json), "$url/$low";
+ $lcat = json_utf8->decode($lei_out);
+ is($lcat->[1], undef, 'only one result for lcat');
+ is($lcat->[0]->{blob}, $x->{$url}->{$low},
+ 'lcat showed correct blob');
+ lei_ok qw(lcat -f json), "$url/$low-$end";
+ $lcat = json_utf8->decode($lei_out);
+ pop @$lcat;
+ for ($low..$end) {
+ my $tip = shift @$lcat;
+ is($x->{$url}->{$_}, $tip->{blob}, "blob matches art #$_");
+ }
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article range not stored as folder');
lei_ok qw(q z:0..); my $start = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 44%]
* [PATCH 02/12] lei inspect: support NNTP URLs
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
@ 2021-09-21 7:41 42% ` Eric Wong
2021-09-21 7:41 45% ` [PATCH 04/12] lei: simplify internal arg2folder usage Eric Wong
` (8 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
No reason not to support them, since there's more
public-inbox-nntpd instances than -imapd instances,
currently.
---
lib/PublicInbox/LeiInspect.pm | 48 +++++++++++++++++++++++++++-------
lib/PublicInbox/LeiMailSync.pm | 40 +++++++++++++++++++++++++++-
lib/PublicInbox/TestCommon.pm | 11 ++++++--
t/lei-import-nntp.t | 21 +++++++++++++++
4 files changed, 108 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 48da826b..ab2c98d9 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -11,6 +11,7 @@ use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
use PublicInbox::MID qw(mids);
+use PublicInbox::NetReader qw(imap_uri nntp_uri);
sub inspect_blob ($$) {
my ($lei, $oidhex) = @_;
@@ -32,13 +33,33 @@ sub inspect_imap_uid ($$) {
my $ent = {};
my $lms = $lei->lms or return $ent;
my $oidhex = $lms->imap_oid($lei, $uid_uri);
- if (ref(my $err = $oidhex)) { # art2folder error
+ if (ref(my $err = $oidhex)) { # arg2folder error
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
$ent->{$$uid_uri} = $oidhex;
$ent;
}
+sub inspect_nntp_range {
+ my ($lei, $uri) = @_;
+ my ($ng, $beg, $end) = $uri->group;
+ $uri = $uri->clone;
+ $uri->group($ng);
+ my $ent = {};
+ my $ret = { "$uri" => $ent };
+ my $lms = $lei->lms or return $ret;
+ my $err = $lms->arg2folder($lei, my $folders = [ $$uri ]);
+ if ($err) {
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ $end //= $beg;
+ for my $art ($beg..$end) {
+ my $oidbin = $lms->imap_oidbin($folders->[0], $art);
+ $ent->{$art} = $oidbin ? unpack('H*', $oidbin) : undef;
+ }
+ $ret;
+}
+
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
@@ -161,14 +182,6 @@ sub inspect1 ($$$) {
my $ent;
if ($item =~ /\Ablob:(.+)/) {
$ent = inspect_blob($lei, $1);
- } elsif ($item =~ m!\Aimaps?://!i) {
- require PublicInbox::URIimap;
- my $uri = PublicInbox::URIimap->new($item);
- if (defined($uri->uid)) {
- $ent = inspect_imap_uid($lei, $uri);
- } else {
- $ent = inspect_sync_folder($lei, $item);
- }
} elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
} elsif ($item =~ m!\Adocid:([0-9]+)\z!) {
@@ -177,6 +190,23 @@ sub inspect1 ($$$) {
$ent = inspect_num($lei, $1 + 0);
} elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
$ent = inspect_mid($lei, $1);
+ } elsif (my $iuri = imap_uri($item)) {
+ if (defined($iuri->uid)) {
+ $ent = inspect_imap_uid($lei, $iuri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ } elsif (my $nuri = nntp_uri($item)) {
+ if (defined(my $mid = $nuri->message)) {
+ $ent = inspect_mid($lei, $mid);
+ } else {
+ my ($group, $beg, $end) = $nuri->group;
+ if (defined($beg)) {
+ $ent = inspect_nntp_range($lei, $nuri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ }
} else { # TODO: more things
return $lei->fail("$item not understood");
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index f185b585..d9b9e117 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -247,12 +247,14 @@ sub location_stats {
SELECT COUNT(name) FROM blob2name WHERE fid = ?
$ret->{'name.count'} = $row if $row;
+ my $ntype = ($folder =~ m!\A(?:nntps?|s?news)://!i) ? 'article' :
+ (($folder =~ m!\Aimaps?://!i) ? 'uid' : "TODO<$folder>");
for my $op (qw(count min max)) {
($row) = $dbh->selectrow_array(<<"", undef, $fid);
SELECT $op(uid) FROM blob2num WHERE fid = ?
$row or last;
- $ret->{"uid.$op"} = $row;
+ $ret->{"$ntype.$op"} = $row;
}
$ret;
}
@@ -369,6 +371,30 @@ sub match_imap_url {
"E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
}
+sub match_nntp_url ($$$) {
+ my ($self, $url, $all) = @_; # $all = [ $lms->folders ];
+ $all //= [ $self->folders ];
+ require PublicInbox::URInntps;
+ my $want = PublicInbox::URInntps->new($url)->canonical;
+ my ($s, $h, $p) = ($want->scheme, $want->host, $want->port);
+ my $ng = $want->group; # force scalar (no article ranges)
+ my @uri = map { PublicInbox::URInntps->new($_)->canonical }
+ grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$ng\E\b!, @$all);
+ my @match;
+ for my $x (@uri) {
+ next if $x->group ne $ng || $x->host ne $h || $x->port != $p;
+ # maybe user was forgotten on CLI:
+ if (defined($x->userinfo) && !defined($want->userinfo)) {
+ push @match, $x;
+ } elsif (($x->userinfo//"\0") eq ($want->userinfo//"\0")) {
+ push @match, $x;
+ }
+ }
+ return @match if wantarray;
+ scalar(@match) <= 1 ? $match[0] :
+ "E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
+}
+
# returns undef on failure, number on success
sub group2folders {
my ($self, $lei, $all, $folders) = @_;
@@ -428,6 +454,18 @@ sub arg2folder {
$_ = $$res;
push(@{$err->{qerr}}, <<EOM);
# using `$res' instead of `$orig'
+EOM
+ } else {
+ $lei->err($res) if defined $res;
+ push @no, $orig;
+ }
+ } elsif (m!\A(?:nntps?|s?news)://!i) {
+ my $orig = $_;
+ my $res = match_nntp_url($self, $orig, \@all);
+ if (ref $res) {
+ $_ = $$res;
+ push(@{$err->{qerr}}, <<EOM);
+# using `$res' instead of `$orig'
EOM
} else {
$lei->err($res) if defined $res;
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 0ee4b228..9e152394 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -18,7 +18,7 @@ BEGIN {
run_script start_script key2sub xsys xsys_e xqx eml_load tick
have_xapian_compact json_utf8 setup_public_inboxes create_inbox
tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt
- test_httpd xbail require_cmd);
+ test_httpd xbail require_cmd is_xdeeply);
require Test::More;
my @methods = grep(!/\W/, @Test::More::EXPORT);
eval(join('', map { "*$_=\\&Test::More::$_;" } @methods));
@@ -520,6 +520,13 @@ sub json_utf8 () {
state $x = ref(PublicInbox::Config->json)->new->utf8->canonical;
}
+sub is_xdeeply ($$$) {
+ my ($x, $y, $desc) = @_;
+ my $ok = is_deeply($x, $y, $desc);
+ diag explain([$x, '!=', $y]) if !$ok;
+ $ok;
+}
+
sub test_lei {
SKIP: {
my ($cb) = pop @_;
@@ -590,7 +597,7 @@ SKIP: {
my $f = "$daemon_xrd/lei/errors.log";
open my $fh, '<', $f or BAIL_OUT "$f: $!";
my @l = <$fh>;
- is_deeply(\@l, [],
+ is_xdeeply(\@l, [],
"$t daemon XDG_RUNTIME_DIR/lei/errors.log empty");
}
}; # SKIP if missing git 2.6+ || Xapian || SQLite || json
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index df0594d4..0b080781 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -49,6 +49,15 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $end = $high - 1;
lei_ok qw(import), "$url/$high";
+ lei_ok('inspect', $url); is_xdeeply(json_utf8->decode($lei_out), {
+ $url => { 'article.count' => 1,
+ 'article.min' => $high,
+ 'article.max' => $high, }
+ }, 'inspect output for URL after single message') or diag $lei_out;
+ lei_ok('inspect', "$url/$high");
+ my $x = json_utf8->decode($lei_out);
+ like($x->{$url}->{$high}, qr/\A[a-f0-9]{40,}\z/, 'inspect shows blob');
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article number not stored as folder');
lei_ok qw(q z:0..); my $one = json_utf8->decode($lei_out);
@@ -57,6 +66,18 @@ test_lei({ tmpdir => $tmpdir }, sub {
local $ENV{HOME} = "$tmpdir/h3";
lei_ok qw(import), "$url/$low-$end";
+ lei_ok('inspect', $url); is_xdeeply(json_utf8->decode($lei_out), {
+ $url => { 'article.count' => $end - $low + 1,
+ 'article.min' => $low,
+ 'article.max' => $end, }
+ }, 'inspect output for URL after range') or diag $lei_out;
+ lei_ok('inspect', "$url/$low-$end");
+ $x = json_utf8->decode($lei_out);
+ is_deeply([ ($low..$end) ], [ sort { $a <=> $b } keys %{$x->{$url}} ],
+ 'inspect range shows range');
+ is(scalar(grep(/\A[a-f0-9]{40,}\z/, values %{$x->{$url}})),
+ $end - $low + 1, 'all values are git blobs');
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article range not stored as folder');
lei_ok qw(q z:0..); my $start = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 42%]
* [PATCH 08/12] lei: various completion improvements
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
` (5 preceding siblings ...)
2021-09-21 7:41 44% ` [PATCH 07/12] lei lcat: support NNTP URLs Eric Wong
@ 2021-09-21 7:41 39% ` Eric Wong
2021-09-21 7:41 63% ` [PATCH 09/12] lei q: show progress on >1s preparation phase Eric Wong
` (3 subsequent siblings)
10 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
"lei export-kw" no longer completes for anonymous sources.
More commands use "lei refresh-mail-sync" as a basis for their
completion work, as well.
";AUTH=ANONYMOUS@" is stripped from completions since it was
preventing bash completion from working on AUTH=ANONYMOUS IMAP
URLs. I'm not sure if there's a better way, but all of our code
works fine without specifying AUTH=ANONYMOUS as a command-line
arg.
Finally, we fallback to using more candidates if none can
be found, allowing multiple URLs to be completed.
---
lib/PublicInbox/LeiExportKw.pm | 8 ++++++--
lib/PublicInbox/LeiExternal.pm | 5 +++++
lib/PublicInbox/LeiForgetMailSync.pm | 5 +++--
lib/PublicInbox/LeiImport.pm | 12 +++++++-----
lib/PublicInbox/LeiInspect.pm | 7 +++----
lib/PublicInbox/LeiLcat.pm | 7 +++----
lib/PublicInbox/LeiLsMailSource.pm | 11 ++++++-----
lib/PublicInbox/LeiMailSync.pm | 13 ++++++++-----
lib/PublicInbox/LeiRefreshMailSync.pm | 12 ++++++++++--
lib/PublicInbox/LeiTag.pm | 14 ++++++++------
lib/PublicInbox/SharedKV.pm | 19 +++++++++++++------
11 files changed, 72 insertions(+), 41 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d5533a2a..cea9beeb 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -127,9 +127,13 @@ EOM
sub _complete_export_kw {
my ($lei, @argv) = @_;
- my $lms = $lei->lms or return;
+ my $lms = $lei->lms or return ();
my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ # filter-out read-only sources:
+ my @k = grep(!m!(?://;AUTH=ANONYMOUS\@|\A(?:nntps?|s?news)://)!,
+ $lms->folders($argv[-1], 1));
+ my @m = map { $match_cb->($_) } @k;
+ @m ? @m : @k;
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index d802f0e2..f8e610ca 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -241,6 +241,11 @@ sub complete_url_prepare {
$re = quotemeta($re);
}
my $match_cb = sub {
+ # the "//;" here (for AUTH=ANONYMOUS) interacts badly with
+ # bash tab completion, strip it out for now since our commands
+ # work w/o it. Not sure if there's a better solution...
+ $_[0] =~ s!//;AUTH=ANONYMOUS\@!//!i;
+ $_[0] =~ s!;!\\;!g;
# only return the part specified on the CLI
# don't duplicate if already 100% completed
$_[0] =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ()
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index d85616cc..762910ed 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -10,7 +10,7 @@
package PublicInbox::LeiForgetMailSync;
use strict;
use v5.10.1;
-use PublicInbox::LeiExportKw;
+use PublicInbox::LeiRefreshMailSync;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
@@ -20,6 +20,7 @@ sub lei_forget_mail_sync {
$lms->forget_folders(@folders);
}
-*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_forget_mail_sync =
+ \&PublicInbox::LeiRefreshMailSync::_complete_refresh_mail_sync;
1;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 3c30db8d..397292d4 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -121,12 +121,14 @@ sub lei_import { # the main "lei import" method
sub _complete_import {
my ($lei, @argv) = @_;
- my $match_cb = $lei->complete_url_prepare(\@argv);
- my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
- my %f = map { $_ => 1 } @m;
+ my ($re, $cur, $match_cb) = $lei->complete_url_prepare(\@argv);
+ my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @m = map { $match_cb->($_) } @k;
+ my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @m = map { $match_cb->($_) } $lms->folders;
- @f{@m} = @m;
+ @k = $lms->folders($argv[-1], 1);
+ @m = map { $match_cb->($_) } @k;
+ if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
keys %f;
}
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 8e128580..2158b996 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -269,10 +269,9 @@ no args allowed on command-line with --stdin
}
sub _complete_inspect {
- my ($lei, @argv) = @_;
- my $lms = $lei->lms or return;
- my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ require PublicInbox::LeiRefreshMailSync;
+ PublicInbox::LeiRefreshMailSync::_complete_refresh_mail_sync(@_);
+ # TODO: message-ids?, blobs? could get expensive...
}
1;
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 0902c213..c13e2153 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -164,10 +164,9 @@ no args allowed on command-line with --stdin
}
sub _complete_lcat {
- my ($lei, @argv) = @_;
- my $lms = $lei->lms or return;
- my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ require PublicInbox::LeiRefreshMailSync;
+ PublicInbox::LeiRefreshMailSync::_complete_refresh_mail_sync(@_);
+ # TODO: message-ids?, blobs? could get expensive...
}
1;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 2265969a..1db15d57 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -107,12 +107,13 @@ sub lei_ls_mail_source {
sub _complete_ls_mail_source {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
- my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
- my %f = map { $_ => 1 } @m;
+ my @k = $lei->url_folder_cache->keys($argv[-1], 1);
+ my @m = map { $match_cb->($_) } @k;
+ my %f = map { $_ => 1 } (@m ? @m : @k);
if (my $lms = $lei->lms) {
- @m = map { $match_cb->($_) } grep(
- m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
- @f{@m} = @m;
+ @k = $lms->folders($argv[-1], 1);
+ @m = map { $match_cb->($_) } grep(m!\A[a-z]+://!, @k);
+ if (@m) { @f{@m} = @m } else { @f{@k} = @k }
}
keys %f;
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 522a5ebc..91cd1c93 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -299,16 +299,19 @@ sub locations_for {
# returns a list of folders used for completion
sub folders {
- my ($self, $pfx) = @_;
- my $dbh = $self->{dbh} //= dbh_new($self);
+ my ($self, @pfx) = @_;
my $sql = 'SELECT loc FROM folders';
- my @pfx;
- if (defined $pfx) {
+ if (defined($pfx[0])) {
$sql .= ' WHERE loc LIKE ? ESCAPE ?';
- @pfx = ($pfx, '\\');
+ my $anywhere = !!$pfx[1];
+ $pfx[1] = '\\';
$pfx[0] =~ s/([%_\\])/\\$1/g; # glob chars
$pfx[0] .= '%';
+ substr($pfx[0], 0, 0, '%') if $anywhere;
+ } else {
+ @pfx = (); # [0] may've been undef
}
+ my $dbh = $self->{dbh} //= dbh_new($self);
map { $_->[0] } @{$dbh->selectall_arrayref($sql, undef, @pfx)};
}
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 51e89b23..eb842843 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -7,7 +7,7 @@ package PublicInbox::LeiRefreshMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
-use PublicInbox::LeiExportKw;
+use PublicInbox::LeiImport;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::Import;
@@ -97,8 +97,16 @@ sub ipc_atfork_child { # needed for PublicInbox::LeiPmdir
undef;
}
+sub _complete_refresh_mail_sync {
+ my ($lei, @argv) = @_;
+ my $lms = $lei->lms or return ();
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ my @k = $lms->folders($argv[-1], 1);
+ my @m = map { $match_cb->($_) } @k;
+ @m ? @m : @k
+}
+
no warnings 'once';
-*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
1;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 9bbf0d79..817b87f8 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -74,7 +74,7 @@ sub ipc_atfork_child {
# Workaround bash word-splitting s to ['kw', ':', 'keyword' ...]
# Maybe there's a better way to go about this in
# contrib/completion/lei-completion.bash
-sub _complete_mark_common ($) {
+sub _complete_tag_common ($) {
my ($argv) = @_;
# Workaround bash word-splitting URLs to ['https', ':', '//' ...]
# Maybe there's a better way to go about this in
@@ -104,16 +104,18 @@ sub _complete_mark_common ($) {
# FIXME: same problems as _complete_forget_external and similar
sub _complete_tag {
my ($self, @argv) = @_;
+ require PublicInbox::LeiImport;
+ my @in = PublicInbox::LeiImport::_complete_import(@_);
my @L = eval { $self->_lei_store->search->all_terms('L') };
- my @all = ((map { ("+kw:$_", "-kw:$_") } @PublicInbox::LeiInput::KW),
+ my @kwL = ((map { ("+kw:$_", "-kw:$_") } @PublicInbox::LeiInput::KW),
(map { ("+L:$_", "-L:$_") } @L));
- return @all if !@argv;
- my ($cur, $re) = _complete_mark_common(\@argv);
- map {
+ my ($cur, $re) = _complete_tag_common(\@argv);
+ my @m = map {
# only return the part specified on the CLI
# don't duplicate if already 100% completed
/\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ();
- } grep(/$re\Q$cur/, @all);
+ } grep(/$re\Q$cur/, @kwL);
+ (@in, (@m ? @m : @kwL));
}
no warnings 'once'; # the following works even when LeiAuth is lazy-loaded
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index 3487e820..645bb57c 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -84,12 +84,19 @@ SELECT k,v FROM kv
}
sub keys {
- my ($self) = @_;
- my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
-SELECT k FROM kv
-
- $sth->execute;
- map { $_->[0] } @{$sth->fetchall_arrayref};
+ my ($self, @pfx) = @_;
+ my $sql = 'SELECT k FROM kv';
+ if (defined $pfx[0]) {
+ $sql .= ' WHERE k LIKE ? ESCAPE ?';
+ my $anywhere = !!$pfx[1];
+ $pfx[1] = '\\';
+ $pfx[0] =~ s/([%_\\])/\\$1/g; # glob chars
+ $pfx[0] .= '%';
+ substr($pfx[0], 0, 0, '%') if $anywhere;
+ } else {
+ @pfx = (); # [0] may've been undef
+ }
+ map { $_->[0] } @{$self->dbh->selectall_arrayref($sql, undef, @pfx)};
}
sub delete_by_val {
^ permalink raw reply related [relevance 39%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 23:06 57% ` Eric Wong
2021-09-16 17:43 71% ` Konstantin Ryabitsev
@ 2021-09-19 21:21 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 21:21 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Eric Wong <e@80x24.org> wrote:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
> # populate ~/.cache/lei/uri_folder/.sqlite3
> lei ls-mail-source nntp://nntp.lore.kernel.org
lei ls-mail-source -l nntp://nntp.lore.kernel.org
I just made "-l" show pretty JSON output in the terminal
to help find NNTP article ranges.
> # if you're using bash:
> . contrib/completion/lei-completion.bash
>
> lei import <TAB><TAB>
>
> lei import will download the entire newsgroup, which may get
> huge, unfortunately. It should support ranges (e.g.
> nntp://example.com/inbox.foo.example/900-1000) but the URI
> package doesn't support it, yet...
I was wrong about the URI package not supporting ranges,
it does; and forgot lei even seemed to have (untested)
support for it. It should be working, now, just pushed out
commit 67fe4d8d90ac77419c8fc41457c849aa7d366a9d
("net_reader: fix single NNTP article fetch, test ranges")
<https://public-inbox.org/meta/20210919125035.6331-11-e@80x24.org/>
Neither Net::NNTP nor Mail::IMAPClient support pipelining
out-of-the-box, though; so network latency is still a problem.
I'm still not sure about doing an interactive UI (my brain
tends to struggle with that stuff :x); but I think extending
tab-completion for epochs and NNTP article ranges would be
good (I'm not particularly good with tab-completion, either :x)
^ permalink raw reply [relevance 71%]
* Re: [PATCH 16/16] doc: lei-config: document various knobs
2021-09-19 16:14 71% ` Kyle Meyer
@ 2021-09-19 20:00 65% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 20:00 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> Eric Wong writes:
> > +Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
>
> Looks like the ')' was unintentionally dropped.
> > +Enable protocol-level compression, this may be incompatible
> > +or broken with some servers.
>
> nit: s/, /. This/
> > +The any per-project .git/config, and global ~/.gitconfig files
>
> s/The any/Any/ ?
Thanks, will squash the below. I'll also clarify the sentence
on imap.debug vs imap.compress
diff --git a/Documentation/lei-config.pod b/Documentation/lei-config.pod
index daf66fe4..c91f36ee 100644
--- a/Documentation/lei-config.pod
+++ b/Documentation/lei-config.pod
@@ -8,7 +8,7 @@ lei config [OPTIONS]
=head1 DESCRIPTION
-Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
+Call L<git-config(1)> with C<$XDG_CONFIG_HOME/lei/config> as the
configuration file. All C<OPTIONS> are passed through, but those that
override the configuration file are not permitted.
@@ -48,7 +48,7 @@ connecting to a Tor .onion or localhost.
=item nntp.compress
-Enable protocol-level compression, this may be incompatible
+Enable protocol-level compression. This may be incompatible
or broken with some servers.
Note: L<Net::NNTP> compression support is pending:
@@ -64,7 +64,7 @@ If using L<imap.proxy> or L<nntp.proxy> point to a SOCKS proxy,
debugging output for L<IO::Socket::Socks> will be enabled, as
well.
-Disabling L<imap.compress> may be required to improve output.
+Disabling L<imap.compress> may be required for readability.
=item imap.timeout
@@ -96,7 +96,7 @@ C<quoted>, C<hdrdefault>, C<status>, C<attachment> color slots
are supported for the C<-f text> and C<-f reply> output formats
of L<lei-lcat(1)> and L<lei-q(1)>.
-The any per-project .git/config, and global ~/.gitconfig files
+Any per-project .git/config, and global ~/.gitconfig files
will also be parsed for diff coloring. git diff color slots
(C<color.diff.SLOT>) supported are C<new>, C<old>, C<meta>,
C<frag>, C<func>, and C<context>.
^ permalink raw reply related [relevance 65%]
* Re: [PATCH 16/16] doc: lei-config: document various knobs
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
@ 2021-09-19 16:14 71% ` Kyle Meyer
2021-09-19 20:00 65% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-19 16:14 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> diff --git a/Documentation/lei-config.pod b/Documentation/lei-config.pod
> index a64045ef..daf66fe4 100644
> --- a/Documentation/lei-config.pod
> +++ b/Documentation/lei-config.pod
> @@ -8,10 +8,99 @@ lei config [OPTIONS]
>
> =head1 DESCRIPTION
>
> -Call git-config(1) with C<$XDG_CONFIG_HOME/lei/config> as the
> +Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
Looks like the ')' was unintentionally dropped.
> +=item imap.compress
> +
> +=item nntp.compress
> +
> +Enable protocol-level compression, this may be incompatible
> +or broken with some servers.
nit: s/, /. This/
> +=item color.SLOT
> +
> +C<quoted>, C<hdrdefault>, C<status>, C<attachment> color slots
> +are supported for the C<-f text> and C<-f reply> output formats
> +of L<lei-lcat(1)> and L<lei-q(1)>.
> +
> +The any per-project .git/config, and global ~/.gitconfig files
s/The any/Any/ ?
> +will also be parsed for diff coloring. git diff color slots
> +(C<color.diff.SLOT>) supported are C<new>, C<old>, C<meta>,
> +C<frag>, C<func>, and C<context>.
^ permalink raw reply [relevance 71%]
* [PATCH 16/16] doc: lei-config: document various knobs
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (5 preceding siblings ...)
2021-09-19 12:50 36% ` [PATCH 14/16] lei config --edit: use controlling terminal Eric Wong
@ 2021-09-19 12:50 57% ` Eric Wong
2021-09-19 16:14 71% ` Kyle Meyer
6 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
It's still a work-in-progress, but the basic debug knob
comes in handy for new users; as does proxy support.
---
Documentation/lei-config.pod | 91 +++++++++++++++++++++++++++++++++++-
1 file changed, 90 insertions(+), 1 deletion(-)
diff --git a/Documentation/lei-config.pod b/Documentation/lei-config.pod
index a64045ef..daf66fe4 100644
--- a/Documentation/lei-config.pod
+++ b/Documentation/lei-config.pod
@@ -8,10 +8,99 @@ lei config [OPTIONS]
=head1 DESCRIPTION
-Call git-config(1) with C<$XDG_CONFIG_HOME/lei/config> as the
+Call L<git-config(1> with C<$XDG_CONFIG_HOME/lei/config> as the
configuration file. All C<OPTIONS> are passed through, but those that
override the configuration file are not permitted.
+All C<imap> and C<nntp> options may be specified per-host or
+(if using git 2.26+) with wildcards:
+
+ [imap "imap://*.onion"]
+ proxy = socks5h://127.0.0.1:9050
+
+ [nntp "nntp://example.com"]
+ proxy = socks5h://127.0.0.1:1080
+
+=head2 VARIABLES
+
+=over 8
+
+=item external.*
+
+Managed by L<lei-add-external(1)> and L<lei-forget-external(1)>
+
+=item imap.proxy
+
+=item nntp.proxy
+
+The C<socks5h://> proxy address. Older versions of SOCKS may
+be supported if there is user demand.
+
+=item imap.starttls
+
+=item nntp.starttls
+
+Enable or disable STARTTLS on non-imaps:// and non-nntps://
+hosts. By default, STARTTLS is enabled if available unless
+connecting to a Tor .onion or localhost.
+
+=item imap.compress
+
+=item nntp.compress
+
+Enable protocol-level compression, this may be incompatible
+or broken with some servers.
+
+Note: L<Net::NNTP> compression support is pending:
+L<https://rt.cpan.org/Ticket/Display.html?id=129967>
+
+=item imap.debug
+
+=item nntp.debug
+
+Enable debugging output of underlying IMAP and NNTP libraries,
+currently L<Mail::IMAPClient> and L<Net::NNTP>, respectively.
+If using L<imap.proxy> or L<nntp.proxy> point to a SOCKS proxy,
+debugging output for L<IO::Socket::Socks> will be enabled, as
+well.
+
+Disabling L<imap.compress> may be required to improve output.
+
+=item imap.timeout
+
+=item nntp.timeout
+
+The read timeout for responses.
+
+Default: 600 seconds (IMAP); 120 seconds (NNTP)
+
+=item imap.fetchBatchSize
+
+Number of full messages to fetch at once. Larger values reduce
+network round trips at the cost of higher memory use, especially
+when retrieving large messages.
+
+Small responses for IMAP flags are fetched at 10000 times this value.
+
+Default: 1
+
+=item imap.ignoreSizeErrors
+
+Ignore size mismatches from broken IMAP server implementations.
+
+Default: false
+
+=item color.SLOT
+
+C<quoted>, C<hdrdefault>, C<status>, C<attachment> color slots
+are supported for the C<-f text> and C<-f reply> output formats
+of L<lei-lcat(1)> and L<lei-q(1)>.
+
+The any per-project .git/config, and global ~/.gitconfig files
+will also be parsed for diff coloring. git diff color slots
+(C<color.diff.SLOT>) supported are C<new>, C<old>, C<meta>,
+C<frag>, C<func>, and C<context>.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
^ permalink raw reply related [relevance 57%]
* [PATCH 14/16] lei config --edit: use controlling terminal
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (4 preceding siblings ...)
2021-09-19 12:50 62% ` [PATCH 09/16] lei ls-mail-source: pretty JSON support Eric Wong
@ 2021-09-19 12:50 36% ` Eric Wong
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
As with "lei edit-search", "lei config --edit" may
spawn an interactive editor which works best from
the terminal running script/lei.
So implement LeiConfig as a superclass of LeiEditSearch
so the two commands can share the same verification
hooks and retry logic.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 18 +++++-----
lib/PublicInbox/LeiConfig.pm | 42 ++++++++++++++++++++++
lib/PublicInbox/LeiEditSearch.pm | 60 +++++++++++--------------------
lib/PublicInbox/LeiExternal.pm | 2 +-
lib/PublicInbox/LeiInit.pm | 4 +--
lib/PublicInbox/LeiSavedSearch.pm | 16 +++------
t/lei.t | 3 ++
8 files changed, 82 insertions(+), 64 deletions(-)
create mode 100644 lib/PublicInbox/LeiConfig.pm
diff --git a/MANIFEST b/MANIFEST
index 2df743f8..8c2e964b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiALE.pm
lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
lib/PublicInbox/LeiBlob.pm
+lib/PublicInbox/LeiConfig.pm
lib/PublicInbox/LeiConvert.pm
lib/PublicInbox/LeiCurl.pm
lib/PublicInbox/LeiDedupe.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b468a32c..148a5b1e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -278,7 +278,7 @@ our %CMD = ( # sorted in order of importance/use:
'config' => [ '[...]', sub {
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
- qw(c=s@ C=s@), pass_through('git config') ],
+ qw(edit|e c=s@ C=s@), pass_through('git config') ],
'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
qw(stdin| pretty ascii dir=s), @c_opt ],
@@ -870,14 +870,6 @@ sub _config {
waitpid(spawn($cmd, \%env, \%rdr), 0);
}
-sub lei_config {
- my ($self, @argv) = @_;
- $self->{opt}->{'config-file'} and return fail $self,
- "config file switches not supported by `lei config'";
- _config(@_);
- x_it($self, $?) if $?;
-}
-
sub lei_daemon_pid { puts shift, $$ }
sub lei_daemon_kill {
@@ -1504,4 +1496,12 @@ sub sto_done_request {
$lei->err($@) if $@;
}
+sub cfg_dump ($$) {
+ my ($lei, $f) = @_;
+ my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
+ return $ret if !$@;
+ $lei->err($@);
+ undef;
+}
+
1;
diff --git a/lib/PublicInbox/LeiConfig.pm b/lib/PublicInbox/LeiConfig.pm
new file mode 100644
index 00000000..23be9aaf
--- /dev/null
+++ b/lib/PublicInbox/LeiConfig.pm
@@ -0,0 +1,42 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::LeiConfig;
+use strict;
+use v5.10.1;
+use PublicInbox::PktOp;
+
+sub cfg_do_edit ($;$) {
+ my ($self, $reason) = @_;
+ my $lei = $self->{lei};
+ $lei->pgr_err($reason) if defined $reason;
+ my $cmd = [ qw(git config --edit -f), $self->{-f} ];
+ my $env = { GIT_CONFIG => $self->{-f} };
+ $self->cfg_edit_begin if $self->can('cfg_edit_begin');
+ # run in script/lei foreground
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ # $op_p will EOF when $EDITOR is done
+ $op_c->{ops} = { '' => [\&cfg_edit_done, $self] };
+ $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], $cmd, $env);
+}
+
+sub cfg_edit_done { # PktOp
+ my ($self) = @_;
+ eval {
+ my $cfg = $self->{lei}->cfg_dump($self->{-f}, $self->{lei}->{2})
+ // return cfg_do_edit($self, "\n");
+ $self->cfg_verify($cfg) if $self->can('cfg_verify');
+ };
+ $self->{lei}->fail($@) if $@;
+}
+
+sub lei_config {
+ my ($lei, @argv) = @_;
+ $lei->{opt}->{'config-file'} and return $lei->fail(
+ "config file switches not supported by `lei config'");
+ return $lei->_config(@argv) unless $lei->{opt}->{edit};
+ my $f = $lei->_lei_cfg(1)->{-f};
+ my $self = bless { lei => $lei, -f => $f }, __PACKAGE__;
+ cfg_do_edit($self);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 47166ce7..bcf7c105 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -7,48 +7,32 @@ use strict;
use v5.10.1;
use PublicInbox::LeiSavedSearch;
use PublicInbox::LeiUp;
+use parent qw(PublicInbox::LeiConfig);
-sub edit_begin {
- my ($lss, $lei) = @_;
- if (ref($lss->{-cfg}->{'lei.q.output'})) {
- delete $lss->{-cfg}->{'lei.q.output'}; # invalid
- $lei->pgr_err(<<EOM);
-$lss->{-f} has multiple values of lei.q.output
+sub cfg_edit_begin {
+ my ($self) = @_;
+ if (ref($self->{lss}->{-cfg}->{'lei.q.output'})) {
+ delete $self->{lss}->{-cfg}->{'lei.q.output'}; # invalid
+ $self->{lei}->pgr_err(<<EOM);
+$self->{lss}->{-f} has multiple values of lei.q.output
please remove redundant ones
EOM
}
- $lei->{-lss_for_edit} = $lss;
}
-sub do_edit ($$;$) {
- my ($lss, $lei, $reason) = @_;
- $lei->pgr_err($reason) if defined $reason;
- my @cmd = (qw(git config --edit -f), $lss->{'-f'});
- $lei->qerr("# spawning @cmd");
- edit_begin($lss, $lei);
- # run in script/lei foreground
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- # $op_p will EOF when $EDITOR is done
- $op_c->{ops} = { '' => [\&op_edit_done, $lss, $lei] };
- $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], \@cmd, {});
-}
-
-sub _edit_done {
- my ($lss, $lei) = @_;
- my $cfg = $lss->can('cfg_dump')->($lei, $lss->{'-f'}) //
- return do_edit($lss, $lei, <<EOM);
-$lss->{-f} is unparseable
-EOM
+sub cfg_verify {
+ my ($self, $cfg) = @_;
my $new_out = $cfg->{'lei.q.output'} // '';
- return do_edit($lss, $lei, <<EOM) if ref $new_out;
-$lss->{-f} has multiple values of lei.q.output
+ return $self->cfg_do_edit(<<EOM) if ref $new_out;
+$self->{-f} has multiple values of lei.q.output
EOM
- return do_edit($lss, $lei, <<EOM) if $new_out eq '';
-$lss->{-f} needs lei.q.output
+ return $self->cfg_do_edit(<<EOM) if $new_out eq '';
+$self->{-f} needs lei.q.output
EOM
+ my $lss = $self->{lss};
my $old_out = $lss->{-cfg}->{'lei.q.output'} // return;
return if $old_out eq $new_out;
+ my $lei = $self->{lei};
my $old_path = $old_out;
my $new_path = $new_out;
s!$PublicInbox::LeiSavedSearch::LOCAL_PFX!! for ($old_path, $new_path);
@@ -57,10 +41,10 @@ EOM
return if $dir_new eq $dir_old;
($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
- return do_edit($lss, $lei, <<EOM);
+ return $self->cfg_do_edit(<<EOM);
conversions from/to v2 inboxes not supported at this time
EOM
- return do_edit($lss, $lei, <<EOM) if -e $dir_new;
+ return $self->cfg_do_edit(<<EOM) if -e $dir_new;
lei.q.output changed from `$old_out' to `$new_out'
However, $dir_new exists
EOM
@@ -79,16 +63,12 @@ E: rename($dir_old, $dir_new) error: $!
EOM
}
-sub op_edit_done { # PktOp
- my ($lss, $lei) = @_;
- eval { _edit_done($lss, $lei) };
- $lei->fail($@) if $@;
-}
-
sub lei_edit_search {
my ($lei, $out) = @_;
my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
- do_edit($lss, $lei);
+ my $f = $lss->{-f};
+ my $self = bless { lei => $lei, lss => $lss, -f => $f }, __PACKAGE__;
+ $self->cfg_do_edit;
}
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 6fd3efef..d802f0e2 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -139,7 +139,7 @@ sub add_external_finish {
my $key = "external.$location.boost";
my $cur_boost = $cfg->{$key};
return if defined($cur_boost) && $cur_boost == $new_boost; # idempotent
- $self->lei_config($key, $new_boost);
+ $self->_config($key, $new_boost);
}
sub lei_add_external {
diff --git a/lib/PublicInbox/LeiInit.pm b/lib/PublicInbox/LeiInit.pm
index 6558ac0a..27ce8169 100644
--- a/lib/PublicInbox/LeiInit.pm
+++ b/lib/PublicInbox/LeiInit.pm
@@ -23,7 +23,7 @@ sub lei_init {
# some folks like symlinks and bind mounts :P
if (@dir && "@cur[1,0]" eq "@dir[1,0]") {
- $self->lei_config('leistore.dir', $dir);
+ $self->_config('leistore.dir', $dir);
$self->_lei_store(1)->done;
return $self->qerr("$exists (as $cur)");
}
@@ -31,7 +31,7 @@ sub lei_init {
E: leistore.dir=$cur already initialized and it is not $dir
}
- $self->lei_config('leistore.dir', $dir);
+ $self->_config('leistore.dir', $dir);
$self->_lei_store(1)->done;
$exists //= "# leistore.dir=$dir newly initialized";
$self->qerr($exists);
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 754f8294..89f5c359 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -29,14 +29,6 @@ sub BOOL_FIELDS () {
qw(external local remote import-remote import-before threads)
}
-sub cfg_dump ($$) {
- my ($lei, $f) = @_;
- my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
- return $ret if !$@;
- $lei->err($@);
- undef;
-}
-
sub lss_dir_for ($$;$) {
my ($lei, $dstref, $on_fs) = @_;
my $pfx;
@@ -64,7 +56,7 @@ sub lss_dir_for ($$;$) {
for my $g ("$pfx-*", '*') {
my @maybe = glob("$lss_dir$g/lei.saved-search");
for my $f (@maybe) {
- $c = cfg_dump($lei, $f) // next;
+ $c = $lei->cfg_dump($f) // next;
$o = $c->{'lei.q.output'} // next;
$o =~ s!$LOCAL_PFX!! or next;
@st = stat($o) or next;
@@ -88,7 +80,7 @@ sub list {
print $fh "\tpath = ", cquote_val($p), "\n";
}
close $fh or die "close $f: $!";
- my $cfg = cfg_dump($lei, $f);
+ my $cfg = $lei->cfg_dump($f);
unlink($f);
my $out = $cfg ? $cfg->get_all('lei.q.output') : [];
map {;
@@ -113,7 +105,7 @@ sub up { # updating existing saved search via "lei up"
output2lssdir($self, $lei, \$dir, \$f) or
return $lei->fail("--save was not used with $dst cwd=".
$lei->rel2abs('.'));
- $self->{-cfg} = cfg_dump($lei, $f) // return $lei->fail;
+ $self->{-cfg} = $lei->cfg_dump($f) // return $lei->fail;
$self->{-ovf} = "$dir/over.sqlite3";
$self->{'-f'} = $f;
$self->{lock_path} = "$self->{-f}.flock";
@@ -276,7 +268,7 @@ sub output2lssdir {
my $dir = lss_dir_for($lei, \$dst, 1);
my $f = "$dir/lei.saved-search";
if (-f $f && -r _) {
- $self->{-cfg} = cfg_dump($lei, $f) // return;
+ $self->{-cfg} = $lei->cfg_dump($f) // return;
$$dir_ref = $dir;
$$fn_ref = $f;
return 1;
diff --git a/t/lei.t b/t/lei.t
index c8f47576..53fc43fb 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -100,6 +100,9 @@ my $test_config = sub {
is($lei_out, "tr00\n", "-c string value passed as-is");
lei_ok(qw(-c imap.debug=a -c imap.debug=b config --get-all imap.debug));
is($lei_out, "a\nb\n", '-c and --get-all work together');
+
+ lei_ok([qw(config -e)], { VISUAL => 'cat' });
+ is($lei_out, "[a]\n\tb = c\n", '--edit works');
};
my $test_completion = sub {
^ permalink raw reply related [relevance 36%]
* [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (2 preceding siblings ...)
2021-09-19 12:50 68% ` [PATCH 07/16] lei: clamp internal worker processes to 4 Eric Wong
@ 2021-09-19 12:50 71% ` Eric Wong
2021-09-19 12:50 62% ` [PATCH 09/16] lei ls-mail-source: pretty JSON support Eric Wong
` (2 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
The meanings of "hwm" and "lwm" may not be obvious abbreviations
for (high|low) water mark descriptions used by RFC 3977.
"high" and "low" should be obvious to anyone.
---
lib/PublicInbox/LeiLsMailSource.pm | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index bcb1838e..a2e75e94 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -59,10 +59,10 @@ sub input_path_url { # overrides LeiInput version
# <https://rt.cpan.org/Ticket/Display.html?id=129966>
$desc =~ s/\r\z//;
- my ($hwm, $lwm, $status) = @{$all->{$ng}};
+ my ($high, $low, $status) = @{$all->{$ng}};
push @x, { name => $ng, url => "$sec/$ng",
- lwm => $lwm + 0,
- hwm => $hwm + 0, status => $status,
+ low => $low + 0,
+ high => $high + 0, status => $status,
description => $desc };
}
@f = map { "$sec/$_" } keys %$all;
^ permalink raw reply related [relevance 71%]
* [PATCH 07/16] lei: clamp internal worker processes to 4
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
2021-09-19 12:50 58% ` [PATCH 04/16] lei: simplify sto_done_request Eric Wong
@ 2021-09-19 12:50 68% ` Eric Wong
2021-09-19 12:50 71% ` [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP Eric Wong
` (3 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
"All" my CPUs is only 4, but it's probably ridiculous for
somebody with a 16-core system to have 16 processes for
accessing SQLite DBs.
We do the same thing in Pmdir for parallel Maildir access
(and V2Writable).
---
lib/PublicInbox/LeiImportKw.pm | 4 +++-
lib/PublicInbox/LeiNoteEvent.pm | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 2863d17f..379101c2 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -11,7 +11,9 @@ use parent qw(PublicInbox::IPC);
sub new {
my ($cls, $lei) = @_;
my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
- my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
+ my $j = $self->detect_nproc // 4;
+ $j = 4 if $j > 4;
+ my ($op_c, $ops) = $lei->workers_start($self, $j);
$op_c->{ops} = $ops; # for PktOp->event_step
$self->{lei_sock} = $lei->{sock};
$lei->{ikw} = $self;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 5f692e75..a0591a09 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -80,8 +80,9 @@ sub lei_note_event {
my $self = $cfg->{-lei_note_event} //= do {
my $wq = bless { lms => $lms }, __PACKAGE__;
# MUAs such as mutt can trigger massive rename() storms so
- # use all CPU power available:
+ # use some CPU, but don't overwhelm slower storage, either
my $jobs = $wq->detect_nproc // 1;
+ $jobs = 4 if $jobs > 4; # same default as V2Writable
my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
$lei->wait_wq_events($op_c, $ops);
note_event_arm_done($lei);
^ permalink raw reply related [relevance 68%]
* [PATCH 09/16] lei ls-mail-source: pretty JSON support
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
` (3 preceding siblings ...)
2021-09-19 12:50 71% ` [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP Eric Wong
@ 2021-09-19 12:50 62% ` Eric Wong
2021-09-19 12:50 36% ` [PATCH 14/16] lei config --edit: use controlling terminal Eric Wong
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
As with other commands, we enable pretty JSON by default if
stdout is a terminal or if --pretty is specified. While the
->pretty JSON output has excessive vertical whitespace, too many
lines is preferable to having everything on one line.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 19 ++++++++++---------
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index def85ef1..b468a32c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -214,7 +214,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
- qw(z|0 ascii l url), @c_opt ],
+ qw(z|0 ascii l pretty url), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index a2e75e94..2265969a 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -12,15 +12,9 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_path_url { # overrides LeiInput version
my ($self, $url) = @_;
# TODO: support ndjson and other JSONs we support elsewhere
- my $json;
my $lei = $self->{lei};
- my $ORS = "\n";
- if ($self->{lei}->{opt}->{l}) {
- $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
- $json->ascii(1) if $lei->{opt}->{ascii};
- } elsif ($self->{lei}->{opt}->{z}) {
- $ORS = "\0";
- }
+ my $json = $lei->{json};
+ my $ORS = $self->{lei}->{opt}->{z} ? "\0" : "\n";
my @f;
if ($url =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($url);
@@ -93,7 +87,14 @@ sub lei_ls_mail_source {
my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
$self->{cfg} = $lei->_lei_cfg; # may be undef
$self->prepare_inputs($lei, [ $url ]) or return;
- $lei->start_pager if -t $lei->{1};
+ my $isatty = -t $lei->{1};
+ if ($lei->{opt}->{l}) {
+ my $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
+ $lei->{json} = $json;
+ $json->ascii(1) if $lei->{opt}->{ascii};
+ $json->pretty(1)->indent(2) if $isatty || $lei->{opt}->{pretty};
+ }
+ $lei->start_pager if $isatty;
my $ops = {};
$lei->{auth}->op_merge($ops, $self);
(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
^ permalink raw reply related [relevance 62%]
* [PATCH 00/16] lei IPC overhaul, NNTP fixes
@ 2021-09-19 12:50 61% Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
` (6 more replies)
0 siblings, 7 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
11/16 is a bit worrying for saved search dedupe over HTTP(S),
and I can't seem to reproduce it reliably, either..
ls-mail-source and import use is far nicer, as it provides a
good avenue for doing partial fetches.
lei/store IPC got a massive overhaul, and the sto_done_request
simplification is nice. This will probably simplify automatic
export-kw support to IMAP folders.
I also noticed "lei config --edit" was wonky, so
I made it share code with "lei edit-search".
Starting to document config knobs, too.
Eric Wong (16):
ipc: wq_do: support synchronous waits and responses
ipc: allow disabling broadcast for wq_workers
lei/store: use SOCK_SEQPACKET rather than pipe
lei: simplify sto_done_request
lei_xsearch: drop Data::Dumper use
ipc: drop dynamic WQ process counts
lei: clamp internal worker processes to 4
lei ls-mail-source: use "high"/"low" for NNTP
lei ls-mail-source: pretty JSON support
net_reader: fix single NNTP article fetch, test ranges
xt: add fsck script over over.sqlite3
watch: use net_reader->mic_new wrapper for SOCKS+TLS
net_reader: no STARTTLS for IMAP localhost or onions
lei config --edit: use controlling terminal
net_reader: disallow imap.fetchBatchSize=0
doc: lei-config: document various knobs
Documentation/lei-config.pod | 91 +++++++++++++++++++-
MANIFEST | 2 +
lib/PublicInbox/IPC.pm | 117 +++++++++++---------------
lib/PublicInbox/LEI.pm | 32 +++----
lib/PublicInbox/LeiConfig.pm | 42 +++++++++
lib/PublicInbox/LeiEditSearch.pm | 60 +++++--------
lib/PublicInbox/LeiExternal.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 2 +-
lib/PublicInbox/LeiImportKw.pm | 6 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInit.pm | 4 +-
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 25 +++---
lib/PublicInbox/LeiNoteEvent.pm | 11 +--
lib/PublicInbox/LeiRefreshMailSync.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 4 +-
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 16 +---
lib/PublicInbox/LeiStore.pm | 22 ++---
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 22 ++---
lib/PublicInbox/LeiXSearch.pm | 9 +-
lib/PublicInbox/NetReader.pm | 39 +++++----
lib/PublicInbox/WQWorker.pm | 9 +-
lib/PublicInbox/Watch.pm | 3 +-
t/imapd-tls.t | 11 ++-
t/ipc.t | 19 ++---
t/lei-import-nntp.t | 26 ++++++
t/lei.t | 3 +
t/nntpd-tls.t | 8 ++
t/uri_nntps.t | 3 +
xt/over-fsck.perl | 44 ++++++++++
32 files changed, 403 insertions(+), 239 deletions(-)
create mode 100644 lib/PublicInbox/LeiConfig.pm
create mode 100644 xt/over-fsck.perl
^ permalink raw reply [relevance 61%]
* [PATCH 04/16] lei: simplify sto_done_request
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
@ 2021-09-19 12:50 58% ` Eric Wong
2021-09-19 12:50 68% ` [PATCH 07/16] lei: clamp internal worker processes to 4 Eric Wong
` (4 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
With the switch from pipes to sockets for lei-daemon =>
lei/store IPC, we can send the script/lei client socket to the
lei/store process and rely on reference counting in both Perl
and the kernel to persist the script/lei.
---
lib/PublicInbox/LEI.pm | 13 ++-----------
lib/PublicInbox/LeiRefreshMailSync.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 13 +------------
3 files changed, 4 insertions(+), 24 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 549b855b..f62e82dc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -40,7 +40,6 @@ $GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => [ ino watches ] }
-our %LIVE_SOCK; # "GLOB(0x....)" => $lei->{sock}
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -580,7 +579,6 @@ sub _lei_atfork_child {
$dir_idle->force_close if $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
- %LIVE_SOCK = ();
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
undef $errors_log;
$quit = \&CORE::exit;
@@ -619,7 +617,6 @@ sub pkt_ops {
$ops->{x_it} = [ \&x_it, $lei ];
$ops->{child_error} = [ \&child_error, $lei ];
$ops->{incr} = [ \&incr, $lei ];
- $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
$ops;
}
@@ -1496,12 +1493,11 @@ sub lms {
(-f $f || $rw) ? PublicInbox::LeiMailSync->new($f) : undef;
}
-sub sto_done_request { # only call this from lei-daemon process (not workers)
+sub sto_done_request {
my ($lei, $sock) = @_;
eval {
if ($sock //= $lei->{sock}) { # issue, async wait
- $LIVE_SOCK{"$sock"} = $sock;
- $lei->{sto}->wq_do('done', "$sock");
+ $lei->{sto}->wq_io_do('done', [ $sock ]);
} else { # forcibly wait
my $wait = $lei->{sto}->wq_do('done');
}
@@ -1509,9 +1505,4 @@ sub sto_done_request { # only call this from lei-daemon process (not workers)
$lei->err($@) if $@;
}
-sub sto_done_complete { # called in lei-daemon when LeiStore->done is complete
- my ($sock_str) = @_;
- delete $LIVE_SOCK{$sock_str}; # frees {sock} for waiting lei clients
-}
-
1;
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 72b8fe63..2f105005 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -60,7 +60,7 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
$self->folder_missing($$uri);
}
} else { die "BUG: $input not supported" }
- $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
+ $self->{lei}->sto_done_request;
}
sub lei_refresh_mail_sync {
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 4ec63699..164a9f2d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -534,10 +534,6 @@ sub done {
$self->{priv_eidx}->done; # V2Writable::done
xchg_stderr($self);
die $err if $err;
-
- # notify clients ->done has been issued
- defined($sock_ref) and
- $self->{s2d_op_p}->pkt_do('sto_done_complete', $sock_ref);
}
sub ipc_atfork_child {
@@ -562,9 +558,6 @@ sub write_prepare {
my ($self, $lei) = @_;
$lei // die 'BUG: $lei not passed';
unless ($self->{-ipc_req}) {
- # s2d => store-to-daemon messages
- require PublicInbox::PktOp;
- my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair;
my $dir = $lei->store_path;
substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
@@ -576,14 +569,10 @@ sub write_prepare {
$self->wq_workers_start("lei/store $dir", 1, $lei->oldset, {
lei => $lei,
-err_wr => $w,
- to_close => [ $r, $s2d_op_c->{sock} ],
- s2d_op_p => $s2d_op_p,
+ to_close => [ $r ],
});
require PublicInbox::LeiStoreErr;
PublicInbox::LeiStoreErr->new($r, $lei);
- $s2d_op_c->{ops} = {
- sto_done_complete => [ $lei->can('sto_done_complete') ]
- };
}
$lei->{sto} = $self;
}
^ permalink raw reply related [relevance 58%]
* [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
@ 2021-09-19 12:50 34% ` Eric Wong
2021-09-19 12:50 58% ` [PATCH 04/16] lei: simplify sto_done_request Eric Wong
` (5 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
This has several advantages:
* no need to use ipc.lock to protect a pipe for non-atomic writes
* ability to pass FDs. In another commit, this will let us
simplify lei->sto_done_request and pass newly-created
sockets to lei/store directly.
disadvantages:
- an extra pipe is required for rare messages over several
hundred KB, this is probably a non-issue, though
The performance delta is unknown, but I expect shards
(which remain pipes) to be the primary bottleneck IPC-wise
for lei/store.
---
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiImport.pm | 2 +-
lib/PublicInbox/LeiImportKw.pm | 2 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiNoteEvent.pm | 8 ++++----
lib/PublicInbox/LeiRemote.pm | 4 ++--
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 10 ++++++++--
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 22 ++++++++++++----------
lib/PublicInbox/LeiXSearch.pm | 6 +++---
12 files changed, 37 insertions(+), 29 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8b0614f2..549b855b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1501,9 +1501,9 @@ sub sto_done_request { # only call this from lei-daemon process (not workers)
eval {
if ($sock //= $lei->{sock}) { # issue, async wait
$LIVE_SOCK{"$sock"} = $sock;
- $lei->{sto}->ipc_do('done', "$sock");
+ $lei->{sto}->wq_do('done', "$sock");
} else { # forcibly wait
- my $wait = $lei->{sto}->ipc_do('done');
+ my $wait = $lei->{sto}->wq_do('done');
}
};
$lei->err($@) if $@;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 9084d771..40530914 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -16,7 +16,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
if (my $all_vmd = $self->{all_vmd}) {
@$vmd{keys %$all_vmd} = values %$all_vmd;
}
- $self->{lei}->{sto}->ipc_do('set_eml', $eml, $vmd, $xoids);
+ $self->{lei}->{sto}->wq_do('set_eml', $eml, $vmd, $xoids);
}
sub input_mbox_cb { # MboxReader callback
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 402125cf..2863d17f 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -37,7 +37,7 @@ sub ck_update_kw { # via wq_io_do
$self->{lse}->kw_changed(undef, $kw, \@docids) or return;
$self->{verbose} and
$self->{lei}->qerr('# '.unpack('H*', $oidbin)." => @$kw\n");
- $self->{sto}->ipc_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
+ $self->{sto}->wq_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
}
sub ikw_done_wait {
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 1b327a2c..b3f3e1a0 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -16,7 +16,7 @@ sub input_eml_cb { # used by input_maildir_cb and input_net_cb
if (my $all_vmd = $self->{all_vmd}) {
@$vmd{keys %$all_vmd} = values %$all_vmd;
}
- $self->{lei}->{sto}->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+ $self->{lei}->{sto}->wq_do('index_eml_only', $eml, $vmd, $xoids);
}
sub input_fh { # overrides PublicInbox::LeiInput::input_fh
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index fe736981..22bedba6 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -378,7 +378,7 @@ sub process_inputs {
}
# always commit first, even on error partial work is acceptable for
# lei <import|tag|convert>
- my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto};
+ my $wait = $self->{lei}->{sto}->wq_do('done') if $self->{lei}->{sto};
$self->{lei}->fail($err) if $err;
}
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 18313359..5f692e75 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -36,18 +36,18 @@ sub eml_event ($$$$) {
my ($self, $eml, $vmd, $state) = @_;
my $sto = $self->{lei}->{sto};
if ($state =~ /\Aimport-(?:rw|ro)\z/) {
- $sto->ipc_do('set_eml', $eml, $vmd);
+ $sto->wq_do('set_eml', $eml, $vmd);
} elsif ($state =~ /\Aindex-(?:rw|ro)\z/) {
my $xoids = $self->{lei}->ale->xoids_for($eml);
- $sto->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+ $sto->wq_do('index_eml_only', $eml, $vmd, $xoids);
} elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
my $docids = [];
my $c = $self->{lse}->kw_changed($eml, $vmd->{kw}, $docids);
if (scalar @$docids) { # already in lei/store
- $sto->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+ $sto->wq_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
# it's in an external, only set kw, here
- $sto->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+ $sto->wq_do('set_xvmd', $xoids, $eml, $vmd);
} # else { totally unknown: ignore
} else {
warn "unknown state: $state (in $self->{lei}->{cfg}->{'-f'})\n";
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 8d4ffed0..346aa6a4 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -28,7 +28,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
my $xoids = $lei->{ale}->xoids_for($eml, 1);
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($lei->{sto} && !$xoids) { # memoize locally
- my $res = $lei->{sto}->ipc_do('add_eml', $eml);
+ my $res = $lei->{sto}->wq_do('add_eml', $eml);
$smsg = $res if ref($res) eq ref($smsg);
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
@@ -56,7 +56,7 @@ sub mset {
my $err = waitpid($pid, 0) == $pid ? undef
: "BUG: waitpid($cmd): $!";
@$reap = (); # cancel OnDestroy
- my $wait = $self->{lei}->{sto}->ipc_do('done');
+ my $wait = $self->{lei}->{sto}->wq_do('done');
die $err if $err;
$self; # we are the mset (and $ibx, and $self)
}
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 3371f3ed..97b1c5c1 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -10,7 +10,7 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
- $self->{lei}->{sto}->ipc_do('remove_eml', $eml);
+ $self->{lei}->{sto}->wq_do('remove_eml', $eml);
}
sub input_mbox_cb { # MboxReader callback
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 08add8f5..4ec63699 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -552,6 +552,12 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
+sub recv_and_run {
+ my ($self, @args) = @_;
+ local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
+ $self->SUPER::recv_and_run(@args);
+}
+
sub write_prepare {
my ($self, $lei) = @_;
$lei // die 'BUG: $lei not passed';
@@ -560,14 +566,14 @@ sub write_prepare {
require PublicInbox::PktOp;
my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair;
my $dir = $lei->store_path;
- $self->ipc_lock_init("$dir/ipc.lock");
substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
$w->autoflush(1);
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
- $self->ipc_worker_spawn("lei/store $dir", $lei->oldset, {
+ $self->{-wq_no_bcast} = 1;
+ $self->wq_workers_start("lei/store $dir", 1, $lei->oldset, {
lei => $lei,
-err_wr => $w,
to_close => [ $r, $s2d_op_c->{sock} ],
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index c4f5ecff..9bbf0d79 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -12,7 +12,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
$self->{lei}->{ale}->xoids_for($eml)) {
- $self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
+ $self->{lei}->{sto}->wq_do('update_xvmd', $xoids, $eml,
$self->{vmd_mod});
} else {
++$self->{unimported};
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 9f7171fb..a419b83f 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -215,14 +215,14 @@ sub update_kw_maybe ($$$$) {
my $c = $lse->kw_changed($eml, $kw, my $docids = []);
my $vmd = { kw => $kw };
if (scalar @$docids) { # already in lei/store
- $lei->{sto}->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+ $lei->{sto}->wq_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $lei->{ale}->xoids_for($eml)) {
# it's in an external, only set kw, here
- $lei->{sto}->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+ $lei->{sto}->wq_do('set_xvmd', $xoids, $eml, $vmd);
} else { # never-before-seen, import the whole thing
# XXX this is critical in protecting against accidental
# data loss without --augment
- $lei->{sto}->ipc_do('set_eml', $eml, $vmd);
+ $lei->{sto}->wq_do('set_eml', $eml, $vmd);
}
}
@@ -296,7 +296,7 @@ sub _maildir_write_cb ($$) {
$lse->xsmsg_vmd($smsg) if $lse;
my $n = _buf2maildir($dst, $bref // \($eml->as_string),
$smsg, $dir);
- $sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
+ $sto->wq_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
++$lei->{-nr_write};
}
}
@@ -326,7 +326,7 @@ sub _imap_write_cb ($$) {
}
# imap_append returns UID if IMAP server has UIDPLUS extension
($sto && $uid =~ /\A[0-9]+\z/) and
- $sto->ipc_do('set_sync_info',
+ $sto->wq_do('set_sync_info',
$smsg->{blob}, $$uri, $uid + 0);
++$lei->{-nr_write};
}
@@ -360,7 +360,7 @@ sub _v2_write_cb ($$) {
my ($bref, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($bref);
return if $dedupe && $dedupe->is_dup($eml, $smsg);
- $lei->{v2w}->ipc_do('add', $eml); # V2Writable->add
+ $lei->{v2w}->wq_do('add', $eml); # V2Writable->add
++$lei->{-nr_write};
}
}
@@ -658,9 +658,10 @@ sub _pre_augment_v2 {
}
PublicInbox::InboxWritable->new($ibx, @creat);
$ibx->init_inbox if @creat;
- my $v2w = $lei->{v2w} = $ibx->importer;
- $v2w->ipc_lock_init("$dir/ipc.lock");
- $v2w->ipc_worker_spawn("lei/v2w $dir", $lei->oldset, { lei => $lei });
+ my $v2w = $ibx->importer;
+ $v2w->{-wq_no_bcast} = 1;
+ $v2w->wq_workers_start("lei/v2w $dir", 1, $lei->oldset, {lei => $lei});
+ $lei->{v2w} = $v2w;
return if !$lei->{opt}->{shared};
my $d = "$lei->{ale}->{git}->{git_dir}/objects";
my $al = "$dir/git/0.git/objects/info/alternates";
@@ -689,7 +690,7 @@ sub do_augment { # slow, runs in wq worker
sub post_augment {
my ($self, $lei, @args) = @_;
my $wait = $lei->{opt}->{'import-before'} ?
- $lei->{sto}->ipc_do('checkpoint', 1) : 0;
+ $lei->{sto}->wq_do('checkpoint', 1) : 0;
# _post_augment_mbox
my $m = $self->can("_post_augment_$self->{base_type}") or return;
$m->($self, $lei, @args);
@@ -774,6 +775,7 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
+ local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
my $lei = $self->{lei};
delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 1d49da3d..4583b067 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -269,7 +269,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
my $xoids = $lei->{ale}->xoids_for($eml, 1);
my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+ my $res = $self->{import_sto}->wq_do('add_eml', $eml);
if (ref($res) eq ref($smsg)) { # totally new message
$smsg = $res;
$smsg->{kw} = []; # short-circuit xsmsg_vmd
@@ -369,7 +369,7 @@ sub query_remote_mboxrd {
@$reap_curl = (); # cancel OnDestroy
die $err if $err;
my $nr = $lei->{-nr_remote_eml};
- my $wait = $lei->{sto}->ipc_do('done') if $nr && $lei->{sto};
+ my $wait = $lei->{sto}->wq_do('done') if $nr && $lei->{sto};
if ($? == 0) {
# don't update if no results, maybe MTA is down
$key && $nr and
@@ -413,7 +413,7 @@ sub query_done { # EOF callback for main daemon
warn "BUG: {sto} missing with --mail-sync";
}
$lei->sto_done_request if $lei->{sto};
- my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+ my $wait = $lei->{v2w} ? $lei->{v2w}->wq_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
^ permalink raw reply related [relevance 34%]
* [PATCH] t/lei-refresh-mail-sync: improve test reliability
@ 2021-09-18 22:38 66% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-18 22:38 UTC (permalink / raw)
To: meta
We can't assume -imapd will be ready by the time we try to
connect to it after restart when using "-l $ADDR". So recreate
the (closed-for-testing) listen socket in the parent and hand it
off to -imapd as we do normally
---
t/lei-refresh-mail-sync.t | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
index 90356b57..43fbc50a 100644
--- a/t/lei-refresh-mail-sync.t
+++ b/t/lei-refresh-mail-sync.t
@@ -4,6 +4,7 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
require_mods(qw(lei));
use File::Path qw(remove_tree);
+require Socket;
my $stop_daemon = sub { # needed since we don't have inotify
lei_ok qw(daemon-pid);
@@ -81,8 +82,10 @@ SKIP: {
my $cfg_path2 = "$home/cfg2";
File::Copy::cp($cfg_path, $cfg_path2);
my $env = { PI_CONFIG => $cfg_path2 };
+ my $sock_cls;
for my $x (qw(imapd)) {
my $s = tcp_server;
+ $sock_cls //= ref($s);
my $cmd = [ "-$x", '-W0', "--stdout=$home/$x.out",
"--stderr=$home/$x.err" ];
my $td = start_script($cmd, $env, { 3 => $s}) or xbail("-$x");
@@ -124,10 +127,17 @@ SKIP: {
ok(!(lei 'refresh-mail-sync', '--all'), '--all fails on dead -imapd');
# restart server (somewhat dangerous since we released the socket)
+ my $listen = $sock_cls->new(
+ ReuseAddr => 1,
+ Proto => 'tcp',
+ Type => Socket::SOCK_STREAM(),
+ Listen => 1024,
+ Blocking => 0,
+ LocalAddr => $srv->{imapd}->{addr},
+ ) or xbail "$sock_cls->new: $!";
my $cmd = $srv->{imapd}->{cmd};
- push @$cmd, '-l', $srv->{imapd}->{addr};
- $srv->{imapd}->{td} = start_script($cmd, $env) or xbail "@$cmd";
-
+ $srv->{imapd}->{td} = start_script($cmd, $env, { 3 => $listen }) or
+ xbail "@$cmd";
lei_ok 'refresh-mail-sync', '--all';
lei_ok 'inspect', "blob:$oid";
is($lei_out, $before, 'no changes when server was down');
^ permalink raw reply related [relevance 66%]
* Re: [PATCH 9/9] lei up: automatically use dt: for remote externals
2021-09-18 14:31 71% ` Kyle Meyer
@ 2021-09-18 20:30 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-18 20:30 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> Eric Wong writes:
> > +interval reduces the likelyhood missing a result due to MTA
> > +delays or downtime.
>
> s/likelyhood/likelihood of/
Thanks, pushed as commit f3d0e746c6a35c8600b91af99958a52cbc114a4b
^ permalink raw reply [relevance 71%]
* Re: [PATCH 9/9] lei up: automatically use dt: for remote externals
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
@ 2021-09-18 14:31 71% ` Kyle Meyer
2021-09-18 20:30 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-18 14:31 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
> index e5d97f43..4dc6d3b8 100644
> --- a/Documentation/lei-up.pod
> +++ b/Documentation/lei-up.pod
> @@ -22,6 +22,21 @@ C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
> C<--all=local> only updates local mailboxes, C<--all=remote> only
> updates remote mailboxes (currently C<imap://> and C<imaps://>).
>
> +=item --remote-fudge-time=INTERVAL
> +
> +Look for mail older than the time of the last successful query.
> +Using a small interval will reduce bandwidth use. A larger
> +interval reduces the likelyhood missing a result due to MTA
> +delays or downtime.
s/likelyhood/likelihood of/
^ permalink raw reply [relevance 71%]
* [PATCH 9/9] lei up: automatically use dt: for remote externals
2021-09-18 9:33 67% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
2021-09-18 9:33 51% ` [PATCH 1/9] lei: lock worker counts Eric Wong
@ 2021-09-18 9:33 39% ` Eric Wong
2021-09-18 14:31 71% ` Kyle Meyer
1 sibling, 1 reply; 200+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
Since we can't use maxuid for remote externals, automatically
maintaining the last time we got results and appending a dt:
range to the query will prevent HTTP(S) responses from getting
too big.
We could be using "rt:", but no stable release of public-inbox
supports it, yet, so we'll use dt:, instead.
By default, there's a two day fudge factor to account for MTA
downtime and delays; which is hopefully enough. The fudge
factor may be changed per-invocation with the
--remote-fudge-factor=INTERVAL option
Since different externals can have different message transport
routes, "lastresult" entries are stored on a per-external basis.
---
Documentation/lei-up.pod | 15 ++++++++++
lib/PublicInbox/LEI.pm | 7 +++--
lib/PublicInbox/LeiSavedSearch.pm | 1 +
lib/PublicInbox/LeiToMail.pm | 4 ++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 50 ++++++++++++++++++++++++++-----
t/lei-q-remote-import.t | 4 +++
7 files changed, 71 insertions(+), 12 deletions(-)
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index e5d97f43..4dc6d3b8 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -22,6 +22,21 @@ C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
C<--all=local> only updates local mailboxes, C<--all=remote> only
updates remote mailboxes (currently C<imap://> and C<imaps://>).
+=item --remote-fudge-time=INTERVAL
+
+Look for mail older than the time of the last successful query.
+Using a small interval will reduce bandwidth use. A larger
+interval reduces the likelyhood missing a result due to MTA
+delays or downtime.
+
+The time(s) of the last successful queries are the C<lastresult>
+values visible from L<lei-edit-search(1)>.
+
+Date formats understood by L<git-rev-parse(1)> may be used.
+e.g C<1.hour> or C<3.days>
+
+Default: 2.days
+
=back
The following options, described in L<lei-q(1)>, are supported.
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 053b6174..8b0614f2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -183,7 +183,8 @@ our %CMD = ( # sorted in order of importance/use:
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT...|--all', 'update saved search',
- qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
+ qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+
+ remote-fudge-time=s all:s), @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
@@ -420,7 +421,9 @@ my %OPTDESC = (
'remote' => 'limit operations to those requiring network access',
'remote!' => 'prevent operations requiring network access',
-'all:s up' => ['local', 'update all (local) saved searches' ],
+'all:s up' => ['local|remote', 'update all remote or local saved searches' ],
+'remote-fudge-time=s' => [ 'INTERVAL',
+ 'look for mail INTERVAL older than the last successful query' ],
'mid=s' => 'specify the Message-ID of a message',
'oid=s' => 'specify the git object ID of a message',
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 96ff816e..754f8294 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -149,6 +149,7 @@ sub new { # new saved search "lei q --save"
$dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i;
print $fh <<EOM;
; to refresh with new results, run: lei up $sq_dst
+; `maxuid' and `lastresult' lines are maintained by "lei up" for optimization
[lei]
$q
[lei "q"]
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d3253d9b..9f7171fb 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -437,7 +437,9 @@ sub new {
($lei->{opt}->{save} ? 'LeiSavedSearch' : 'LeiDedupe');
eval "require $dd_cls";
die "$dd_cls: $@" if $@;
- $dd_cls->new($lei);
+ my $dd = $dd_cls->new($lei);
+ $lei->{lss} //= $dd if $dd && $dd->can('cfg_set');
+ $dd;
};
$self;
}
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 4637cb46..abb05d46 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -45,7 +45,7 @@ sub up1 ($$) {
ref($v) and return $lei->fail("multiple values of $c in $f");
$lei->{opt}->{$k} = $v;
}
- $lei->{lss} = $lss; # for LeiOverview->new
+ $lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
my $lxs = $lei->lxs_prepare or return;
$lei->ale->refresh_externals($lxs, $lei);
$lei->_start_query;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 50cadb5e..1d49da3d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -18,6 +18,7 @@ use PublicInbox::Smsg;
use PublicInbox::Eml;
use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
use PublicInbox::ContentHash qw(git_sha);
+use POSIX qw(strftime);
sub new {
my ($class) = @_;
@@ -168,8 +169,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $can_kw = !!$ibxish->can('msg_keywords');
my $threads = $lei->{opt}->{threads} // 0;
my $fl = $threads > 1 ? 1 : undef;
- my $lss = $lei->{dedupe};
- $lss = undef unless $lss && $lss->can('cfg_set'); # saved search
+ my $lss = $lei->{lss};
my $maxk = "external.$dir.maxuid";
my $stop_at = $lss ? $lss->{-cfg}->{$maxk} : undef;
if (defined $stop_at) {
@@ -292,6 +292,37 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
$each_smsg->($smsg, undef, $eml);
}
+sub fudge_qstr_time ($$$) {
+ my ($lei, $uri, $qstr) = @_;
+ return ($qstr, undef) unless $lei->{lss};
+ my $cfg = $lei->{lss}->{-cfg} // die 'BUG: no lss->{-cfg}';
+ my $cfg_key = "external.$uri.lastresult";
+ my $lr = $cfg->{$cfg_key} or return ($qstr, $cfg_key);
+ if ($lr !~ /\A\-?[0-9]+\z/) {
+ $lei->child_error(0,
+ "$cfg->{-f}: $cfg_key=$lr not an integer, ignoring");
+ return ($qstr, $cfg_key);
+ }
+ my $rft = $lei->{opt}->{'remote-fudge-time'};
+ if ($rft && $rft !~ /\A-?[0-9]+\z/) {
+ my @t = $lei->{lss}->git->date_parse($rft);
+ my $diff = time - $t[0];
+ $lei->qerr("# $rft => $diff seconds");
+ $rft = $diff;
+ }
+ $lr -= ($rft || (48 * 60 * 60));
+ $lei->qerr("# $uri limiting to ".
+ strftime('%Y-%m-%d %k:%M', gmtime($lr)). ' and newer');
+ # this should really be rt: (received-time), but no stable
+ # public-inbox releases support it, yet.
+ my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..';
+ if ($qstr =~ /\S/) {
+ substr($qstr, 0, 0, '(');
+ $qstr .= ') AND ';
+ }
+ ($qstr .= $dt, $cfg_key);
+}
+
sub query_remote_mboxrd {
my ($self, $uris) = @_;
local $0 = "$0 query_remote_mboxrd";
@@ -300,7 +331,7 @@ sub query_remote_mboxrd {
my $opt = $lei->{opt};
my $qstr = $lei->{mset_opt}->{qstr};
$qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
- my @qform = (q => $qstr, x => 'm');
+ my @qform = (x => 'm');
push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
my ($reap_tail, $reap_curl);
@@ -323,7 +354,9 @@ sub query_remote_mboxrd {
for my $uri (@$uris) {
$lei->{-current_url} = $uri->as_string;
$lei->{-nr_remote_eml} = 0;
- $uri->query_form(@qform);
+ my $start = time;
+ my ($q, $key) = fudge_qstr_time($lei, $uri, $qstr);
+ $uri->query_form(@qform, q => $q);
my $cmd = $curl->for_uri($lei, $uri);
$lei->qerr("# $cmd");
my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
@@ -336,10 +369,11 @@ sub query_remote_mboxrd {
@$reap_curl = (); # cancel OnDestroy
die $err if $err;
my $nr = $lei->{-nr_remote_eml};
- if ($nr && $lei->{sto}) {
- my $wait = $lei->{sto}->ipc_do('done');
- }
+ my $wait = $lei->{sto}->ipc_do('done') if $nr && $lei->{sto};
if ($? == 0) {
+ # don't update if no results, maybe MTA is down
+ $key && $nr and
+ $lei->{lss}->cfg_set($key, $start);
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
}
@@ -353,7 +387,7 @@ sub query_remote_mboxrd {
$lei->err("truncate($cmd stderr): $!");
}
next if (($? >> 8) == 22 && $err =~ /\b404\b/);
- $uri->query_form(q => $lei->{mset_opt}->{qstr});
+ $uri->query_form(q => $qstr);
$lei->child_error($?, "E: <$uri> $err");
}
undef $each_smsg;
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 9131c01b..fdf6a11e 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -99,5 +99,9 @@ EOF
lei_ok('up', "$ENV{HOME}/md");
is_deeply(\@f, [ glob("$ENV{HOME}/md/*/*") ],
'lei up remote dedupe works on maildir');
+ my $edit_env = { VISUAL => 'cat' };
+ lei_ok([qw(edit-search), "$ENV{HOME}/md"], $edit_env);
+ like($lei_out, qr/^\Q[external "$url"]\E\n\s*lastresult = \d+/sm,
+ 'lastresult set');
});
done_testing;
^ permalink raw reply related [relevance 39%]
* [PATCH 1/9] lei: lock worker counts
2021-09-18 9:33 67% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
@ 2021-09-18 9:33 51% ` Eric Wong
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
It doesn't seem worthwhile to change worker counts dynamically
on a per-command-basis with lei, and I don't know how such an
interface would even work...
---
lib/PublicInbox/LEI.pm | 3 ++-
lib/PublicInbox/LeiExportKw.pm | 1 -
lib/PublicInbox/LeiImport.pm | 1 -
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiRefreshMailSync.pm | 4 +---
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 3 +--
8 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9794497b..41e761f8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -258,7 +258,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
+ qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
@@ -627,6 +627,7 @@ sub workers_start {
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
$flds->{lei} = $lei;
+ $wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d37f3768..8b8aa373 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -124,7 +124,6 @@ EOM
my $ops = {};
$sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7c563bd8..b1cb3940 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -101,7 +101,6 @@ sub do_import_index ($$@) {
}
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
$lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index f012e10e..bcb1838e 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -96,8 +96,7 @@ sub lei_ls_mail_source {
$lei->start_pager if -t $lei->{1};
my $ops = {};
$lei->{auth}->op_merge($ops, $self);
- my $j = $self->{-wq_nr_workers} = 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 70136135..aebf0184 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -71,7 +71,7 @@ sub do_ls_search_long {
sub bg_worker ($$$) {
my ($lei, $pfx, $json) = @_;
- my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
+ my $self = bless { json => $json }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 09a7ead0..cdd99725 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -84,11 +84,9 @@ EOM
my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
- my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 778fa1de..3371f3ed 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -32,7 +32,7 @@ sub lei_rm {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
$lei->{opt}->{'in-format'} //= 'eml';
- my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$self->prepare_inputs($lei, \@inputs) or return;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 44d77b88..c4f5ecff 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -50,8 +50,7 @@ sub lei_tag { # the "lei tag" method
return $lei->fail('no keywords or labels specified');
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- my $j = $self->{-wq_nr_workers} = 1; # locked for now
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
^ permalink raw reply related [relevance 51%]
* [PATCH 0/9] lei: a bunch of random stuff
@ 2021-09-18 9:33 67% Eric Wong
2021-09-18 9:33 51% ` [PATCH 1/9] lei: lock worker counts Eric Wong
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
The unique timers stuff will be used for "lei up" polling,
as will 9/9 to improve "lei up" usability.
The net_reader changes were noticed while getting imaps://
to work with socks5h:// (not just imap://).
There's still a lot of mail_sync stuff going on, but it's
getting closer...
Eric Wong (9):
lei: lock worker counts
lei_mail_sync: rely on flock(2), avoid IPC
lei_mail_sync: set nodatacow on btrfs
ds: support add unique timers
net_reader: tie SocksDebug to {imap,nntp}.Debug
net_reader: detect IMAP failures earlier
net_reader: support imaps:// w/ socks5h:// proxy
net_reader: set SO_KEEPALIVE on all Net::NNTP sockets
lei up: automatically use dt: for remote externals
Documentation/lei-up.pod | 15 ++++
lib/PublicInbox/DS.pm | 100 +++++++++++++-------------
lib/PublicInbox/LEI.pm | 40 +++++------
lib/PublicInbox/LeiExportKw.pm | 32 ++++-----
lib/PublicInbox/LeiForgetMailSync.pm | 6 +-
lib/PublicInbox/LeiImport.pm | 8 +--
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiInspect.pm | 5 +-
lib/PublicInbox/LeiLsMailSource.pm | 3 +-
lib/PublicInbox/LeiLsMailSync.pm | 3 +-
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMailSync.pm | 51 ++++++++++---
lib/PublicInbox/LeiNoteEvent.pm | 31 ++++----
lib/PublicInbox/LeiRefreshMailSync.pm | 35 ++++-----
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 1 +
lib/PublicInbox/LeiStore.pm | 39 +---------
lib/PublicInbox/LeiTag.pm | 3 +-
lib/PublicInbox/LeiToMail.pm | 10 ++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 50 ++++++++++---
lib/PublicInbox/NetReader.pm | 26 ++++---
t/lei-q-remote-import.t | 4 ++
23 files changed, 259 insertions(+), 211 deletions(-)
^ permalink raw reply [relevance 67%]
* [PATCH] doc: add lei-security(7) manpage
@ 2021-09-17 12:38 38% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 12:38 UTC (permalink / raw)
To: meta
It seems like a good idea to have a manpage where somebody
can quickly look up and address their concerns as to what
to put on encrypted device/filesystem.
And I probably would've designed lei around make(1) for
parallelization if I didn't have to keep credentials off
the FS :P
---
Documentation/lei-add-external.pod | 4 +-
Documentation/lei-security.pod | 134 +++++++++++++++++++++++++++++
Documentation/txt2pre | 8 +-
MANIFEST | 1 +
Makefile.PL | 3 +-
5 files changed, 146 insertions(+), 4 deletions(-)
create mode 100644 Documentation/lei-security.pod
diff --git a/Documentation/lei-add-external.pod b/Documentation/lei-add-external.pod
index 71229865..9c8bde0f 100644
--- a/Documentation/lei-add-external.pod
+++ b/Documentation/lei-add-external.pod
@@ -10,8 +10,8 @@ lei add-external [OPTIONS] LOCATION
Configure lei to search against an external (an inbox or external
index). When C<LOCATION> is an existing local path, it should point
-to a directory that is a C<public.<name>.inboxdir> or
-C<extindex.<name>.topdir> value in ~/.public-inbox/config.
+to a directory that is a C<publicinbox.$NAME.inboxdir> or
+C<extindex.$NAME.topdir> value in ~/.public-inbox/config.
=head1 OPTIONS
diff --git a/Documentation/lei-security.pod b/Documentation/lei-security.pod
new file mode 100644
index 00000000..4b712c2d
--- /dev/null
+++ b/Documentation/lei-security.pod
@@ -0,0 +1,134 @@
+=head1 NAME
+
+lei - security information
+
+=head1 SYNOPSIS
+
+L<lei(1)> is intended for use with both publicly-archived
+and "private" mail in personal mailboxes. This document is
+intended to give an overview of security implications and
+lower^Wmanage user expectations.
+
+=head1 DESCRIPTION
+
+lei expects to be run as a regular user on a Unix-like system.
+It expects a case-sensitive filesystem with standard Unix
+permissions support.
+
+It does not use POSIX ACLs, extended attributes, nor any other
+security-related functions which require non-standard Perl modules.
+
+=head1 INTERNAL FILES
+
+lei runs with a umask of 077 to prevent other users on the
+system from accessing each other's mail.
+
+The git storage and Xapian databases are located at
+C<$XDG_DATA_HOME/lei/store> (typically C<~/.local/share/lei/store>).
+Any personal mail imported will reside here, so this should
+be on an encrypted filesystem or block device.
+
+C<$XDG_RUNTIME_DIR/lei> (typically C</run/user/$UID/lei> or
+C</tmp/lei-$UID>) contain the socket used to access the lei
+daemon. It must only be accessible to the owner (mode 0700).
+
+C<$XDG_CACHE_HOME/lei> (typically C<~/.cache/lei>) will
+contain IMAP and Maildir folder names which could leak sensitive
+information as well as git repository names.
+
+C<$XDG_DATA_HOME/lei/saved-searches> (typically
+C<~/.local/share/lei/saved-searches>) will contain aforementioned
+folder names as well as (removable) search history.
+
+The configuration for lei resides at C<$XDG_CONFIG_HOME/lei/config>
+(typically C<~/.config/lei/config>). It may contain sensitive pathnames
+and hostnames in the config if a user chooses to configure them.
+
+lei itself will never write credentials to the
+filesystem. However, L<git-credential(1)> may be
+configured to do so. lei will only read C<~/.netrc> if
+C<--netrc> is used (and it will never write to C<~/.netrc>).
+
+C<$XDG_CACHE_HOME/public-inbox> (typically C<~/.cache/public-inbox>)
+can contain data and L<Inline::C>-built modules which can be
+shared with public-facing L<public-inbox-daemon(8)> instances;
+so no private data should be in "public-inbox" paths.
+
+=head1 EXTERNAL FILES
+
+Locations set by L<lei-add-external(1)> can be shared with
+public-facing L<public-inbox-daemon(8)> processes. They may
+reside on shared storage and may be made world-readable to
+other users on the local system.
+
+=head1 NETWORK ACCESS
+
+lei currently uses the L<curl(1)> and L<git(1)> executables in
+C<$PATH> for HTTP and HTTPS network access. Interactive
+authentication for HTTP and HTTPS is not-yet-supported since all
+currently supported HTTP/HTTPS sources are L<PublicInbox::WWW>
+instances.
+
+The L<Mail::IMAPClient> library is used for IMAP and IMAPS.
+L<Net::NNTP> (standard library) is used for NNTP and NNTPS.
+
+L<Mail::IMAPClient> and L<Net::NNTP> will use L<IO::Socket::SSL>
+for TLS if available. In turn, L<IO::Socket::SSL> uses the
+widely-installed OpenSSL library.
+
+STARTTLS will be attempted if advertised by the server
+unless IMAPS or NNTPS are used. C<-c imap.starttls=0>
+and C<-c nntp.startls=0> may be used to disable STARTTLS.
+
+L<IO::Socket::Socks> will be used if C<-c imap.proxy> or
+C<-c nntp.proxy> point to a C<socks5h://$HOST:$PORT>
+address (common for Tor).
+
+The C<--netrc> switch may be passed to curl and used for
+NNTP/IMAP access (via L<Net::Netrc>).
+
+=head1 CREDENTIAL DATA
+
+lei uses L<git-credential(1)> to prompt users for IMAP and NNTP
+usernames and passwords. These passwords are not encrypted in
+memory and get transferred across processes via anonymous UNIX
+sockets and pipes. They may be exposed via syscall tracing
+tools (e.g. L<strace(1)>).
+
+While credentials are not written to the filesystem by default,
+it is possible for them to end up on disk if processes are
+swapped out. Use of an encrypted swap partition is recommended.
+
+=head1 DENIAL-OF-SERVICE VECTORS
+
+lei uses the same MIME parsing library as L<public-inbox-mda(1)>
+with limits header sizes, parts, nesting and boundary limits
+similar to those found in SpamAssassin and postfix.
+
+Email address parsing is handled by L<Email::Address::XS> if
+available, but may fall back to regular expressions which favor
+speed and predictable execution times over correctness.
+
+=head1 ENCRYPTED EMAILS
+
+Not yet supported, but it should eventually be possible to
+configure decryption and indexing of encrypted messages and
+attachments. When supported, decrypted terms will be stored
+in Xapian DBs under C<$XDG_DATA_HOME/lei/store>.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-overview(7)>, L<lei(1)>
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 6367374d..7bc31d23 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -21,6 +21,7 @@ for (qw[lei(1)
lei-forget-external(1)
lei-forget-search(1)
lei-import(1)
+ lei-index(1)
lei-init(1)
lei-lcat(1)
lei-ls-external(1)
@@ -31,14 +32,19 @@ for (qw[lei(1)
lei-p2q(1)
lei-q(1)
lei-rediff(1)
+ lei-rm(1)
+ lei-store-format(5)
+ lei-security(7)
lei-tag(1)
lei-up(1)
public-inbox.cgi(1)
- public-inbox-compact(1)
+ public-inbox-clone(1)
+ public-inbox-config(5)
public-inbox-config(5)
public-inbox-convert(1)
public-inbox-daemon(8)
public-inbox-edit(1)
+ public-inbox-fetch(1)
public-inbox-glossary(7)
public-inbox-httpd(1)
public-inbox-imapd(1)
diff --git a/MANIFEST b/MANIFEST
index 9f11f2f9..218e20e9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -44,6 +44,7 @@ Documentation/lei-p2q.pod
Documentation/lei-q.pod
Documentation/lei-rediff.pod
Documentation/lei-rm.pod
+Documentation/lei-security.pod
Documentation/lei-store-format.pod
Documentation/lei-tag.pod
Documentation/lei-up.pod
diff --git a/Makefile.PL b/Makefile.PL
index 1e935012..f56445ae 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -54,7 +54,8 @@ $v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
) ];
-$v->{-m7} = [ qw(lei-overview public-inbox-overview public-inbox-tuning
+$v->{-m7} = [ qw(lei-overview lei-security
+ public-inbox-overview public-inbox-tuning
public-inbox-glossary) ];
$v->{-m8} = [ qw(public-inbox-daemon) ];
my @sections = (1, 5, 7, 8);
^ permalink raw reply related [relevance 38%]
* [PATCH] script/lei: umask(077) before execve
@ 2021-09-17 12:12 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 12:12 UTC (permalink / raw)
To: meta
While my MUA also runs umask(077) unconditionally, not all
MUAs do. Additionally, pagers may support writing its buffer
to disk, so ensure anything else we spawn has umask(077).
---
script/lei | 1 +
1 file changed, 1 insertion(+)
diff --git a/script/lei b/script/lei
index 2d84487a..591013e3 100755
--- a/script/lei
+++ b/script/lei
@@ -53,6 +53,7 @@ my $exec_cmd = sub {
}
my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
@ENV{keys %env} = values %env;
+ umask 077;
exec(@argv);
warn "exec: @argv: $!\n";
POSIX::_exit(1);
^ permalink raw reply related [relevance 71%]
* Re: [PATCH] lei: sqlite: do not forcibly enable WAL
2021-09-16 11:04 66% [PATCH] lei: sqlite: do not forcibly enable WAL Eric Wong
@ 2021-09-17 2:16 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 2:16 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> It could be a problem with my ancient SSD, improper use on our
> end,
Yes, EDOOFUS :x
I suspect it was fixed in v2 of <20210917015644.32746-2-e@80x24.org>
("lei refresh-mail-sync: replace prune-mail-sync")
So I think lei will keep using WAL (and maybe start using mmap, too)
^ permalink raw reply [relevance 71%]
* [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
` (3 preceding siblings ...)
2021-09-17 1:56 61% ` [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders Eric Wong
@ 2021-09-17 1:56 56% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
Like with Maildir, IMAP folders can be deleted entirely.
Ensure they can be eliminated, but don't be fooled into
removing them if they're temporarily unreachable.
---
lib/PublicInbox/LeiRefreshMailSync.pm | 11 +++--
t/lei-refresh-mail-sync.t | 60 +++++++++++++++++++++++++++
2 files changed, 67 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 19f64b58..09a7ead0 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -53,10 +53,13 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
PublicInbox::LeiInput::input_path_url($self, $input);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
- my $mic = $self->{lei}->{net}->mic_for_folder($uri);
- my $uids = $mic->search('UID 1:*');
- $uids = +{ map { $_ => undef } @$uids };
- $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
+ if (my $mic = $self->{lei}->{net}->mic_for_folder($uri)) {
+ my $uids = $mic->search('UID 1:*');
+ $uids = +{ map { $_ => undef } @$uids };
+ $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri)
+ } else {
+ $self->folder_missing($$uri);
+ }
} else { die "BUG: $input not supported" }
$self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
}
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
index d3438011..90356b57 100644
--- a/t/lei-refresh-mail-sync.t
+++ b/t/lei-refresh-mail-sync.t
@@ -72,6 +72,66 @@ test_lei({ daemon_only => 1 }, sub {
is($lei_out, '{}', 'no known locations after "removal"');
lei_ok 'ls-mail-sync';
is($lei_out, '', 'no sync left when folder is gone');
+
+SKIP: {
+ require_mods(qw(-imapd -nntpd Mail::IMAPClient Net::NNTP), 1);
+ require File::Copy; # stdlib
+ my $home = $ENV{HOME};
+ my $srv;
+ my $cfg_path2 = "$home/cfg2";
+ File::Copy::cp($cfg_path, $cfg_path2);
+ my $env = { PI_CONFIG => $cfg_path2 };
+ for my $x (qw(imapd)) {
+ my $s = tcp_server;
+ my $cmd = [ "-$x", '-W0', "--stdout=$home/$x.out",
+ "--stderr=$home/$x.err" ];
+ my $td = start_script($cmd, $env, { 3 => $s}) or xbail("-$x");
+ $srv->{$x} = {
+ addr => (my $scalar = tcp_host_port($s)),
+ td => $td,
+ cmd => $cmd,
+ };
+ }
+ my $url = "imap://$srv->{imapd}->{addr}/t.v1.0";
+ lei_ok 'import', $url, '+L:v1';
+ lei_ok 'inspect', "blob:$oid";
+ $before = json_utf8->decode($lei_out);
+ my @f = grep(m!\Aimap://;AUTH=ANONYMOUS\@\Q$srv->{imapd}->{addr}\E!,
+ keys %{$before->{'mail-sync'}});
+ is(scalar(@f), 1, 'got IMAP folder') or xbail(\@f);
+ xsys([qw(git config), '-f', $cfg_path2,
+ qw(--unset publicinbox.t1.newsgroup)]) and
+ xbail "git config $?";
+ $stop_daemon->(); # drop IMAP IDLE
+ $srv->{imapd}->{td}->kill('HUP');
+ tick; # wait for HUP
+ lei_ok 'refresh-mail-sync', $url;
+ lei_ok 'inspect', "blob:$oid";
+ my $after = json_utf8->decode($lei_out);
+ ok(!$after->{'mail-sync'}, 'no sync info for non-existent mailbox');
+ lei_ok 'ls-mail-sync';
+ unlike $lei_out, qr!^\Q$f[0]\E!, 'IMAP folder gone from mail_sync';
+
+ # simulate server downtime
+ $url = "imap://$srv->{imapd}->{addr}/t.v2.0";
+ lei_ok 'import', $url, '+L:v2';
+
+ lei_ok 'inspect', "blob:$oid";
+ $before = $lei_out;
+ delete $srv->{imapd}->{td}; # kill + join daemon
+
+ ok(!(lei 'refresh-mail-sync', $url), 'URL fails on dead -imapd');
+ ok(!(lei 'refresh-mail-sync', '--all'), '--all fails on dead -imapd');
+
+ # restart server (somewhat dangerous since we released the socket)
+ my $cmd = $srv->{imapd}->{cmd};
+ push @$cmd, '-l', $srv->{imapd}->{addr};
+ $srv->{imapd}->{td} = start_script($cmd, $env) or xbail "@$cmd";
+
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, $before, 'no changes when server was down');
+}; # imapd+nntpd stuff
});
done_testing;
^ permalink raw reply related [relevance 56%]
* [PATCH v2 0/6] lei refresh-mail-sync: another try...
@ 2021-09-17 1:56 71% Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
OK, so a bunch of things were broken/annoying in my initial
patch @ <20210916094116.11457-3-e@80x24.org>
2/6 was only exposed in real-world usage with giant folders;
and a testcase is still pending for that...
Eric Wong (6):
lei refresh-mail-sync: replace prune-mail-sync
lei_mail_sync: don't hold statement handle into callback
lei refresh-mail-sync: remove "gone" notices
lei refresh-mail-sync: drop unused {verify} code path
lei refresh-mail-sync: implicitly remove missing folders
lei refresh-mail-sync: drop old IMAP folder info
MANIFEST | 3 +-
lib/PublicInbox/LEI.pm | 3 +-
lib/PublicInbox/LeiInput.pm | 11 +-
lib/PublicInbox/LeiMailSync.pm | 41 ++++--
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 79 +++++-----
lib/PublicInbox/LeiStore.pm | 5 +
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 137 ++++++++++++++++++
8 files changed, 231 insertions(+), 49 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (52%)
create mode 100644 t/lei-refresh-mail-sync.t
^ permalink raw reply [relevance 71%]
* [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
@ 2021-09-17 1:56 71% ` Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
Those stderr messages are not useful at all, and harmful with
the noise they cause.
---
lib/PublicInbox/LeiRefreshMailSync.pm | 2 --
1 file changed, 2 deletions(-)
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 3c083965..71fc348c 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -31,14 +31,12 @@ sub prune_mdir { # lms->each_src callback
}
}
# both tries failed
- $self->{lei}->qerr("# maildir:$mdir $$id gone");
$self->{lei}->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
}
sub prune_imap { # lms->each_src callback
my ($oidbin, $uid, $self, $uids, $url) = @_;
return if exists $uids->{$uid};
- $self->{lei}->qerr("# $url $uid gone");
$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
}
^ permalink raw reply related [relevance 71%]
* [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices Eric Wong
@ 2021-09-17 1:56 71% ` Eric Wong
2021-09-17 1:56 61% ` [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders Eric Wong
2021-09-17 1:56 56% ` [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
That option was never wired up, and probably not needed...
---
lib/PublicInbox/LeiRefreshMailSync.pm | 17 +----------------
1 file changed, 1 insertion(+), 16 deletions(-)
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 71fc348c..4cae1536 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -9,27 +9,12 @@ use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
-use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::Import;
-sub eml_match ($$) {
- my ($eml, $oidbin) = @_;
- $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
- $oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
-}
-
sub prune_mdir { # lms->each_src callback
my ($oidbin, $id, $self, $mdir) = @_;
my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur);
- for my $d (@try) {
- my $src = "$mdir/$d/$$id";
- if ($self->{verify}) {
- my $eml = eml_from_path($src) // next;
- return if eml_match($eml, $oidbin);
- } elsif (-f $src) {
- return;
- }
- }
+ for (@try) { return if -f "$mdir/$_/$$id" }
# both tries failed
$self->{lei}->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
}
^ permalink raw reply related [relevance 71%]
* [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
` (2 preceding siblings ...)
2021-09-17 1:56 71% ` [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path Eric Wong
@ 2021-09-17 1:56 61% ` Eric Wong
2021-09-17 1:56 56% ` [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
There's no point in keeping mail_sync.sqlite3 entries around
if the folder is gone. We do keep saved-search configs around,
however, since somebody may decide to blow away a search and
start over.
---
lib/PublicInbox/LeiInput.pm | 11 ++++++++++-
lib/PublicInbox/LeiRefreshMailSync.pm | 5 +++++
t/lei-refresh-mail-sync.t | 10 ++++++++++
3 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 8ce445c8..372e0fe1 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -124,7 +124,11 @@ sub input_path_url {
handle_http_input($self, $input, @args);
return;
}
+
+ # local-only below
+ my $ifmt_pfx = '';
if ($input =~ s!\A([a-z0-9]+):!!i) {
+ $ifmt_pfx = "$1:";
$ifmt = lc($1);
} elsif ($input =~ /\.(?:patch|eml)\z/i) {
$ifmt = 'eml';
@@ -172,11 +176,16 @@ EOM
$self->can('input_maildir_cb'),
$self, @args);
}
+ } elsif ($self->{missing_ok} && !-e $input) { # don't ->fail
+ $self->folder_missing("$ifmt:$input");
} else {
- $lei->fail("$input unsupported (TODO)");
+ $lei->fail("$ifmt_pfx$input unsupported (TODO)");
}
}
+# subclasses should overrride this (see LeiRefreshMailSync)
+sub folder_missing { die "BUG: ->folder_missing undefined for $_[0]" }
+
sub bad_http ($$;$) {
my ($lei, $url, $alt) = @_;
my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 4cae1536..19f64b58 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -11,6 +11,11 @@ use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::Import;
+sub folder_missing {
+ my ($self, $folder) = @_;
+ $self->{lei}->{sto}->ipc_do('lms_forget_folders', $folder);
+}
+
sub prune_mdir { # lms->each_src callback
my ($oidbin, $id, $self, $mdir) = @_;
my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur);
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
index ff558277..d3438011 100644
--- a/t/lei-refresh-mail-sync.t
+++ b/t/lei-refresh-mail-sync.t
@@ -3,6 +3,7 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
require_mods(qw(lei));
+use File::Path qw(remove_tree);
my $stop_daemon = sub { # needed since we don't have inotify
lei_ok qw(daemon-pid);
@@ -62,6 +63,15 @@ test_lei({ daemon_only => 1 }, sub {
lei_ok 'inspect', "blob:$oid";
is_deeply(json_utf8->decode($lei_out), $exp1,
'replaced file noted again');
+
+ $stop_daemon->();
+
+ remove_tree($d);
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, '{}', 'no known locations after "removal"');
+ lei_ok 'ls-mail-sync';
+ is($lei_out, '', 'no sync left when folder is gone');
});
done_testing;
^ permalink raw reply related [relevance 61%]
* [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
@ 2021-09-17 1:56 36% ` Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-17 1:56 UTC (permalink / raw)
To: meta
Merely pruning mail synchronization information was
insufficient for Maildir: renames are common in Maildir
and we need to detect them after-the-fact when lei-daemon
isn't running.
Running this command could make "lei index" far more
useful...
v2: close R/O mail_sync.sqlite3 dbh before fork
Keeping the DB file handle open across fork can cause bad things
to happen even if we don't use it since sqlite3 itself still knows
about it (but doesn't know Perl code doesn't know about it).
---
MANIFEST | 3 +-
lib/PublicInbox/LEI.pm | 3 +-
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 54 +++++++++------
lib/PublicInbox/LeiStore.pm | 5 ++
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 67 +++++++++++++++++++
6 files changed, 111 insertions(+), 22 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (62%)
create mode 100644 t/lei-refresh-mail-sync.t
diff --git a/MANIFEST b/MANIFEST
index 640eabd1..9f11f2f9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -235,9 +235,9 @@ lib/PublicInbox/LeiNoteEvent.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiPmdir.pm
-lib/PublicInbox/LeiPruneMailSync.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
+lib/PublicInbox/LeiRefreshMailSync.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiRmWatch.pm
@@ -450,6 +450,7 @@ t/lei-q-kw.t
t/lei-q-remote-import.t
t/lei-q-save.t
t/lei-q-thread.t
+t/lei-refresh-mail-sync.t
t/lei-sigpipe.t
t/lei-tag.t
t/lei-up.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ec103231..9794497b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -263,7 +263,7 @@ our %CMD = ( # sorted in order of importance/use:
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
-'prune-mail-sync' => [ 'LOCATION...|--all',
+'refresh-mail-sync' => [ 'LOCATION...|--all',
'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
@@ -616,6 +616,7 @@ sub pkt_ops {
$ops->{x_it} = [ \&x_it, $lei ];
$ops->{child_error} = [ \&child_error, $lei ];
$ops->{incr} = [ \&incr, $lei ];
+ $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
$ops;
}
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
similarity index 62%
rename from lib/PublicInbox/LeiPruneMailSync.pm
rename to lib/PublicInbox/LeiRefreshMailSync.pm
index 3678bd04..3c083965 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -1,16 +1,20 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# "lei prune-mail-sync" drops dangling sync information
-package PublicInbox::LeiPruneMailSync;
+# "lei refresh-mail-sync" drops dangling sync information
+# and attempts to detect moved files
+package PublicInbox::LeiRefreshMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub eml_match ($$) {
my ($eml, $oidbin) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
$oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
}
@@ -20,7 +24,7 @@ sub prune_mdir { # lms->each_src callback
for my $d (@try) {
my $src = "$mdir/$d/$$id";
if ($self->{verify}) {
- my $eml = eml_from_path($src) or next;
+ my $eml = eml_from_path($src) // next;
return if eml_match($eml, $oidbin);
} elsif (-f $src) {
return;
@@ -38,12 +42,27 @@ sub prune_imap { # lms->each_src callback
$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
}
+# detects missed file moves
+sub pmdir_cb { # called via LeiPmdir->each_mdir_fn
+ my ($self, $f, $fl) = @_;
+ my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+ die "BUG: $f was not from a Maildir?";
+ substr($folder, 0, 0) = 'maildir:'; # add prefix
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
+ return if defined($lms->name_oidbin($folder, $bn));
+ my $eml = eml_from_path($f) // return;
+ my $oidbin = $self->{lei}->git_oid($eml)->digest;
+ $self->{lei}->{sto}->ipc_do('lms_set_src', $oidbin, $folder, \$bn);
+}
+
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
- my $mdir = $1;
- $lms->each_src($input, \&prune_mdir, $self, $mdir);
+ $lms->each_src($input, \&prune_mdir, $self, my $mdir = $1);
+ $self->{lse} //= $self->{lei}->{sto}->search;
+ # call pmdir_cb (via maildir_each_file -> each_mdir_fn)
+ PublicInbox::LeiInput::input_path_url($self, $input);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{lei}->{net}->mic_for_folder($uri);
@@ -51,34 +70,31 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
$uids = +{ map { $_ => undef } @$uids };
$lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
} else { die "BUG: $input not supported" }
- my $wait = $self->{lei}->{sto}->ipc_do('done');
+ $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
}
-sub lei_prune_mail_sync {
+sub lei_refresh_mail_sync {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
EOM
- if (my $lms = $lei->lms) {
- if (defined(my $all = $lei->{opt}->{all})) {
- $lms->group2folders($lei, $all, \@folders) or return;
- } else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
- }
- } else {
- return $lei->fail(<<EOM);
+ my $lms = $lei->lms or return $lei->fail(<<EOM);
lei mail_sync.sqlite3 uninitialized, see lei-import(1)
EOM
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
}
+ undef $lms; # must be done before fork
$sto->write_prepare($lei);
my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
- $sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
@@ -89,7 +105,7 @@ EOM
}
no warnings 'once';
-*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e8bcb04e..32f55abd 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -293,6 +293,11 @@ sub set_sync_info {
_lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
}
+sub lms_set_src {
+ my ($self, $oidbin, $folder, $id) = @_;
+ _lms_rw($self)->set_src($oidbin, $folder, $id);
+}
+
sub _remove_if_local { # git->cat_async arg
my ($bref, $oidhex, $type, $size, $self) = @_;
$self->{im}->remove($bref) if $bref;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
index 9531949a..1fe940bb 100644
--- a/t/lei-export-kw.t
+++ b/t/lei-export-kw.t
@@ -6,7 +6,6 @@ use File::Copy qw(cp);
use File::Path qw(make_path);
require_mods(qw(lei -imapd Mail::IMAPClient));
my ($tmpdir, $for_destroy) = tmpdir;
-my ($ro_home, $cfg_path) = setup_public_inboxes;
my $expect = eml_load('t/data/0001.patch');
test_lei({ tmpdir => $tmpdir }, sub {
my $home = $ENV{HOME};
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
new file mode 100644
index 00000000..ff558277
--- /dev/null
+++ b/t/lei-refresh-mail-sync.t
@@ -0,0 +1,67 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+
+my $stop_daemon = sub { # needed since we don't have inotify
+ lei_ok qw(daemon-pid);
+ chomp(my $pid = $lei_out);
+ $pid > 0 or xbail "bad pid: $pid";
+ kill('TERM', $pid) or xbail "kill: $!";
+ for (0..10) {
+ tick;
+ kill(0, $pid) or last;
+ }
+ kill(0, $pid) and xbail "daemon still running (PID:$pid)";
+};
+
+test_lei({ daemon_only => 1 }, sub {
+ my $d = "$ENV{HOME}/d";
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
+ lei_ok qw(daemon-pid);
+ lei_ok qw(add-external), "$ro_home/t2";
+ lei_ok qw(q mid:testmessage@example.com -o), "Maildir:$d";
+ my (@o) = glob("$d/*/*");
+ scalar(@o) == 1 or xbail('multiple results', \@o);
+ my ($bn0) = ($o[0] =~ m!/([^/]+)\z!);
+
+ my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+ lei_ok 'inspect', "blob:$oid";
+ my $before = json_utf8->decode($lei_out);
+ my $exp0 = { 'mail-sync' => { "maildir:$d" => [ $bn0 ] } };
+ is_deeply($before, $exp0, 'inspect shows expected');
+
+ $stop_daemon->();
+ my $dst = $o[0];
+ $dst =~ s/:2,.*\z// and $dst =~ s!/cur/!/new/! and
+ rename($o[0], $dst) or xbail "rename($o[0] => $dst): $!";
+
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out),
+ $before, 'inspect unchanged immediately after restart');
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ my ($bn1) = ($dst =~ m!/([^/]+)\z!);
+ my $exp1 = { 'mail-sync' => { "maildir:$d" => [ $bn1 ] } };
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'refresh-mail-sync updated location');
+
+ $stop_daemon->();
+ rename($dst, "$d/unwatched") or xbail "rename $dst out-of-the-way $!";
+
+ lei_ok 'refresh-mail-sync', $d;
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, '{}', 'no known locations after "removal"');
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+
+ $stop_daemon->();
+ rename("$d/unwatched", $dst) or xbail "rename $dst back";
+
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'replaced file noted again');
+});
+
+done_testing;
^ permalink raw reply related [relevance 36%]
* Re: lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-17 0:22 89% ` Eric Wong
@ 2021-09-17 0:48 90% ` Luis Chamberlain
0 siblings, 0 replies; 200+ results
From: Luis Chamberlain @ 2021-09-17 0:48 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 16, 2021 at 5:22 PM Eric Wong <e@80x24.org> wrote:
> Since epochs expose the gittyness of lei, perhaps a limited form
> of commitish-like ranges used by git would work:
>
> lei add-external --mirror $URL --epoch=0..9
> lei add-external --mirror $URL --epoch=5..
> lei add-external --mirror $URL --epoch=~2.. # like HEAD~2..
That looks sexy.
I mentioned the query option so that I could end up displaying on
'make menuconfig' the epochs dynamically, it would be a script that
wants the list of epochs, and I'd use this to dynamically populate
kconfig. Dynamic population of kconfig is not a yet well understood
thing, but I've figured out a way to do it :)
> That only works for contiguous ranges, though; but I'm also
> unsure if discontiguous ranges make sense to anyone.
>
> There's other syntaxes (e.g. (sed|Perl)-like), of course,
> but I think git-like here would be most natural.
>
> Thoughts?
I think the above would suffice for sure!
Luis
^ permalink raw reply [relevance 90%]
* Re: lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-16 23:53 90% ` Luis Chamberlain
@ 2021-09-17 0:22 89% ` Eric Wong
2021-09-17 0:48 90% ` Luis Chamberlain
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-17 0:22 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> On Thu, Sep 16, 2021 at 09:09:02PM +0000, Eric Wong wrote:
> > Luis Chamberlain <mcgrof@kernel.org> wrote:
> > > # The next two don't work
> > > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> >
> > Btw, extra "import" aside; I never intended to support the above
> > case. However, your post got me thinking we could and probably
> > should... :>
> >
> > (epochs are git storage with a loose limit of ~1G packed data,
> > "slices" are public-inbox-imapd folders capped at 50K to
> > accomodate limitations of existing clients/filesystems)
>
> So... by default all epochs are pulled? If so, yeah geesh.
> I don't want to pull all linux-kernel epochs. Only if doing
> R&D on that list would I need it.
Right now, "lei import" doesn't read nor understand git repos at all.
If you meant add-external, yes, all epochs are pulled when using
"lei add-external --mirror" (and public-inbox-clone). Yes,
there should be a way to do partial mirrors, just haven't
gotten around to it...
"lei import" currently reads IMAP, NNTP, Maildir, mbox*, etc.
It also understands various https?://*/(raw,t.mbox.gz) URLs used by
PublicInbox::WWW, since they're (gzipped) mboxrd.
But none of that is reading from git repos directly...
> Maybe a --query-epochs and then a --use-epoch 0 or whatever
> as well?
Since epochs expose the gittyness of lei, perhaps a limited form
of commitish-like ranges used by git would work:
lei add-external --mirror $URL --epoch=0..9
lei add-external --mirror $URL --epoch=5..
lei add-external --mirror $URL --epoch=~2.. # like HEAD~2..
That only works for contiguous ranges, though; but I'm also
unsure if discontiguous ranges make sense to anyone.
There's other syntaxes (e.g. (sed|Perl)-like), of course,
but I think git-like here would be most natural.
Thoughts?
^ permalink raw reply [relevance 89%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-16 17:38 71% ` Konstantin Ryabitsev
@ 2021-09-16 23:54 71% ` Luis Chamberlain
0 siblings, 0 replies; 200+ results
From: Luis Chamberlain @ 2021-09-16 23:54 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
On Thu, Sep 16, 2021 at 01:38:32PM -0400, Konstantin Ryabitsev wrote:
> On Wed, Sep 15, 2021 at 02:34:40PM -0700, Luis Chamberlain wrote:
> > My use case is I'm subscribed to a few kernel mailign lists and I use
> > mutt with Maildir. I had configured recently pi-piper and grokmirror
> > so that I get only the last 1 year of email from a few set of mailing
> > lists. For this I needed to know the commit IDs for those emails on
> > the public-inbox git mirror for each mailing list.
>
> The pi-piper bit was really only useful for this until lei showed up. I wrote
> it mostly so we could pipe things to patchwork straight from public-inbox git
> archives.
OK got it! In lei we trust.
> > I was hinted using lei would be better though. But I'm stuck:
>
> FYI, I'm giving a talk about that on Monday.
> https://linuxplumbersconf.org/event/11/contributions/983/
>
> Assuming I finish the prep work by then.
>
> Hopefully, you don't live on the US West Coast and don't have to wake up at
> 7AM to attend.
I do.. but hey I sometimes have 5am meetings so I might be able to make it!
Luis
^ permalink raw reply [relevance 71%]
* Re: lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
@ 2021-09-16 23:53 90% ` Luis Chamberlain
2021-09-17 0:22 89% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Luis Chamberlain @ 2021-09-16 23:53 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 16, 2021 at 09:09:02PM +0000, Eric Wong wrote:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
> > # The next two don't work
> > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
>
> Btw, extra "import" aside; I never intended to support the above
> case. However, your post got me thinking we could and probably
> should... :>
>
> (epochs are git storage with a loose limit of ~1G packed data,
> "slices" are public-inbox-imapd folders capped at 50K to
> accomodate limitations of existing clients/filesystems)
So... by default all epochs are pulled? If so, yeah geesh.
I don't want to pull all linux-kernel epochs. Only if doing
R&D on that list would I need it.
Maybe a --query-epochs and then a --use-epoch 0 or whatever
as well?
Luis
^ permalink raw reply [relevance 90%]
* lei import on epochs [was: make menuconfig interface for lei / grok-pull]
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
` (2 preceding siblings ...)
2021-09-16 17:38 71% ` Konstantin Ryabitsev
@ 2021-09-16 21:09 90% ` Eric Wong
2021-09-16 23:53 90% ` Luis Chamberlain
3 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-16 21:09 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> # The next two don't work
> lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
Btw, extra "import" aside; I never intended to support the above
case. However, your post got me thinking we could and probably
should... :>
(epochs are git storage with a loose limit of ~1G packed data,
"slices" are public-inbox-imapd folders capped at 50K to
accomodate limitations of existing clients/filesystems)
^ permalink raw reply [relevance 90%]
* IMAP vs NNTP [was: make menuconfig interface for lei / grok-pull]
2021-09-16 17:43 71% ` Konstantin Ryabitsev
@ 2021-09-16 20:04 70% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 20:04 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: Luis Chamberlain, meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Wed, Sep 15, 2021 at 11:06:05PM +0000, Eric Wong wrote:
> > Does lore.kernel.org run public-inbox-imapd?
>
> I'm still not convinced it's useful for huge collections, especially
> considering how chatty IMAP is. Is there any benefit to enable it for lei use?
lei tries to treat IMAP and NNTP the same for (import|tag).
One benefit of IMAP over NNTP is it doesn't add Xref or
Newsgroups headers that could throw off OID-based deduplication.
The other is compression (see below).
lei will be able to use IMAP IDLE to detect new messages for
auto-import; AFAIK there's nothing like it in NNTP.
In general, client-side support for read-only IMAP folders is
lacking, but lei isn't too different in that regard.
On the Perl side (which affects lei):
* the Mail::IMAPClient maintainer has been responsive to
bugfixes I've sent, while the Net::NNTP (libnet) maintainer
hasn't been.
* as a result, Net::NNTP doesn't have DEFLATE support, yet
https://rt.cpan.org/Ticket/Display.html?id=129966
https://rt.cpan.org/Ticket/Display.html?id=129967
* Net::NNTP is bundled with Perl, but Mail::IMAPClient is a
separate package (but widely-packaged by distros)
^ permalink raw reply [relevance 70%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 23:06 57% ` Eric Wong
@ 2021-09-16 17:43 71% ` Konstantin Ryabitsev
2021-09-16 20:04 70% ` IMAP vs NNTP [was: make menuconfig interface for lei / grok-pull] Eric Wong
2021-09-19 21:21 71% ` make menuconfig interface for lei / grok-pull Eric Wong
1 sibling, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-16 17:43 UTC (permalink / raw)
To: Eric Wong; +Cc: Luis Chamberlain, meta
On Wed, Sep 15, 2021 at 11:06:05PM +0000, Eric Wong wrote:
> Does lore.kernel.org run public-inbox-imapd?
I'm still not convinced it's useful for huge collections, especially
considering how chatty IMAP is. Is there any benefit to enable it for lei use?
-K
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 23:06 57% ` Eric Wong
@ 2021-09-16 17:38 71% ` Konstantin Ryabitsev
2021-09-16 23:54 71% ` Luis Chamberlain
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
3 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-16 17:38 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
On Wed, Sep 15, 2021 at 02:34:40PM -0700, Luis Chamberlain wrote:
> My use case is I'm subscribed to a few kernel mailign lists and I use
> mutt with Maildir. I had configured recently pi-piper and grokmirror
> so that I get only the last 1 year of email from a few set of mailing
> lists. For this I needed to know the commit IDs for those emails on
> the public-inbox git mirror for each mailing list.
The pi-piper bit was really only useful for this until lei showed up. I wrote
it mostly so we could pipe things to patchwork straight from public-inbox git
archives.
> I was hinted using lei would be better though. But I'm stuck:
FYI, I'm giving a talk about that on Monday.
https://linuxplumbersconf.org/event/11/contributions/983/
Assuming I finish the prep work by then.
Hopefully, you don't live on the US West Coast and don't have to wake up at
7AM to attend.
-K
^ permalink raw reply [relevance 71%]
* [PATCH] lei: sqlite: do not forcibly enable WAL
@ 2021-09-16 11:04 66% Eric Wong
2021-09-17 2:16 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-16 11:04 UTC (permalink / raw)
To: meta
I'm not sure why and I can no longer reproduce it; but I've
triggered a SIGBUS in the lei/store inside the WAL commit code
while writing to mail_sync.sqlite3 on my Debian (oldstable)
buster system.
It could be a problem with my ancient SSD, improper use on our
end, a bug fixed in a newer SQLite version, etc.. In any case,
lets minimize the risk and keep using TRUNCATE like we do with
the time-tested public-inbox-* stuff, but respect WAL if a user
wants to set it themselves.
---
lib/PublicInbox/LeiMailSync.pm | 5 ++++-
lib/PublicInbox/LeiSavedSearch.pm | 5 +----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 5a10c127..dc4395f0 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -21,8 +21,11 @@ sub dbh_new {
sqlite_use_immediate_transaction => 1,
});
# no sqlite_unicode, here, all strings are binary
+ # TRUNCATE reduces I/O compared to the default (DELETE).
+ # Allow and preserve user-overridden WAL, but don't force it.
+ my $jm = $dbh->selectrow_array('PRAGMA journal_mode');
+ $dbh->do('PRAGMA journal_mode = TRUNCATE') if $jm ne 'wal';
create_tables($dbh) if $rw;
- $dbh->do('PRAGMA journal_mode = WAL') if $creat;
$dbh->do('PRAGMA case_sensitive_like = ON');
$dbh;
}
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 96ff816e..8fcfa752 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -229,10 +229,7 @@ sub prepare_dedupe {
my $oidx = PublicInbox::OverIdx->new($self->{-ovf});
$oidx->{-no_fsync} = 1;
$oidx->dbh;
- if ($creat) {
- $oidx->{dbh}->do('PRAGMA journal_mode = WAL');
- $oidx->eidx_prep; # for xref3
- }
+ $oidx->eidx_prep if $creat; # for xref3
$oidx
};
}
^ permalink raw reply related [relevance 66%]
* [PATCH 0/3] lei refresh-mail-sync
@ 2021-09-16 9:41 71% Eric Wong
2021-09-16 9:41 59% ` [PATCH 1/3] lei: git_oid: replace git_blob_id Eric Wong
2021-09-16 9:41 39% ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-16 9:41 UTC (permalink / raw)
To: meta
The command I need because lei-daemon keeps dying so inotify
ain't useful :P Well, actually, I just restart lei-daemon
frequently because I'm always changing code.
1/3 is a nice cleanup, 2/3 makes the change
AFAIK, refresh-mail-sync (and prune-mail-sync before it)
were the only which could trigger the bug fixed in PATCH 3/3
Eric Wong (3):
lei: git_oid: replace git_blob_id
lei refresh-mail-sync: replace prune-mail-sync
net_reader: ensure IO::Socket::Socks is loaded in all workers
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 13 ++--
lib/PublicInbox/LeiInspect.pm | 3 +-
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 36 +++++++---
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 14 ++--
lib/PublicInbox/LeiXSearch.pm | 3 +-
lib/PublicInbox/NetReader.pm | 5 ++
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 67 +++++++++++++++++++
10 files changed, 121 insertions(+), 25 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (70%)
create mode 100644 t/lei-refresh-mail-sync.t
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei: git_oid: replace git_blob_id
2021-09-16 9:41 71% [PATCH 0/3] lei refresh-mail-sync Eric Wong
@ 2021-09-16 9:41 59% ` Eric Wong
2021-09-16 9:41 39% ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 9:41 UTC (permalink / raw)
To: meta
We'll be using binary SHA-1 and SHA-256 in-memory since that's
what mail_sync.sqlite3 stores.
---
lib/PublicInbox/LEI.pm | 10 +++++++---
lib/PublicInbox/LeiInspect.pm | 3 ++-
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 9 +--------
lib/PublicInbox/LeiXSearch.pm | 3 ++-
5 files changed, 13 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0a30bc36..ec103231 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -24,6 +24,8 @@ use PublicInbox::DS qw(now dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
use PublicInbox::Eml;
+use PublicInbox::Import;
+use PublicInbox::ContentHash qw(git_sha);
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
@@ -1479,9 +1481,11 @@ sub refresh_watches {
}
}
-sub git_blob_id {
- my ($lei, $eml) = @_;
- ($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
+# TODO: support SHA-256
+sub git_oid {
+ my $eml = $_[-1];
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+ git_sha(1, $eml);
}
sub lms { # read-only LeiMailSync
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 25bd47e7..2385f7f8 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -202,7 +202,8 @@ sub ins_add { # InputPipe->consume callback
my $str = delete $lei->{istr};
$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
my $eml = PublicInbox::Eml->new(\$str);
- _inspect_argv($lei, [ 'blob:'.$lei->git_blob_id($eml),
+ _inspect_argv($lei, [
+ 'blob:'.$lei->git_oid($eml)->hexdigest,
map { "mid:$_" } @{mids($eml)} ]);
};
$lei->{istr} .= $_[1];
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 580787c0..8d4ffed0 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -32,7 +32,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
$smsg = $res if ref($res) eq ref($smsg);
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : $lei->git_blob_id($eml);
+ : $lei->git_oid($eml)->hexdigest;
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
push @{$self->{smsg}}, $smsg;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 42f574f2..e8bcb04e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -20,7 +20,7 @@ use PublicInbox::Eml;
use PublicInbox::Import;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash git_sha);
+use PublicInbox::ContentHash qw(content_hash);
use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
@@ -603,13 +603,6 @@ sub write_prepare {
$lei->{sto} = $self;
}
-# TODO: support SHA-256
-sub git_blob_id { # called via LEI->git_blob_id
- my ($self, $eml) = @_;
- $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
- git_sha(1, $eml)->hexdigest;
-}
-
# called by lei-daemon before lei->refresh_watches
sub add_sync_folders {
my ($self, @folders) = @_;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 556ffd58..50cadb5e 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -275,7 +275,8 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
$smsg->{kw} = []; # short-circuit xsmsg_vmd
}
}
- $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
+ : $lei->git_oid($eml)->hexdigest;
_smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
^ permalink raw reply related [relevance 59%]
* [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync
2021-09-16 9:41 71% [PATCH 0/3] lei refresh-mail-sync Eric Wong
2021-09-16 9:41 59% ` [PATCH 1/3] lei: git_oid: replace git_blob_id Eric Wong
@ 2021-09-16 9:41 39% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 9:41 UTC (permalink / raw)
To: meta
Merely pruning mail synchronization information was
insufficient for Maildir: renames are common in Maildir
and we need to detect them after-the-fact when lei-daemon
isn't running.
Running this command could make "lei index" far more
useful...
---
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 3 +-
...PruneMailSync.pm => LeiRefreshMailSync.pm} | 36 +++++++---
lib/PublicInbox/LeiStore.pm | 5 ++
t/lei-export-kw.t | 1 -
t/lei-refresh-mail-sync.t | 67 +++++++++++++++++++
6 files changed, 103 insertions(+), 11 deletions(-)
rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (70%)
create mode 100644 t/lei-refresh-mail-sync.t
diff --git a/MANIFEST b/MANIFEST
index 640eabd1..221cb992 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -238,6 +238,7 @@ lib/PublicInbox/LeiPmdir.pm
lib/PublicInbox/LeiPruneMailSync.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
+lib/PublicInbox/LeiRefreshMailSync.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiRmWatch.pm
@@ -450,6 +451,7 @@ t/lei-q-kw.t
t/lei-q-remote-import.t
t/lei-q-save.t
t/lei-q-thread.t
+t/lei-refresh-mail-sync.t
t/lei-sigpipe.t
t/lei-tag.t
t/lei-up.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ec103231..9794497b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -263,7 +263,7 @@ our %CMD = ( # sorted in order of importance/use:
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
-'prune-mail-sync' => [ 'LOCATION...|--all',
+'refresh-mail-sync' => [ 'LOCATION...|--all',
'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
@@ -616,6 +616,7 @@ sub pkt_ops {
$ops->{x_it} = [ \&x_it, $lei ];
$ops->{child_error} = [ \&child_error, $lei ];
$ops->{incr} = [ \&incr, $lei ];
+ $ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
$ops;
}
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
similarity index 70%
rename from lib/PublicInbox/LeiPruneMailSync.pm
rename to lib/PublicInbox/LeiRefreshMailSync.pm
index 3678bd04..07b0aa52 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -1,16 +1,20 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# "lei prune-mail-sync" drops dangling sync information
-package PublicInbox::LeiPruneMailSync;
+# "lei refresh-mail-sync" drops dangling sync information
+# and attempts to detect moved files
+package PublicInbox::LeiRefreshMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiExportKw;
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub eml_match ($$) {
my ($eml, $oidbin) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
$oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
}
@@ -20,7 +24,7 @@ sub prune_mdir { # lms->each_src callback
for my $d (@try) {
my $src = "$mdir/$d/$$id";
if ($self->{verify}) {
- my $eml = eml_from_path($src) or next;
+ my $eml = eml_from_path($src) // next;
return if eml_match($eml, $oidbin);
} elsif (-f $src) {
return;
@@ -38,12 +42,27 @@ sub prune_imap { # lms->each_src callback
$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
}
+# detects missed file moves
+sub pmdir_cb { # called via LeiPmdir->each_mdir_fn
+ my ($self, $f, $fl) = @_;
+ my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+ die "BUG: $f was not from a Maildir?";
+ substr($folder, 0, 0) = 'maildir:'; # add prefix
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
+ return if defined($lms->name_oidbin($folder, $bn));
+ my $eml = eml_from_path($f) // return;
+ my $oidbin = $self->{lei}->git_oid($eml)->digest;
+ $self->{lei}->{sto}->ipc_do('lms_set_src', $oidbin, $folder, \$bn);
+}
+
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
- my $mdir = $1;
- $lms->each_src($input, \&prune_mdir, $self, $mdir);
+ $lms->each_src($input, \&prune_mdir, $self, my $mdir = $1);
+ $self->{lse} //= $self->{lei}->{sto}->search;
+ # call pmdir_cb (via maildir_each_file -> each_mdir_fn)
+ PublicInbox::LeiInput::input_path_url($self, $input);
} elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{lei}->{net}->mic_for_folder($uri);
@@ -51,10 +70,10 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
$uids = +{ map { $_ => undef } @$uids };
$lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
} else { die "BUG: $input not supported" }
- my $wait = $self->{lei}->{sto}->ipc_do('done');
+ $self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
}
-sub lei_prune_mail_sync {
+sub lei_refresh_mail_sync {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
@@ -78,7 +97,6 @@ EOM
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
- $sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
@@ -89,7 +107,7 @@ EOM
}
no warnings 'once';
-*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e8bcb04e..32f55abd 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -293,6 +293,11 @@ sub set_sync_info {
_lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
}
+sub lms_set_src {
+ my ($self, $oidbin, $folder, $id) = @_;
+ _lms_rw($self)->set_src($oidbin, $folder, $id);
+}
+
sub _remove_if_local { # git->cat_async arg
my ($bref, $oidhex, $type, $size, $self) = @_;
$self->{im}->remove($bref) if $bref;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
index 9531949a..1fe940bb 100644
--- a/t/lei-export-kw.t
+++ b/t/lei-export-kw.t
@@ -6,7 +6,6 @@ use File::Copy qw(cp);
use File::Path qw(make_path);
require_mods(qw(lei -imapd Mail::IMAPClient));
my ($tmpdir, $for_destroy) = tmpdir;
-my ($ro_home, $cfg_path) = setup_public_inboxes;
my $expect = eml_load('t/data/0001.patch');
test_lei({ tmpdir => $tmpdir }, sub {
my $home = $ENV{HOME};
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
new file mode 100644
index 00000000..ff558277
--- /dev/null
+++ b/t/lei-refresh-mail-sync.t
@@ -0,0 +1,67 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+
+my $stop_daemon = sub { # needed since we don't have inotify
+ lei_ok qw(daemon-pid);
+ chomp(my $pid = $lei_out);
+ $pid > 0 or xbail "bad pid: $pid";
+ kill('TERM', $pid) or xbail "kill: $!";
+ for (0..10) {
+ tick;
+ kill(0, $pid) or last;
+ }
+ kill(0, $pid) and xbail "daemon still running (PID:$pid)";
+};
+
+test_lei({ daemon_only => 1 }, sub {
+ my $d = "$ENV{HOME}/d";
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
+ lei_ok qw(daemon-pid);
+ lei_ok qw(add-external), "$ro_home/t2";
+ lei_ok qw(q mid:testmessage@example.com -o), "Maildir:$d";
+ my (@o) = glob("$d/*/*");
+ scalar(@o) == 1 or xbail('multiple results', \@o);
+ my ($bn0) = ($o[0] =~ m!/([^/]+)\z!);
+
+ my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+ lei_ok 'inspect', "blob:$oid";
+ my $before = json_utf8->decode($lei_out);
+ my $exp0 = { 'mail-sync' => { "maildir:$d" => [ $bn0 ] } };
+ is_deeply($before, $exp0, 'inspect shows expected');
+
+ $stop_daemon->();
+ my $dst = $o[0];
+ $dst =~ s/:2,.*\z// and $dst =~ s!/cur/!/new/! and
+ rename($o[0], $dst) or xbail "rename($o[0] => $dst): $!";
+
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out),
+ $before, 'inspect unchanged immediately after restart');
+ lei_ok 'refresh-mail-sync', '--all';
+ lei_ok 'inspect', "blob:$oid";
+ my ($bn1) = ($dst =~ m!/([^/]+)\z!);
+ my $exp1 = { 'mail-sync' => { "maildir:$d" => [ $bn1 ] } };
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'refresh-mail-sync updated location');
+
+ $stop_daemon->();
+ rename($dst, "$d/unwatched") or xbail "rename $dst out-of-the-way $!";
+
+ lei_ok 'refresh-mail-sync', $d;
+ lei_ok 'inspect', "blob:$oid";
+ is($lei_out, '{}', 'no known locations after "removal"');
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+
+ $stop_daemon->();
+ rename("$d/unwatched", $dst) or xbail "rename $dst back";
+
+ lei_ok 'refresh-mail-sync', "Maildir:$d";
+ lei_ok 'inspect', "blob:$oid";
+ is_deeply(json_utf8->decode($lei_out), $exp1,
+ 'replaced file noted again');
+});
+
+done_testing;
^ permalink raw reply related [relevance 39%]
* [PATCH] doc: lei-mail-formats: add "eml" and expand on git things
@ 2021-09-16 7:45 63% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 7:45 UTC (permalink / raw)
To: meta
While "eml" is not an output format, it seems worthy
to document, here, since users are likely to have experience
with *.patch files from "git format-patch".
---
Documentation/lei-mail-formats.pod | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/Documentation/lei-mail-formats.pod b/Documentation/lei-mail-formats.pod
index da723d2b..3c37c880 100644
--- a/Documentation/lei-mail-formats.pod
+++ b/Documentation/lei-mail-formats.pod
@@ -51,7 +51,9 @@ with any number of C<E<gt>> characters and is thus fully
reversible.
This format is emitted by L<PublicInbox::WWW(3pm)> with gzip.
-It is supported by L<git-am(1)> since git 2.10.
+Since git 2.10, C<git am --patch-format=mboxrd> reads this
+format. C<git log> and C<git format-patch --stdout> can also
+generate this format with the C<--pretty=mboxrd> switch.
As with uncompressed L</mboxo>, uncompressed mboxrd are vulnerable
to undetectable truncation.
@@ -96,6 +98,22 @@ Depending on the IMAP server software and configuration, IMAP
servers may use any (or combination) of the aforementioned
formats or a non-standard database backend.
+=head1 eml
+
+A single raw message file. C<eml> is not an output format for lei,
+but accepted by as an C<--input-format> (C<-F>) for read-only
+commands such as L<lei-tag(1)> and L<lei-import(1)>.
+
+Since C<eml> is the suffix for the C<message/rfc822> MIME type
+(according to the C<mime.types> file), lei will infer the type
+based on the C<.eml> suffix if C<--input-format> is unspecified
+
+C<.patch>-suffixed files generated by L<git-format-patch(1)>
+(without C<--stdout>) are C<eml> files with the addition of an
+mbox C<From > header. L<lei(1)> removes C<From > lines to treat
+them as C<eml> when reading these for compatibility with
+C<git-am(1)> and similar tools.
+
=head1 COPYRIGHT
Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
^ permalink raw reply related [relevance 63%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
2021-09-15 22:36 71% ` Kyle Meyer
@ 2021-09-16 2:59 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 2:59 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Btw, without having any externals on disk:
lei q [OPTIONS] --only https://lore.kernel.org/linux-fsdevel/ SEARCH_TERMS...
works, and I use it often.
Perl^Wlei: there's more than one way to do it :>
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei ls-mail-source: sort IMAP folder names
@ 2021-09-16 2:19 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-16 2:19 UTC (permalink / raw)
To: meta
Otherwise, public-inbox-imapd will emit mailboxes in random
order (as IMAP servers do not need to guarantee any sort of
ordering). We'll take into account numeric slice numbers
generated by -imapd if they exist, so slice "80" doesn't show up
next to "8".
---
lib/PublicInbox/LeiLsMailSource.pm | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 71e253d9..f012e10e 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -27,6 +27,12 @@ sub input_path_url { # overrides LeiInput version
my $sec = $lei->{net}->can('uri_section')->($uri);
my $mic = $lei->{net}->mic_get($uri);
my $l = $mic->folders_hash($uri->path); # server-side filter
+ @$l = map { $_->[2] } # undo Schwartzian transform below:
+ sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
+ map { # prepare to sort -imapd slices numerically
+ $_->{name} =~ /\A(.+?)\.([0-9]+)\z/ ?
+ [ $1, $2 + 0, $_ ] : [ $_->{name}, -1, $_ ];
+ } @$l;
@f = map { "$sec/$_->{name}" } @$l;
if ($json) {
$_->{url} = "$sec/$_->{name}" for @$l;
^ permalink raw reply related [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 22:36 71% ` Kyle Meyer
@ 2021-09-15 23:31 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-15 23:31 UTC (permalink / raw)
To: Kyle Meyer; +Cc: Luis Chamberlain, meta
Kyle Meyer <kyle@kyleam.com> wrote:
> A good chunk of the current documentation was fumbled together by me, so
> I'm sure there are many spots for improvement but...
No worries, my brain is a disorganized mess (always has been,
but pandemic has made it worse) and I've realized a lot of my
design choices around public-inbox + lei is based around that :x
Btw, I'm thinking about replacing prune-mail-sync with
"refresh-mail-sync [--prune]", so probably no point
in documenting it just yet....
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:57 71% ` Luis Chamberlain
@ 2021-09-15 23:12 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-15 23:12 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> On Wed, Sep 15, 2021 at 09:44:20PM +0000, Eric Wong wrote:
> > Luis Chamberlain <mcgrof@kernel.org> wrote:
> > > # The next two don't work
> > > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> > > lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
> >
> > Apologies, I need to document this better. This should work:
> >
> > lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
>
> Groovy thanks! I'll start with that!
>
> Any reason we need to specify SOME_FS_PATH ? Can't we assume a sensible
> default somewhere in .local/lei or whatever in the future?
Probably OK to have a default path for --mirror, the
add-external w/o --mirror works for existing inboxes (which I
also serve via -httpd/-imapd/-nntpd).
I think ~/.local/public-inbox/ would be a better default location
for mirrors, since I assume anything mirrored is public (and
therefor to rehost, as well).
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
@ 2021-09-15 23:06 57% ` Eric Wong
2021-09-16 17:43 71% ` Konstantin Ryabitsev
2021-09-19 21:21 71% ` make menuconfig interface for lei / grok-pull Eric Wong
2021-09-16 17:38 71% ` Konstantin Ryabitsev
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
3 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-15 23:06 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> Greetings,
>
> I unsubscribed to most mailing lists thanks to public-inbox. Thanks
> to public-inbox my inbox went on quite a bit of diet.
>
> Now, the way I work though is... I like to hop onto different systems
> every now and then, and not rely on one central place for everything.
> This is because when I've done that before I get outages / network
> outages / etc.
>
> So I often have to start from scratch.
Same here, this goes for project code repos, as well...
At the same time, I don't want too much synchronization since
the systems I hop on to vary hugely in storage capacity.
<snip replied parts>
> lei q --only=linux-fsdevel <search-term>
>
> but its not clear if a date is supported as a search term.
All the search prefixes documented in
https://public-inbox.org/meta/_/text/help/ work for lei.
dt: (date-time), d: (date (mairix-compatible) have been there a while,
rt: (Received-time) is new.
They gained support for git approxidate recently, so "rt:last.week.."
gives you all the messages that were received from last week til now.
Anything w/o spaces which git understands should work.
> Is my intended use case better with grok-pull or is there
> hopes I can end up using lei for all what I have intentions for?
I think introducing support for "lei update-external [--all]"
could be useful combined with my previous note about
add-external.
> My vision is, kernel deveoper would 'make menuconfig' and select
> a few mailing lists they want to import, then when one is selected
> new options appear so you can pick the starting year from where
> you want the import to occur for. Then for each mailing list there
> is also a refresh thing, so that we can grok-pull differently
> for each mailing list, which will be handled by systemd timers
> and a service for each. The reason for differnt times for refresh
> is updating linux-kernel is not a priority to me, but linux-fsdevel
> and linux-block are, so I want linux-fsdevel and linux-block to
> refresh once every 10 minutes, while linux-kernel can update...
> I don't know once every 8 hours. Anyway, this should all be
> configurable. And my hope is that developers would not have to
> know what's going on behind the scenes. All they need to know
> is that their ~Mail/linux/ directory will eventually be stuffed
> with Maildir directories for the mailing lists they enabled and
> they'll be updated as often as specified in their configuration.
Interesting... I suppose it could work. I don't know much
about curses or interactive UIs, but "make menuconfig" isn't
a bad UI :) I've been halfway envisioning something else
that would use curses (more on that some other time).
Does lore.kernel.org run public-inbox-imapd?
Well, at least nntp works for lore:
# populate ~/.cache/lei/uri_folder/.sqlite3
lei ls-mail-source nntp://nntp.lore.kernel.org
# if you're using bash:
. contrib/completion/lei-completion.bash
lei import <TAB><TAB>
lei import will download the entire newsgroup, which may get
huge, unfortunately. It should support ranges (e.g.
nntp://example.com/inbox.foo.example/900-1000) but the URI
package doesn't support it, yet...
public-inbox-imapd IMAP mailbox slices are limited to 50K
messages since typical MUAs can't handle much more, so it's
easier to import just the most recent .[0-9]+ slices.
If you have Tor (on port 9050), libio-socket-socks (IO::Socket::Socks)
# long Tor v3 URL :<
url=imap://rskvuqcfnfizkjg6h5jvovwb3wkikzcwskf54lfpymus6mxrzw67b5ad.onion
# configure IMAP to use Tor on port 9050:
lei config imap.$url.proxy socks5h://0:9050
# git 2.26+ can use a wildcard for all .onions:
# lei config 'imap.imap://*.onion.proxy' socks5h://0:9050
# I also suggest enabling compression for public-inbox-imapd
# (some IMAP servers can't handle it, our -imapd does):
lei config imap.$url.compress 1
# Same as above with NNTP:
lei ls-mail-source $url
lei import <TAB-COMPLETE>
For public-inbox IMAP folders, make sure the \.[0-9]+ is at the
end of the mailbox to get the slice you want (there isn't
recursive import, yet).
Right now, "lei import" can be run periodically. The plan is to
support IMAP IDLE and auto-imports. For read/write IMAP
folders, there'd be auto keyword (FLAG) synchronization
(currently manual: lei export-kw imaps://...)
It mostly works for Maildirs at the moment...
...Which reminds me, I need to add an imapserver config variable
so the WWW interface can advertise IMAP servers like it does
NNTP servers
^ permalink raw reply [relevance 57%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
@ 2021-09-15 22:36 71% ` Kyle Meyer
2021-09-15 23:31 71% ` Eric Wong
2021-09-16 2:59 71% ` Eric Wong
2 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-09-15 22:36 UTC (permalink / raw)
To: Eric Wong; +Cc: Luis Chamberlain, meta
Eric Wong writes:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
>> # The next two don't work
>> lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
>> lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
>
> Apologies, I need to document this better. This should work:
A good chunk of the current documentation was fumbled together by me, so
I'm sure there are many spots for improvement but...
> lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
... hey, the equivalent of the snippet above is one of the few things
that made it into lei-overview :)
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:44 71% ` Eric Wong
@ 2021-09-15 21:57 71% ` Luis Chamberlain
2021-09-15 23:12 71% ` Eric Wong
2021-09-15 22:36 71% ` Kyle Meyer
2021-09-16 2:59 71% ` Eric Wong
2 siblings, 1 reply; 200+ results
From: Luis Chamberlain @ 2021-09-15 21:57 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Wed, Sep 15, 2021 at 09:44:20PM +0000, Eric Wong wrote:
> Luis Chamberlain <mcgrof@kernel.org> wrote:
> > # The next two don't work
> > lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> > lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
>
> Apologies, I need to document this better. This should work:
>
> lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
Groovy thanks! I'll start with that!
Any reason we need to specify SOME_FS_PATH ? Can't we assume a sensible
default somewhere in .local/lei or whatever in the future?
> > Also how about doing updates?
>
> public-inbox-fetch -C SOME_FS_PATH
> public-inbox-index SOME_FS_PATH
Neat!
> And I just posted some patches to simplify updates to:
>
> make -C SOME_FS_PATH
Oh, pretty nice.
> In case you reorganize your file system:
>
> lei forget-external OLD_FS_PATH
> lei add-external NEW_FS_PATH
>
> I'll read the rest and reply later. "lei index" is still
> half-baked, but the above should be fine.
Thanks!
Luis
^ permalink raw reply [relevance 71%]
* Re: make menuconfig interface for lei / grok-pull
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
@ 2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
` (2 more replies)
2021-09-15 23:06 57% ` Eric Wong
` (2 subsequent siblings)
3 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-15 21:44 UTC (permalink / raw)
To: Luis Chamberlain; +Cc: meta
Luis Chamberlain <mcgrof@kernel.org> wrote:
> # The next two don't work
> lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
> lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
Apologies, I need to document this better. This should work:
lei add-external --mirror https://lore.kernel.org/linux-fsdevel SOME_FS_PATH
> Also how about doing updates?
public-inbox-fetch -C SOME_FS_PATH
public-inbox-index SOME_FS_PATH
And I just posted some patches to simplify updates to:
make -C SOME_FS_PATH
In case you reorganize your file system:
lei forget-external OLD_FS_PATH
lei add-external NEW_FS_PATH
I'll read the rest and reply later. "lei index" is still
half-baked, but the above should be fine.
^ permalink raw reply [relevance 71%]
* make menuconfig interface for lei / grok-pull
@ 2021-09-15 21:34 52% Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Luis Chamberlain @ 2021-09-15 21:34 UTC (permalink / raw)
To: meta; +Cc: Luis Chamberlain
Greetings,
I unsubscribed to most mailing lists thanks to public-inbox. Thanks
to public-inbox my inbox went on quite a bit of diet.
Now, the way I work though is... I like to hop onto different systems
every now and then, and not rely on one central place for everything.
This is because when I've done that before I get outages / network
outages / etc.
So I often have to start from scratch.
My use case is I'm subscribed to a few kernel mailign lists and I use
mutt with Maildir. I had configured recently pi-piper and grokmirror
so that I get only the last 1 year of email from a few set of mailing
lists. For this I needed to know the commit IDs for those emails on
the public-inbox git mirror for each mailing list.
To start from scratch I could go into my logs and figure out what that
was again, and start from scratch... but I'm inpsired to just provide
a 'make menuconfig' interface for this so that all I have to do is
select the mailing list I want to git clone and then dump into Maildir
the last 1 year of content for each. Since this time frame of 1 year
can be variable, it would seem a good candidate would be to use kconfig
to allow those interested to associate the content year by year to git
tree and respective commit IDs on each mailing list epoch/git tree.
I was hinted using lei would be better though. But I'm stuck:
lei init
# The next two don't work
lei import import https://lore.kernel.org/linux-fsdevel/git/0.git
lei add-external git://lore.kernel.org/linux-fsdevel/git/0.git
I can use nttp but jeesh, really?
Anyway, once the above works, if one didn't want to have the git
entire git tree around I gather that I can use something like
lei index <same-path-as-above>
Then in so far as the date idea I mentioned, I take it the way to go
there might be after this to use something like:
lei q --only=linux-fsdevel <search-term>
but its not clear if a date is supported as a search term.
Also how about doing updates?
Is my intended use case better with grok-pull or is there
hopes I can end up using lei for all what I have intentions for?
My vision is, kernel deveoper would 'make menuconfig' and select
a few mailing lists they want to import, then when one is selected
new options appear so you can pick the starting year from where
you want the import to occur for. Then for each mailing list there
is also a refresh thing, so that we can grok-pull differently
for each mailing list, which will be handled by systemd timers
and a service for each. The reason for differnt times for refresh
is updating linux-kernel is not a priority to me, but linux-fsdevel
and linux-block are, so I want linux-fsdevel and linux-block to
refresh once every 10 minutes, while linux-kernel can update...
I don't know once every 8 hours. Anyway, this should all be
configurable. And my hope is that developers would not have to
know what's going on behind the scenes. All they need to know
is that their ~Mail/linux/ directory will eventually be stuffed
with Maildir directories for the mailing lists they enabled and
they'll be updated as often as specified in their configuration.
Luis
^ permalink raw reply [relevance 52%]
* Re: RFC: lei-daemon and auto-up
2021-09-14 20:42 71% RFC: lei-daemon and auto-up Konstantin Ryabitsev
@ 2021-09-14 22:32 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 22:32 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Hello:
>
> Since lei-daemon is already up and running, would it be possible to tell it to
> automatically "lei up" things at certain intervals?
Yes, something along those lines... It should at least fall
back to refresh intervals if network connectivity is lost or
HTTP servers w/o support for long-polling.
The ideal thing would be to POST HTTP requests that do
long-polling against -httpd and let curl wait on the response;
similar to IMAP IDLE.
-httpd will probably trickle zero-byte gzipped responses at a
regular intervals to keep connections alive if there is no data,
yet.
There's still a lot of things I want to do w/ inotify and
IDLE(-like) real-time notifications (including the new
public-inbox-fetch command). I don't like polling as a first
choice.
Also. I'm not sure what ways exist to be notified on network
up/down events on Linux or *BSDs w/o polling.
^ permalink raw reply [relevance 71%]
* RFC: lei-daemon and auto-up
@ 2021-09-14 20:42 71% Konstantin Ryabitsev
2021-09-14 22:32 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-14 20:42 UTC (permalink / raw)
To: meta
Hello:
Since lei-daemon is already up and running, would it be possible to tell it to
automatically "lei up" things at certain intervals?
Maybe something like:
[lei]
q = [...]
[lei "q"]
output = [...]
include = https://lore.kernel.org/all/
external = 1
local = 1
remote = 1
refresh = 300
That would allow folks to automatically get updated info without needing to
set up cronjobs or systemd timers.
-K
^ permalink raw reply [relevance 71%]
* [PATCH 1/5] lei: warn on event loop errors
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
@ 2021-09-14 2:39 71% ` Eric Wong
2021-09-14 2:39 71% ` [PATCH 2/5] lei: sto_done_request: add eval guard Eric Wong
2021-09-14 2:39 56% ` [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
This should help us notice (and fix) bugs more easily.
---
lib/PublicInbox/LEI.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 784e679d..f0caac03 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1347,7 +1347,8 @@ sub lazy_start {
open STDERR, '>&STDIN' or die "redirect stderr failed: $!";
open STDOUT, '>&STDIN' or die "redirect stdout failed: $!";
# $daemon pipe to `lei' closed, main loop begins:
- PublicInbox::DS->EventLoop;
+ eval { PublicInbox::DS->EventLoop };
+ warn "event loop error: $@\n" if $@;
dump_and_clear_log();
exit($exit_code // 0);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes
@ 2021-09-14 2:39 71% Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
The 5/5 "lei up" fix has real-world implications if you're
using it in parallel. Otherwise, it's test-only stuff, and
`make check-run TEST_LEI_DAEMON_PERSIST=1 N=$(nproc)`
seems to fail or stall less often than before (but it
still can...).
(persisting the lei-daemon is around 5% faster with check-run)
Eric Wong (5):
lei: warn on event loop errors
lei: sto_done_request: add eval guard
t/run: TEST_LEI_DAEMON_PERSIST: die if pid changes
test_common: remove non-hidden files, first
lei up: fix env/cwd mismatches with multiple folders
lib/PublicInbox/LEI.pm | 33 ++++++++++++++++-----------------
lib/PublicInbox/LeiUp.pm | 15 +++++++--------
lib/PublicInbox/TestCommon.pm | 5 ++++-
t/run.perl | 3 +++
4 files changed, 30 insertions(+), 26 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/5] lei: sto_done_request: add eval guard
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
@ 2021-09-14 2:39 71% ` Eric Wong
2021-09-14 2:39 56% ` [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
Failures here can cause the lei-daemon event loop to break
since PktOp doesn't guard dispatch. Add a guard here (and
not deeper in the stack) so we can use the $lei object to
report errors.
---
lib/PublicInbox/LEI.pm | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f0caac03..e529c86a 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1500,12 +1500,15 @@ sub lms { # read-only LeiMailSync
sub sto_done_request { # only call this from lei-daemon process (not workers)
my ($lei, $sock) = @_;
- if ($sock //= $lei->{sock}) {
- $LIVE_SOCK{"$sock"} = $sock;
- $lei->{sto}->ipc_do('done', "$sock"); # issue, async wait
- } else { # forcibly wait
- my $wait = $lei->{sto}->ipc_do('done');
- }
+ eval {
+ if ($sock //= $lei->{sock}) { # issue, async wait
+ $LIVE_SOCK{"$sock"} = $sock;
+ $lei->{sto}->ipc_do('done', "$sock");
+ } else { # forcibly wait
+ my $wait = $lei->{sto}->ipc_do('done');
+ }
+ };
+ $lei->err($@) if $@;
}
sub sto_done_complete { # called in lei-daemon when LeiStore->done is complete
^ permalink raw reply related [relevance 71%]
* [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
2021-09-14 2:39 71% ` [PATCH 2/5] lei: sto_done_request: add eval guard Eric Wong
@ 2021-09-14 2:39 56% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-14 2:39 UTC (permalink / raw)
To: meta
By moving %ENV localization and fchdir into ->dispatch,
we can maintain a consistent environment across multiple
dispatches while having different clients.
---
lib/PublicInbox/LEI.pm | 15 +++++----------
lib/PublicInbox/LeiUp.pm | 15 +++++++--------
2 files changed, 12 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e529c86a..0a30bc36 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -769,6 +769,8 @@ sub lazy_cb ($$$) {
sub dispatch {
my ($self, $cmd, @argv) = @_;
+ fchdir($self) or return;
+ local %ENV = %{$self->{env}};
local $current_lei = $self; # for __WARN__
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
return _help($self, 'no command given') unless defined($cmd);
@@ -1104,14 +1106,9 @@ sub accept_dispatch { # Listener {post_accept} callback
my ($argc, @argv) = split(/\0/, $buf, -1);
undef $buf;
my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
- if (chdir($self->{3})) {
- local %ENV = %env;
- $self->{env} = \%env;
- eval { dispatch($self, @argv) };
- send($sock, $@, MSG_EOR) if $@;
- } else {
- send($sock, "fchdir: $!", MSG_EOR); # implicit close
- }
+ $self->{env} = \%env;
+ eval { dispatch($self, @argv) };
+ send($sock, $@, MSG_EOR) if $@;
}
sub dclose {
@@ -1181,7 +1178,6 @@ sub cfg2lei ($) {
open($lei->{1}, '>>&', \*STDOUT) or die "dup 1: $!";
open($lei->{2}, '>>&', \*STDERR) or die "dup 2: $!";
open($lei->{3}, '/') or die "open /: $!";
- chdir($lei->{3}) or die "chdir /': $!";
my ($x, $y);
socketpair($x, $y, AF_UNIX, SOCK_SEQPACKET, 0) or die "socketpair: $!";
$lei->{sock} = $x;
@@ -1199,7 +1195,6 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
my $cfg = $PATH2CFG{$f} // next;
eval {
- local %ENV = %{$cfg->{-env}};
my $lei = cfg2lei($cfg);
$lei->dispatch('note-event',
"maildir:$mdir", $nc, $bn, $fn);
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 53f06dbc..4637cb46 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -11,7 +11,6 @@ use PublicInbox::LeiSavedSearch;
use PublicInbox::DS;
use PublicInbox::PktOp;
use PublicInbox::LeiFinmsg;
-use PublicInbox::LEI;
my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
sub up1 ($$) {
@@ -60,19 +59,16 @@ sub up1_redispatch {
} else { # multiple outputs
$l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} }; # deep copy
+ delete $l->{opt}->{all};
delete $l->{sock}; # do not close
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or
return $l->child_error(0, "dup: $!");
$l->{1} = $fh;
+ $l->qerr("# updating $out");
}
- local $PublicInbox::LEI::current_lei = $l;
- local %ENV = %{$l->{env}};
$l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
- eval {
- $l->qerr("# updating $out");
- up1($l, $out);
- };
+ eval { $l->dispatch('up', $out) };
$lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
}
@@ -94,7 +90,6 @@ sub lei_up {
my ($lei, @outs) = @_;
my $opt = $lei->{opt};
my $self = bless { -mail_sync => 1 }, __PACKAGE__;
- $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
defined($opt->{mua}) and return
@@ -110,10 +105,14 @@ sub lei_up {
} else {
$lei->fail("only --all=$all not understood");
}
+ } elsif ($lei->{lse}) {
+ scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
+ return up1($lei, $outs[0]);
} else {
$self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
+ $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
defined($opt->{mua})) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
^ permalink raw reply related [relevance 56%]
* [PATCH 4/6] lei up: localize %ENV in redispatch
2021-09-13 20:53 67% ` [PATCH 2/6] lei: stop_pager: restore stdout when done Eric Wong
@ 2021-09-13 20:53 71% ` Eric Wong
2021-09-13 20:53 62% ` [PATCH 6/6] lei up: fix --mua with single output Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-13 20:53 UTC (permalink / raw)
To: meta
We need to restore %ENV of script/lei in case another
lei client has a different %ENV than what daemon has.
---
lib/PublicInbox/LeiUp.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index be476427..a16117c9 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -55,13 +55,14 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
my $l = bless { %$lei }, ref($lei);
- $l->{opt} = { %{$l->{opt}} };
+ $l->{opt} = { %{$l->{opt}} }; # deep copy
delete $l->{sock}; # do not close
$l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
local $PublicInbox::LEI::current_lei = $l;
+ local %ENV = %{$l->{env}};
$l->{1} = $fh;
eval {
$l->qerr("# updating $out");
^ permalink raw reply related [relevance 71%]
* [PATCH 2/6] lei: stop_pager: restore stdout when done
@ 2021-09-13 20:53 67% ` Eric Wong
2021-09-13 20:53 71% ` [PATCH 4/6] lei up: localize %ENV in redispatch Eric Wong
2021-09-13 20:53 62% ` [PATCH 6/6] lei up: fix --mua with single output Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-13 20:53 UTC (permalink / raw)
To: meta
The reason(s) we had for not restoring stdout haven't been valid
since script/lei (and not lei-daemon) spawns the pager, nowadays.
---
lib/PublicInbox/LEI.pm | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6d5d3c03..784e679d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -464,11 +464,11 @@ sub x_it ($$) {
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
- if ($self->{pkt_op_p}) { # to top lei-daemon
+ if ($self->{pkt_op_p}) { # worker => lei-daemon
$self->{pkt_op_p}->pkt_do('x_it', $code);
- } elsif ($self->{sock}) { # to lei(1) client
+ } elsif ($self->{sock}) { # lei->daemon => lei(1) client
send($self->{sock}, "x_it $code", MSG_EOR);
- } elsif ($quit == \&CORE::exit) { # an admin command
+ } elsif ($quit == \&CORE::exit) { # an admin (one-shot) command
exit($code >> 8);
} # else ignore if client disconnected
}
@@ -1065,9 +1065,7 @@ sub pgr_err {
start_pager($self, { LESS => 'RX' }); # no 'F' so we prompt
print { $self->{2} } @msg;
$self->{2}->autoflush(1);
- my $pgr = delete($self->{pgr}) or return;
- $self->{2} = $pgr->[2];
- $self->{1} = $pgr->[1];
+ stop_pager($self);
send($self->{sock}, 'wait', MSG_EOR); # wait for user to quit pager
}
@@ -1075,8 +1073,8 @@ sub stop_pager {
my ($self) = @_;
my $pgr = delete($self->{pgr}) or return;
$self->{2} = $pgr->[2];
- # do not restore original stdout, just close it so we error out
close(delete($self->{1})) if $self->{1};
+ $self->{1} = $pgr->[1];
}
sub accept_dispatch { # Listener {post_accept} callback
^ permalink raw reply related [relevance 67%]
* [PATCH 6/6] lei up: fix --mua with single output
2021-09-13 20:53 67% ` [PATCH 2/6] lei: stop_pager: restore stdout when done Eric Wong
2021-09-13 20:53 71% ` [PATCH 4/6] lei up: localize %ENV in redispatch Eric Wong
@ 2021-09-13 20:53 62% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-13 20:53 UTC (permalink / raw)
To: meta
Oops :x
Fixes: b584a53f053a7629 ("lei up: support --all for IMAP folders")
---
lib/PublicInbox/LeiUp.pm | 25 +++++++++++++++----------
t/lei-up.t | 4 ++++
2 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index a16117c9..53f06dbc 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -54,16 +54,21 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
- my $l = bless { %$lei }, ref($lei);
- $l->{opt} = { %{$l->{opt}} }; # deep copy
- delete $l->{sock}; # do not close
- $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
-
- # make close($l->{1}) happy in lei->dclose
- open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ my $l;
+ if (defined($lei->{opt}->{mua})) { # single output
+ $l = $lei;
+ } else { # multiple outputs
+ $l = bless { %$lei }, ref($lei);
+ $l->{opt} = { %{$l->{opt}} }; # deep copy
+ delete $l->{sock}; # do not close
+ # make close($l->{1}) happy in lei->dclose
+ open my $fh, '>&', $l->{1} or
+ return $l->child_error(0, "dup: $!");
+ $l->{1} = $fh;
+ }
local $PublicInbox::LEI::current_lei = $l;
local %ENV = %{$l->{env}};
- $l->{1} = $fh;
+ $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
eval {
$l->qerr("# updating $out");
up1($l, $out);
@@ -92,7 +97,7 @@ sub lei_up {
$lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
- length($opt->{mua}//'') and return
+ defined($opt->{mua}) and return
$lei->fail('--all and --mua= are incompatible');
@outs = PublicInbox::LeiSavedSearch::list($lei);
if ($all eq 'local') {
@@ -110,7 +115,7 @@ sub lei_up {
$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
- length($opt->{mua} // '')) and return $lei->fail(<<EOM);
+ defined($opt->{mua})) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
if ($self->{remote}) { # setup lei->{auth}
diff --git a/t/lei-up.t b/t/lei-up.t
index 6b34774d..8937cfb1 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -34,6 +34,10 @@ test_lei(sub {
open $fh, "$ENV{HOME}/b" or xbail "open: $!";
$uc = do { local $/; <$fh> };
is($uc, $exp, 'uncompressed both match');
+
+ lei_ok [ 'up', "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
+ undef, { run_mode => 0 };
+ ok(-f "$ENV{HOME}/c", '--mua works with single output');
});
done_testing;
^ permalink raw reply related [relevance 62%]
* [PATCH 1/3] t/lei-*.t: guard setup_public_inboxes with test_lei
@ 2021-09-12 8:42 67% ` Eric Wong
2021-09-12 8:42 65% ` [PATCH 2/3] lei sucks: allow it to work without SQLite Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-12 8:42 UTC (permalink / raw)
To: meta
This ensures tests are skipped properly if SQLite or Xapian
are missing and don't bail out.
---
t/lei-auto-watch.t | 3 +--
t/lei-up.t | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index 3b0c1b10..e5e132eb 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -3,14 +3,13 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Basename qw(basename);
-my ($ro_home, $cfg_path) = setup_public_inboxes;
my $have_fast_inotify = eval { require Linux::Inotify2 } ||
eval { require IO::KQueue };
-
$have_fast_inotify or
diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
test_lei(sub {
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
my $x = "$ENV{HOME}/x";
my $y = "$ENV{HOME}/y";
lei_ok qw(add-external), "$ro_home/t1";
diff --git a/t/lei-up.t b/t/lei-up.t
index c6f31c74..6b34774d 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -2,9 +2,9 @@
# Copyright all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
-my ($ro_home, $cfg_path) = setup_public_inboxes;
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
test_lei(sub {
+ my ($ro_home, $cfg_path) = setup_public_inboxes;
my $s = eml_load('t/plack-qp.eml')->as_string;
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a.mbox.gz";
^ permalink raw reply related [relevance 67%]
* [PATCH 2/3] lei sucks: allow it to work without SQLite
2021-09-12 8:42 67% ` [PATCH 1/3] t/lei-*.t: guard setup_public_inboxes with test_lei Eric Wong
@ 2021-09-12 8:42 65% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-12 8:42 UTC (permalink / raw)
To: meta
And try to improve the message about Inline::C while we're at
it, since Socket::Msghdr isn't widely-packaged, yet.
---
lib/PublicInbox/LeiSucks.pm | 4 ++--
lib/PublicInbox/TestCommon.pm | 12 +++++++-----
t/lei.t | 4 ++++
3 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index 3e945d0b..e832f95e 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -11,7 +11,6 @@ use Digest::SHA ();
use Config;
use POSIX ();
use PublicInbox::Config;
-use PublicInbox::Search;
sub lei_sucks {
my ($lei, @argv) = @_;
@@ -41,7 +40,8 @@ sub lei_sucks {
} else {
push @out, "Unable to load DBI / DBD::SQLite: $@\n";
}
- if (PublicInbox::Search::load_xapian()) {
+ if (eval { require PublicInbox::Search } &&
+ PublicInbox::Search::load_xapian()) {
push @out, 'Xapian '.
join('.', map {
$PublicInbox::Search::Xap->can($_)->();
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 14dac03f..d28b32b0 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -521,7 +521,8 @@ SKIP: {
local $lei_cwdfh;
opendir $lei_cwdfh, '.' or xbail "opendir .: $!";
require_git(2.6, 1) or skip('git 2.6+ required for lei test', 2);
- require_mods(qw(json DBD::SQLite Search::Xapian), 2);
+ my $mods = $test_opt->{mods} // [ 'lei' ];
+ require_mods(@$mods, 2);
require PublicInbox::Config;
require File::Path;
local %ENV = %ENV;
@@ -534,10 +535,11 @@ SKIP: {
require PublicInbox::Spawn;
state $lei_daemon = PublicInbox::Spawn->can('send_cmd4') ||
eval { require Socket::MsgHdr; 1 };
- # XXX fix and move this inside daemon-only before 1.7 release
- skip <<'EOM', 1 unless $lei_daemon;
-Socket::MsgHdr missing or Inline::C is unconfigured/missing
-EOM
+ unless ($lei_daemon) {
+ skip('Inline::C unconfigured/missing '.
+'(mkdir -p ~/.cache/public-inbox/inline-c) OR Socket::MsgHdr missing',
+ 1);
+ }
$lei_opt = { 1 => \$lei_out, 2 => \$lei_err };
my ($daemon_pid, $for_destroy, $daemon_xrd);
my $tmpdir = $test_opt->{tmpdir};
diff --git a/t/lei.t b/t/lei.t
index dfbcb1f3..d1f1cbc0 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -176,4 +176,8 @@ test_lei(sub {
$test_fail->();
});
+test_lei({ mods => [] }, sub {
+ lei_ok('sucks', \'no optional modules required');
+});
+
done_testing;
^ permalink raw reply related [relevance 65%]
* [PATCH 0/1] lei q -f reply ... perhaps my new favorite feature
@ 2021-09-11 8:33 71% Eric Wong
2021-09-11 8:33 40% ` [PATCH 1/1] lei q|lcat: support "-f reply" output format Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-11 8:33 UTC (permalink / raw)
To: meta
I think this deserves a cover-letter even if it's just one patch :>
Eric Wong (1):
lei q|lcat: support "-f reply" output format
Documentation/lei-lcat.pod | 10 +++++
Documentation/lei-q.pod | 11 ++++-
lib/PublicInbox/LeiToMail.pm | 4 +-
lib/PublicInbox/LeiViewText.pm | 82 +++++++++++++++++++++++++++++-----
t/lei-lcat.t | 14 ++++++
5 files changed, 106 insertions(+), 15 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/1] lei q|lcat: support "-f reply" output format
2021-09-11 8:33 71% [PATCH 0/1] lei q -f reply ... perhaps my new favorite feature Eric Wong
@ 2021-09-11 8:33 40% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 8:33 UTC (permalink / raw)
To: meta
When composing replies in "git format-patch" cover letters,
I'd been relying on "lei q -f text ...", but that still requires
several steps to make it suitable for composing a reply:
* s/^/> / to quote the body
* drop existing In-Reply-To+References
* s/^Message-ID:/In-Reply-To:/;
* add an attribute line
...
"lei q -f reply" takes care of most of that and users will
only have to trim "From " lines, unnecessary results and
over-quoted text (and trimming is likely less error-prone
than doing all the steps above manually).
This should also be a good replacement for
"git format-patch --in-reply-to=...", since copying long
Message-IDs can be error-prone (and this lets you include
quoted text in replies).
---
Documentation/lei-lcat.pod | 10 +++++
Documentation/lei-q.pod | 11 ++++-
lib/PublicInbox/LeiToMail.pm | 4 +-
lib/PublicInbox/LeiViewText.pm | 82 +++++++++++++++++++++++++++++-----
t/lei-lcat.t | 14 ++++++
5 files changed, 106 insertions(+), 15 deletions(-)
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
index 656df489..b7887b6c 100644
--- a/Documentation/lei-lcat.pod
+++ b/Documentation/lei-lcat.pod
@@ -20,9 +20,19 @@ Message-ID or link from surrounding text (e.g., a "Link: $URL" line).
=head1 OPTIONS
The following options, described in L<lei-q(1)>, are supported.
+One deviation from L<lei-q(1)> is the default output format is
+C<-f text> when writing to stdout.
=over
+=item --format=FORMAT
+
+=item -f FORMAT
+
+Most commonly C<text> (the default) or C<reply> to
+display the message(s) in a format suitable for trimming
+and sending as a email reply.
+
=item --[no-]remote
=item --no-local
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 69a6cdf2..1d9e66cd 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -60,7 +60,7 @@ Default: C<-> (stdout)
Format of results to stdout. This option exists as a convenient
way to specify the format for the default stdout destination.
-C<text>, C<json>, C<jsonl>, or C<concatjson> are all supported,
+C<reply>, C<text>, C<json>, C<jsonl>, or C<concatjson> are all supported,
as are the various mbox variants described in L</--output>.
When a format isn't specified, it's chosen based on the
@@ -72,7 +72,7 @@ preferred when not writing to stdout.
=item --no-color
-Disable color (for C<--format=text>).
+Disable color (for C<-f reply> and C<-f text>).
=item --pretty
@@ -241,6 +241,13 @@ Default: C<auto>
=back
+=head1 TIPS
+
+C<-f reply> is intended to aid in turning a cover letter
+into a reply (since using C<git format-patch --in-reply-to=...>
+is tedious). Results (including "From " lines) should be edited
+and trimmed in your favorite C<$EDITOR> before sending.
+
=head1 CONTACT
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index dbf58df9..15729bda 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -410,9 +410,9 @@ sub new {
$lei->{net} = $net;
$self->{base_type} = 'imap';
$lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
- } elsif ($fmt eq 'text') {
+ } elsif ($fmt eq 'text' || $fmt eq 'reply') {
require PublicInbox::LeiViewText;
- $lei->{lvt} = PublicInbox::LeiViewText->new($lei);
+ $lei->{lvt} = PublicInbox::LeiViewText->new($lei, $fmt);
$self->{base_type} = 'text';
@conflict = qw(mua save);
} elsif ($fmt eq 'v2') {
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 340a6648..34612711 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -13,6 +13,8 @@ use PublicInbox::Hval;
use PublicInbox::ViewDiff;
use PublicInbox::Spawn qw(popen_rd);
use Term::ANSIColor;
+use POSIX ();
+use PublicInbox::Address;
sub _xs {
# xhtml_map works since we don't search for HTML ([&<>'"])
@@ -66,8 +68,9 @@ sub my_colored {
sub uncolored { ${$_[0]->{obuf}} .= $_[2] }
sub new {
- my ($cls, $lei) = @_;
+ my ($cls, $lei, $fmt) = @_;
my $self = bless { %{$lei->{opt}}, -colored => \&uncolored }, $cls;
+ $self->{-quote_reply} = 1 if $fmt eq 'reply';
return $self unless $self->{color} //= -t $lei->{1};
my $cmd = [ qw(git config -z --includes -l) ];
my ($r, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
@@ -83,6 +86,45 @@ sub new {
$self;
}
+sub quote_hdr_buf ($$) {
+ my ($self, $eml) = @_;
+ my $hbuf = '';
+ my $to = $eml->header_raw('Reply-To') //
+ $eml->header_raw('From') //
+ $eml->header_raw('Sender');
+ my $cc = '';
+ for my $f (qw(To Cc)) {
+ for my $v ($eml->header_raw($f)) {
+ next if $v !~ /\S/;
+ $cc .= $v;
+ $to //= $v;
+ }
+ }
+ PublicInbox::View::fold_addresses($to);
+ PublicInbox::View::fold_addresses($cc);
+ _xs($to);
+ _xs($cc);
+ $hbuf .= "To: $to\n" if defined $to && $to =~ /\S/;
+ $hbuf .= "Cc: $cc\n" if $cc =~ /\S/;
+ my $s = $eml->header_str('Subject') // 'your mail';
+ _xs($s);
+ substr($s, 0, 0, 'Re: ') if $s !~ /\bRe:/i;
+ $hbuf .= "Subject: $s\n";
+ if (defined(my $irt = $eml->header_raw('Message-ID'))) {
+ _xs($irt);
+ $hbuf .= "In-Reply-To: $irt\n";
+ }
+ $self->{-colored}->($self, 'hdrdefault', $hbuf);
+ my ($n) = PublicInbox::Address::names($eml->header_str('From') //
+ $eml->header_str('Sender') //
+ $eml->header_str('Reply-To') //
+ 'unknown sender');
+ my $d = $eml->header_raw('Date') // 'some unknown date';
+ _xs($d);
+ _xs($n);
+ ${delete $self->{obuf}} . "\nOn $d, $n wrote:\n";
+}
+
sub hdr_buf ($$) {
my ($self, $eml) = @_;
my $hbuf = '';
@@ -224,25 +266,43 @@ sub add_text_buf { # callback for Eml->each_part
}
}
-# returns an arrayref suitable for $lei->out or print
+# returns a stringref suitable for $lei->out or print
sub eml_to_text {
my ($self, $smsg, $eml) = @_;
local $Term::ANSIColor::EACHLINE = "\n";
$self->{obuf} = \(my $obuf = '');
$self->{-smsg} = $smsg;
$self->{-max_cols} = ($self->{columns} //= 80) - 8; # for header wrap
- my @h = ();
- for my $f (qw(blob pct)) {
- push @h, "$f:$smsg->{$f}" if defined $smsg->{$f};
+ my $h = [];
+ if ($self->{-quote_reply}) {
+ my $blob = $smsg->{blob} // 'unknown-blob';
+ my $pct = $smsg->{pct} // 'unknown';
+ my $t = POSIX::asctime(gmtime($smsg->{ts} // $smsg->{ds} // 0));
+ $h->[0] = "From $blob\@$pct $t";
+ } else {
+ for my $f (qw(blob pct)) {
+ push @$h, "$f:$smsg->{$f}" if defined $smsg->{$f};
+ }
+ @$h = ("# @$h\n") if @$h;
+ for my $f (qw(kw L)) {
+ my $v = $smsg->{$f} or next;
+ push @$h, "# $f:".join(',', @$v)."\n" if @$v;
+ }
}
- @h = ("# @h\n") if @h;
- for my $f (qw(kw L)) {
- my $v = $smsg->{$f} or next;
- push @h, "# $f:".join(',', @$v)."\n" if @$v;
+ $h = join('', @$h);
+ $self->{-colored}->($self, 'status', $h);
+ my $quote_hdr;
+ if ($self->{-quote_reply}) {
+ $quote_hdr = ${delete $self->{obuf}};
+ $quote_hdr .= quote_hdr_buf($self, $eml);
+ } else {
+ hdr_buf($self, $eml);
}
- $self->{-colored}->($self, 'status', join('', @h));
- hdr_buf($self, $eml);
$eml->each_part(\&add_text_buf, $self, 1);
+ if (defined $quote_hdr) {
+ ${$self->{obuf}} =~ s/^/> /sgm;
+ substr(${$self->{obuf}}, 0, 0, $quote_hdr);
+ }
delete $self->{obuf};
}
diff --git a/t/lei-lcat.t b/t/lei-lcat.t
index e5f00706..31a84744 100644
--- a/t/lei-lcat.t
+++ b/t/lei-lcat.t
@@ -11,6 +11,20 @@ test_lei(sub {
lei_ok('import', 't/plack-qp.eml');
lei_ok([qw(lcat --stdin)], undef, { 0 => \$in, %$lei_opt });
like($lei_out, qr/qp\@example\.com/, 'got a result');
+
+ # test Link:, -f reply, and implicit --stdin:
+ my $prev = $lei_out;
+ $in = "\nLink: https://example.com/foo/qp\@example.com/\n";
+ lei_ok([qw(lcat -f reply)], undef, { 0 => \$in, %$lei_opt});
+ my $exp = <<'EOM';
+To: qp@example.com
+Subject: Re: QP
+In-Reply-To: <qp@example.com>
+
+On some unknown date, qp wrote:
+> hi = bye
+EOM
+ like($lei_out, qr/\AFrom [^\n]+\n\Q$exp\E/sm, '-f reply works');
});
done_testing;
^ permalink raw reply related [relevance 40%]
* [PATCH 0/3] lei saved-search fixes
2021-09-10 14:11 65% RFC: normalize whitespace in lei queries Konstantin Ryabitsev
@ 2021-09-11 0:19 71% ` Eric Wong
2021-09-11 0:19 53% ` [PATCH 1/3] lei: fix handling of broken lei.saved-search config files Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta
Konstantin Ryabitsev wrote:
> I think it's reasonable to normalize \s+ into a single space for all queries,
> right?
Yes, 3/3 does it for HTTP(S). Don't think it's needed for local
queries. I'm leaving non-ASCII whitespace alone for now since I
don't think it'd be a real problem.
> Another observation is that when I missed a \ on one of the lines, I managed
> to make lei unusable. :)
That's a far more important problem fixed in 1/3 :>
Eric Wong (3):
lei: fix handling of broken lei.saved-search config files
lei: pass client stderr to git-config in more places
lei: normalize whitespace in remote queries
lib/PublicInbox/Config.pm | 10 ++--
lib/PublicInbox/LEI.pm | 23 +++++++--
lib/PublicInbox/LeiEditSearch.pm | 79 +++++++++++++++++++++++++++++--
lib/PublicInbox/LeiMirror.pm | 4 +-
lib/PublicInbox/LeiSavedSearch.pm | 77 ++++++------------------------
lib/PublicInbox/LeiXSearch.pm | 4 +-
script/lei | 2 +
t/lei-q-save.t | 17 +++++++
8 files changed, 136 insertions(+), 80 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei: normalize whitespace in remote queries
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
2021-09-11 0:19 53% ` [PATCH 1/3] lei: fix handling of broken lei.saved-search config files Eric Wong
2021-09-11 0:19 63% ` [PATCH 2/3] lei: pass client stderr to git-config in more places Eric Wong
@ 2021-09-11 0:19 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
Having redundant "+" in URLs is ugly and can hurt cacheability
of queries. Even with "quoted phrase searches", Xapian seems
unaffected by redundant spaces, so just normalize the ASCII
white spaces to ' ' (%20) when fed via STDIN or saved-search
config file.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210910141157.6u5adehpx7wftkor@meerkat.local/
---
lib/PublicInbox/LeiXSearch.pm | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 709a3b3a..9f7f3885 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -297,7 +297,9 @@ sub query_remote_mboxrd {
local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
my $lei = $self->{lei};
my $opt = $lei->{opt};
- my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
+ my $qstr = $lei->{mset_opt}->{qstr};
+ $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
+ my @qform = (q => $qstr, x => 'm');
push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
my ($reap_tail, $reap_curl);
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] lei: pass client stderr to git-config in more places
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
2021-09-11 0:19 53% ` [PATCH 1/3] lei: fix handling of broken lei.saved-search config files Eric Wong
@ 2021-09-11 0:19 63% ` Eric Wong
2021-09-11 0:19 71% ` [PATCH 3/3] lei: normalize whitespace in remote queries Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta
This should improve the users' chances of seeing errors in
various git config files we use.
---
lib/PublicInbox/Config.pm | 4 ++--
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 74a1a6f5..ee5322fe 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -19,7 +19,7 @@ sub _array ($) { ref($_[0]) eq 'ARRAY' ? $_[0] : [ $_[0] ] }
# returns key-value pairs of config directives in a hash
# if keys may be multi-value, the value is an array ref containing all values
sub new {
- my ($class, $file) = @_;
+ my ($class, $file, $errfh) = @_;
$file //= default_file();
my $self;
if (ref($file) eq 'SCALAR') { # used by some tests
@@ -27,7 +27,7 @@ sub new {
$self = config_fh_parse($fh, "\n", '=');
bless $self, $class;
} else {
- $self = git_config_dump($class, $file);
+ $self = git_config_dump($class, $file, $errfh);
$self->{'-f'} = $file;
}
# caches
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0b4c99dc..aff2bf19 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -820,7 +820,7 @@ sub _lei_cfg ($;$) {
$cur_st = pack('dd', $st[10], $st[7]);
qerr($self, "# $f created") if $self->{cmd} ne 'config';
}
- my $cfg = PublicInbox::Config->git_config_dump($f);
+ my $cfg = PublicInbox::Config->git_config_dump($f, $self->{2});
$cfg->{-st} = $cur_st;
$cfg->{'-f'} = $f;
if ($sto && canonpath_harder($sto_dir // store_path($self))
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 638add42..8689b825 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -110,7 +110,7 @@ sub _try_config {
}
return $lei->err("# @$cmd failed (non-fatal)") if $cerr;
rename($f, $ce) or return $lei->err("link($f, $ce): $! (non-fatal)");
- my $cfg = PublicInbox::Config->git_config_dump($f);
+ my $cfg = PublicInbox::Config->git_config_dump($f, $lei->{2});
my $ibx = $self->{ibx} = {};
for my $sec (grep(/\Apublicinbox\./, @{$cfg->{-section_order}})) {
for (qw(address newsgroup nntpmirror)) {
@@ -136,7 +136,7 @@ sub index_cloned_inbox {
}
# force synchronous dwaitpid for v2:
local $PublicInbox::DS::in_loop = 0;
- my $cfg = PublicInbox::Config->new;
+ my $cfg = PublicInbox::Config->new(undef, $lei->{2});
my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
local %ENV = (%ENV, %$env) if $env;
PublicInbox::Admin::progress_prepare($opt, $lei->{2});
^ permalink raw reply related [relevance 63%]
* [PATCH 1/3] lei: fix handling of broken lei.saved-search config files
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
@ 2021-09-11 0:19 53% ` Eric Wong
2021-09-11 0:19 63% ` [PATCH 2/3] lei: pass client stderr to git-config in more places Eric Wong
2021-09-11 0:19 71% ` [PATCH 3/3] lei: normalize whitespace in remote queries Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
lei shouldn't become unusable if a config file is invalid.
Instead, show the "git config" stderr and attempt to continue
gracefully.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210910141157.6u5adehpx7wftkor@meerkat.local/
---
lib/PublicInbox/Config.pm | 6 +--
lib/PublicInbox/LEI.pm | 21 ++++++--
lib/PublicInbox/LeiEditSearch.pm | 79 +++++++++++++++++++++++++++++--
lib/PublicInbox/LeiSavedSearch.pm | 77 ++++++------------------------
script/lei | 2 +
t/lei-q-save.t | 17 +++++++
6 files changed, 128 insertions(+), 74 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index b3e00ae0..74a1a6f5 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -164,12 +164,12 @@ sub config_fh_parse ($$$) {
}
sub git_config_dump {
- my ($class, $file) = @_;
+ my ($class, $file, $errfh) = @_;
return bless {}, $class unless -e $file;
my $cmd = [ qw(git config -z -l --includes), "--file=$file" ];
- my $fh = popen_rd($cmd);
+ my $fh = popen_rd($cmd, undef, { 2 => $errfh // 2 });
my $rv = config_fh_parse($fh, "\0", "\n");
- close $fh or die "failed to close (@$cmd) pipe: $?";
+ close $fh or die "@$cmd failed: \$?=$?\n";
bless $rv, $class;
}
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bbb6ab7e..0b4c99dc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1029,13 +1029,15 @@ sub path_to_fd {
# caller needs to "-t $self->{1}" to check if tty
sub start_pager {
- my ($self) = @_;
+ my ($self, $new_env) = @_;
my $fh = popen_rd([qw(git var GIT_PAGER)]);
chomp(my $pager = <$fh> // '');
close($fh) or warn "`git var PAGER' error: \$?=$?";
return if $pager eq 'cat' || $pager eq '';
- my $new_env = { LESS => 'FRX', LV => '-c' };
- $new_env->{MORE} = 'FRX' if $^O eq 'freebsd';
+ $new_env //= {};
+ $new_env->{LESS} //= 'FRX';
+ $new_env->{LV} //= '-c';
+ $new_env->{MORE} = $new_env->{LESS} if $^O eq 'freebsd';
pipe(my ($r, $wpager)) or return warn "pipe: $!";
my $rdr = { 0 => $r, 1 => $self->{1}, 2 => $self->{2} };
my $pgr = [ undef, @$rdr{1, 2} ];
@@ -1052,6 +1054,19 @@ sub start_pager {
$self->{pgr} = $pgr;
}
+# display a message for user before spawning full-screen $VISUAL
+sub pgr_err {
+ my ($self, @msg) = @_;
+ return $self->err(@msg) unless $self->{sock} && -t $self->{2};
+ start_pager($self, { LESS => 'RX' }); # no 'F' so we prompt
+ print { $self->{2} } @msg;
+ $self->{2}->autoflush(1);
+ my $pgr = delete($self->{pgr}) or return;
+ $self->{2} = $pgr->[2];
+ $self->{1} = $pgr->[1];
+ send($self->{sock}, 'wait', MSG_EOR); # wait for user to quit pager
+}
+
sub stop_pager {
my ($self) = @_;
my $pgr = delete($self->{pgr}) or return;
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 82dfbf63..47166ce7 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -8,20 +8,89 @@ use v5.10.1;
use PublicInbox::LeiSavedSearch;
use PublicInbox::LeiUp;
-sub lei_edit_search {
- my ($lei, $out) = @_;
- my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
+sub edit_begin {
+ my ($lss, $lei) = @_;
+ if (ref($lss->{-cfg}->{'lei.q.output'})) {
+ delete $lss->{-cfg}->{'lei.q.output'}; # invalid
+ $lei->pgr_err(<<EOM);
+$lss->{-f} has multiple values of lei.q.output
+please remove redundant ones
+EOM
+ }
+ $lei->{-lss_for_edit} = $lss;
+}
+
+sub do_edit ($$;$) {
+ my ($lss, $lei, $reason) = @_;
+ $lei->pgr_err($reason) if defined $reason;
my @cmd = (qw(git config --edit -f), $lss->{'-f'});
$lei->qerr("# spawning @cmd");
- $lss->edit_begin($lei);
+ edit_begin($lss, $lei);
# run in script/lei foreground
require PublicInbox::PktOp;
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
# $op_p will EOF when $EDITOR is done
- $op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
+ $op_c->{ops} = { '' => [\&op_edit_done, $lss, $lei] };
$lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], \@cmd, {});
}
+sub _edit_done {
+ my ($lss, $lei) = @_;
+ my $cfg = $lss->can('cfg_dump')->($lei, $lss->{'-f'}) //
+ return do_edit($lss, $lei, <<EOM);
+$lss->{-f} is unparseable
+EOM
+ my $new_out = $cfg->{'lei.q.output'} // '';
+ return do_edit($lss, $lei, <<EOM) if ref $new_out;
+$lss->{-f} has multiple values of lei.q.output
+EOM
+ return do_edit($lss, $lei, <<EOM) if $new_out eq '';
+$lss->{-f} needs lei.q.output
+EOM
+ my $old_out = $lss->{-cfg}->{'lei.q.output'} // return;
+ return if $old_out eq $new_out;
+ my $old_path = $old_out;
+ my $new_path = $new_out;
+ s!$PublicInbox::LeiSavedSearch::LOCAL_PFX!! for ($old_path, $new_path);
+ my $dir_old = $lss->can('lss_dir_for')->($lei, \$old_path, 1);
+ my $dir_new = $lss->can('lss_dir_for')->($lei, \$new_path);
+ return if $dir_new eq $dir_old;
+
+ ($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
+ return do_edit($lss, $lei, <<EOM);
+conversions from/to v2 inboxes not supported at this time
+EOM
+ return do_edit($lss, $lei, <<EOM) if -e $dir_new;
+lei.q.output changed from `$old_out' to `$new_out'
+However, $dir_new exists
+EOM
+ # start the conversion asynchronously
+ my $old_sq = PublicInbox::Config::squote_maybe($old_out);
+ my $new_sq = PublicInbox::Config::squote_maybe($new_out);
+ $lei->puts("lei.q.output changed from $old_sq to $new_sq");
+ $lei->qerr("# lei convert $old_sq -o $new_sq");
+ my $v = !$lei->{opt}->{quiet};
+ $lei->{opt} = { output => $new_out, verbose => $v };
+ require PublicInbox::LeiConvert;
+ PublicInbox::LeiConvert::lei_convert($lei, $old_out);
+
+ $lei->fail(<<EOM) if -e $dir_old && !rename($dir_old, $dir_new);
+E: rename($dir_old, $dir_new) error: $!
+EOM
+}
+
+sub op_edit_done { # PktOp
+ my ($lss, $lei) = @_;
+ eval { _edit_done($lss, $lei) };
+ $lei->fail($@) if $@;
+}
+
+sub lei_edit_search {
+ my ($lei, $out) = @_;
+ my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
+ do_edit($lss, $lei);
+}
+
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index fd51fe38..8ceaaf01 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -14,7 +14,7 @@ use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::MID qw(mids_for_index);
use Digest::SHA qw(sha256_hex);
-my $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
+our $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
# move this to PublicInbox::Config if other things use it:
my %cquote = ("\n" => '\\n', "\t" => '\\t', "\b" => '\\b');
@@ -29,6 +29,14 @@ sub BOOL_FIELDS () {
qw(external local remote import-remote import-before threads)
}
+sub cfg_dump ($$) {
+ my ($lei, $f) = @_;
+ my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
+ return $ret if !$@;
+ $lei->err($@);
+ undef;
+}
+
sub lss_dir_for ($$;$) {
my ($lei, $dstref, $on_fs) = @_;
my @n;
@@ -56,7 +64,7 @@ sub lss_dir_for ($$;$) {
for my $g ("$n[0]-*", '*') {
my @maybe = glob("$lss_dir$g/lei.saved-search");
for my $f (@maybe) {
- $c = PublicInbox::Config->git_config_dump($f);
+ $c = cfg_dump($lei, $f) // next;
$o = $c->{'lei.q.output'} // next;
$o =~ s!$LOCAL_PFX!! or next;
@st = stat($o) or next;
@@ -80,9 +88,9 @@ sub list {
print $fh "\tpath = ", cquote_val($p), "\n";
}
close $fh or die "close $f: $!";
- my $cfg = PublicInbox::Config->git_config_dump($f);
+ my $cfg = cfg_dump($lei, $f);
unlink($f);
- my $out = $cfg->get_all('lei.q.output') or return ();
+ my $out = $cfg ? $cfg->get_all('lei.q.output') : [];
map {;
s!$LOCAL_PFX!!;
$_;
@@ -105,7 +113,7 @@ sub up { # updating existing saved search via "lei up"
output2lssdir($self, $lei, \$dir, \$f) or
return $lei->fail("--save was not used with $dst cwd=".
$lei->rel2abs('.'));
- $self->{-cfg} = PublicInbox::Config->git_config_dump($f);
+ $self->{-cfg} = cfg_dump($lei, $f) // return $lei->fail;
$self->{-ovf} = "$dir/over.sqlite3";
$self->{'-f'} = $f;
$self->{lock_path} = "$self->{-f}.flock";
@@ -267,7 +275,7 @@ sub output2lssdir {
my $dir = lss_dir_for($lei, \$dst, 1);
my $f = "$dir/lei.saved-search";
if (-f $f && -r _) {
- $self->{-cfg} = PublicInbox::Config->git_config_dump($f);
+ $self->{-cfg} = cfg_dump($lei, $f) // return;
$$dir_ref = $dir;
$$fn_ref = $f;
return 1;
@@ -275,63 +283,6 @@ sub output2lssdir {
undef;
}
-sub edit_begin {
- my ($self, $lei) = @_;
- if (ref($self->{-cfg}->{'lei.q.output'})) {
- delete $self->{-cfg}->{'lei.q.output'}; # invalid
- $lei->err(<<EOM);
-$self->{-f} has multiple values of lei.q.output
-please remove redundant ones
-EOM
- }
- $lei->{-lss_for_edit} = $self;
-}
-
-sub edit_done {
- my ($self, $lei) = @_;
- my $cfg = PublicInbox::Config->git_config_dump($self->{'-f'});
- my $new_out = $cfg->{'lei.q.output'} // '';
- return $lei->fail(<<EOM) if ref $new_out;
-$self->{-f} has multiple values of lei.q.output
-please edit again
-EOM
- return $lei->fail(<<EOM) if $new_out eq '';
-$self->{-f} needs lei.q.output
-please edit again
-EOM
- my $old_out = $self->{-cfg}->{'lei.q.output'} // '';
- return if $old_out eq $new_out;
- my $old_path = $old_out;
- my $new_path = $new_out;
- s!$LOCAL_PFX!! for ($old_path, $new_path);
- my $dir_old = lss_dir_for($lei, \$old_path, 1);
- my $dir_new = lss_dir_for($lei, \$new_path);
- return if $dir_new eq $dir_old; # no change, likely
-
- ($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
- return $lei->fail(<<EOM);
-conversions from/to v2 inboxes not supported at this time
-EOM
-
- return $lei->fail(<<EOM) if -e $dir_new;
-lei.q.output changed from `$old_out' to `$new_out'
-However, $dir_new exists
-EOM
- # start the conversion asynchronously
- my $old_sq = PublicInbox::Config::squote_maybe($old_out);
- my $new_sq = PublicInbox::Config::squote_maybe($new_out);
- $lei->puts("lei.q.output changed from $old_sq to $new_sq");
- $lei->qerr("# lei convert $old_sq -o $new_sq");
- my $v = !$lei->{opt}->{quiet};
- $lei->{opt} = { output => $new_out, verbose => $v };
- require PublicInbox::LeiConvert;
- PublicInbox::LeiConvert::lei_convert($lei, $old_out);
-
- $lei->fail(<<EOM) if -e $dir_old && !rename($dir_old, $dir_new);
-E: rename($dir_old, $dir_new) error: $!
-EOM
-}
-
# cf. LeiDedupe->has_entries
sub has_entries {
my $oidx = $_[0]->{oidx} // die 'BUG: no {oidx}';
diff --git a/script/lei b/script/lei
index 99d94b4e..2d84487a 100755
--- a/script/lei
+++ b/script/lei
@@ -127,6 +127,8 @@ while (1) {
last;
} elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
$x_it_code ||= $1 + 0;
+ } elsif ($buf eq 'wait') {
+ $sigchld->();
} else {
$sigchld->();
die $buf;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 743a7b70..9c17a011 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -215,5 +215,22 @@ test_lei(sub {
'absolute path appears in ls-search';
lei_ok qw(up ../s -C), "$home/v2s", \'relative lei up';
lei_ok qw(up), "$home/s", \'absolute lei up';
+
+ # mess up a config file
+ my @lss = glob("$home/" .
+ '.local/share/lei/saved-searches/*/lei.saved-search');
+ my $out = xqx([qw(git config -f), $lss[0], 'lei.q.output']);
+ xsys($^X, qw(-i -p -e), "s/\\[/\\0/", $lss[0])
+ and xbail "-ipe $lss[0]: $?";
+ lei_ok qw(ls-search);
+ like($lei_err, qr/bad config line.*?\Q$lss[0]\E/,
+ 'git config parse error shown w/ lei ls-search');
+ lei_ok qw(up --all), \'up works with bad config';
+ like($lei_err, qr/bad config line.*?\Q$lss[0]\E/,
+ 'git config parse error shown w/ lei up');
+ xsys($^X, qw(-i -p -e), "s/\\0/\\[/", $lss[0])
+ and xbail "-ipe $lss[0]: $?";
+ lei_ok qw(ls-search);
+ is($lei_err, '', 'no errors w/ fixed config');
});
done_testing;
^ permalink raw reply related [relevance 53%]
* Re: Using lei with podman + toolbox
2021-09-10 13:56 71% ` Eric Wong
@ 2021-09-10 14:48 71% ` Konstantin Ryabitsev
0 siblings, 0 replies; 200+ results
From: Konstantin Ryabitsev @ 2021-09-10 14:48 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Fri, Sep 10, 2021 at 01:56:53PM +0000, Eric Wong wrote:
> > Seems to have it as perl-Search-Xapian, which provides Xapian.pm.
>
> Oh I meant the newer SWIG Xapian.pm. Search::Xapian is
> Search/Xapian.pm which uses XS and isn't getting new features.
Ah, no, doesn't look like xapian-bindings-perl is built.
I may open an RFE with the packager for this once I get around to that.
> > All noted -- I may be the one who packages things for Fedora at some point, so
> > this is useful info.
>
> Cool. I'm wondering how to better arrange INSTALL to suit
> different usages (lei-only vs daemons vs mda/watch-only).
I mean, it's not like it pulls in huge dependency trees. Perl cpan packages
are a few KB in size, so pulling in some extras isn't going to inconvenience a
lot of people or anything. :)
-K
^ permalink raw reply [relevance 71%]
* RFC: normalize whitespace in lei queries
@ 2021-09-10 14:11 65% Konstantin Ryabitsev
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-10 14:11 UTC (permalink / raw)
To: meta
Just a quickie note. I was trying to make my query more readable, so I
switched my test example to the following:
[lei]
q = (dfn:Documentation/security/landlock.rst \
OR dfn:Documentation/userspace-api/landlock.rst \
OR dfn:include/uapi/linux/landlock.h \
OR dfn:samples/landlock/ \
OR dfn:security/landlock/ \
OR dfn:tools/testing/selftests/landlock/ \
OR dfhh:landlock OR dfctx:landlock) \
AND rt:2.months.ago..
However, I noticed that curl queries ended up gaining a lot of ++++:
# /usr/bin/curl -Sf -s -d '' https://lore.kernel.org/all/?q=(dfn%3ADocumentation%2Fsecurity%2Flandlock.rst++++++++++OR+dfn%3ADocumentation%2Fuserspace-api%2Flandlock.rst++++++++++OR+dfn%3Ainclude%2Fuapi%2Flinux%2Flandlock.h++++++++++OR+dfn%3Asamples%2Flandlock%2F++++++++++OR+dfn%3Asecurity%2Flandlock%2F++++++++++OR+dfn%3Atools%2Ftesting%2Fselftests%2Flandlock%2F++++++++++OR+dfhh%3Alandlock+OR+dfctx%3Alandlock)+++++++++AND+rt%3A1625943784..&x=m&t=1
I think it's reasonable to normalize \s+ into a single space for all queries,
right?
Another observation is that when I missed a \ on one of the lines, I managed
to make lei unusable. :)
$ lei up --all
failed to close (git config -z -l --includes --file=/home/user/.cache/lei/saved-tmp.1581950.1631282889.config) pipe: 32768 at /usr/local/share/perl/5.32.1/PublicInbox/Config.pm line 172.
$ lei edit-search ~/work/temp/lei/landlock
failed to close (git config -z -l --includes --file=/home/user/.local/share/lei/saved-searches/landlock-1804cfad691a409f55598a8528566d5f1539b2632e1db7e206cb147396582631/lei.saved-search) pipe: 32768 at /usr/local/share/perl/5.32.1/PublicInbox/Config.pm line 172.
I fixed it by manually editing lei.saved-search and adding the missing
backslash. Not sure what the proper solution for this is -- perhaps attempting
to parse the edited config file and refusing to save it if it doesn't work?
-K
^ permalink raw reply [relevance 65%]
* Re: Using lei with podman + toolbox
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
@ 2021-09-10 13:56 71% ` Eric Wong
2021-09-10 14:48 71% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-10 13:56 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Thu, Sep 09, 2021 at 11:36:14PM +0000, Eric Wong wrote:
> > > These are my quickie instructions for how to use lei in a toolbox environment
> > > if you are running a distribution like Fedora and don't want to install a lot
> > > of perl dependencies into your main OS.
> >
> > Off the top of my head, I think Search::Xapian // Xapian.pm was
> > the main thing that was missing from CentOS 7. Does Fedora have
> > that?
>
> Seems to have it as perl-Search-Xapian, which provides Xapian.pm.
Oh I meant the newer SWIG Xapian.pm. Search::Xapian is
Search/Xapian.pm which uses XS and isn't getting new features.
The XS version is far better-tested, but the SWIG version gives
access to some newer APIs. AFAIK neither lets us do custom
query parsing like notmuch does in C++ (our approxidate
rt:/dt:/d: handling is a huge hack)
> > (disclaimer: I don't care for Docker, seems like a giant waste
> > of space and bandwidth compared to just using the distro)
>
> Well, this is for toolbox which uses podman, not docker. Toolbox is actually
> the preferred mechanism in Fedora for setting up quickie work environments,
> especially on something like Fedora Silverblue with its immutable root
> partition.
Ah, still seems like a waste of space and bandwidth :>
(It just took me several hours to download upgrades from
buster => bullseye on a small dev VM)
> I don't intend these instructions as the preferred mechanism for getting lei
> up and running, just to be clear. Eventually, it will be packaged for most
> distros -- but for now it's a convenient way to get the latest version on the
> platform most likely to be most tested (Debian).
I actually test everything on FreeBSD to force myself into doing
things portably. Probably 95% of my "git push" is done on
FreeBSD (IOW, whenever I have connectivity to that VM).
> All noted -- I may be the one who packages things for Fedora at some point, so
> this is useful info.
Cool. I'm wondering how to better arrange INSTALL to suit
different usages (lei-only vs daemons vs mda/watch-only).
^ permalink raw reply [relevance 71%]
* Re: Using lei with podman + toolbox
2021-09-09 23:36 68% ` Eric Wong
2021-09-09 23:51 71% ` native C++ Xapian wrapper [was: Using lei with podman + toolbox] Eric Wong
@ 2021-09-10 12:42 70% ` Konstantin Ryabitsev
2021-09-10 13:56 71% ` Eric Wong
1 sibling, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-10 12:42 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 09, 2021 at 11:36:14PM +0000, Eric Wong wrote:
> > These are my quickie instructions for how to use lei in a toolbox environment
> > if you are running a distribution like Fedora and don't want to install a lot
> > of perl dependencies into your main OS.
>
> Off the top of my head, I think Search::Xapian // Xapian.pm was
> the main thing that was missing from CentOS 7. Does Fedora have
> that?
Seems to have it as perl-Search-Xapian, which provides Xapian.pm.
> (disclaimer: I don't care for Docker, seems like a giant waste
> of space and bandwidth compared to just using the distro)
Well, this is for toolbox which uses podman, not docker. Toolbox is actually
the preferred mechanism in Fedora for setting up quickie work environments,
especially on something like Fedora Silverblue with its immutable root
partition.
I don't intend these instructions as the preferred mechanism for getting lei
up and running, just to be clear. Eventually, it will be packaged for most
distros -- but for now it's a convenient way to get the latest version on the
platform most likely to be most tested (Debian).
> > RUN apt-get update && \
> > apt-get -y install sudo libcap2-bin locales vim \
> > git liburi-perl libemail-mime-perl libplack-perl libtimedate-perl \
>
> Email::MIME isn't used at all outside of tests (but it's widely packaged).
> No idea why libcap2-bin and vim are explicit dependencies (any
> editor will do). Don't need Plack for lei, either.
Yeah, some of these were mostly for "make test" runs and others are adding
basic packages to the container image to make it slightly more usable (I don't
like nano or whatever is the default editor in the container image).
> No need for Net::Server nor Parse::RecDescent for lei. I don't
> use Net::Server at all outside of tests, since I use systemd.
>
> Email::Address::XS and TimeDate can be useful for messed up
> messages, but low importance (I think they're widely packaged).
> E:A:X and P:RD are required for -imapd but nothing else.
>
> Socket::Msghdr makes lei a teeny bit faster, but I don't think
> it's worth using another distro or running a compiler to get
> since Inline::C is already available in all distros. Everything
> else should be in Fedora...
>
> > sqlite3 libgit2-dev make eatmydata man-db pkg-config
>
> eatmydata shouldn't be useful outside of development, and
> libgit2+pkg-config isn't used by lei, yet
> (it is for -httpd/-imapd/-nntpd)
All noted -- I may be the one who packages things for Fedora at some point, so
this is useful info.
-K
^ permalink raw reply [relevance 70%]
* [PATCH] lei up: only delay non-zero "# $NR written to ..."
@ 2021-09-10 11:46 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 11:46 UTC (permalink / raw)
To: meta
"# 0 written to $FOLDER" messages aren't important to the
user, so we can show them in real time and allow them to
be lost in the terminal scroll. When >0 messages are
written to a folder, we'll show them last so a user
will know which folders to open with their MUA.
---
lib/PublicInbox/LeiXSearch.pm | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index b6d7bf2b..709a3b3a 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -400,8 +400,9 @@ Error closing $lei->{ovv}->{dst}: $!
my $tot = $lei->{-mset_total} // 0;
my $nr = $lei->{-nr_write} // 0;
if ($l2m) {
- $lei->qfin("# $nr written to " .
- "$lei->{ovv}->{dst} ($tot matches)");
+ my $m = "# $nr written to " .
+ "$lei->{ovv}->{dst} ($tot matches)";
+ $nr ? $lei->qfin($m) : $lei->qerr($m);
} else {
$lei->qerr("# $tot matches");
}
^ permalink raw reply related [relevance 71%]
* [PATCH] lei add-external --mirror: quiet unlink error on ENOENT
@ 2021-09-10 9:15 68% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:15 UTC (permalink / raw)
To: meta
If the mirror.done file doesn't exist for unlink, it's because
we already got another error, so don't confuse users by noting
an unlink error since the ENOENT is expected in the face of
other errors.
---
lib/PublicInbox/LeiMirror.pm | 2 +-
t/lei-mirror.t | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index fca11ccf..638add42 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -16,7 +16,7 @@ sub do_finish_mirror { # dwaitpid callback
if ($?) {
$lei->child_error($?);
} elsif (!unlink($f)) {
- $lei->err("unlink($f): $!");
+ $lei->err("unlink($f): $!") unless $!{ENOENT};
} else {
$lei->add_external_finish($mrr->{dst});
$lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 65b6068c..a61a7565 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -47,6 +47,14 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('add-external', "$t1-pfx", '--mirror', "$http/pfx/t1/",
\'--mirror v1 w/ PSGI prefix');
+ my $d = "$home/404";
+ ok(!lei(qw(add-external --mirror), "$http/404", $d), 'mirror 404');
+ unlike($lei_err, qr!unlink.*?404/mirror\.done!,
+ 'no unlink failure message');
+ ok(!-d $d, "`404' dir not created");
+ lei_ok('ls-external');
+ unlike($lei_out, qr!\Q$d\E!s, 'not added to ls-external');
+
my %phail = (
HTTPS => 'https://public-inbox.org/' . 'phail',
ONION =>
^ permalink raw reply related [relevance 68%]
* [PATCH 0/4] lei: some net-related things
@ 2021-09-10 9:08 71% Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
After some consideration, ~/.netrc will no longer be read by
default to match the behavior of existing IMAP/NNTP clients.
And lei-index is pretty limited, but still useful for Maildir
users, so it's documented (mainly for its limitations).
Eric Wong (4):
lei_query: fix comment about %lei2curl commands
lei: split out @net_opt for curl/torsocks use
lei: do not read ~/.netrc by default
doc: lei-index manpage
Documentation/lei-index.pod | 69 ++++++++++++++++++++++++++++++++
MANIFEST | 1 +
Makefile.PL | 2 +-
lib/PublicInbox/GitCredential.pm | 8 +++-
lib/PublicInbox/LEI.pm | 34 ++++++++--------
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/NetReader.pm | 4 +-
7 files changed, 96 insertions(+), 24 deletions(-)
create mode 100644 Documentation/lei-index.pod
^ permalink raw reply [relevance 71%]
* [PATCH 3/4] lei: do not read ~/.netrc by default
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
@ 2021-09-10 9:08 65% ` Eric Wong
2021-09-10 9:08 57% ` [PATCH 4/4] doc: lei-index manpage Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
Since ~/.netrc isn't widely used by most (if any) NNTP and IMAP
clients, we won't read it by default for lei. AFAIK, ~/.netrc
is mainly by FTP clients (e.g. ftp(1) and lftp(1)). wget uses
it by default for HTTP(S) (and FTP), but curl does not.
To avoid breaking stable release use cases, public-inbox-watch
continues to read ~/.netrc by default.
The --netrc switch is supported by all existing lei commands
which may use curl.
---
lib/PublicInbox/GitCredential.pm | 8 ++++++--
lib/PublicInbox/NetReader.pm | 4 ++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/GitCredential.pm b/lib/PublicInbox/GitCredential.pm
index c83fed43..b18bba1e 100644
--- a/lib/PublicInbox/GitCredential.pm
+++ b/lib/PublicInbox/GitCredential.pm
@@ -31,8 +31,12 @@ sub run ($$;$) {
close $out_r or die "`git credential $op' failed: \$!=$! \$?=$?\n";
}
-sub check_netrc ($) {
- my ($self) = @_;
+sub check_netrc {
+ my ($self, $lei) = @_;
+
+ # n.b. lei doesn't load ~/.netrc by default, public-inbox-watch does,
+ # which may've been a mistake, but we have to live with it.
+ return if ($lei && !$lei->{opt}->{netrc});
# part of the standard library, but distributions may split it out
eval { require Net::Netrc };
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index a0e52fc5..f0f56431 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -96,7 +96,7 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
$cred = undef;
}
if ($cred) {
- my $p = $cred->{password} // $cred->check_netrc;
+ my $p = $cred->{password} // $cred->check_netrc($lei);
$cred->fill($lei) unless defined($p); # may prompt user here
$mic->User($mic_arg->{User} = $cred->{username});
$mic->Password($mic_arg->{Password} = $cred->{password});
@@ -191,7 +191,7 @@ sub nn_for ($$$$) { # nn = Net::NNTP
}, 'PublicInbox::GitCredential';
($u, $p) = split(/:/, $ui, 2);
($cred->{username}, $cred->{password}) = ($u, $p);
- $p //= $cred->check_netrc;
+ $p //= $cred->check_netrc($lei);
}
my $common = $nn_common->{$sec} // {};
my $nn_arg = {
^ permalink raw reply related [relevance 65%]
* [PATCH 4/4] doc: lei-index manpage
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
2021-09-10 9:08 65% ` [PATCH 3/4] lei: do not read ~/.netrc by default Eric Wong
@ 2021-09-10 9:08 57% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
It's a pretty incomplete command, so it's important to document
its incompleteness.
---
Documentation/lei-index.pod | 69 +++++++++++++++++++++++++++++++++++++
MANIFEST | 1 +
Makefile.PL | 2 +-
3 files changed, 71 insertions(+), 1 deletion(-)
create mode 100644 Documentation/lei-index.pod
diff --git a/Documentation/lei-index.pod b/Documentation/lei-index.pod
new file mode 100644
index 00000000..bd125bcc
--- /dev/null
+++ b/Documentation/lei-index.pod
@@ -0,0 +1,69 @@
+=head1 NAME
+
+lei-index - index messages without importing them into lei/store
+
+=head1 SYNOPSIS
+
+lei index [OPTIONS] FOLDER
+
+lei index [OPTIONS] --stdin
+
+=head1 DESCRIPTION
+
+Similar to L<lei-import(1)>, but does not store a copy of
+messages into C<lei/store>.
+
+This command only makes sense for messages stored in Maildir
+folders. Other folder types may be supported in the future
+(they can all be indexed, but the message isn't automatically
+retrieved by L<lei-q(1)> or L<lei-lcat(1)>).
+
+Combined with L<lei-q(1)>, C<lei index> allows Maildir users to
+have similar functionality to L<mairix(1)> by not duplicating
+messages into C<lei/store>.
+
+=head1 OPTIONS
+
+=over
+
+=item -
+
+=item --stdin
+
+Read input from standard input. This is the default if standard
+input is a pipe or regular file and there are no arguments on
+the command-line.
+
+=item -F MAIL_FORMAT
+
+=item --in-format=MAIL_FORMAT
+
+Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
+C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+
+Default: C<eml> when reading from stdin
+
+=item -q
+
+=item --quiet
+
+Suppress feedback messages.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-store-format(5)>, L<lei-import(1)>
diff --git a/MANIFEST b/MANIFEST
index 531f8c46..c0e3e855 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -31,6 +31,7 @@ Documentation/lei-edit-search.pod
Documentation/lei-forget-external.pod
Documentation/lei-forget-search.pod
Documentation/lei-import.pod
+Documentation/lei-index.pod
Documentation/lei-init.pod
Documentation/lei-lcat.pod
Documentation/lei-ls-external.pod
diff --git a/Makefile.PL b/Makefile.PL
index 82b50543..bfabb171 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -47,7 +47,7 @@ $v->{-m1} = [ map {
qw(
lei-add-external lei-blob lei-config lei-convert lei-edit-search
lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
- lei-import lei-init lei-lcat lei-ls-external lei-ls-label
+ lei-import lei-index lei-init lei-lcat lei-ls-external lei-ls-label
lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-rm lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
^ permalink raw reply related [relevance 57%]
* [PATCH 2/4] lei: split out @net_opt for curl/torsocks use
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
@ 2021-09-10 9:08 47% ` Eric Wong
2021-09-10 9:08 65% ` [PATCH 3/4] lei: do not read ~/.netrc by default Eric Wong
2021-09-10 9:08 57% ` [PATCH 4/4] doc: lei-index manpage Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
IMAP and NNTP connections share some curl(1) options for TLS,
IPv4/IPv6, or netrc, etc...
---
lib/PublicInbox/LEI.pm | 34 ++++++++++++++++------------------
1 file changed, 16 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3dce0236..bbb6ab7e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -147,9 +147,9 @@ sub index_opt {
}
my @c_opt = qw(c=s@ C=s@ quiet|q);
-my @lxs_opt = (qw(remote! local! external! include|I=s@ exclude=s@ only=s@
- import-remote! no-torsocks torsocks=s),
- PublicInbox::LeiQuery::curl_opt());
+my @net_opt = (qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt());
+my @lxs_opt = qw(remote! local! external! include|I=s@ exclude=s@ only=s@
+ import-remote!);
# we don't support -C as an alias for --find-copies since it's already
# used for chdir
@@ -174,7 +174,7 @@ our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
our %CMD = ( # sorted in order of importance/use:
'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms',
'stdin|', # /|\z/ must be first for lone dash
- @lxs_opt,
+ @lxs_opt, @net_opt,
qw(save! output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
@@ -186,26 +186,26 @@ our %CMD = ( # sorted in order of importance/use:
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
# some of these options are ridiculous for lcat
- @lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+ @lxs_opt, @net_opt,
+ qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'blob' => [ 'OID', 'show a git blob, reconstructing from mail if necessary',
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
- @lxs_opt, @c_opt ],
+ @lxs_opt, @net_opt, @c_opt ],
'rediff' => [ '--stdin|LOCATION...',
'regenerate a diff with different options',
'stdin|', # /|\z/ must be first for lone dash
qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
- @diff_opt, @lxs_opt, @c_opt ],
+ @diff_opt, @lxs_opt, @net_opt, @c_opt ],
'add-external' => [ 'LOCATION',
'add/set priority of a publicinbox|extindex for extra matches',
- qw(boost=i mirror=s no-torsocks torsocks=s inbox-version=i
- verbose|v+), @c_opt, index_opt(),
- PublicInbox::LeiQuery::curl_opt() ],
+ qw(boost=i mirror=s inbox-version=i verbose|v+),
+ @c_opt, index_opt(), @net_opt ],
'ls-external' => [ '[FILTER]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
@@ -226,15 +226,14 @@ our %CMD = ( # sorted in order of importance/use:
'rm' => [ '--stdin|LOCATION...',
'remove a message from the index and prevent reindexing',
'stdin|', # /|\z/ must be first for lone dash
- qw(in-format|F=s lock=s@), @c_opt ],
+ qw(in-format|F=s lock=s@), @net_opt, @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
'tag' => [ 'KEYWORDS...',
'set/unset keywords and/or labels on message(s)',
qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
- qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt,
- pass_through('-kw:foo for delete') ],
+ @net_opt, @c_opt, pass_through('-kw:foo for delete') ],
'purge-mailsource' => [ 'LOCATION|--all',
'remove imported messages from IMAP, Maildirs, and MH',
@@ -253,25 +252,24 @@ our %CMD = ( # sorted in order of importance/use:
'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
- verbose|v+ incremental!),
- PublicInbox::LeiQuery::curl_opt(), # mainly for --proxy=
+ verbose|v+ incremental!), @net_opt, # mainly for --proxy=
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
- qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+ @net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
'prune-mail-sync' => [ 'LOCATION...|--all',
'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
- qw(all:s mode=s), @c_opt ],
+ qw(all:s mode=s), @net_opt, @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
- qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+ @net_opt, @c_opt ],
'p2q' => [ 'FILE|COMMIT_OID|--stdin',
"use a patch to generate a query for `lei q --stdin'",
qw(stdin| want|w=s@ uri debug), @c_opt ],
^ permalink raw reply related [relevance 47%]
* [PATCH] lei add-external --mirror: deduce paths for PSGI mount prefixes
@ 2021-09-10 5:51 51% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 5:51 UTC (permalink / raw)
To: meta
The current manifest.js.gz generation in WWW doesn't account for
PSGI mount prefixes (and grokmirror 1.x appears to work fine).
In other words, <https://yhbt.net/lore/lkml/manifest.js.gz>
currently has keys like "/lkml/git/0.git" and not
"/lore/lkml/git/0.git" where "/lore" is the PSGI mount prefix.
This works fine with the prefix accounted for in my grokmirror
(1.x) repos.conf like this:
site = https://yhbt.net/lore/
manifest = https://yhbt.net/lore/manifest.js.gz
Adding the PSGI mount prefix in manifest.js.gz is probably not
desirable since it would force the prefix into the locally
cloned path by grokmirror, and all the cloned directories
would have the remote PSGI mount prefix prepended to the
toplevel.
So, "lei add-external --mirror" needs to account for PSGI
mount prefixes by deducing the prefix based on available keys
in the manifest.js.gz hash table.
---
MANIFEST | 1 +
lib/PublicInbox/LeiMirror.pm | 28 +++++++++++++++++++++-------
t/lei-mirror.psgi | 9 +++++++++
t/lei-mirror.t | 6 +++++-
4 files changed, 36 insertions(+), 8 deletions(-)
create mode 100644 t/lei-mirror.psgi
diff --git a/MANIFEST b/MANIFEST
index 531f8c46..a22672e7 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -436,6 +436,7 @@ t/lei-import-nntp.t
t/lei-import.t
t/lei-index.t
t/lei-lcat.t
+t/lei-mirror.psgi
t/lei-mirror.t
t/lei-p2q.t
t/lei-q-kw.t
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 39671f90..fca11ccf 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -200,6 +200,19 @@ failed to extract epoch number from $src
index_cloned_inbox($self, 2);
}
+# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+sub deduce_epochs ($$) {
+ my ($m, $path) = @_;
+ my ($v1_bare, @v2_epochs);
+ my $path_pfx = '';
+ do {
+ $v1_bare = $m->{$path};
+ @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+ } while (!defined($v1_bare) && !@v2_epochs &&
+ $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
+ ($path_pfx, $v1_bare, @v2_epochs);
+}
+
sub try_manifest {
my ($self) = @_;
my $uri = URI->new($self->{src});
@@ -229,8 +242,7 @@ sub try_manifest {
die "$uri: error decoding `$js': $@" if $@;
ref($m) eq 'HASH' or die "$uri unknown type: ".ref($m);
- my $v1_bare = $m->{$path};
- my @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+ my ($path_pfx, $v1_bare, @v2_epochs) = deduce_epochs($m, $path);
if (@v2_epochs) {
# It may be possible to have v1 + v2 in parallel someday:
$lei->err(<<EOM) if defined $v1_bare;
@@ -238,14 +250,16 @@ sub try_manifest {
# @v2_epochs
# ignoring $v1_bare (use --inbox-version=1 to force v1 instead)
EOM
- @v2_epochs = map { $uri->path($_); $uri->clone } @v2_epochs;
+ @v2_epochs = map {
+ $uri->path($path_pfx.$_);
+ $uri->clone
+ } @v2_epochs;
clone_v2($self, \@v2_epochs);
- } elsif ($v1_bare) {
+ } elsif (defined $v1_bare) {
clone_v1($self);
- } elsif (my @maybe = grep(m!\Q$path\E!, keys %$m)) {
- die "E: confused by <$uri>, possible matches:\n@maybe";
} else {
- die "E: confused by <$uri>";
+ die "E: confused by <$uri>, possible matches:\n\t",
+ join(', ', sort keys %$m), "\n";
}
}
diff --git a/t/lei-mirror.psgi b/t/lei-mirror.psgi
new file mode 100644
index 00000000..6b4bbfec
--- /dev/null
+++ b/t/lei-mirror.psgi
@@ -0,0 +1,9 @@
+use Plack::Builder;
+use PublicInbox::WWW;
+my $www = PublicInbox::WWW->new;
+$www->preload;
+builder {
+ enable 'Head';
+ mount '/pfx' => builder { sub { $www->call(@_) } };
+ mount '/' => builder { sub { $www->call(@_) } };
+};
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 80bc6ed5..65b6068c 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -7,7 +7,8 @@ my $sock = tcp_server();
my ($tmpdir, $for_destroy) = tmpdir();
my $http = 'http://'.tcp_host_port($sock);
my ($ro_home, $cfg_path) = setup_public_inboxes;
-my $cmd = [ qw(-httpd -W0), "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
+my $cmd = [ qw(-httpd -W0 ./t/lei-mirror.psgi),
+ "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
my $td = start_script($cmd, { PI_CONFIG => $cfg_path }, { 3 => $sock });
test_lei({ tmpdir => $tmpdir }, sub {
my $home = $ENV{HOME};
@@ -43,6 +44,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('ls-external');
unlike($lei_out, qr!\Q$t2-fail\E!, 'not added to ls-external');
+ lei_ok('add-external', "$t1-pfx", '--mirror', "$http/pfx/t1/",
+ \'--mirror v1 w/ PSGI prefix');
+
my %phail = (
HTTPS => 'https://public-inbox.org/' . 'phail',
ONION =>
^ permalink raw reply related [relevance 51%]
* native C++ Xapian wrapper [was: Using lei with podman + toolbox]
2021-09-09 23:36 68% ` Eric Wong
@ 2021-09-09 23:51 71% ` Eric Wong
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 23:51 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Eric Wong <e@80x24.org> wrote:
> Off the top of my head, I think Search::Xapian // Xapian.pm was
> the main thing that was missing from CentOS 7. Does Fedora have
> that?
Btw, I've been considering a Just-Ahead-Of-Time C++ wrapper for
Xapian. SWIG-or-not, bindings always seem behind and limited in
functionality compared to the native Xapian API. I still need
to learn the "++" parts of C++...
^ permalink raw reply [relevance 71%]
* Re: Using lei with podman + toolbox
2021-09-09 21:39 64% Using lei with podman + toolbox Konstantin Ryabitsev
@ 2021-09-09 23:36 68% ` Eric Wong
2021-09-09 23:51 71% ` native C++ Xapian wrapper [was: Using lei with podman + toolbox] Eric Wong
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-09 23:36 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Hi, all:
>
> These are my quickie instructions for how to use lei in a toolbox environment
> if you are running a distribution like Fedora and don't want to install a lot
> of perl dependencies into your main OS.
Off the top of my head, I think Search::Xapian // Xapian.pm was
the main thing that was missing from CentOS 7. Does Fedora have
that?
(disclaimer: I don't care for Docker, seems like a giant waste
of space and bandwidth compared to just using the distro)
> 1. Grab the dockerfile:
> https://gist.github.com/mricon/046ba7c8b03bd92176dbe83e04f2466c
>
> Right now, it's as below, though it may change in the future:
> --- start: public-inbox.dockerfile ---
> # Podman/Toolbox container for public-inbox
> FROM docker.io/library/debian
>
> LABEL com.github.containers.toolbox="true" \
> com.github.debarshiray.toolbox="true"
>
> RUN apt-get update && \
> apt-get -y install sudo libcap2-bin locales vim \
> git liburi-perl libemail-mime-perl libplack-perl libtimedate-perl \
Email::MIME isn't used at all outside of tests (but it's widely packaged).
No idea why libcap2-bin and vim are explicit dependencies (any
editor will do). Don't need Plack for lei, either.
> libdbd-sqlite3-perl libsearch-xapian-perl libnet-server-perl \
> libinline-c-perl libemail-address-xs-perl libparse-recdescent-perl \
No need for Net::Server nor Parse::RecDescent for lei. I don't
use Net::Server at all outside of tests, since I use systemd.
Email::Address::XS and TimeDate can be useful for messed up
messages, but low importance (I think they're widely packaged).
E:A:X and P:RD are required for -imapd but nothing else.
> xapian-tools libencode-perl libdbi-perl liblinux-inotify2-perl \
> libio-compress-perl curl libmail-imapclient-perl libsocket-msghdr-perl \
Socket::Msghdr makes lei a teeny bit faster, but I don't think
it's worth using another distro or running a compiler to get
since Inline::C is already available in all distros. Everything
else should be in Fedora...
> sqlite3 libgit2-dev make eatmydata man-db pkg-config
eatmydata shouldn't be useful outside of development, and
libgit2+pkg-config isn't used by lei, yet
(it is for -httpd/-imapd/-nntpd)
^ permalink raw reply [relevance 68%]
* Using lei with podman + toolbox
@ 2021-09-09 21:39 64% Konstantin Ryabitsev
2021-09-09 23:36 68% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-09 21:39 UTC (permalink / raw)
To: meta
Hi, all:
These are my quickie instructions for how to use lei in a toolbox environment
if you are running a distribution like Fedora and don't want to install a lot
of perl dependencies into your main OS.
1. Grab the dockerfile:
https://gist.github.com/mricon/046ba7c8b03bd92176dbe83e04f2466c
Right now, it's as below, though it may change in the future:
--- start: public-inbox.dockerfile ---
# Podman/Toolbox container for public-inbox
FROM docker.io/library/debian
LABEL com.github.containers.toolbox="true" \
com.github.debarshiray.toolbox="true"
RUN apt-get update && \
apt-get -y install sudo libcap2-bin locales vim \
git liburi-perl libemail-mime-perl libplack-perl libtimedate-perl \
libdbd-sqlite3-perl libsearch-xapian-perl libnet-server-perl \
libinline-c-perl libemail-address-xs-perl libparse-recdescent-perl \
xapian-tools libencode-perl libdbi-perl liblinux-inotify2-perl \
libio-compress-perl curl libmail-imapclient-perl libsocket-msghdr-perl \
sqlite3 libgit2-dev make eatmydata man-db pkg-config
# Change this to your locale, if you're not en_CA
RUN echo "en_CA.UTF-8 UTF-8" >> /etc/locale.gen && \
locale-gen && \
sed -i -e 's/ ALL$/ NOPASSWD:ALL/' /etc/sudoers && \
touch /etc/localtime && \
echo VARIANT_ID=container >> /etc/os-release
RUN git clone https://public-inbox.org /usr/local/public-inbox && \
cd /usr/local/public-inbox && \
perl Makefile.PL && \
make && \
make install && \
make clean
CMD /bin/bash
--- end: public-inbox.dockerfile ---
2. podman build -t public-inbox -f public-inbox.dockerfile
3. toolbox create --image localhost/public-inbox:latest lei
4. toolbox enter lei
That should let you use "lei" commands right after entering the container. To
update public-inbox, just run "git pull" in /usr/local/public-inbox and build
it again.
-K
^ permalink raw reply [relevance 64%]
* Re: Tracking one-off threads with lei
2021-09-09 20:56 71% ` Konstantin Ryabitsev
@ 2021-09-09 21:06 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 21:06 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Thu, Sep 09, 2021 at 08:06:28PM +0000, Eric Wong wrote:
> > Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> > > Eric:
> > >
> > > Is there a way to "tag" a single thread that isn't matching a saved search and
> > > have it be followed for any new updates? E.g. someone pings a developer on IRC
> > > and says "you may be interested in following this discussion" -- what's the
> > > best course of action for them to pull that into their MFOLDER and get all the
> > > new updates?
> >
> > -t includes every message in the thread strictly (same with mairix):
> >
> > lei q -t -o $MFOLDER mid:$MSGID
>
> Ah, sorry, I wasn't clear -- this would be into an existing $MFOLDER with a
> regular q already defined. E.g. I have ~/Maildir/foofunc with:
>
> q = dfhh:foofunc
>
> However, I am now suddenly interested in a thread with msgid foo@bar. I can
> modify the q= parameter to be "dfhh:foofunc OR mid:foo@bar", or I can define a
> new $MFOLDER just for mid:foo@bar, but it doesn't look like I can just one-off
> cherry-pick a thread into an existing ~/Maildir/foofunc, right?
You can modify existing searches with:
lei edit-search $MFOLDER
And possibly reset the external.$FOO.maxuid for local externals
to so old messages aren't excluded (dedupe should still work).
> Is there a way to feed multiple saved searches into the same local maildir?
> (Not saying there should be a way, just trying to get a clear picture in my
> head.)
Not sure, maybe --augment works, but I haven't thought about how
it interacts with saved searches at all
Of course Xapian OR lets you combine as many queries as you want
(but things like -t and the external list used is global across
all subqueries).
^ permalink raw reply [relevance 71%]
* Re: Tracking one-off threads with lei
2021-09-09 20:06 71% ` Eric Wong
@ 2021-09-09 20:56 71% ` Konstantin Ryabitsev
2021-09-09 21:06 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-09 20:56 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 09, 2021 at 08:06:28PM +0000, Eric Wong wrote:
> Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> > Eric:
> >
> > Is there a way to "tag" a single thread that isn't matching a saved search and
> > have it be followed for any new updates? E.g. someone pings a developer on IRC
> > and says "you may be interested in following this discussion" -- what's the
> > best course of action for them to pull that into their MFOLDER and get all the
> > new updates?
>
> -t includes every message in the thread strictly (same with mairix):
>
> lei q -t -o $MFOLDER mid:$MSGID
Ah, sorry, I wasn't clear -- this would be into an existing $MFOLDER with a
regular q already defined. E.g. I have ~/Maildir/foofunc with:
q = dfhh:foofunc
However, I am now suddenly interested in a thread with msgid foo@bar. I can
modify the q= parameter to be "dfhh:foofunc OR mid:foo@bar", or I can define a
new $MFOLDER just for mid:foo@bar, but it doesn't look like I can just one-off
cherry-pick a thread into an existing ~/Maildir/foofunc, right?
Is there a way to feed multiple saved searches into the same local maildir?
(Not saying there should be a way, just trying to get a clear picture in my
head.)
Thanks,
-K
^ permalink raw reply [relevance 71%]
* Re: Tracking one-off threads with lei
2021-09-09 15:19 71% Tracking one-off threads with lei Konstantin Ryabitsev
@ 2021-09-09 20:06 71% ` Eric Wong
2021-09-09 20:56 71% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-09 20:06 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Eric:
>
> Is there a way to "tag" a single thread that isn't matching a saved search and
> have it be followed for any new updates? E.g. someone pings a developer on IRC
> and says "you may be interested in following this discussion" -- what's the
> best course of action for them to pull that into their MFOLDER and get all the
> new updates?
-t includes every message in the thread strictly (same with mairix):
lei q -t -o $MFOLDER mid:$MSGID
For a fuzzy subject match (like /t/ and /T/ in WWW), add ` OR s:"..." '
lei q -t -o $MFOLDER mid:$MSGID OR s:"subject phrase"
And the usual: lei up $MFOLDER
^ permalink raw reply [relevance 71%]
* Tracking one-off threads with lei
@ 2021-09-09 15:19 71% Konstantin Ryabitsev
2021-09-09 20:06 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-09 15:19 UTC (permalink / raw)
To: meta
Eric:
Is there a way to "tag" a single thread that isn't matching a saved search and
have it be followed for any new updates? E.g. someone pings a developer on IRC
and says "you may be interested in following this discussion" -- what's the
best course of action for them to pull that into their MFOLDER and get all the
new updates?
-K
^ permalink raw reply [relevance 71%]
* lei refresh-mail-sync [vs. prune-mail-sync]
@ 2021-09-09 5:47 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 5:47 UTC (permalink / raw)
To: meta
I'm thinking "lei prune-mail-sync" should be replaced with
"lei refresh-mail-sync". Some things move around in
Maildir and get pruned from mail_sync.sqlite3 instead
of updated if lei-daemon wasn't running (or the inotify
queue overflows)
^ permalink raw reply [relevance 71%]
* [PATCH] lei up: print messages before disconnecting
@ 2021-09-09 5:34 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 5:34 UTC (permalink / raw)
To: meta
Closing the socket for script/lei needs to be done AFTER the
final message(s) are printed.
---
lib/PublicInbox/LeiFinmsg.pm | 9 +++++----
lib/PublicInbox/LeiUp.pm | 2 +-
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiFinmsg.pm b/lib/PublicInbox/LeiFinmsg.pm
index 395e7d3c..7ed58c24 100644
--- a/lib/PublicInbox/LeiFinmsg.pm
+++ b/lib/PublicInbox/LeiFinmsg.pm
@@ -8,14 +8,15 @@ use strict;
use v5.10.1;
sub new {
- my ($cls, $io) = @_;
- bless [ $io, $$ ], $cls;
+ my ($cls, $lei) = @_;
+ bless [ @$lei{qw(2 sock)}, $$ ], $cls;
}
sub DESTROY {
my ($self) = @_;
- my $io = shift @$self;
- shift(@$self) == $$ and print $io @$self;
+ my ($stderr, $sock, $pid) = splice(@$self, 0, 3);
+ print $stderr @$self if $pid == $$;
+ # script/lei disconnects when $sock SvREFCNT drops to zero
}
1;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 30358e9d..be476427 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -73,7 +73,7 @@ sub up1_redispatch {
sub redispatch_all ($$) {
my ($self, $lei) = @_;
# re-dispatch into our event loop w/o creating an extra fork-level
- $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei->{2});
+ $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei);
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
PublicInbox::DS::requeue(sub {
^ permalink raw reply related [relevance 70%]
* [PATCH] lei prune-mail-sync: ignore missing locations
@ 2021-09-08 19:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-08 19:04 UTC (permalink / raw)
To: meta
"lei prune-mail-sync --all" shouldn't abort if a location
isn't available, and maybe it should prune harder...
---
lib/PublicInbox/LeiPruneMailSync.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
index 1a277122..3678bd04 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -73,7 +73,7 @@ lei mail_sync.sqlite3 uninitialized, see lei-import(1)
EOM
}
$sto->write_prepare($lei);
- my $self = bless {}, __PACKAGE__;
+ my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
^ permalink raw reply related [relevance 71%]
* [PATCH] lei-rm: add man page, support LeiInput args
@ 2021-09-08 18:48 54% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-08 18:48 UTC (permalink / raw)
To: meta
-F/--in-format and --lock=TYPE(S) are easily supported by
all classes using LeiInput.
---
Documentation/lei-rm.pod | 72 ++++++++++++++++++++++++++++++++++++++++
MANIFEST | 1 +
Makefile.PL | 2 +-
lib/PublicInbox/LEI.pm | 2 +-
4 files changed, 75 insertions(+), 2 deletions(-)
create mode 100644 Documentation/lei-rm.pod
diff --git a/Documentation/lei-rm.pod b/Documentation/lei-rm.pod
new file mode 100644
index 00000000..f2a0c0f0
--- /dev/null
+++ b/Documentation/lei-rm.pod
@@ -0,0 +1,72 @@
+=head1 NAME
+
+lei-rm - unindex a message in lei/store
+
+=head1 SYNOPSIS
+
+lei rm [OPTIONS] (-|--stdin)
+
+lei rm [OPTIONS] LOCATION
+
+=head1 DESCRIPTION
+
+Removes message(s) and associated private metadata from lei/store
+indices. It does not affect messages stored in externals, so it's
+still possible to get "removed" messages from externals in L<lei-q>
+search results.
+
+This does not remove the message from underlying git storage nor
+does it remove messages from Maildir/mbox/IMAP/etc. sources.
+
+=head1 OPTIONS
+
+=over
+
+=item -
+
+=item --stdin
+
+Read input from standard input. This is the default if standard
+input is a pipe or regular file and there are no arguments on
+the command-line.
+
+=item -F MAIL_FORMAT
+
+=item --in-format=MAIL_FORMAT
+
+Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
+C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+
+Default: C<eml> when reading from stdin
+
+=item --lock=METHOD
+
+L<mbox(5)> locking method(s) to use: C<dotlock>, C<fcntl>, C<flock> or
+C<none>.
+
+Default: fcntl,dotlock
+
+=item -q
+
+=item --quiet
+
+Suppress feedback messages.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-store-format(5)>
diff --git a/MANIFEST b/MANIFEST
index fad29622..531f8c46 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -42,6 +42,7 @@ Documentation/lei-overview.pod
Documentation/lei-p2q.pod
Documentation/lei-q.pod
Documentation/lei-rediff.pod
+Documentation/lei-rm.pod
Documentation/lei-store-format.pod
Documentation/lei-tag.pod
Documentation/lei-up.pod
diff --git a/Makefile.PL b/Makefile.PL
index 2af8c2f1..82b50543 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -48,7 +48,7 @@ $v->{-m1} = [ map {
lei-add-external lei-blob lei-config lei-convert lei-edit-search
lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
lei-import lei-init lei-lcat lei-ls-external lei-ls-label
- lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-tag
+ lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-rm lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a258722e..3dce0236 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -226,7 +226,7 @@ our %CMD = ( # sorted in order of importance/use:
'rm' => [ '--stdin|LOCATION...',
'remove a message from the index and prevent reindexing',
'stdin|', # /|\z/ must be first for lone dash
- @c_opt ],
+ qw(in-format|F=s lock=s@), @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
^ permalink raw reply related [relevance 54%]
* Re: Showcasing lei at Linux Plumbers
2021-09-08 17:17 67% ` Konstantin Ryabitsev
@ 2021-09-08 17:32 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-08 17:32 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Wed, Sep 08, 2021 at 02:49:48PM +0000, Eric Wong wrote:
> > I've been thinking of making lei storage accessible as Maildirs
> > via FUSE, as well.
>
> That's a pretty cool idea, actually -- would that be readonly or with full
> deletes/renames support?
Renames for sure. Likely deletes, at least on a per-label basis.
Haven't thought much about deletes/purge...
^ permalink raw reply [relevance 71%]
* Re: Showcasing lei at Linux Plumbers
2021-09-08 14:49 66% ` Eric Wong
@ 2021-09-08 17:17 67% ` Konstantin Ryabitsev
2021-09-08 17:32 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-08 17:17 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Wed, Sep 08, 2021 at 02:49:48PM +0000, Eric Wong wrote:
> > On the other hand, a service that offers full search-based imap/pop3 folders
> > is going to be an easy sell:
> >
> > - it works with any imap client as a simple extra account
> > - it can be mirrored locally and synced two-ways via mbsync
>
> POP3 would be significantly easier to support server-side with
> multiple users since it won't need to store per-user keywords.
Okay, then perhaps I should sit on my hands for a bit. I'll showcase lei with
remote searches as a feature preview, but buffer it with the following
statements:
- We're working on making it easy to add search-based inboxes that would allow
developers to closely match subsystem MAINTAINERS entries. In fact, we can
probably automate the creation of such feeds by watching the MAINTAINERS
file and automatically converting F:/X: lines into queries (not so easily
done for K: and N: lines unless they aren't using actual regex expressions).
- Developers will be able to easily access these feeds via multiple ways, e.g:
- read-only imap folders
- pseudo mailing list subscriptions
- nntp groups
- pop3 mailboxes (coming in the future)
The goal is to solve the following several problems:
- remove content-mangling corporate mail gateways out of the picture
- make it unnecessary for patch submitters to know where they should send the
patches ("just send them to patches@linux.dev").
- reduce the need for new mailing lists as new subsystems are introduced
("just send email to discuss@linux.dev with somekeyword: in the subject")
I think that sounds pretty reasonable and I can get most of it done by EOY.
> > - I can do clever things like suspend "lei up" runs if there was no access to
> > the folder for over N weeks
> > - we can use FS dedupe features since all messages are going to be
> > identical after writing them out to maildirs
>
> I've been thinking of making lei storage accessible as Maildirs
> via FUSE, as well.
That's a pretty cool idea, actually -- would that be readonly or with full
deletes/renames support?
-K
^ permalink raw reply [relevance 67%]
* Re: Showcasing lei at Linux Plumbers
2021-09-08 13:36 65% ` Konstantin Ryabitsev
@ 2021-09-08 14:49 66% ` Eric Wong
2021-09-08 17:17 67% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-08 14:49 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Tue, Sep 07, 2021 at 10:14:04PM +0000, Eric Wong wrote:
> > > One of the
> > > options I want to investigate is making IMAP/POP3 accessible individual
> > > mailboxes fed by lei, such that a new subsystem maintainer could have a
> > > ready-made mailbox available to them without needing to subscribe/unsubscribe
> > > to a bunch of mailing lists. (This would be different from read-only imap
> > > mailboxes offered by public-inbox-imapd, since we'll be tracking individual
> > > message state. The POP3 bit would allow them to plug it into something like
> > > Gmail which allows sucking down remote POPs.)
> >
> > I think using the "-o v2:..." option for now would be the way to
> > go for making a v2 inbox available via -imapd (and it'll get
> > JMAP/POP3 support in the future).
>
> I'm worried that read-only imap folders are going to cause problems for dumber
> imap clients, including mbsync. My goal is to make it easy for folks to use
> existing tools to which they are already accustomed, since my experience is
> that if the learning curve is too steep or requires too much fiddling to
> configure, the uptake is going to be extremely limited.
Agreed with read-only IMAP being a problem for existing clients.
lei is gradual in that approach in you can pick and choose which
parts to use. It's actually close to being able to offer
<mbsync||offlineimap> functionality, but it's a bit clumsy
usage-wise atm:
lei import imaps://example.com/folder
# lei <q|lcat> results dumped to Maildir
# inotify reads Maildir keyword updates done by MUA
lei export-kw imaps://example.com/folder
I'm working on making the "export-kw" part transparent like it
mostly is with Maildirs.
The one thing lei doesn't do right now is deleting messages
from IMAP folders (unless it's overwriting search results).
That will probably be a separate command:
lei prune-mfolder [--expire=...]
I hope to stop using <mbsync||offlineimap> myself, soon...
> On the other hand, a service that offers full search-based imap/pop3 folders
> is going to be an easy sell:
>
> - it works with any imap client as a simple extra account
> - it can be mirrored locally and synced two-ways via mbsync
POP3 would be significantly easier to support server-side with
multiple users since it won't need to store per-user keywords.
Since lei is a daemon and can support multiple users, it could
have an R/W JMAP||IMAP front-end, though...
> - it can be incorporated into existing services like gmail, so people can
> monitor things on the go
POP3 seems excellent for integrating into large mail providers.
I mainly haven't gotten around to implementing it, nor figuring
out how to deal with account management...
> - I can do clever things like suspend "lei up" runs if there was no access to
> the folder for over N weeks
> - we can use FS dedupe features since all messages are going to be
> identical after writing them out to maildirs
I've been thinking of making lei storage accessible as Maildirs
via FUSE, as well.
> The slightly harder part is making it easy for people to configure their
> search parameters, but I'm hoping to expose this via a git repo.
*shrug* I've been trying to keep the learning curve as low as
possible by using most of the same prefixes as mairix (lei only
adds L: and kw: for labels and keywords).
^ permalink raw reply [relevance 66%]
* Re: Showcasing lei at Linux Plumbers
2021-09-07 22:14 71% ` Eric Wong
@ 2021-09-08 13:36 65% ` Konstantin Ryabitsev
2021-09-08 14:49 66% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-08 13:36 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Tue, Sep 07, 2021 at 10:14:04PM +0000, Eric Wong wrote:
> > One of the
> > options I want to investigate is making IMAP/POP3 accessible individual
> > mailboxes fed by lei, such that a new subsystem maintainer could have a
> > ready-made mailbox available to them without needing to subscribe/unsubscribe
> > to a bunch of mailing lists. (This would be different from read-only imap
> > mailboxes offered by public-inbox-imapd, since we'll be tracking individual
> > message state. The POP3 bit would allow them to plug it into something like
> > Gmail which allows sucking down remote POPs.)
>
> I think using the "-o v2:..." option for now would be the way to
> go for making a v2 inbox available via -imapd (and it'll get
> JMAP/POP3 support in the future).
I'm worried that read-only imap folders are going to cause problems for dumber
imap clients, including mbsync. My goal is to make it easy for folks to use
existing tools to which they are already accustomed, since my experience is
that if the learning curve is too steep or requires too much fiddling to
configure, the uptake is going to be extremely limited.
On the other hand, a service that offers full search-based imap/pop3 folders
is going to be an easy sell:
- it works with any imap client as a simple extra account
- it can be mirrored locally and synced two-ways via mbsync
- it can be incorporated into existing services like gmail, so people can
monitor things on the go
- I can do clever things like suspend "lei up" runs if there was no access to
the folder for over N weeks
- we can use FS dedupe features since all messages are going to be
identical after writing them out to maildirs
The slightly harder part is making it easy for people to configure their
search parameters, but I'm hoping to expose this via a git repo.
I'm not implementing this right away, but I'm going to float this idea at
plumbers to see what the reception is going to be. I believe this will be of
interest to many devs, since this would allow them to no longer depend on
their corporate mail servers and their mail-mangling ways.
-K
^ permalink raw reply [relevance 65%]
* [PATCH] lei q|up: fix write counter for v2
@ 2021-09-07 22:41 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 22:41 UTC (permalink / raw)
To: meta
It's a bit confusing to see "0 written to ..." when we actually
wrote something.
---
lib/PublicInbox/LeiToMail.pm | 1 +
t/lei-q-save.t | 2 ++
2 files changed, 3 insertions(+)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 01f08384..dbf58df9 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -361,6 +361,7 @@ sub _v2_write_cb ($$) {
$eml //= PublicInbox::Eml->new($bref);
return if $dedupe && $dedupe->is_dup($eml, $smsg);
$lei->{v2w}->ipc_do('add', $eml); # V2Writable->add
+ ++$lei->{-nr_write};
}
}
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 7aa9b84e..743a7b70 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -179,6 +179,8 @@ test_lei(sub {
my (@before, @after);
require PublicInbox::MboxReader;
lei_ok(qw(q z:0.. -o), "v2:$v2");
+ like($lei_err, qr/^# ([1-9][0-9]*) written to \Q$v2\E/sm,
+ 'non-zero write output to stderr');
lei_ok(qw(q z:0.. -o), "mboxrd:$home/before", '--only', $v2, '-j1,1');
open my $fh, '<', "$home/before";
PublicInbox::MboxReader->mboxrd($fh, sub { push @before, $_[0] });
^ permalink raw reply related [relevance 71%]
* Re: Showcasing lei at Linux Plumbers
2021-09-07 21:33 71% ` Konstantin Ryabitsev
@ 2021-09-07 22:14 71% ` Eric Wong
2021-09-08 13:36 65% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-07 22:14 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Thu, Sep 02, 2021 at 09:58:50PM +0000, Eric Wong wrote:
> > # the destination, could be Maildir
> > MFOLDER=imaps://user@example.com/INBOX.landlock
> >
> > # initial search:
> > lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
>
> If I had a local mirror with extindex and I wanted to do the same thing, would
> I just modify the -I flag to point at the extindex location?
Yes. For local stuff that's permanently mounted, I tend to do
"lei add-external $PATHNAME" so it's included by default.
> One of the
> options I want to investigate is making IMAP/POP3 accessible individual
> mailboxes fed by lei, such that a new subsystem maintainer could have a
> ready-made mailbox available to them without needing to subscribe/unsubscribe
> to a bunch of mailing lists. (This would be different from read-only imap
> mailboxes offered by public-inbox-imapd, since we'll be tracking individual
> message state. The POP3 bit would allow them to plug it into something like
> Gmail which allows sucking down remote POPs.)
I think using the "-o v2:..." option for now would be the way to
go for making a v2 inbox available via -imapd (and it'll get
JMAP/POP3 support in the future).
We don't have POP3 support in client nor server form, yet. Not
sure how account/state management would work, nor how to
prioritize it vs JMAP support. I'm thinking POP3 takes priority
since there's more clients for it...
Existing POP3 servers would work, too; since lei can output
to Maildir/mbox* which can work with them.
On a side note, I'm not aware of IMAP sync tools accounting for
read-only IMAP servers well, since they attempt bidirectional
sync. "lei import" seems alright in that regard, treating IMAP
the same way it will (eventually) treat POP3.
^ permalink raw reply [relevance 71%]
* Re: Showcasing lei at Linux Plumbers
2021-09-02 21:58 65% ` Eric Wong
2021-09-03 15:15 71% ` Konstantin Ryabitsev
@ 2021-09-07 21:33 71% ` Konstantin Ryabitsev
2021-09-07 22:14 71% ` Eric Wong
1 sibling, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-07 21:33 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 02, 2021 at 09:58:50PM +0000, Eric Wong wrote:
> OK, there's two main commands, "lei q" and "lei up".
> Both of which may prompt for passwords depending on how
> git-credential is set up:
>
> # the destination, could be Maildir
> MFOLDER=imaps://user@example.com/INBOX.landlock
>
> # initial search:
> lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
> (
> dfn:Documentation/security/landlock.rst OR
> dfn:Documentation/userspace-api/landlock.rst OR
> dfn:include/uapi/linux/landlock.h OR
> dfn:samples/landlock/ OR
> dfn:security/landlock/ OR
> dfn:tools/testing/selftests/landlock/ OR
> dfhh:landlock
> ) AND rt:2.months.ago..
> EOF
>
> # update whenever, may prompt for IMAP password, but could be
> # cron-ed or similar if passwords are cached
> lei up $MFOLDER
>
> # Optional: tweaking the search parameters can be done via
> lei edit-search $MFOLDER
If I had a local mirror with extindex and I wanted to do the same thing, would
I just modify the -I flag to point at the extindex location? One of the
options I want to investigate is making IMAP/POP3 accessible individual
mailboxes fed by lei, such that a new subsystem maintainer could have a
ready-made mailbox available to them without needing to subscribe/unsubscribe
to a bunch of mailing lists. (This would be different from read-only imap
mailboxes offered by public-inbox-imapd, since we'll be tracking individual
message state. The POP3 bit would allow them to plug it into something like
Gmail which allows sucking down remote POPs.)
-K
^ permalink raw reply [relevance 71%]
* [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
` (2 preceding siblings ...)
2021-09-07 11:32 38% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
@ 2021-09-07 11:32 52% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
We missed a few when new documentation came in, and there's no
going back to v2 onions.
Followup-to: 0b15dfc58ceaecdc ("treewide: update to v3 Tor onions")
---
Documentation/lei-convert.pod | 2 +-
Documentation/lei-edit-search.pod | 2 +-
Documentation/lei-forget-search.pod | 2 +-
Documentation/lei-lcat.pod | 2 +-
Documentation/lei-ls-mail-sync.pod | 2 +-
Documentation/lei-ls-search.pod | 2 +-
Documentation/lei-rediff.pod | 2 +-
Documentation/lei-up.pod | 2 +-
8 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/Documentation/lei-convert.pod b/Documentation/lei-convert.pod
index e8a71393..7f372327 100644
--- a/Documentation/lei-convert.pod
+++ b/Documentation/lei-convert.pod
@@ -59,7 +59,7 @@ L<lei-q(1)>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-edit-search.pod b/Documentation/lei-edit-search.pod
index 7908b5a2..21cb11aa 100644
--- a/Documentation/lei-edit-search.pod
+++ b/Documentation/lei-edit-search.pod
@@ -15,7 +15,7 @@ Invoke C<git config --edit> to edit the saved search at C<OUTPUT>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
index 49bc1d68..f3f043f9 100644
--- a/Documentation/lei-forget-search.pod
+++ b/Documentation/lei-forget-search.pod
@@ -15,7 +15,7 @@ Forget a saved search at C<OUTPUT>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
index 5a2bdb5a..656df489 100644
--- a/Documentation/lei-lcat.pod
+++ b/Documentation/lei-lcat.pod
@@ -66,7 +66,7 @@ The following options, described in L<lei-q(1)>, are supported.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-ls-mail-sync.pod b/Documentation/lei-ls-mail-sync.pod
index 37aa910f..86aede40 100644
--- a/Documentation/lei-ls-mail-sync.pod
+++ b/Documentation/lei-ls-mail-sync.pod
@@ -42,7 +42,7 @@ Use C<\0> (NUL) instead of newline (CR) to delimit lines.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-ls-search.pod b/Documentation/lei-ls-search.pod
index 138dbbff..a56611bf 100644
--- a/Documentation/lei-ls-search.pod
+++ b/Documentation/lei-ls-search.pod
@@ -51,7 +51,7 @@ is incompatible with C<--format>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index e968fb20..c7db6c1e 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -67,7 +67,7 @@ The options below, described in L<lei-q(1)>, are also supported.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index ca4cf4fe..e5d97f43 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -43,7 +43,7 @@ This option is incompatible with C<--all>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
^ permalink raw reply related [relevance 52%]
* [PATCH 0/4] lei up --all support for IMAP
@ 2021-09-07 11:32 71% Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
It's not efficient since it wastes an IMAP connection for
auth-only (unlike every other lei command which reuses the
connection once authed). However, there may be ways we can
work around it in the future.
Eric Wong (4):
xt/net_writer_imap: test "lei up" on single IMAP output
lei: dump and clear log at exit
lei up: support --all for IMAP folders
doc: lei-*.pod: update to Tor v3 .onion address
Documentation/lei-convert.pod | 2 +-
Documentation/lei-edit-search.pod | 2 +-
Documentation/lei-forget-search.pod | 2 +-
Documentation/lei-lcat.pod | 2 +-
Documentation/lei-ls-mail-sync.pod | 2 +-
Documentation/lei-ls-search.pod | 2 +-
Documentation/lei-rediff.pod | 2 +-
Documentation/lei-up.pod | 17 +++--
lib/PublicInbox/LEI.pm | 20 +++---
lib/PublicInbox/LeiAuth.pm | 15 ++--
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 104 +++++++++++++++++-----------
xt/net_writer-imap.t | 4 ++
13 files changed, 114 insertions(+), 63 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
@ 2021-09-07 11:32 71% ` Eric Wong
2021-09-07 11:32 71% ` [PATCH 2/4] lei: dump and clear log at exit Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
That's the minimum, at least...
---
xt/net_writer-imap.t | 3 +++
1 file changed, 3 insertions(+)
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index ec8f80d1..aeed3144 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -239,6 +239,9 @@ EOM
lei_ok qw(q m:forwarded@test.example.com);
is_deeply(json_utf8->decode($lei_out)->[0]->{kw}, ['forwarded'],
'forwarded kw imported from IMAP');
+
+ lei_ok qw(q m:testmessage --no-external -o), $folder_url;
+ lei_ok qw(up), $folder_url;
});
undef $cleanup; # remove temporary folder
^ permalink raw reply related [relevance 71%]
* [PATCH 2/4] lei: dump and clear log at exit
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
@ 2021-09-07 11:32 71% ` Eric Wong
2021-09-07 11:32 38% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
2021-09-07 11:32 52% ` [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
This may be helpful for diagnosing errors in case we missed any.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 098a45ba..bd44cfae 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1330,6 +1330,7 @@ sub lazy_start {
open STDOUT, '>&STDIN' or die "redirect stdout failed: $!";
# $daemon pipe to `lei' closed, main loop begins:
PublicInbox::DS->EventLoop;
+ dump_and_clear_log();
exit($exit_code // 0);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 3/4] lei up: support --all for IMAP folders
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
2021-09-07 11:32 71% ` [PATCH 2/4] lei: dump and clear log at exit Eric Wong
@ 2021-09-07 11:32 38% ` Eric Wong
2021-09-07 11:32 52% ` [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
Since "lei up" is expected to be a heavily-used command,
better support for IMAP seems like a reasonable idea.
This is inefficient since we waste an IMAP(S) TCP connection
since it dies when an auth-only LeiUp worker process dies, but
it's better than not working at all, right now.
---
Documentation/lei-up.pod | 15 ++++-
lib/PublicInbox/LEI.pm | 19 ++++---
lib/PublicInbox/LeiAuth.pm | 15 +++--
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 104 ++++++++++++++++++++++-------------
xt/net_writer-imap.t | 1 +
6 files changed, 102 insertions(+), 55 deletions(-)
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index cea0f619..ca4cf4fe 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -6,15 +6,24 @@ lei-up - update a saved search
lei up [OPTIONS] OUTPUT
-lei up [OPTIONS] --all=TYPE
+lei up [OPTIONS] --all[=<local|remote>]
=head1 DESCRIPTION
-Update the saved search at C<OUTPUT> or all saved searches of C<TYPE>
-(currently C<local> is the only supported value).
+Update the saved search at C<OUTPUT> or all saved searches.
=head1 OPTIONS
+=over
+
+=item --all[=<local|remote>]
+
+C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
+C<--all=local> only updates local mailboxes, C<--all=remote> only
+updates remote mailboxes (currently C<imap://> and C<imaps://>).
+
+=back
+
The following options, described in L<lei-q(1)>, are supported.
=over
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bd44cfae..a258722e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -605,16 +605,19 @@ sub incr {
$self->{counters}->{$field} += $nr;
}
+sub pkt_ops {
+ my ($lei, $ops) = @_;
+ $ops->{'!'} = [ \&fail_handler, $lei ];
+ $ops->{'|'} = [ \&sigpipe_handler, $lei ];
+ $ops->{x_it} = [ \&x_it, $lei ];
+ $ops->{child_error} = [ \&child_error, $lei ];
+ $ops->{incr} = [ \&incr, $lei ];
+ $ops;
+}
+
sub workers_start {
my ($lei, $wq, $jobs, $ops, $flds) = @_;
- $ops = {
- '!' => [ \&fail_handler, $lei ],
- '|' => [ \&sigpipe_handler, $lei ],
- 'x_it' => [ \&x_it, $lei ],
- 'child_error' => [ \&child_error, $lei ],
- 'incr' => [ \&incr, $lei ],
- ($ops ? %$ops : ()),
- };
+ $ops = pkt_ops($lei, { ($ops ? %$ops : ()) });
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm
index 465a2758..9b09cecf 100644
--- a/lib/PublicInbox/LeiAuth.pm
+++ b/lib/PublicInbox/LeiAuth.pm
@@ -30,10 +30,16 @@ sub do_auth_atfork { # used by IPC WQ workers
return if $wq->{-wq_worker_nr} != 0; # only first worker calls this
my $lei = $wq->{lei};
my $net = $lei->{net};
+ if ($net->{-auth_done}) { # from previous worker... (ugly)
+ $lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
+ $lei->fail("pkt_do net_merge_continue: $!");
+ return;
+ }
eval { # fill auth info (may prompt user or read netrc)
my $mics = $net->imap_common_init($lei);
my $nn = $net->nntp_common_init($lei);
# broadcast successful auth info to lei-daemon:
+ $net->{-auth_done} = 1;
$lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
die "pkt_do net_merge_continue: $!";
$net->{mics_cached} = $mics if $mics;
@@ -51,14 +57,15 @@ sub net_merge_all { # called in wq worker via wq_broadcast
# called by top-level lei-daemon when first worker is done with auth
# passes updated net auth info to current workers
sub net_merge_continue {
- my ($wq, $net_new) = @_;
+ my ($wq, $lei, $net_new) = @_;
+ $wq->{-net_new} = $net_new; # for "lei up"
$wq->wq_broadcast('PublicInbox::LeiAuth::net_merge_all', $net_new);
- $wq->net_merge_all_done; # defined per-WQ
+ $wq->net_merge_all_done($lei); # defined per-WQ
}
sub op_merge { # prepares PktOp->pair ops
- my ($self, $ops, $wq) = @_;
- $ops->{net_merge_continue} = [ \&net_merge_continue, $wq ];
+ my ($self, $ops, $wq, $lei) = @_;
+ $ops->{net_merge_continue} = [ \&net_merge_continue, $wq, $lei ];
}
sub new { bless \(my $x), __PACKAGE__ }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 2c7a92de..01f08384 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -396,7 +396,8 @@ sub new {
} elsif ($fmt =~ /\Aimaps?\z/) {
require PublicInbox::NetWriter;
require PublicInbox::URIimap;
- my $net = PublicInbox::NetWriter->new;
+ # {net} may exist from "lei up" for auth
+ my $net = $lei->{net} // PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
$net->add_url($$uri);
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index a39d6047..30358e9d 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -5,8 +5,14 @@
package PublicInbox::LeiUp;
use strict;
use v5.10.1;
+# n.b. we use LeiInput to setup IMAP auth
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiSavedSearch;
-use parent qw(PublicInbox::IPC);
+use PublicInbox::DS;
+use PublicInbox::PktOp;
+use PublicInbox::LeiFinmsg;
+use PublicInbox::LEI;
+my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
sub up1 ($$) {
my ($lei, $out) = @_;
@@ -48,75 +54,95 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
- require PublicInbox::LeiFinmsg;
- $lei->{fmsg} //= PublicInbox::LeiFinmsg->new($lei->{2});
my $l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} };
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
+ delete $l->{sock}; # do not close
+ $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ local $PublicInbox::LEI::current_lei = $l;
$l->{1} = $fh;
eval {
$l->qerr("# updating $out");
up1($l, $out);
- $l->qerr("# $out done");
};
- $l->child_error(0, $@) if $@;
+ $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
+}
+
+sub redispatch_all ($$) {
+ my ($self, $lei) = @_;
+ # re-dispatch into our event loop w/o creating an extra fork-level
+ $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei->{2});
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
+ PublicInbox::DS::requeue(sub {
+ up1_redispatch($lei, $o, $op_p);
+ });
+ }
+ $lei->event_step_init;
+ $lei->pkt_ops($op_c->{ops} = { '' => [$lei->can('dclose'), $lei] });
}
sub lei_up {
my ($lei, @outs) = @_;
- $lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
- my @local;
- if (defined $opt->{all}) {
+ my $self = bless { -mail_sync => 1 }, __PACKAGE__;
+ $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
+ if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
length($opt->{mua}//'') and return
$lei->fail('--all and --mua= are incompatible');
-
- # supporting IMAP outputs is more involved due to
- # git-credential prompts. TODO: add this in 1.8
- $opt->{all} eq 'local' or return
- $lei->fail('only --all=local works at the moment');
- my @all = PublicInbox::LeiSavedSearch::list($lei);
- @local = grep(!m!\Aimaps?://!i, @all);
+ @outs = PublicInbox::LeiSavedSearch::list($lei);
+ if ($all eq 'local') {
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq 'remote') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq '') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } else {
+ $lei->fail("only --all=$all not understood");
+ }
} else {
- @local = @outs;
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
- if (scalar(@outs) > 1) {
- length($opt->{mua}//'') and return $lei->fail(<<EOM);
+ ((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
+ length($opt->{mua} // '')) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
- # TODO:
- return $lei->fail(<<EOM) if grep(m!\Aimaps?://!i, @outs);
-multiple destinations only supported for local outputs (FIXME)
-EOM
+ if ($self->{remote}) { # setup lei->{auth}
+ $self->prepare_inputs($lei, $self->{remote}) or return;
}
- if (scalar(@local) > 1) {
- $lei->_lei_store->write_prepare($lei); # share early
- # daemon mode, re-dispatch into our event loop w/o
- # creating an extra fork-level
- require PublicInbox::DS;
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@local) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
- }
- $lei->event_step_init;
- $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
+ if ($lei->{auth}) { # start auth worker
+ require PublicInbox::NetWriter;
+ bless $lei->{net}, 'PublicInbox::NetWriter';
+ $lei->{auth}->op_merge(my $ops = {}, $self, $lei);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
+ $lei->{wq1} = $self;
+ $lei->wait_wq_events($op_c, $ops);
+ # net_merge_all_done will fire when auth is done
} else {
- up1($lei, $local[0]);
+ redispatch_all($self, $lei); # see below
}
}
+# called in top-level lei-daemon when LeiAuth is done
+sub net_merge_all_done {
+ my ($self, $lei) = @_;
+ $lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
+ $self->wq_close(1);
+ redispatch_all($self, $lei);
+}
+
sub _complete_up {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
}
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+
1;
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index aeed3144..0a4cea68 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -242,6 +242,7 @@ EOM
lei_ok qw(q m:testmessage --no-external -o), $folder_url;
lei_ok qw(up), $folder_url;
+ lei_ok qw(up --all=remote);
});
undef $cleanup; # remove temporary folder
^ permalink raw reply related [relevance 38%]
* Re: Showcasing lei at Linux Plumbers
2021-09-02 21:58 65% ` Eric Wong
@ 2021-09-03 15:15 71% ` Konstantin Ryabitsev
2021-09-07 21:33 71% ` Konstantin Ryabitsev
1 sibling, 0 replies; 200+ results
From: Konstantin Ryabitsev @ 2021-09-03 15:15 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 02, 2021 at 09:58:50PM +0000, Eric Wong wrote:
> Fwiw, most of the functionality works much better with Maildir
> because of potential password prompts needed for IMAP and
> interactivity required.
Okay, I'll try this out with maildir for now -- it's easy to hook mbsync into
the process if desired.
> OK, there's two main commands, "lei q" and "lei up".
> Both of which may prompt for passwords depending on how
> git-credential is set up:
>
> # the destination, could be Maildir
> MFOLDER=imaps://user@example.com/INBOX.landlock
>
> # initial search:
> lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
> (
> dfn:Documentation/security/landlock.rst OR
> dfn:Documentation/userspace-api/landlock.rst OR
> dfn:include/uapi/linux/landlock.h OR
> dfn:samples/landlock/ OR
> dfn:security/landlock/ OR
> dfn:tools/testing/selftests/landlock/ OR
> dfhh:landlock
> ) AND rt:2.months.ago..
> EOF
>
> # update whenever, may prompt for IMAP password, but could be
> # cron-ed or similar if passwords are cached
> lei up $MFOLDER
>
> # Optional: tweaking the search parameters can be done via
> lei edit-search $MFOLDER
Yep, that seems to work fine. Question -- I noticed that lei just issues a
regular query, retrieves results with curl and then parses the output. Is
there a danger of potentially running into issues with parsing the regular
HTML output if it changes in the future?
-K
^ permalink raw reply [relevance 71%]
* [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (2 preceding siblings ...)
2021-09-03 8:54 52% ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
@ 2021-09-03 8:54 83% ` Eric Wong
2021-09-03 8:54 58% ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
` (2 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
We can golf out some code and refcounts this way.
---
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiPruneMailSync.pm | 26 +++++++++++++-------------
2 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2d2ff1a0..2ade17af 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -206,8 +206,7 @@ sub lei_inspect {
sub _complete_inspect {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } $lms->folders;
}
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
index 79f3325d..98239a13 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -40,7 +40,7 @@ sub prune_imap { # lms->each_src callback
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
- my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
my $mdir = $1;
$lms->each_src($input, \&prune_mdir, $self, $mdir);
@@ -59,24 +59,24 @@ sub lei_prune_mail_sync {
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
EOM
- my $lse = $sto->search;
- my $lms = $lse->lms or return $lei->fail(<<EOM);
-lei mail_sync uninitialized, see lei-import(1)
-EOM
- if (defined(my $all = $lei->{opt}->{all})) {
- $lms->group2folders($lei, $all, \@folders) or return;
+ if (my $lms = $lei->lms) {
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
+ }
} else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ return $lei->fail(<<EOM);
+lei mail_sync.sqlite3 uninitialized, see lei-import(1)
+EOM
}
- delete $lms->{dbh};
$sto->write_prepare($lei);
- my $self = bless { lse => $lse }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
- undef $lms; # for fork
my $ops = {};
$sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
^ permalink raw reply related [relevance 83%]
* [PATCH 2/8] lei/store: quiet down link(2) warnings
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
@ 2021-09-03 8:54 71% ` Eric Wong
2021-09-03 8:54 52% ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
` (4 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
ENOENT can be too common due to timing and concurrent access
from MUAs and "lei export-kw", and other mail synchronization
tools (e.g. mbsync and offlineimap).
---
lib/PublicInbox/LeiStore.pm | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index a91b30f7..f81a8dae 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -215,14 +215,10 @@ sub export1_kw_md ($$$$$) {
return;
} elsif ($! == EEXIST) { # lost race with "lei export-kw"?
return;
- } elsif ($! == ENOENT) {
- syslog('warning', "link($src -> $dst): $!")
- } # else loop @try
+ } elsif ($! != ENOENT) {
+ syslog('warning', "link($src -> $dst): $!");
+ }
}
- my $e = $!;
- my $src = "$mdir/{".join(',', @try)."}/$orig";
- my $oidhex = unpack('H*', $oidbin);
- syslog('warning', "link($src -> $dst) ($oidhex): $e");
for (@try) { return if -e "$mdir/$_/$orig" };
lms_clear_src($self, "maildir:$mdir", \$orig);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives
@ 2021-09-03 8:54 66% Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
` (6 more replies)
0 siblings, 7 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
Back to using syslog more for errors, since it's still less
instrusive. IMAP R/W is fixed (ugh, lack of automatic tests
since we only have a read-only IMAP server for testing).
Some error/warning fixes, and false-positives on L: and kw:
searches on local (but not remote) externals are avoided.
"lei inspect --stdin" was somewhat useful for figuring out why
L: was false-positiving an external result on me.
Eric Wong (8):
lei: dump errors to syslog, and not to CLI
lei/store: quiet down link(2) warnings
lei: ->child_error less error-prone
lei: use lei->lms in place of lse->lms in a few places
lei up --all: avoid double-close on shared STDOUT
lei inspect: support reading eml from --stdin
lei_xsearch: avoid false-positives on externals w/ L: and kw:
lei: fix read/write IMAP access
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 27 ++++++++++--------
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 4 +--
lib/PublicInbox/LeiInspect.pm | 43 ++++++++++++++++++++++++-----
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiPruneMailSync.pm | 26 ++++++++---------
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 10 ++-----
lib/PublicInbox/LeiStoreErr.pm | 4 +--
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 6 +++-
lib/PublicInbox/LeiXSearch.pm | 1 +
lib/PublicInbox/NetReader.pm | 5 +++-
lib/PublicInbox/NetWriter.pm | 2 ++
lib/PublicInbox/Watch.pm | 2 ++
t/lei-daemon.t | 5 ----
t/lei-up.t | 39 ++++++++++++++++++++++++++
19 files changed, 130 insertions(+), 55 deletions(-)
create mode 100644 t/lei-up.t
^ permalink raw reply [relevance 66%]
* [PATCH 8/8] lei: fix read/write IMAP access
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (5 preceding siblings ...)
2021-09-03 8:54 61% ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
@ 2021-09-03 8:54 63% ` Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
xt/net_writer-imap.t was completely broken in recent months and
I completely forgot this test. net->add_url still only accepts
bare scalars (and not scalar refs), so we must set that up
properly. Furthermore, our changes to do FLAGS-only
synchronization in lei of old messages was causing us to not
handle FLAGS properly for the test.
---
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/NetReader.pm | 5 ++++-
lib/PublicInbox/NetWriter.pm | 2 ++
lib/PublicInbox/Watch.pm | 2 ++
4 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index be6e178f..6e102a1d 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -406,7 +406,7 @@ sub new {
my $net = PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
- $net->add_url($uri);
+ $net->add_url($$uri);
my $err = $net->errors($lei);
return $lei->fail($err) if $err;
$uri->mailbox or return $lei->fail("No mailbox: $dst");
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 23445e7a..c050c60f 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -493,6 +493,9 @@ sub perm_fl_ok ($) {
undef;
}
+# may be overridden in NetWriter or Watch
+sub folder_select { $_[0]->{each_old} ? 'select' : 'examine' }
+
sub _imap_fetch_all ($$$) {
my ($self, $mic, $orig_uri) = @_;
my $sec = uri_section($orig_uri);
@@ -501,7 +504,7 @@ sub _imap_fetch_all ($$$) {
# we need to check for mailbox writability to see if we care about
# FLAGS from already-imported messages.
- my $cmd = $self->{each_old} ? 'select' : 'examine';
+ my $cmd = $self->folder_select;
$mic->$cmd($mbx) or return "E: \U$cmd\E $mbx ($sec) failed: $!";
my ($r_uidval, $r_uidnext, $perm_fl);
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 82288e6b..629a752a 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,6 +26,8 @@ sub imap_append {
die "APPEND $folder: $@";
}
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
sub imap_delete_all {
my ($self, $uri) = @_;
my $mic = $self->mic_for_folder($uri) or return;
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 86dae91f..482d35c4 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -682,4 +682,6 @@ EOF
undef;
}
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
1;
^ permalink raw reply related [relevance 63%]
* [PATCH 6/8] lei inspect: support reading eml from --stdin
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (4 preceding siblings ...)
2021-09-03 8:54 58% ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
@ 2021-09-03 8:54 61% ` Eric Wong
2021-09-03 8:54 63% ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
This can be useful inside mutt since I was diagnosing why
a label ("L:$FOO") search was giving me a false-positive
search result...
---
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiInspect.pm | 40 ++++++++++++++++++++++++++++++-----
2 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8b6c1d36..098a45ba 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -279,8 +279,8 @@ our %CMD = ( # sorted in order of importance/use:
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
qw(c=s@ C=s@), pass_through('git config') ],
-'inspect' => [ 'ITEMS...', 'inspect lei/store and/or local external',
- qw(pretty ascii dir=s), @c_opt ],
+'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
+ qw(stdin| pretty ascii dir=s), @c_opt ],
'init' => [ '[DIRNAME]', sub {
"initialize storage, default: ".store_path($_[0]);
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2ade17af..25bd47e7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -9,6 +9,7 @@ package PublicInbox::LeiInspect;
use strict;
use v5.10.1;
use PublicInbox::Config;
+use PublicInbox::MID qw(mids);
sub inspect_blob ($$) {
my ($lei, $oidhex) = @_;
@@ -184,6 +185,32 @@ sub inspect1 ($$$) {
1;
}
+sub _inspect_argv ($$) {
+ my ($lei, $argv) = @_;
+ my $multi = scalar(@$argv) > 1;
+ $lei->out('[') if $multi;
+ while (defined(my $x = shift @$argv)) {
+ inspect1($lei, $x, scalar(@$argv)) or return;
+ }
+ $lei->out(']') if $multi;
+}
+
+sub ins_add { # InputPipe->consume callback
+ my ($lei) = @_; # $_[1] = $rbuf
+ if (defined $_[1]) {
+ $_[1] eq '' and return eval {
+ my $str = delete $lei->{istr};
+ $str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+ my $eml = PublicInbox::Eml->new(\$str);
+ _inspect_argv($lei, [ 'blob:'.$lei->git_blob_id($eml),
+ map { "mid:$_" } @{mids($eml)} ]);
+ };
+ $lei->{istr} .= $_[1];
+ } else {
+ $lei->fail("error reading stdin: $!");
+ }
+}
+
sub lei_inspect {
my ($lei, @argv) = @_;
$lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
@@ -196,12 +223,15 @@ sub lei_inspect {
}
$lei->start_pager if -t $lei->{1};
$lei->{1}->autoflush(0);
- my $multi = scalar(@argv) > 1;
- $lei->out('[') if $multi;
- while (defined(my $x = shift @argv)) {
- inspect1($lei, $x, scalar(@argv)) or return;
+ if ($lei->{opt}->{stdin}) {
+ return $lei->fail(<<'') if @argv;
+no args allowed on command-line with --stdin
+
+ require PublicInbox::InputPipe;
+ PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
+ return;
}
- $lei->out(']') if $multi;
+ _inspect_argv($lei, \@argv);
}
sub _complete_inspect {
^ permalink raw reply related [relevance 61%]
* [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (3 preceding siblings ...)
2021-09-03 8:54 83% ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
@ 2021-09-03 8:54 58% ` Eric Wong
2021-09-03 8:54 61% ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
2021-09-03 8:54 63% ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
This is merely to avoid perl setting errors internally which
were not user visible. The double-close wasn't a problem in
practice since we open a new file hanlde for the mbox or
mbox.gz anyways, so the new t/lei-up.t test case shows no
regressions nor fixes.
---
MANIFEST | 1 +
lib/PublicInbox/LeiUp.pm | 4 ++++
t/lei-up.t | 39 +++++++++++++++++++++++++++++++++++++++
3 files changed, 44 insertions(+)
create mode 100644 t/lei-up.t
diff --git a/MANIFEST b/MANIFEST
index be6ec927..fad29622 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -443,6 +443,7 @@ t/lei-q-save.t
t/lei-q-thread.t
t/lei-sigpipe.t
t/lei-tag.t
+t/lei-up.t
t/lei-watch.t
t/lei.t
t/lei_dedupe.t
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index e1da64aa..a39d6047 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -54,6 +54,10 @@ sub up1_redispatch {
$l->{opt} = { %{$l->{opt}} };
delete $l->{sock};
$l->{''} = $op_p; # daemon only
+
+ # make close($l->{1}) happy in lei->dclose
+ open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ $l->{1} = $fh;
eval {
$l->qerr("# updating $out");
up1($l, $out);
diff --git a/t/lei-up.t b/t/lei-up.t
new file mode 100644
index 00000000..c6f31c74
--- /dev/null
+++ b/t/lei-up.t
@@ -0,0 +1,39 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
+test_lei(sub {
+ my $s = eml_load('t/plack-qp.eml')->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a.mbox.gz";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b.mbox.gz";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b";
+ lei_ok qw(ls-search);
+ $s = eml_load('t/utf8.eml')->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+ lei_ok qw(up --all=local);
+ open my $fh, "$ENV{HOME}/a.mbox.gz" or xbail "open: $!";
+ my $gz = do { local $/; <$fh> };
+ my $uc;
+ gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+ open $fh, "$ENV{HOME}/a" or xbail "open: $!";
+
+ my $exp = do { local $/; <$fh> };
+ is($uc, $exp, 'compressed and uncompressed match (a.gz)');
+ like($exp, qr/testmessage\@example.com/, '2nd message added');
+ open $fh, "$ENV{HOME}/b.mbox.gz" or xbail "open: $!";
+
+ $gz = do { local $/; <$fh> };
+ undef $uc;
+ gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+ is($uc, $exp, 'compressed and uncompressed match (b.gz)');
+
+ open $fh, "$ENV{HOME}/b" or xbail "open: $!";
+ $uc = do { local $/; <$fh> };
+ is($uc, $exp, 'uncompressed both match');
+});
+
+done_testing;
^ permalink raw reply related [relevance 58%]
* [PATCH 1/8] lei: dump errors to syslog, and not to CLI
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
@ 2021-09-03 8:54 54% ` Eric Wong
2021-09-03 8:54 71% ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
` (5 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
Dumping errors from the previous run can often get lost, so just
spew to syslog since it's a standard place to put errors that
don't make it to a client. Note: we don't rely on $SIG{__WARN__}
since some of the Net:: stuff will write directly to STDERR
(as will external processes).
---
lib/PublicInbox/LEI.pm | 16 +++++++++-------
lib/PublicInbox/LeiStoreErr.pm | 4 ++--
t/lei-daemon.t | 5 -----
3 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 41e811ca..9e9aa165 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -27,6 +27,7 @@ use PublicInbox::Eml;
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
+use Sys::Syslog qw(openlog syslog closelog);
our $quit = \&CORE::exit;
our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
$recv_cmd, $send_cmd);
@@ -464,7 +465,6 @@ sub x_it ($$) {
my ($self, $code) = @_;
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
- dump_and_clear_log();
stop_pager($self);
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('x_it', $code);
@@ -765,7 +765,6 @@ sub dispatch {
my ($self, $cmd, @argv) = @_;
local $current_lei = $self; # for __WARN__
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
- dump_and_clear_log("from previous run\n");
return _help($self, 'no command given') unless defined($cmd);
# do not support Getopt bundling for this
while ($cmd eq '-C' || $cmd eq '-c') {
@@ -1147,10 +1146,12 @@ sub oldset { $oldset }
sub dump_and_clear_log {
if (defined($errors_log) && -s STDIN && seek(STDIN, 0, SEEK_SET)) {
- my @pfx = @_;
- unshift(@pfx, "$errors_log ") if @pfx;
- warn @pfx, do { local $/; <STDIN> };
- truncate(STDIN, 0) or warn "ftruncate ($errors_log): $!";
+ openlog('lei-daemon', 'pid,nowait,nofatal,ndelay', 'user');
+ chomp(my @lines = <STDIN>);
+ truncate(STDIN, 0) or
+ syslog('warning', "ftruncate (%s): %m", $errors_log);
+ for my $l (@lines) { syslog('warning', '%s', $l) }
+ closelog(); # don't share across fork
}
}
@@ -1243,7 +1244,7 @@ sub lazy_start {
(-p STDOUT) or die "E: stdout must be a pipe\n";
open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
STDIN->autoflush(1);
- dump_and_clear_log("from previous daemon process:\n");
+ dump_and_clear_log();
POSIX::setsid() > 0 or die "setsid: $!";
my $pid = fork // die "fork: $!";
return if $pid;
@@ -1345,6 +1346,7 @@ sub DESTROY {
}
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
+ dump_and_clear_log();
# preserve $? for ->fail or ->x_it code
}
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index 5f9ba24d..cc085fdc 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -2,7 +2,7 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# forwards stderr from lei/store process to any lei clients using
-# the same store
+# the same store, falls back to syslog if no matching clients exist.
package PublicInbox::LeiStoreErr;
use strict;
use v5.10.1;
@@ -31,7 +31,7 @@ sub event_step {
print $err $$rbuf and $printed = 1;
}
if (!$printed) {
- openlog('lei-store', 'pid,nowait,nofatal,ndelay', 'user');
+ openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
closelog(); # don't share across fork
}
diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index a7c4b799..b0c94a87 100644
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -21,14 +21,9 @@ test_lei({ daemon_only => 1 }, sub {
ok(kill(0, $pid), 'pid is valid');
ok(-S $sock, 'sock created');
is(-s $err_log, 0, 'nothing in errors.log');
- open my $efh, '>>', $err_log or BAIL_OUT $!;
- print $efh "phail\n" or BAIL_OUT $!;
- close $efh or BAIL_OUT $!;
-
lei_ok('daemon-pid');
chomp(my $pid_again = $lei_out);
is($pid, $pid_again, 'daemon-pid idempotent');
- like($lei_err, qr/phail/, 'got mock "phail" error previous run');
SKIP: {
skip 'only testing open files on Linux', 1 if $^O ne 'linux';
^ permalink raw reply related [relevance 54%]
* [PATCH 3/8] lei: ->child_error less error-prone
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
2021-09-03 8:54 71% ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
@ 2021-09-03 8:54 52% ` Eric Wong
2021-09-03 8:54 83% ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
` (3 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
I was calling "child_error(1, ...)" in a few places where I meant
to be calling "child_error(1 << 8, ...)" and inadvertantly
triggering SIGHUP in script/lei. Since giving a zero exit code
to child_error makes no sense, just allow falsy values to
default to 1 << 8.
---
lib/PublicInbox/LEI.pm | 7 ++++---
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 4 ++--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 +-
7 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9e9aa165..8b6c1d36 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -532,6 +532,7 @@ sub puts ($;@) { out(shift, map { "$_\n" } @_) }
sub child_error { # passes non-fatal curl exit codes to user
my ($self, $child_error, $msg) = @_; # child_error is $?
+ $child_error ||= 1 << 8;
$self->err($msg) if $msg;
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('child_error', $child_error);
@@ -1341,7 +1342,7 @@ sub DESTROY {
if (my $counters = delete $self->{counters}) {
for my $k (sort keys %$counters) {
my $nr = $counters->{$k};
- $self->child_error(1 << 8, "$nr $k messages");
+ $self->child_error(0, "$nr $k messages");
}
}
$self->{1}->autoflush(1) if $self->{1};
@@ -1417,7 +1418,7 @@ sub refresh_watches {
add_maildir_watch($d, $cfg_f);
}
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1, "E: watch $url not supported, yet")
+ $lei->child_error(0, "E: watch $url not supported, yet")
}
}
@@ -1452,7 +1453,7 @@ sub refresh_watches {
my $d = canonpath_harder($1);
cancel_maildir_watch($d, $cfg_f);
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1, "E: watch $url TODO");
+ $lei->child_error(0, "E: watch $url TODO");
}
}
}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 21003894..b94f67a0 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -32,7 +32,7 @@ sub solver_user_cb { # called by solver when done
my $lei = $self->{lei};
my $log_buf = delete $lei->{'log_buf'};
$$log_buf =~ s/^/# /sgm;
- ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+ ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
$lei->qerr($$log_buf);
my ($git, $oid, $type, $size, $di) = @$res;
my $gd = $git->{git_dir};
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 5b545998..ef3e4d0b 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -21,7 +21,7 @@ sub input_eml_cb { # used by input_maildir_cb and input_net_cb
sub input_fh { # overrides PublicInbox::LeiInput::input_fh
my ($self, $ifmt, $fh, $input, @args) = @_;
- $self->{lei}->child_error(1<<8, <<EOM);
+ $self->{lei}->child_error(0, <<EOM);
$input ($ifmt) not yet supported, try `lei import'
EOM
}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 1b28f36f..cb71e97c 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -63,7 +63,7 @@ sub input_fh {
my ($self, $ifmt, $fh, $name, @args) = @_;
if ($ifmt eq 'eml') {
my $buf = do { local $/; <$fh> } //
- return $self->{lei}->child_error(1 << 8, <<"");
+ return $self->{lei}->child_error(0, <<"");
error reading $name: $!
# mutt pipes single RFC822 messages with a "From " line,
@@ -104,7 +104,7 @@ sub handle_http_input ($$@) {
my $err = $@;
waitpid($pid, 0);
$? || $err and
- $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+ $lei->child_error($?, "@$cmd failed".$err ? " $err" : '');
}
sub input_path_url {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 9d95e899..1e54c3bf 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -18,7 +18,7 @@ sub lcat_folder ($$$) {
my $err = $lms->arg2folder($lei, $folders);
$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
if ($err && $err->{fail}) {
- $lei->child_error(1 << 8, "# unknown folder: $folder");
+ $lei->child_error(0, "# unknown folder: $folder");
} else {
for my $f (@$folders) {
my $fid = $lms->fid_for($f);
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 0ba5897c..60286b06 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -23,7 +23,7 @@ sub rediff_user_cb { # called by solver when done
my $lei = $self->{lei};
my $log_buf = delete $lei->{log_buf};
$$log_buf =~ s/^/# /sgm;
- ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+ ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
$lei->qerr($$log_buf);
my ($git, $oid, $type, $size, $di) = @$res;
my $oid_want = delete $self->{cur_oid_want};
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 85efd9f5..e1da64aa 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -59,7 +59,7 @@ sub up1_redispatch {
up1($l, $out);
$l->qerr("# $out done");
};
- $l->child_error(1 << 8, $@) if $@;
+ $l->child_error(0, $@) if $@;
}
sub lei_up {
^ permalink raw reply related [relevance 52%]
* [PATCH 1/2] t/lei-auto-watch: improve test reliability
@ 2021-09-02 22:36 68% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-02 22:36 UTC (permalink / raw)
To: meta
On slower systems, even a 100ms delay may not be enough;
so loop and retry in hopes of an early exit for faster
systems.
---
t/lei-auto-watch.t | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index 146402a6..3b0c1b10 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -4,9 +4,8 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Basename qw(basename);
my ($ro_home, $cfg_path) = setup_public_inboxes;
-my $tick = 2.1;
-my $have_fast_inotify = eval { require Linux::Inotify2; $tick = 0.1 } ||
- eval { require IO::KQueue; $tick = 0.5 };
+my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+ eval { require IO::KQueue };
$have_fast_inotify or
diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
@@ -31,9 +30,13 @@ test_lei(sub {
lei_ok qw(add-watch), $x;
my $dst = $x[0] . 'S';
rename($x[0], $dst) or xbail "rename($x[0], $dst): $!";
- tick($tick); # wait for inotify or kevent
+ my $ys = "$y[0]S";
+ for (0..50) {
+ last if -f $ys;
+ tick; # wait for inotify or kevent
+ }
my @y2 = glob("$y/*/*");
- is_deeply(\@y2, [ "$y[0]S" ], "`seen' kw propagated to `y' dir");
+ is_deeply(\@y2, [ $ys ], "`seen' kw propagated to `y' dir");
lei_ok qw(note-event done);
lei_ok qw(inspect), "blob:$oid";
$ins = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 68%]
* Re: Showcasing lei at Linux Plumbers
2021-09-02 21:12 61% Showcasing lei at Linux Plumbers Konstantin Ryabitsev
@ 2021-09-02 21:58 65% ` Eric Wong
2021-09-03 15:15 71% ` Konstantin Ryabitsev
2021-09-07 21:33 71% ` Konstantin Ryabitsev
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-02 21:58 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Eric:
>
> I am getting ready for my presentation to the Linux Plumbers (happening in a
> few weeks, eek), which is based around lore, lei (I see what you did there)
> and search-based subscriptions. I want to make it hands-on with practical
> examples, which is what developers would appreciate more than just dry
> manpages.
>
> I am in the process of wrapping my head around lei tooling, but I may have
> some questions in the process, so I wanted to start this thread as a record of
> my poking at it. :)
Yeah, I'm still trying to figure out how some things are
supposed to work myself...
> What I currently have:
>
> - an imap mailbox
> - lei configured and installed locally (in a debian container)
Fwiw, most of the functionality works much better with Maildir
because of potential password prompts needed for IMAP and
interactivity required.
> The goal is to illustrate how to use this to start "receiving" mail for a
> subsystem without subscribing to any of the lists. The example I have in mind
> is the LANDLOCK subsystem, and the reason I picked it is because it already
> has a well-defined set of search criteria we can use:
> https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/MAINTAINERS#n10462
>
> LANDLOCK SECURITY MODULE
> ...
> F: Documentation/security/landlock.rst
> F: Documentation/userspace-api/landlock.rst
> F: include/uapi/linux/landlock.h
> F: samples/landlock/
> F: security/landlock/
> F: tools/testing/selftests/landlock/
> K: landlock
> K: LANDLOCK
>
> This means we want to configure lei to grab any mail from lore.kernel.org/all/
> that matches this query:
>
> dfn:Documentation/security/landlock.rst OR
> dfn:Documentation/userspace-api/landlock.rst OR
> dfn:include/uapi/linux/landlock.h OR
> dfn:samples/landlock/ OR
> dfn:security/landlock/ OR
> dfn:tools/testing/selftests/landlock/ OR
> dfhh:landlock
>
> https://lore.kernel.org/all/?q=dfn%3ADocumentation%2Fsecurity%2Flandlock.rst+OR+dfn%3ADocumentation%2Fuserspace-api%2Flandlock.rst+OR+dfn%3Ainclude%2Fuapi%2Flinux%2Flandlock.h+OR+dfn%3Asamples%2Flandlock%2F+OR+dfn%3Asecurity%2Flandlock%2F+OR+dfn%3Atools%2Ftesting%2Fselftests%2Flandlock%2F+OR+dfhh%3Alandlock
For HTTP(S)-based queries, I would add rt: (received-time)
around the whole thing and maybe use "lei edit-search" to tweak
for subsequent runs. Not sure if the rt: handling should be
automatic for HTTP(S) (local Xapian searches track max docid, instead)
> I'll want to retrieve any threads and follow-ups and upload them to my imap
> landlock folder -- and then run in the background and just continuously update
> things as more mail comes in, so I don't have to remember to run anything
> manually.
>
> What succession of lei commands would accomplish this?
OK, there's two main commands, "lei q" and "lei up".
Both of which may prompt for passwords depending on how
git-credential is set up:
# the destination, could be Maildir
MFOLDER=imaps://user@example.com/INBOX.landlock
# initial search:
lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
(
dfn:Documentation/security/landlock.rst OR
dfn:Documentation/userspace-api/landlock.rst OR
dfn:include/uapi/linux/landlock.h OR
dfn:samples/landlock/ OR
dfn:security/landlock/ OR
dfn:tools/testing/selftests/landlock/ OR
dfhh:landlock
) AND rt:2.months.ago..
EOF
# update whenever, may prompt for IMAP password, but could be
# cron-ed or similar if passwords are cached
lei up $MFOLDER
# Optional: tweaking the search parameters can be done via
lei edit-search $MFOLDER
For Maildirs, "lei up --all=local" works as it should.
"lei up --all" and "lei up --all=remote" don't work, yet,
because prompting for multiple IMAP folders (with potentially
different accounts) can get a bit complicated. But
"lei up $ONE_IMAP_FOLDER" already works.
> Thanks for your continued help.
No problem, thanks for your patience since everything seems
overwhelming :<
^ permalink raw reply [relevance 65%]
* Showcasing lei at Linux Plumbers
@ 2021-09-02 21:12 61% Konstantin Ryabitsev
2021-09-02 21:58 65% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-02 21:12 UTC (permalink / raw)
To: meta
Eric:
I am getting ready for my presentation to the Linux Plumbers (happening in a
few weeks, eek), which is based around lore, lei (I see what you did there)
and search-based subscriptions. I want to make it hands-on with practical
examples, which is what developers would appreciate more than just dry
manpages.
I am in the process of wrapping my head around lei tooling, but I may have
some questions in the process, so I wanted to start this thread as a record of
my poking at it. :)
What I currently have:
- an imap mailbox
- lei configured and installed locally (in a debian container)
The goal is to illustrate how to use this to start "receiving" mail for a
subsystem without subscribing to any of the lists. The example I have in mind
is the LANDLOCK subsystem, and the reason I picked it is because it already
has a well-defined set of search criteria we can use:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/MAINTAINERS#n10462
LANDLOCK SECURITY MODULE
...
F: Documentation/security/landlock.rst
F: Documentation/userspace-api/landlock.rst
F: include/uapi/linux/landlock.h
F: samples/landlock/
F: security/landlock/
F: tools/testing/selftests/landlock/
K: landlock
K: LANDLOCK
This means we want to configure lei to grab any mail from lore.kernel.org/all/
that matches this query:
dfn:Documentation/security/landlock.rst OR
dfn:Documentation/userspace-api/landlock.rst OR
dfn:include/uapi/linux/landlock.h OR
dfn:samples/landlock/ OR
dfn:security/landlock/ OR
dfn:tools/testing/selftests/landlock/ OR
dfhh:landlock
https://lore.kernel.org/all/?q=dfn%3ADocumentation%2Fsecurity%2Flandlock.rst+OR+dfn%3ADocumentation%2Fuserspace-api%2Flandlock.rst+OR+dfn%3Ainclude%2Fuapi%2Flinux%2Flandlock.h+OR+dfn%3Asamples%2Flandlock%2F+OR+dfn%3Asecurity%2Flandlock%2F+OR+dfn%3Atools%2Ftesting%2Fselftests%2Flandlock%2F+OR+dfhh%3Alandlock
I'll want to retrieve any threads and follow-ups and upload them to my imap
landlock folder -- and then run in the background and just continuously update
things as more mail comes in, so I don't have to remember to run anything
manually.
What succession of lei commands would accomplish this?
Thanks for your continued help.
-K
^ permalink raw reply [relevance 61%]
* [SQUASH 4/3] t/lei-auto-watch: workaround for FreeBSD kevent
2021-09-02 10:17 51% ` [PATCH 3/3] lei: propagate keyword changes from lei/store Eric Wong
@ 2021-09-02 10:25 67% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-02 10:25 UTC (permalink / raw)
To: meta
My FreeBSD VM seems to need longer for this test than inotify
under Linux, likely because the kevent support code is more
complicated in userspace and needs extra file handles.
And drop unnecessary tick delay after "note-event done" since
that seems unneeded with transactions eliminated for
mail_sync.sqlite3
---
t/lei-auto-watch.t | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index 321c0ab3..146402a6 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -4,8 +4,9 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Basename qw(basename);
my ($ro_home, $cfg_path) = setup_public_inboxes;
-my $have_fast_inotify = eval { require Linux::Inotify2 } ||
- eval { require IO::KQueue };
+my $tick = 2.1;
+my $have_fast_inotify = eval { require Linux::Inotify2; $tick = 0.1 } ||
+ eval { require IO::KQueue; $tick = 0.5 };
$have_fast_inotify or
diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
@@ -30,11 +31,10 @@ test_lei(sub {
lei_ok qw(add-watch), $x;
my $dst = $x[0] . 'S';
rename($x[0], $dst) or xbail "rename($x[0], $dst): $!";
- tick($have_fast_inotify ? undef : 2.1); # wait for inotify
+ tick($tick); # wait for inotify or kevent
my @y2 = glob("$y/*/*");
is_deeply(\@y2, [ "$y[0]S" ], "`seen' kw propagated to `y' dir");
lei_ok qw(note-event done);
- tick; # XXX why is this needed?
lei_ok qw(inspect), "blob:$oid";
$ins = json_utf8->decode($lei_out);
$exp = { "maildir:$x" => [ map { basename($_) } glob("$x/*/*") ],
^ permalink raw reply related [relevance 67%]
* [PATCH 3/3] lei: propagate keyword changes from lei/store
2021-09-02 10:17 71% [PATCH 0/3] lei: auto keyword propagation to Maildirs Eric Wong
@ 2021-09-02 10:17 51% ` Eric Wong
2021-09-02 10:25 67% ` [SQUASH 4/3] t/lei-auto-watch: workaround for FreeBSD kevent Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-02 10:17 UTC (permalink / raw)
To: meta
This works with existing inotify/EVFILT_VNODE functionality to
propagate changes made from one Maildir to another Maildir.
I chose the lei/store worker process to handle this since
propagating changes back into lei-daemon on a massive scale
could lead to dead-locking while both processes are attempting
to write to each other. Eliminating IPC overhead is a nice
side effect, but could hurt performance if Maildirs are slow.
The code for "lei export-kw" is significantly revamped to match
the new code used in the "lei/store" daemon. It should be more
correct w.r.t. corner-cases and stale entries, but perhaps
better tests need to be written.
---
MANIFEST | 1 +
lib/PublicInbox/LeiExportKw.pm | 24 ++++----
lib/PublicInbox/LeiNoteEvent.pm | 14 ++---
lib/PublicInbox/LeiStore.pm | 105 ++++++++++++++++++++++++++++++--
t/lei-auto-watch.t | 45 ++++++++++++++
5 files changed, 165 insertions(+), 24 deletions(-)
create mode 100644 t/lei-auto-watch.t
diff --git a/MANIFEST b/MANIFEST
index cf7268ed..be6ec927 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -423,6 +423,7 @@ t/init.t
t/ipc.t
t/iso-2202-jp.eml
t/kqnotify.t
+t/lei-auto-watch.t
t/lei-convert.t
t/lei-daemon.t
t/lei-export-kw.t
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 42a5ff22..78c6c6f9 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -25,12 +25,11 @@ sub export_kw_md { # LeiMailSync->each_src callback
}
$bn .= ':2,'.
PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown);
+ return if $bn eq $$id;
my $dst = "$mdir/cur/$bn";
- my @fail;
my $lei = $self->{lei};
for my $d (@try) {
my $src = "$mdir/$d/$$id";
- next if $src eq $dst;
# we use link(2) + unlink(2) since rename(2) may
# inadvertently clobber if the "uniquefilename" part wasn't
@@ -44,20 +43,19 @@ sub export_kw_md { # LeiMailSync->each_src callback
$lei->{sto}->ipc_do('lms_mv_src', "maildir:$mdir",
$oidbin, $id, $bn);
return; # success anyways if link(2) worked
- }
- if ($! == ENOENT && !-e $src) { # some other process moved it
- $lei->{sto}->ipc_do('lms_clear_src',
- "maildir:$mdir", $id);
- next;
- }
- push @fail, $src if $! != EEXIST;
+ } elsif ($! == EEXIST) { # lost race with lei/store?
+ return;
+ } elsif ($! != ENOENT) {
+ $lei->child_error(1, "E: link($src -> $dst): $!");
+ } # else loop @try
}
- return unless @fail;
- # both tries failed
my $e = $!;
- my $orig = '['.join('|', @fail).']';
+ # both tries failed
my $oidhex = unpack('H*', $oidbin);
- $lei->child_error(1, "link($orig, $dst) ($oidhex): $e");
+ my $src = "$mdir/{".join(',', @try)."}/$$id";
+ $lei->child_error(1, "link($src -> $dst) ($oidhex): $e");
+ for (@try) { return if -e "$mdir/$_/$$id" }
+ $lei->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
}
sub export_kw_imap { # LeiMailSync->each_src callback
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 6a40ba39..41415346 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -36,32 +36,31 @@ sub note_event_arm_done ($) {
}
sub eml_event ($$$$) {
- my ($self, $eml, $kw, $state) = @_;
+ my ($self, $eml, $vmd, $state) = @_;
my $sto = $self->{lei}->{sto};
my $lse = $self->{lse} //= $sto->search;
- my $vmd = { kw => $kw };
if ($state =~ /\Aimport-(?:rw|ro)\z/) {
$sto->ipc_do('set_eml', $eml, $vmd);
} elsif ($state =~ /\Aindex-(?:rw|ro)\z/) {
my $xoids = $self->{lei}->ale->xoids_for($eml);
$sto->ipc_do('index_eml_only', $eml, $vmd, $xoids);
} elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
- my $c = $lse->kw_changed($eml, $kw, my $docids = []);
+ my $c = $lse->kw_changed($eml, $vmd->{kw}, my $docids = []);
if (scalar @$docids) { # already in lei/store
$sto->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
# it's in an external, only set kw, here
$sto->ipc_do('set_xvmd', $xoids, $eml, $vmd);
- } # else { totally unknown
+ } # else { totally unknown: ignore
} else {
warn "unknown state: $state (in $self->{lei}->{cfg}->{'-f'})\n";
}
}
sub maildir_event { # via wq_io_do
- my ($self, $fn, $kw, $state) = @_;
+ my ($self, $fn, $vmd, $state) = @_;
my $eml = PublicInbox::InboxWritable::eml_from_path($fn) // return;
- eml_event($self, $eml, $kw, $state);
+ eml_event($self, $eml, $vmd, $state);
}
sub lei_note_event {
@@ -98,7 +97,8 @@ sub lei_note_event {
// return;
return if index($fl, 'T') >= 0;
my $kw = PublicInbox::MdirReader::flags2kw($fl);
- $self->wq_io_do('maildir_event', [], $fn, $kw, $state);
+ my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] };
+ $self->wq_io_do('maildir_event', [], $fn, $vmd, $state);
} # else: TODO: imap
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 0fa2d3c0..a91b30f7 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -25,10 +25,14 @@ use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
use PublicInbox::Spawn qw(spawn);
+use PublicInbox::MdirReader;
+use PublicInbox::LeiToMail;
use List::Util qw(max);
use File::Temp ();
use POSIX ();
use IO::Handle (); # ->autoflush
+use Sys::Syslog qw(syslog openlog);
+use Errno qw(EEXIST ENOENT);
sub new {
my (undef, $dir, $opt) = @_;
@@ -165,12 +169,92 @@ sub _docids_for ($$) {
sort { $a <=> $b } values %docids;
}
+# n.b. similar to LeiExportKw->export_kw_md, but this is for a single eml
+sub export1_kw_md ($$$$$) {
+ my ($self, $mdir, $bn, $oidbin, $vmdish) = @_; # vmd/vmd_mod
+ my $orig = $bn;
+ my (@try, $unkn, $kw);
+ if ($bn =~ s/:2,([a-zA-Z]*)\z//) {
+ ($kw, $unkn) = PublicInbox::MdirReader::flags2kw($1);
+ if (my $set = $vmdish->{kw}) {
+ $kw = $set;
+ } elsif (my $add = $vmdish->{'+kw'}) {
+ @$kw{@$add} = ();
+ } elsif (my $del = $vmdish->{-kw}) {
+ delete @$kw{@$del};
+ } # else no changes...
+ @try = qw(cur new);
+ } else { # no keywords, yet, could be in new/
+ @try = qw(new cur);
+ $unkn = [];
+ if (my $set = $vmdish->{kw}) {
+ $kw = $set;
+ } elsif (my $add = $vmdish->{'+kw'}) {
+ @$kw{@$add} = (); # auto-vivify
+ } else { # ignore $vmdish->{-kw}
+ $kw = [];
+ }
+ }
+ $kw = [ keys %$kw ] if ref($kw) eq 'HASH';
+ $bn .= ':2,'. PublicInbox::LeiToMail::kw2suffix($kw, @$unkn);
+ return if $orig eq $bn; # no change
+
+ # we use link(2) + unlink(2) since rename(2) may
+ # inadvertently clobber if the "uniquefilename" part wasn't
+ # actually unique.
+ my $dst = "$mdir/cur/$bn";
+ for my $d (@try) {
+ my $src = "$mdir/$d/$orig";
+ if (link($src, $dst)) {
+ if (!unlink($src) and $! != ENOENT) {
+ syslog('warning', "unlink($src): $!");
+ }
+ # TODO: verify oidbin?
+ lms_mv_src($self, "maildir:$mdir",
+ $oidbin, \$orig, $bn);
+ return;
+ } elsif ($! == EEXIST) { # lost race with "lei export-kw"?
+ return;
+ } elsif ($! == ENOENT) {
+ syslog('warning', "link($src -> $dst): $!")
+ } # else loop @try
+ }
+ my $e = $!;
+ my $src = "$mdir/{".join(',', @try)."}/$orig";
+ my $oidhex = unpack('H*', $oidbin);
+ syslog('warning', "link($src -> $dst) ($oidhex): $e");
+ for (@try) { return if -e "$mdir/$_/$orig" };
+ lms_clear_src($self, "maildir:$mdir", \$orig);
+}
+
+sub sto_export_kw ($$$) {
+ my ($self, $docid, $vmdish) = @_; # vmdish (vmd or vmd_mod)
+ my ($eidx, $tl) = eidx_init($self);
+ my $lms = _lms_rw($self) // return;
+ my $xr3 = $eidx->{oidx}->get_xref3($docid, 1);
+ for my $row (@$xr3) {
+ my (undef, undef, $oidbin) = @$row;
+ my $locs = $lms->locations_for($oidbin) // next;
+ while (my ($loc, $ids) = each %$locs) {
+ if ($loc =~ s!\Amaildir:!!i) {
+ for my $id (@$ids) {
+ export1_kw_md($self, $loc, $id,
+ $oidbin, $vmdish);
+ }
+ }
+ # TODO: IMAP
+ }
+ }
+}
+
+# vmd = { kw => [ qw(seen ...) ], L => [ qw(inbox ...) ] }
sub set_eml_vmd {
my ($self, $eml, $vmd, $docids) = @_;
my ($eidx, $tl) = eidx_init($self);
$docids //= [ _docids_for($self, $eml) ];
for my $docid (@$docids) {
$eidx->idx_shard($docid)->ipc_do('set_vmd', $docid, $vmd);
+ sto_export_kw($self, $docid, $vmd);
}
$docids;
}
@@ -284,6 +368,12 @@ EOF
$docid;
}
+sub _add_vmd ($$$$) {
+ my ($self, $idx, $docid, $vmd) = @_;
+ $idx->ipc_do('add_vmd', $docid, $vmd);
+ sto_export_kw($self, $docid, $vmd);
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
@@ -310,7 +400,7 @@ sub add_eml {
@$vivify_xvmd = sort { $a <=> $b } keys(%docids);
}
}
- if (@$vivify_xvmd) {
+ if (@$vivify_xvmd) { # docids list
$xoids //= {};
$xoids->{$smsg->{blob}} = 1;
for my $docid (@$vivify_xvmd) {
@@ -327,7 +417,7 @@ sub add_eml {
for my $oid (keys %$xoids) {
$oidx->add_xref3($docid, -1, $oid, '.');
}
- $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd;
+ _add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
$vivify_xvmd;
} elsif (my @docids = _docids_for($self, $eml)) {
@@ -337,7 +427,7 @@ sub add_eml {
$oidx->add_xref3($docid, -1, $smsg->{blob}, '.');
# add_eidx_info for List-Id
$idx->ipc_do('add_eidx_info', $docid, '.', $eml);
- $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd;
+ _add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
\@docids;
} else { # totally new message
@@ -347,7 +437,7 @@ sub add_eml {
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
my $idx = $eidx->idx_shard($smsg->{num});
$idx->index_eml($eml, $smsg);
- $idx->ipc_do('add_vmd', $smsg->{num}, $vmd) if $vmd;
+ _add_vmd($self, $idx, $smsg->{num}, $vmd) if $vmd;
$smsg;
}
}
@@ -365,6 +455,7 @@ sub index_eml_only {
set_eml($self, $eml, $vmd, $xoids);
}
+# store {kw} / {L} info for a message which is only in an external
sub _external_only ($$$) {
my ($self, $xoids, $eml) = @_;
my $eidx = $self->{priv_eidx};
@@ -398,6 +489,7 @@ sub update_xvmd {
next if $seen{$docid}++;
my $idx = $eidx->idx_shard($docid);
$idx->ipc_do('update_vmd', $docid, $vmd_mod);
+ sto_export_kw($self, $docid, $vmd_mod);
}
return unless scalar(keys(%$xoids));
@@ -410,12 +502,14 @@ sub update_xvmd {
}
my $idx = $eidx->idx_shard($docid);
$idx->ipc_do('update_vmd', $docid, $vmd_mod);
+ sto_export_kw($self, $docid, $vmd_mod);
}
return;
}
# totally unseen
my ($smsg, $idx) = _external_only($self, $xoids, $eml);
$idx->ipc_do('update_vmd', $smsg->{num}, $vmd_mod);
+ sto_export_kw($self, $smsg->{num}, $vmd_mod);
}
# set or update keywords for external message, called via ipc_do
@@ -433,6 +527,7 @@ sub set_xvmd {
next if $seen{$docid}++;
my $idx = $eidx->idx_shard($docid);
$idx->ipc_do('set_vmd', $docid, $vmd);
+ sto_export_kw($self, $docid, $vmd);
}
return unless scalar(keys(%$xoids));
@@ -443,6 +538,7 @@ sub set_xvmd {
# totally unseen:
my ($smsg, $idx) = _external_only($self, $xoids, $eml);
$idx->ipc_do('add_vmd', $smsg->{num}, $vmd);
+ sto_export_kw($self, $smsg->{num}, $vmd);
}
sub checkpoint {
@@ -497,6 +593,7 @@ sub ipc_atfork_child {
if (my $to_close = delete($self->{to_close})) {
close($_) for @$to_close;
}
+ openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
$self->SUPER::ipc_atfork_child;
}
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
new file mode 100644
index 00000000..321c0ab3
--- /dev/null
+++ b/t/lei-auto-watch.t
@@ -0,0 +1,45 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Basename qw(basename);
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+ eval { require IO::KQueue };
+
+$have_fast_inotify or
+ diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
+
+test_lei(sub {
+ my $x = "$ENV{HOME}/x";
+ my $y = "$ENV{HOME}/y";
+ lei_ok qw(add-external), "$ro_home/t1";
+ lei_ok qw(q mid:testmessage@example.com -o), $x;
+ lei_ok qw(q mid:testmessage@example.com -o), $y;
+ my @x = glob("$x/cur/*");
+ my @y = glob("$y/cur/*");
+ scalar(@x) == 1 or xbail 'expected 1 file', \@x;
+ scalar(@y) == 1 or xbail 'expected 1 file', \@y;
+
+ my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+ lei_ok qw(inspect), "blob:$oid";
+ my $ins = json_utf8->decode($lei_out);
+ my $exp = { "maildir:$x" => [ map { basename($_) } @x ],
+ "maildir:$y" => [ map { basename($_) } @y ] };
+ is_deeply($ins->{'mail-sync'}, $exp, 'inspect as expected');
+ lei_ok qw(add-watch), $x;
+ my $dst = $x[0] . 'S';
+ rename($x[0], $dst) or xbail "rename($x[0], $dst): $!";
+ tick($have_fast_inotify ? undef : 2.1); # wait for inotify
+ my @y2 = glob("$y/*/*");
+ is_deeply(\@y2, [ "$y[0]S" ], "`seen' kw propagated to `y' dir");
+ lei_ok qw(note-event done);
+ tick; # XXX why is this needed?
+ lei_ok qw(inspect), "blob:$oid";
+ $ins = json_utf8->decode($lei_out);
+ $exp = { "maildir:$x" => [ map { basename($_) } glob("$x/*/*") ],
+ "maildir:$y" => [ map { basename($_) } glob("$y/*/*") ] };
+ is_deeply($ins->{'mail-sync'}, $exp, 'mail_sync matches FS');
+});
+
+done_testing;
^ permalink raw reply related [relevance 51%]
* [PATCH 0/3] lei: auto keyword propagation to Maildirs
@ 2021-09-02 10:17 71% Eric Wong
2021-09-02 10:17 51% ` [PATCH 3/3] lei: propagate keyword changes from lei/store Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-02 10:17 UTC (permalink / raw)
To: meta
At least the tests pass, and getting t/lei-export-kw.t to pass
after 3/3 was no small feat, but I believe everything is more
correct now (especially after the 10-patch series posted
yesterday-ish).
Patches 1 and 2 were developed while fixing 3/3, since the stuff
in t/lei-auto-watch.t happened to work right away while
preserving the behavior of t/lei-export-kw.t (unchanged) proved
extremely challenging in my current mental state.
Eric Wong (3):
lei_mail_sync: do not use transactions
lei_input: set and prepare watches early
lei: propagate keyword changes from lei/store
MANIFEST | 1 +
lib/PublicInbox/LeiExportKw.pm | 24 +++----
lib/PublicInbox/LeiInput.pm | 20 ++++--
lib/PublicInbox/LeiMailSync.pm | 14 +---
lib/PublicInbox/LeiNoteEvent.pm | 14 ++--
lib/PublicInbox/LeiStore.pm | 122 +++++++++++++++++++++++++++++---
t/lei-auto-watch.t | 45 ++++++++++++
t/lei_mail_sync.t | 18 ++---
8 files changed, 196 insertions(+), 62 deletions(-)
create mode 100644 t/lei-auto-watch.t
^ permalink raw reply [relevance 71%]
* [PATCH] lei up: only show finmsg in top-level lei-daemon
@ 2021-08-31 19:38 90% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 19:38 UTC (permalink / raw)
To: meta
->DESTROY can get triggered in child processes, which
unnecessarily duplicates messages queued up for display
when lei spawns extra workers.
---
lib/PublicInbox/LeiFinmsg.pm | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiFinmsg.pm b/lib/PublicInbox/LeiFinmsg.pm
index 0ef5f070..395e7d3c 100644
--- a/lib/PublicInbox/LeiFinmsg.pm
+++ b/lib/PublicInbox/LeiFinmsg.pm
@@ -9,13 +9,13 @@ use v5.10.1;
sub new {
my ($cls, $io) = @_;
- bless [ $io ], $cls;
+ bless [ $io, $$ ], $cls;
}
sub DESTROY {
my ($self) = @_;
my $io = shift @$self;
- print $io @$self;
+ shift(@$self) == $$ and print $io @$self;
}
1;
^ permalink raw reply related [relevance 90%]
* [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
` (3 preceding siblings ...)
2021-08-31 11:21 71% ` [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir Eric Wong
@ 2021-08-31 11:21 94% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 10/10] lei/store: correctly delete entries from over Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
We must set autoflush to ensure timely notification of clients;
and lei-daemon must not block when waiting on reads in case of
spurious wakeups.
Finally, if no clients are connected to lei-daemon, write to
syslog to ensure the error is visible.
---
lib/PublicInbox/LeiStore.pm | 2 ++
lib/PublicInbox/LeiStoreErr.pm | 14 ++++++++++++--
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 5a48c064..0652137e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -28,6 +28,7 @@ use PublicInbox::Spawn qw(spawn);
use List::Util qw(max);
use File::Temp ();
use POSIX ();
+use IO::Handle (); # ->autoflush
sub new {
my (undef, $dir, $opt) = @_;
@@ -514,6 +515,7 @@ sub write_prepare {
$self->ipc_lock_init("$dir/ipc.lock");
substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
+ $w->autoflush(1);
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index 68ce96d6..5f9ba24d 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -8,22 +8,32 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::DS);
use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use Sys::Syslog qw(openlog syslog closelog);
+use IO::Handle (); # ->blocking
sub new {
my ($cls, $rd, $lei) = @_;
my $self = bless { sock => $rd, store_path => $lei->store_path }, $cls;
+ $rd->blocking(0);
$self->SUPER::new($rd, EPOLLIN | EPOLLONESHOT);
}
sub event_step {
my ($self) = @_;
- $self->do_read(\(my $rbuf), 4096) or return;
+ my $rbuf = $self->{rbuf} // \(my $x = '');
+ $self->do_read($rbuf, 8192, length($$rbuf)) or return;
my $cb;
+ my $printed;
for my $lei (values %PublicInbox::DS::DescriptorMap) {
$cb = $lei->can('store_path') // next;
next if $cb->($lei) ne $self->{store_path};
my $err = $lei->{2} // next;
- print $err $rbuf;
+ print $err $$rbuf and $printed = 1;
+ }
+ if (!$printed) {
+ openlog('lei-store', 'pid,nowait,nofatal,ndelay', 'user');
+ for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
+ closelog(); # don't share across fork
}
}
^ permalink raw reply related [relevance 94%]
* [PATCH 05/10] t/lei-watch: avoid race between glob + readlink
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 06/10] lei note-event: always flush changes on daemon exit Eric Wong
` (3 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Open file handles in lei-daemon may be unstable so we need to
account for readlink() returning undef.
---
t/lei-watch.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 86fa6649..a881fbb9 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -25,7 +25,7 @@ test_lei(sub {
lei_ok 'daemon-pid'; chomp(my $pid = $lei_out);
skip 'missing /proc/$PID/fd', 1 if !-d "/proc/$pid/fd";
my @ino = grep {
- readlink($_) =~ /\binotify\b/
+ (readlink($_) // '') =~ /\binotify\b/
} glob("/proc/$pid/fd/*");
is(scalar(@ino), 1, 'only one inotify FD');
my $ino_fd = (split('/', $ino[0]))[-1];
^ permalink raw reply related [relevance 71%]
* [PATCH 06/10] lei note-event: always flush changes on daemon exit
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
2021-08-31 11:21 71% ` [PATCH 05/10] t/lei-watch: avoid race between glob + readlink Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir Eric Wong
` (2 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Because the timer may not fire in time before daemon shutdown.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 28fe0c83..520fb519 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1254,6 +1254,7 @@ sub lazy_start {
my (undef, $eof_p) = PublicInbox::PktOp->pair;
sub {
$exit_code //= shift;
+ eval 'PublicInbox::LeiNoteEvent::flush_task()';
my $lis = $pil or exit($exit_code);
# closing eof_p triggers \&noop wakeup
$listener = $eof_p = $pil = $path = undef;
^ permalink raw reply related [relevance 71%]
* [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
` (2 preceding siblings ...)
2021-08-31 11:21 71% ` [PATCH 06/10] lei note-event: always flush changes on daemon exit Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 94% ` [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon Eric Wong
2021-08-31 11:21 71% ` [PATCH 10/10] lei/store: correctly delete entries from over Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
If we possibly just wrote or created a Maildir, ensure it's
monitored by the lei watch mechanism.
---
lib/PublicInbox/LEI.pm | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 520fb519..41e811ca 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -951,6 +951,9 @@ sub exec_buf ($$) {
sub start_mua {
my ($self) = @_;
+ if ($self->{ovv}->{fmt} =~ /\A(?:maildir)\z/) { # TODO: IMAP
+ refresh_watches($self);
+ }
my $mua = $self->{opt}->{mua} // return;
my $mfolder = $self->{ovv}->{dst};
my (@cmd, $replaced);
^ permalink raw reply related [relevance 71%]
* [PATCH 10/10] lei/store: correctly delete entries from over
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
` (4 preceding siblings ...)
2021-08-31 11:21 94% ` [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Some of these errors were inadvertantly lost due to delayed
error reporting in the past.
---
lib/PublicInbox/LeiStore.pm | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 0652137e..ab39043e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -243,8 +243,8 @@ sub remove_docids ($;@) {
my $eidx = eidx_init($self);
for my $docid (@docids) {
$eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
- $self->{oidx}->delete_by_num($docid);
- $self->{oidx}->{dbh}->do(<<EOF, undef, $docid);
+ $eidx->{oidx}->delete_by_num($docid);
+ $eidx->{oidx}->{dbh}->do(<<EOF, undef, $docid);
DELETE FROM xref3 WHERE docid = ?
EOF
}
^ permalink raw reply related [relevance 71%]
* [PATCH 00/10] lei: several bug fixes and refinements
@ 2021-08-31 11:21 71% Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
` (5 more replies)
0 siblings, 6 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Another pile of things found while working on better
synchronization.
Eric Wong (10):
lei_mail_sync: forget_folder: simplify code
lei prune-mail-sync: handle --all (no args)
lei_mail_sync: simplify group2folders
lei_mail_sync: make rename_folder more robust
t/lei-watch: avoid race between glob + readlink
lei note-event: always flush changes on daemon exit
lei: refresh watches before MUA spawn for Maildir
lei_mail_sync: set_src uses binary OIDs
lei: fix error reporting from lei/store -> lei-daemon
lei/store: correctly delete entries from over
lib/PublicInbox/LEI.pm | 4 ++++
lib/PublicInbox/LeiMailSync.pm | 36 ++++++++++++++++++++++------------
lib/PublicInbox/LeiStore.pm | 8 +++++---
lib/PublicInbox/LeiStoreErr.pm | 14 +++++++++++--
t/lei-watch.t | 2 +-
t/lei_mail_sync.t | 15 +++++++-------
6 files changed, 53 insertions(+), 26 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 02/10] lei prune-mail-sync: handle --all (no args)
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 05/10] t/lei-watch: avoid race between glob + readlink Eric Wong
` (4 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
This still needs tests, but I noticed "--all" w/o "local" or
"remote" was not working correctly since split() returned
an empty array.
---
lib/PublicInbox/LeiMailSync.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 57b56b3c..bf8fb7de 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -330,6 +330,7 @@ sub group2folders {
EOM
my %x = map { $_ => $_ } split(/,/, $all);
my @ok = grep(defined, delete(@x{qw(local remote), ''}));
+ push(@ok, '') if $all eq '';
my @no = keys %x;
if (@no) {
@no = (join(',', @no));
^ permalink raw reply related [relevance 71%]
* [PATCH 1/2] lei up: improve --all=local stderr output
2021-08-25 8:40 71% [PATCH 0/2] minor lei usability tweaks Eric Wong
@ 2021-08-25 8:40 59% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-25 8:40 UTC (permalink / raw)
To: meta
The "# $NR written to $DEST ($total matches)" messages are
arguably the most useful output of "lei up --all=local",
but they get intermixed with progress messages from various
workers. Queue up these finalization messages and only spit
them out on ->DESTROY.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiFinmsg.pm | 21 +++++++++++++++++++++
lib/PublicInbox/LeiUp.pm | 2 ++
lib/PublicInbox/LeiXSearch.pm | 9 ++++++---
5 files changed, 36 insertions(+), 3 deletions(-)
create mode 100644 lib/PublicInbox/LeiFinmsg.pm
diff --git a/MANIFEST b/MANIFEST
index fb9f16bf..cf7268ed 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiDedupe.pm
lib/PublicInbox/LeiEditSearch.pm
lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
+lib/PublicInbox/LeiFinmsg.pm
lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5694e92c..28fe0c83 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -486,6 +486,12 @@ sub err ($;@) {
sub qerr ($;@) { $_[0]->{opt}->{quiet} or err(shift, @_) }
+sub qfin { # show message on finalization (LeiFinmsg)
+ my ($lei, $msg) = @_;
+ return if $lei->{opt}->{quiet};
+ $lei->{fmsg} ? push(@{$lei->{fmsg}}, "$msg\n") : qerr($lei, $msg);
+}
+
sub fail_handler ($;$$) {
my ($lei, $code, $io) = @_;
close($io) if $io; # needed to avoid warnings on SIGPIPE
diff --git a/lib/PublicInbox/LeiFinmsg.pm b/lib/PublicInbox/LeiFinmsg.pm
new file mode 100644
index 00000000..0ef5f070
--- /dev/null
+++ b/lib/PublicInbox/LeiFinmsg.pm
@@ -0,0 +1,21 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Finalization messages, used to queue up a bunch of messages which
+# only get written out on ->DESTROY
+package PublicInbox::LeiFinmsg;
+use strict;
+use v5.10.1;
+
+sub new {
+ my ($cls, $io) = @_;
+ bless [ $io ], $cls;
+}
+
+sub DESTROY {
+ my ($self) = @_;
+ my $io = shift @$self;
+ print $io @$self;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index ba11761a..85efd9f5 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -48,6 +48,8 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
+ require PublicInbox::LeiFinmsg;
+ $lei->{fmsg} //= PublicInbox::LeiFinmsg->new($lei->{2});
my $l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} };
delete $l->{sock};
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 1f83e582..b9f0d692 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -398,9 +398,12 @@ Error closing $lei->{ovv}->{dst}: $!
if ($lei->{-progress}) {
my $tot = $lei->{-mset_total} // 0;
my $nr = $lei->{-nr_write} // 0;
- $lei->qerr($l2m ?
- "# $nr written to $lei->{ovv}->{dst} ($tot matches)" :
- "# $tot matches");
+ if ($l2m) {
+ $lei->qfin("# $nr written to " .
+ "$lei->{ovv}->{dst} ($tot matches)");
+ } else {
+ $lei->qerr("# $tot matches");
+ }
}
$lei->start_mua if $start_mua;
$lei->dclose;
^ permalink raw reply related [relevance 59%]
* [PATCH 0/2] minor lei usability tweaks
@ 2021-08-25 8:40 71% Eric Wong
2021-08-25 8:40 59% ` [PATCH 1/2] lei up: improve --all=local stderr output Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-08-25 8:40 UTC (permalink / raw)
To: meta
Some things I noticed replacing much of my existing mail
stack with lei...
Eric Wong (2):
lei up: improve --all=local stderr output
lei_mail_sync: remove warning message from caller
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiFinmsg.pm | 21 +++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 ++
lib/PublicInbox/LeiXSearch.pm | 9 ++++++---
6 files changed, 37 insertions(+), 4 deletions(-)
create mode 100644 lib/PublicInbox/LeiFinmsg.pm
^ permalink raw reply [relevance 71%]
* [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
` (2 preceding siblings ...)
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
@ 2021-08-24 20:14 71% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-24 20:14 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> 2/3 is probably a fix for a long-standing problem, 3/3 was
> noticed while working on it. If 2/3 doesn't fix it, then maybe
> 1/3 will help us narrow it down.
s/is probably/was almost certainly/ \o/
^ permalink raw reply [relevance 71%]
* [PATCH] lei: non-blocking lei/store->done in lei-daemon
@ 2021-08-24 13:06 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-24 13:06 UTC (permalink / raw)
To: meta
This allows client sockets to wait for "done" commits to
lei/store while the daemon reacts asynchronously. The goal
of this change is to keep the script/lei client alive until
lei/store commits changes to the filesystem, but without
blocking the lei-daemon event loop. It depends on Perl
refcounting to close the socket.
This change also highlighted our over-use of "done" requests to
lei/store processes, which is now corrected so we only issue it
on collective socket EOF rather than upon reaping every single
worker.
This also fixes "lei forget-mail-sync" when it is the initial
command.
This took several iterations and much debugging to arrive at the
current implementation:
1. The initial iteration of this change utilized socket passing
from lei-daemon to lei/store, which necessitated switching
from faster pipes to slower Unix sockets.
2. The second iteration switched to registering notification sockets
independently of "done" requests, but that could lead to early
wakeups when "done" was requested by other workers. This
appeared to work most of the time, but suffered races under
high load which were difficult to track down.
Finally, this iteration passes the stringified socket GLOB ref
to lei/store which is echoed back to lei-daemon upon completion
of that particular "done" request.
---
lib/PublicInbox/LEI.pm | 19 ++++++++++++++-
lib/PublicInbox/LeiForgetMailSync.pm | 4 ++--
lib/PublicInbox/LeiImportKw.pm | 3 ++-
lib/PublicInbox/LeiNoteEvent.pm | 4 ++--
lib/PublicInbox/LeiPmdir.pm | 5 ++--
lib/PublicInbox/LeiStore.pm | 35 +++++++++++++++++++---------
lib/PublicInbox/LeiXSearch.pm | 4 ++--
lib/PublicInbox/PktOp.pm | 9 ++++---
8 files changed, 59 insertions(+), 24 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ea3ec0fe..5694e92c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -37,6 +37,7 @@ $GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => [ ino watches ] }
+our %LIVE_SOCK; # "GLOB(0x....)" => $lei->{sock}
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -565,6 +566,7 @@ sub _lei_atfork_child {
$dir_idle->force_close if $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
+ %LIVE_SOCK = ();
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
undef $errors_log;
$quit = \&CORE::exit;
@@ -1429,7 +1431,7 @@ sub refresh_watches {
add_maildir_watch($cd, $cfg_f);
}
}
- my $wait = $renames ? $sto->ipc_do('done') : undef;
+ $lei->sto_done_request if $renames;
if ($old) { # cull old non-existent entries
for my $url (keys %$old) {
next if exists $seen{$url};
@@ -1463,4 +1465,19 @@ sub lms { # read-only LeiMailSync
$lse ? $lse->lms : undef;
}
+sub sto_done_request { # only call this from lei-daemon process (not workers)
+ my ($lei, $sock) = @_;
+ if ($sock //= $lei->{sock}) {
+ $LIVE_SOCK{"$sock"} = $sock;
+ $lei->{sto}->ipc_do('done', "$sock"); # issue, async wait
+ } else { # forcibly wait
+ my $wait = $lei->{sto}->ipc_do('done');
+ }
+}
+
+sub sto_done_complete { # called in lei-daemon when LeiStore->done is complete
+ my ($sock_str) = @_;
+ delete $LIVE_SOCK{$sock_str}; # frees {sock} for waiting lei clients
+}
+
1;
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 940ca1b6..2b4e58a9 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -16,12 +16,12 @@ sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
my $lms = $lei->lms or return;
my $sto = $lei->_lei_store or return; # may disappear due to race
- $sto->write_prepare;
+ $sto->write_prepare($lei);
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
$sto->ipc_do('lms_forget_folders', @folders);
- my $wait = $sto->ipc_do('done');
+ $lei->sto_done_request;
}
*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 2878cbdf..402125cf 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -13,6 +13,7 @@ sub new {
my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
$op_c->{ops} = $ops; # for PktOp->event_step
+ $self->{lei_sock} = $lei->{sock};
$lei->{ikw} = $self;
}
@@ -42,13 +43,13 @@ sub ck_update_kw { # via wq_io_do
sub ikw_done_wait {
my ($arg, $pid) = @_;
my ($self, $lei) = @$arg;
- my $wait = $lei->{sto}->ipc_do('done');
$lei->can('wq_done_wait')->($arg, $pid);
}
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $ikw = delete $lei->{ikw} or return $lei->fail;
+ $lei->sto_done_request($ikw->{lei_sock});
$ikw->wq_wait_old(\&ikw_done_wait, $lei);
}
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 1cd15296..6a40ba39 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -15,7 +15,7 @@ sub flush_lei ($) {
if (my $lne = delete $lei->{cfg}->{-lei_note_event}) {
$lne->wq_close(1, undef, $lei); # runs _lei_wq_eof;
} elsif ($lei->{sto}) { # lms_clear_src calls only:
- my $wait = $lei->{sto}->ipc_do('done');
+ $lei->sto_done_request;
}
}
@@ -117,7 +117,7 @@ sub lne_done_wait {
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $lne = delete $lei->{lne} or return $lei->fail;
- my $wait = $lei->{sto}->ipc_do('done');
+ $lei->sto_done_request;
$lne->wq_wait_old(\&lne_done_wait, $lei);
}
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index 760f276c..59cf886e 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -25,6 +25,7 @@ sub new {
my ($op_c, $ops) = $lei->workers_start($self, $nproc,
undef, { ipt => $ipt }); # LeiInput subclass
$op_c->{ops} = $ops; # for PktOp->event_step
+ $self->{lei_sock} = $lei->{sock}; # keep client for pmd_done_wait
$lei->{pmd} = $self;
}
@@ -32,7 +33,7 @@ sub ipc_atfork_child {
my ($self) = @_;
my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}';
$ipt->{lei} = $self->{lei};
- $ipt->ipc_atfork_child;
+ $ipt->ipc_atfork_child; # calls _lei_atfork_child;
}
sub each_mdir_fn { # maildir_each_file callback
@@ -48,13 +49,13 @@ sub mdir_iter { # via wq_io_do
sub pmd_done_wait {
my ($arg, $pid) = @_;
my ($self, $lei) = @$arg;
- my $wait = $lei->{sto}->ipc_do('done');
$lei->can('wq_done_wait')->($arg, $pid);
}
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $pmd = delete $lei->{pmd} or return $lei->fail;
+ $lei->sto_done_request($pmd->{lei_sock});
$pmd->wq_wait_old(\&pmd_done_wait, $lei);
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index bbd853e5..28e36e89 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -471,7 +471,7 @@ sub xchg_stderr {
}
sub done {
- my ($self) = @_;
+ my ($self, $sock_ref) = @_;
my $err = '';
if (my $im = delete($self->{im})) {
eval { $im->done };
@@ -486,6 +486,10 @@ sub done {
$self->{priv_eidx}->done; # V2Writable::done
xchg_stderr($self);
die $err if $err;
+
+ # notify clients ->done has been issued
+ defined($sock_ref) and
+ $self->{s2d_op_p}->pkt_do('sto_done_complete', $sock_ref);
}
sub ipc_atfork_child {
@@ -493,28 +497,37 @@ sub ipc_atfork_child {
my $lei = $self->{lei};
$lei->_lei_atfork_child(1) if $lei;
xchg_stderr($self);
- if (my $err = delete($self->{err_pipe})) {
- close $err->[0];
- $self->{-err_wr} = $err->[1];
+ if (my $to_close = delete($self->{to_close})) {
+ close($_) for @$to_close;
}
$self->SUPER::ipc_atfork_child;
}
sub write_prepare {
my ($self, $lei) = @_;
+ $lei // die 'BUG: $lei not passed';
unless ($self->{-ipc_req}) {
- my $d = $lei->store_path;
- $self->ipc_lock_init("$d/ipc.lock");
- substr($d, -length('/lei/store'), 10, '');
+ # s2d => store-to-daemon messages
+ require PublicInbox::PktOp;
+ my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair;
+ my $dir = $lei->store_path;
+ $self->ipc_lock_init("$dir/ipc.lock");
+ substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
- my $err_pipe = [ $r, $w ];
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
- $self->ipc_worker_spawn("lei/store $d", $lei->oldset,
- { lei => $lei, err_pipe => $err_pipe });
+ $self->ipc_worker_spawn("lei/store $dir", $lei->oldset, {
+ lei => $lei,
+ -err_wr => $w,
+ to_close => [ $r, $s2d_op_c->{sock} ],
+ s2d_op_p => $s2d_op_p,
+ });
require PublicInbox::LeiStoreErr;
- PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
+ PublicInbox::LeiStoreErr->new($r, $lei);
+ $s2d_op_c->{ops} = {
+ sto_done_complete => [ $lei->can('sto_done_complete') ]
+ };
}
$lei->{sto} = $self;
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 5e34d864..1f83e582 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -374,8 +374,8 @@ sub query_done { # EOF callback for main daemon
if ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) {
warn "BUG: {sto} missing with --mail-sync";
}
- my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
- $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+ $lei->sto_done_request if $lei->{sto};
+ my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
diff --git a/lib/PublicInbox/PktOp.pm b/lib/PublicInbox/PktOp.pm
index 92e150a4..10942dd1 100644
--- a/lib/PublicInbox/PktOp.pm
+++ b/lib/PublicInbox/PktOp.pm
@@ -56,9 +56,12 @@ sub event_step {
($cmd, @pargs) = split(/ /, $msg);
}
my $op = $self->{ops}->{$cmd //= $msg};
- die "BUG: unknown message: `$cmd'" unless $op;
- my ($sub, @args) = @$op;
- $sub->(@args, @pargs);
+ if ($op) {
+ my ($sub, @args) = @$op;
+ $sub->(@args, @pargs);
+ } elsif ($msg ne '') {
+ die "BUG: unknown message: `$cmd'";
+ }
return $self->close if $msg eq ''; # close on EOF
}
}
^ permalink raw reply related [relevance 70%]
* [PATCH] lei: add missing LeiWatch lazy-load
@ 2021-08-24 13:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-24 13:04 UTC (permalink / raw)
To: meta
I'm not sure if this class will actually be needed, but
we need to load it while we're using it.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 4d5c61fe..ea3ec0fe 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1423,6 +1423,7 @@ sub refresh_watches {
++$renames;
}
next if $watches->{$f}; # may be set to pause
+ require PublicInbox::LeiWatch;
$watches->{$f} = PublicInbox::LeiWatch->new($f);
$seen{$f} = undef;
add_maildir_watch($cd, $cfg_f);
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: implicitly watch all Maildirs it knows about
@ 2021-08-19 9:49 54% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-19 9:49 UTC (permalink / raw)
To: meta
This allows MUA-made flag changes to Maildirs to be instantly
read and acknowledged for future search results.
In the future, it may be used to speed up --augment and
--import-before (the default) with with "lei q".
---
lib/PublicInbox/LEI.pm | 42 +++++++++++++++++++++++++++------
lib/PublicInbox/LeiMailSync.pm | 10 ++++++++
lib/PublicInbox/LeiNoteEvent.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 5 ++++
4 files changed, 51 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 79dc9bf9..4d5c61fe 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1370,6 +1370,14 @@ sub cancel_maildir_watch ($$) {
for my $x (@{$w // []}) { $x->cancel }
}
+sub add_maildir_watch ($$) {
+ my ($d, $cfg_f) = @_;
+ if (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
+ my @w = $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
+ push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
+ }
+}
+
sub refresh_watches {
my ($lei) = @_;
my $cfg = _lei_cfg($lei) or return;
@@ -1380,7 +1388,7 @@ sub refresh_watches {
for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
my $url = substr($w, length('watch.'), -length('.state'));
require PublicInbox::LeiWatch;
- my $lw = $watches->{$url} //= PublicInbox::LeiWatch->new($url);
+ $watches->{$url} //= PublicInbox::LeiWatch->new($url);
$seen{$url} = undef;
my $state = $cfg->get_1("watch.$url", 'state');
if (!watch_state_ok($state)) {
@@ -1391,16 +1399,36 @@ sub refresh_watches {
my $d = canonpath_harder($1);
if ($state eq 'pause') {
cancel_maildir_watch($d, $cfg_f);
- } elsif (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
- my @w = $dir_idle->add_watches(
- ["$d/cur", "$d/new"], 1);
- push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
+ } else {
+ add_maildir_watch($d, $cfg_f);
}
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1,
- "E: watch $url not supported, yet");
+ $lei->child_error(1, "E: watch $url not supported, yet")
+ }
+ }
+
+ # add all known Maildir folders as implicit watches
+ my $sto = $lei->_lei_store;
+ my $renames = 0;
+ if (my $lms = $sto ? $sto->search->lms : undef) {
+ for my $d ($lms->folders('maildir:')) {
+ substr($d, 0, length('maildir:')) = '';
+ my $cd = canonpath_harder($d);
+ my $f = "maildir:$cd";
+
+ # fixup old bugs while we're iterating:
+ if ($d ne $cd) {
+ $sto->ipc_do('lms_rename_folder',
+ "maildir:$d", $f);
+ ++$renames;
+ }
+ next if $watches->{$f}; # may be set to pause
+ $watches->{$f} = PublicInbox::LeiWatch->new($f);
+ $seen{$f} = undef;
+ add_maildir_watch($cd, $cfg_f);
}
}
+ my $wait = $renames ? $sto->ipc_do('done') : undef;
if ($old) { # cull old non-existent entries
for my $url (keys %$old) {
next if exists $seen{$url};
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 6dfa03be..80e1bb9d 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -412,6 +412,16 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
+# only used for changing canonicalization errors
+sub rename_folder {
+ my ($self, $old, $new) = @_;
+ my $fid = delete($self->{fmap}->{$old}) //
+ fid_for($self, $old) // return;
+ $self->{dbh}->do(<<EOM, undef, $new, $fid);
+UPDATE folders SET loc = ? WHERE fid = ?
+EOM
+}
+
sub imap_oidbin ($$$) {
my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY
my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index d6511cf6..1cd15296 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -74,7 +74,7 @@ sub lei_note_event {
my $err = $lms->arg2folder($lei, [ $folder ]);
return if $err->{fail};
undef $lms;
- my $state = $cfg->get_1("watch.$folder", 'state') // 'pause';
+ my $state = $cfg->get_1("watch.$folder", 'state') // 'tag-rw';
return if $state eq 'pause';
$lei->ale; # prepare
$sto->write_prepare($lei);
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e8334187..bbd853e5 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -222,6 +222,11 @@ sub lms_forget_folders {
for my $f (@folders) { $lms->forget_folder($f) }
}
+sub lms_rename_folder {
+ my ($self, $old, $new) = @_;
+ _lms_rw($self)->rename_folder($old, $new);
+}
+
sub set_sync_info {
my ($self, $oidhex, $folder, $id) = @_;
_lms_rw($self)->set_src($oidhex, $folder, $id);
^ permalink raw reply related [relevance 54%]
* [PATCH] lei q: make --save the default
2021-08-18 11:56 71% "lei q --save" - should it be the default? Eric Wong
@ 2021-08-19 1:36 31% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-19 1:36 UTC (permalink / raw)
To: meta
Since "lei up" is more often useful than not and incurs neglible
overhead; enable --save by default and allow --no-save to work.
This also fixes a long-standing when overwriting --output
destinations with saved searches: dedupe data from previous
searches are reset and no longer influences the new (changed)
search, so results no longer go missing if two sequential
invocations of "lei q --save" point to the same --output.
---
Documentation/lei-q.pod | 4 ++--
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiSavedSearch.pm | 10 ++++++++++
lib/PublicInbox/LeiToMail.pm | 31 +++++++++++++++++++++++++++----
lib/PublicInbox/LeiUp.pm | 1 -
t/lei-q-kw.t | 3 ++-
t/lei-q-save.t | 26 ++++++++++++++------------
t/lei.t | 2 +-
t/lei_to_mail.t | 2 +-
9 files changed, 59 insertions(+), 24 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index fbe61920..69a6cdf2 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -26,9 +26,9 @@ TODO: mention curl options?
Read search terms from stdin.
-=item --save
+=item --no-save
-Save a search for L<lei-up(1)>.
+Do not save the search for L<lei-up(1)>.
=item --output=MFOLDER
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e5232f1b..79dc9bf9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -173,7 +173,7 @@ our %CMD = ( # sorted in order of importance/use:
'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms',
'stdin|', # /|\z/ must be first for lone dash
@lxs_opt,
- qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+ qw(save! output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
@@ -337,7 +337,7 @@ my %OPTDESC = (
'torsocks=s' => ['VAL|auto|no|yes',
'whether or not to wrap git and curl commands with torsocks'],
'no-torsocks' => 'alias for --torsocks=no',
-'save' => "save a search for `lei up'",
+'save!' => "do not save a search for `lei up'",
'import-remote!' => 'do not memoize remote messages into local store',
'type=s' => [ 'any|mid|git', 'disambiguate type' ],
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 2a0e9321..fd51fe38 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -243,6 +243,16 @@ sub pause_dedupe {
$oidx->commit_lazy;
}
+sub reset_dedupe {
+ my ($self) = @_;
+ prepare_dedupe($self);
+ my $lk = $self->lock_for_scope_fast;
+ for my $t (qw(xref3 over id2num)) {
+ $self->{oidx}->{dbh}->do("DELETE FROM $t");
+ }
+ pause_dedupe($self);
+}
+
sub mm { undef }
sub altid_map { {} }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d782d4c7..be6e178f 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -390,10 +390,16 @@ sub new {
-e $dst && !-d _ and die
"$dst exists and is not a directory\n";
$lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
+ $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
} elsif (substr($fmt, 0, 4) eq 'mbox') {
require PublicInbox::MboxReader;
$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
$self->{base_type} = 'mbox';
+ if ($lei->{cmd} eq 'q' &&
+ (($lei->path_to_fd($dst) // -1) < 0) &&
+ (-f $dst || !-e _)) {
+ $lei->{opt}->{save} //= \1;
+ }
} elsif ($fmt =~ /\Aimaps?\z/) {
require PublicInbox::NetWriter;
require PublicInbox::URIimap;
@@ -408,6 +414,7 @@ sub new {
$dst = $lei->{ovv}->{dst} = $$uri; # canonicalized
$lei->{net} = $net;
$self->{base_type} = 'imap';
+ $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
} elsif ($fmt eq 'text') {
require PublicInbox::LeiViewText;
$lei->{lvt} = PublicInbox::LeiViewText->new($lei);
@@ -454,9 +461,17 @@ sub _pre_augment_maildir {
open $self->{poke_dh}, '<', "${dst}cur" or die "open ${dst}cur: $!";
}
+sub clobber_dst_prepare ($;$) {
+ my ($lei, $f) = @_;
+ my $wait = (defined($f) && $lei->{sto}) ?
+ $lei->{sto}->ipc_do('lms_forget_folders', $f) : undef;
+ my $dedupe = $lei->{dedupe} or return;
+ $dedupe->reset_dedupe if $dedupe->can('reset_dedupe');
+}
+
sub _do_augment_maildir {
my ($self, $lei) = @_;
- return if ($lei->{opt}->{save} // 0) < 0;
+ return if $lei->{cmd} eq 'up';
my $dst = $lei->{ovv}->{dst};
my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
my $mdr = PublicInbox::MdirReader->new;
@@ -468,9 +483,11 @@ sub _do_augment_maildir {
$dedupe->pause_dedupe;
}
} elsif ($lse) {
+ clobber_dst_prepare($lei, "maildir:$dst");
$mdr->{shard_info} = $self->{shard_info};
$mdr->maildir_each_eml($dst, \&_md_update, $lei, $lse, 1);
} else {# clobber existing Maildir
+ clobber_dst_prepare($lei, "maildir:$dst");
$mdr->maildir_each_file($dst, \&_unlink);
}
}
@@ -487,7 +504,7 @@ sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
sub _do_augment_imap {
my ($self, $lei) = @_;
- return if ($lei->{opt}->{save} // 0) < 0;
+ return if $lei->{cmd} eq 'up';
my $net = $lei->{net};
my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
if ($lei->{opt}->{augment}) {
@@ -499,11 +516,13 @@ sub _do_augment_imap {
}
} elsif ($lse) {
my $delete_mic;
+ clobber_dst_prepare($lei, "$self->{uri}");
$net->imap_each($self->{uri}, \&_imap_augment_or_delete,
$lei, $lse, \$delete_mic);
$delete_mic->expunge if $delete_mic;
} elsif (!$self->{-wq_worker_nr}) { # undef or 0
# clobber existing IMAP folder
+ clobber_dst_prepare($lei, "$self->{uri}");
$net->imap_delete_all($self->{uri});
}
}
@@ -563,6 +582,8 @@ sub _pre_augment_mbox {
if $imp_before && !ref($imp_before);
die "--augment specified but $dst is not seekable\n" if
$lei->{opt}->{augment};
+ die "cannot --save with unseekable $dst\n" if
+ $lei->{dedupe} && $lei->{dedupe}->can('reset_dedupe');
}
if ($self->{zsfx} = PublicInbox::MboxReader::zsfx($dst)) {
pipe(my ($r, $w)) or die "pipe: $!";
@@ -581,10 +602,10 @@ sub _do_augment_mbox {
my ($self, $lei) = @_;
return unless $self->{seekable};
my $opt = $lei->{opt};
- return if ($opt->{save} // 0) < 0;
+ return if $lei->{cmd} eq 'up';
my $out = $lei->{1};
my ($fmt, $dst) = @{$lei->{ovv}}{qw(fmt dst)};
- return unless -s $out;
+ return clobber_dst_prepare($lei) unless -s $out;
unless ($opt->{augment} || $opt->{'import-before'}) {
truncate($out, 0) or die "truncate($dst): $!";
return;
@@ -599,6 +620,8 @@ sub _do_augment_mbox {
if ($opt->{augment}) {
$dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
+ } else {
+ clobber_dst_prepare($lei);
}
if ($opt->{'import-before'}) { # the default
my $lse = $lei->{lse};
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index c5a01527..ba11761a 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -64,7 +64,6 @@ sub lei_up {
my ($lei, @outs) = @_;
$lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
- $opt->{save} = -1;
my @local;
if (defined $opt->{all}) {
return $lei->fail("--all and @outs incompatible") if @outs;
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 2e6be1f0..4edee72a 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -28,7 +28,8 @@ ok(!glob("$o/cur/*"), 'last result cleared after augment-import');
lei_ok(qw(q -o), "maildir:$o", qw(m:qp@example.com));
@fn = glob("$o/cur/*:2,S");
-is(scalar(@fn), 1, "`seen' flag set on Maildir file");
+is(scalar(@fn), 1, "`seen' flag set on Maildir file") or
+ diag "$o contents: ", explain([glob("$o/*/*")]);
# ensure --no-import-before works
my $n = $fn[0];
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index eada2dd4..7aa9b84e 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -25,7 +25,7 @@ test_lei(sub {
my $home = $ENV{HOME};
my $in = $doc1->as_string;
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
- lei_ok qw(q -q --save z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
+ lei_ok qw(q -q z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
my %before = map { $_ => 1 } glob("$home/md/cur/*");
my $f = (keys %before)[0] or xbail({before => \%before});
is_deeply(eml_load($f), $doc1, 'doc1 matches');
@@ -52,8 +52,8 @@ test_lei(sub {
is_deeply(eml_load($f), $doc2, 'doc2 matches');
# check stdin
- lei_ok [qw(q --save - -o), "mboxcl2:mbcl2" ],
- undef, { -C => $home, %$lei_opt, 0 => \'d:last.week..'};
+ lei_ok [qw(q - -o), "mboxcl2:mbcl2" ], undef,
+ { -C => $home, %$lei_opt, 0 => \'d:last.week..'};
@s = glob("$home/.local/share/lei/saved-searches/mbcl2-*");
$cfg = PublicInbox::Config->new("$s[0]/lei.saved-search");
is_deeply $cfg->{'lei.q'}, 'd:last.week..',
@@ -68,7 +68,11 @@ test_lei(sub {
lei_ok([qw(up mbcl2)], undef, { -C => $home, %$lei_opt });
ok(-s "$home/mbcl2" > $size, 'size increased after up');
- ok(!lei(qw(up -q), $home), 'up fails w/o --save');
+ ok(!lei(qw(up -q), $home), 'up fails on unknown dir');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
+
+ lei_ok(qw(q --no-save d:last.week.. -q -o), "$home/no-save");
+ ok(!lei(qw(up -q), "$home/no-save"), 'up fails on --no-save');
like($lei_err, qr/--save was not used/, 'error noted --save');
lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
@@ -94,7 +98,7 @@ test_lei(sub {
open my $mb, '>', "$home/mbrd" or xbail "open $!";
print $mb $pre_existing;
close $mb or xbail "close: $!";
- lei_ok(qw(q --save -o mboxrd:mbrd m:qp@example.com -C), $home);
+ lei_ok(qw(q -o mboxrd:mbrd m:qp@example.com -C), $home);
open $mb, '<', "$home/mbrd" or xbail "open $!";
is_deeply([grep(/pre-existing/, <$mb>)], [],
'pre-existing messsage gone w/o augment');
@@ -107,7 +111,7 @@ test_lei(sub {
open $mb, '>', "$home/mbrd-aug" or xbail "open $!";
print $mb $pre_existing;
close $mb or xbail "close: $!";
- lei_ok(qw(q -a --save -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
+ lei_ok(qw(q -a -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
open $mb, '<', "$home/mbrd-aug" or xbail "open $!";
$mb = do { local $/; <$mb> };
like($mb, qr/pre-existing/, 'pre-existing message preserved w/ -a');
@@ -133,7 +137,7 @@ test_lei(sub {
my $o = "$home/dd-mid";
$in = $doc2->as_string . "\n-------\nappended list sig\n";
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
- lei_ok(qw(q --dedupe=mid --save m:testmessage@example.com -o), $o);
+ lei_ok(qw(q --dedupe=mid m:testmessage@example.com -o), $o);
my @m = glob("$o/cur/*");
is(scalar(@m), 1, '--dedupe=mid w/ --save');
$in = $doc2->as_string . "\n-------\nanother list sig\n";
@@ -143,8 +147,7 @@ test_lei(sub {
for my $dd (qw(content)) {
$o = "$home/dd-$dd";
- lei_ok(qw(q --save m:testmessage@example.com -o), $o,
- "--dedupe=$dd");
+ lei_ok(qw(q m:testmessage@example.com -o), $o, "--dedupe=$dd");
@m = glob("$o/cur/*");
is(scalar(@m), 3, 'all 3 matches with dedupe='.$dd);
}
@@ -153,7 +156,7 @@ test_lei(sub {
$o = "$home/dd-oid";
my $ibx = create_inbox 'ibx', indexlevel => 'medium',
tmpdir => "$home/v1", sub {};
- lei_ok(qw(q --save --dedupe=oid m:qp@example.com -o), $o,
+ lei_ok(qw(q --dedupe=oid m:qp@example.com -o), $o,
'-I', $ibx->{inboxdir});
@m = glob("$o/cur/*");
is(scalar(@m), 1, 'got first result');
@@ -203,8 +206,7 @@ test_lei(sub {
lei_ok([qw(edit-search), $v2s], { VISUAL => 'cat', EDITOR => 'cat' });
like($lei_out, qr/^\[lei/sm, 'edit-search can cat');
- lei_ok('-C', "$home/v2s",
- qw(q -q --save -o ../s m:testmessage@example.com));
+ lei_ok('-C', "$home/v2s", qw(q -q -o ../s m:testmessage@example.com));
lei_ok qw(ls-search);
unlike $lei_out, qr{/\.\./s$}sm, 'relative path not in ls-search';
like $lei_out, qr{^\Q$home\E/s$}sm,
diff --git a/t/lei.t b/t/lei.t
index 8211c01d..dfbcb1f3 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -114,7 +114,7 @@ my $test_completion = sub {
%out = map { $_ => 1 } split(/\s+/s, $lei_out);
for my $sw (qw(-f --format -o --output --mfolder --augment -a
--mua --no-local --local --verbose -v
- --save --no-remote --remote --torsocks
+ --save --no-save --no-remote --remote --torsocks
--reverse -r )) {
ok($out{$sw}, "$sw offered as `lei q' completion");
}
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 35904706..e8958c64 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -75,7 +75,7 @@ for my $mbox (@MBOX) {
my ($tmpdir, $for_destroy) = tmpdir();
local $ENV{TMPDIR} = $tmpdir;
open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!;
-my $lei = bless { 2 => $err }, 'PublicInbox::LEI';
+my $lei = bless { 2 => $err, cmd => 'test' }, 'PublicInbox::LEI';
my $commit = sub {
$_[0] = undef; # wcb
delete $lei->{1};
^ permalink raw reply related [relevance 31%]
* "lei q --save" - should it be the default?
@ 2021-08-18 11:56 71% Eric Wong
2021-08-19 1:36 31% ` [PATCH] lei q: make --save the default Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-08-18 11:56 UTC (permalink / raw)
To: meta
I've often found "lei up" useful after-the-fact, but it requires
"lei q --save" to be used initially. Perhaps "--save" should be
the default when writing to mbox/Maildir/IMAP...
Thoughts?
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei forget-mail-sync: rely on lei/store process
2021-08-17 8:52 71% [PATCH 0/3] lei: some mail sync stuff Eric Wong
2021-08-17 8:52 54% ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
@ 2021-08-17 8:52 88% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
As implied in commit 6ff03ba2be9247f1
("lei export-kw: do not write directly to mail_sync.sqlite3"),
modifying mail_sync.sqlite3 directly can lead to conflicts
and making everything go through lei/store is easier.
---
lib/PublicInbox/LeiForgetMailSync.pm | 8 ++++----
lib/PublicInbox/LeiStore.pm | 6 ++++++
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index c74ba25d..940ca1b6 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -15,13 +15,13 @@ use PublicInbox::LeiExportKw;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
my $lms = $lei->lms or return;
+ my $sto = $lei->_lei_store or return; # may disappear due to race
+ $sto->write_prepare;
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
- delete $lms->{dbh};
- $lms->lms_begin;
- $lms->forget_folder($_) for @folders;
- $lms->lms_commit;
+ $sto->ipc_do('lms_forget_folders', @folders);
+ my $wait = $sto->ipc_do('done');
}
*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 3f33d114..e8334187 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -216,6 +216,12 @@ sub lms_mv_src {
_lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn);
}
+sub lms_forget_folders {
+ my ($self, @folders) = @_;
+ my $lms = _lms_rw($self);
+ for my $f (@folders) { $lms->forget_folder($f) }
+}
+
sub set_sync_info {
my ($self, $oidhex, $folder, $id) = @_;
_lms_rw($self)->set_src($oidhex, $folder, $id);
^ permalink raw reply related [relevance 88%]
* [PATCH 0/3] lei: some mail sync stuff
@ 2021-08-17 8:52 71% Eric Wong
2021-08-17 8:52 54% ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
2021-08-17 8:52 88% ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
Still trying to wrap my head around some other stuff related to
mail_sync.sqlite3, but 3/3 is a necessary fix and 2/3 is an obvious
cleanup. The ipc_do('done') stuff could probably be made
faster, but probably after the 1.7 release...
Eric Wong (3):
lei: add ->lms shortcut for LeiMailSync
ipc: remove WQ_MAX_WORKERS
lei forget-mail-sync: rely on lei/store process
lib/PublicInbox/IPC.pm | 5 -----
lib/PublicInbox/LEI.pm | 9 +++++++++
lib/PublicInbox/LeiBlob.pm | 3 +--
lib/PublicInbox/LeiForgetMailSync.pm | 11 +++++------
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 10 ++++------
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
lib/PublicInbox/LeiStore.pm | 6 ++++++
8 files changed, 27 insertions(+), 23 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync
2021-08-17 8:52 71% [PATCH 0/3] lei: some mail sync stuff Eric Wong
@ 2021-08-17 8:52 54% ` Eric Wong
2021-08-17 8:52 88% ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
We access this read-only in many places (and will in more),
so provide a shortcut to simplify callers.
---
lib/PublicInbox/LEI.pm | 9 +++++++++
lib/PublicInbox/LeiBlob.pm | 3 +--
lib/PublicInbox/LeiForgetMailSync.pm | 3 +--
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 10 ++++------
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
6 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 347dd280..e5232f1b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1425,4 +1425,13 @@ sub git_blob_id {
($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
}
+sub lms { # read-only LeiMailSync
+ my ($lei) = @_;
+ my $lse = $lei->{lse} // do {
+ my $sto = $lei->{sto} // _lei_store($lei);
+ $sto ? $sto->search : undef
+ };
+ $lse ? $lse->lms : undef;
+}
+
1;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 3158ca3b..21003894 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -133,8 +133,7 @@ sub lei_blob {
}
my $ce = $?;
return if $ce == 0;
- my $sto = $lei->_lei_store;
- my $lms = $sto ? $sto->search->lms : undef;
+ my $lms = $lei->lms;
if (my $bref = $lms ? $lms->local_blob($blob, 1) : undef) {
defined($lei->{-attach_idx}) and
return extract_attach($lei, $blob, $bref);
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 46dde1a7..c74ba25d 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -14,8 +14,7 @@ use PublicInbox::LeiExportKw;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index bf7a4836..2d2ff1a0 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -41,8 +41,7 @@ sub inspect_imap_uid ($$) {
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
- my $lse = $lei->{lse} or return $ent;
- my $lms = $lse->lms or return $ent;
+ my $lms = $lei->lms or return $ent;
my $folders = [ $folder ];
my $err = $lms->arg2folder($lei, $folders);
if ($err) {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 4a0c24a9..9d95e899 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -13,7 +13,7 @@ use PublicInbox::MID qw($MID_EXTRACT);
sub lcat_folder ($$$) {
my ($lei, $lms, $folder) = @_;
- $lms //= $lei->{lse}->lms // return;
+ $lms //= $lei->lms or return;
my $folders = [ $folder];
my $err = $lms->arg2folder($lei, $folders);
$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
@@ -29,7 +29,7 @@ sub lcat_folder ($$$) {
sub lcat_imap_uri ($$) {
my ($lei, $uri) = @_;
- my $lms = $lei->{lse}->lms or return;
+ my $lms = $lei->lms or return;
# cf. LeiXsearch->lcat_dump
if (defined $uri->uid) {
my $oidhex = $lms->imap_oid($lei, $uri);
@@ -129,8 +129,7 @@ sub lei_lcat {
my ($lei, @argv) = @_;
my $lxs = $lei->lxs_prepare or return;
$lei->ale->refresh_externals($lxs, $lei);
- my $sto = $lei->_lei_store(1);
- $lei->{lse} = $sto->search;
+ $lei->_lei_store(1);
my $opt = $lei->{opt};
my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
@@ -153,8 +152,7 @@ no args allowed on command-line with --stdin
sub _complete_lcat {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } $lms->folders;
}
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index cadc61ed..2d8913ac 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -103,8 +103,7 @@ sub _complete_ls_mail_source {
my $match_cb = $lei->complete_url_prepare(\@argv);
my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
my %f = map { $_ => 1 } @m;
- my $sto = $lei->_lei_store;
- if (my $lms = $sto ? $sto->search->lms : undef) {
+ if (my $lms = $lei->lms) {
@m = map { $match_cb->($_) } grep(
m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
@f{@m} = @m;
^ permalink raw reply related [relevance 54%]
* [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
@ 2021-08-14 0:29 64% ` Eric Wong
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
2021-08-24 20:14 71% ` [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
This ought to avoid /Document \d+ not found/ errors from Xapian
when seeing a message for the first time by not attempting to
read keywords for totally unseen messages.
---
lib/PublicInbox/LeiRemote.pm | 7 ++++---
lib/PublicInbox/LeiStore.pm | 1 +
lib/PublicInbox/LeiXSearch.pm | 10 +++++++---
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 945d9990..e7deecb8 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -26,11 +26,12 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
my ($eml, $self) = @_;
my $lei = $self->{lei};
my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ my $smsg = bless {}, 'PublicInbox::Smsg';
if ($lei->{sto} && !$xoids) { # memoize locally
- $lei->{sto}->ipc_do('add_eml', $eml);
+ my $res = $lei->{sto}->ipc_do('add_eml', $eml);
+ $smsg = $res if ref($res) eq ref($smsg);
}
- my $smsg = bless {}, 'PublicInbox::Smsg';
- $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e26b622d..ce66014f 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -329,6 +329,7 @@ sub add_eml {
}
\@docids;
} else { # totally new message
+ delete $smsg->{-oidx}; # for IPC-friendliness
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
$oidx->add_overview($eml, $smsg);
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 393f25bf..971f3a06 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -266,11 +266,15 @@ sub _smsg_fill ($$) {
sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- $self->{import_sto}->ipc_do('add_eml', $eml);
+ my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+ if (ref($res) eq ref($smsg)) { # totally new message
+ $smsg = $res;
+ $smsg->{kw} = []; # short-circuit xsmsg_vmd
+ }
}
- my $smsg = bless {}, 'PublicInbox::Smsg';
- $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
_smsg_fill($smsg, $eml);
wait_startq($lei);
^ permalink raw reply related [relevance 64%]
* [PATCH 3/3] lei: hexdigest mocks account for unwanted headers
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
@ 2021-08-14 0:29 58% ` Eric Wong
2021-08-24 20:14 71% ` [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
PublicInbox::Import never imports @UNWANTED_HEADERS, so ensure
our mock blob OIDs do the same. This ought to prevent
duplicates if the PSGI mboxrd download starts setting
"X-Status: F" like "lei q -tt .."
---
lib/PublicInbox/FakeImport.pm | 3 +++
lib/PublicInbox/LEI.pm | 5 +++++
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 9 ++++++++-
lib/PublicInbox/LeiXSearch.pm | 3 +--
6 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/FakeImport.pm b/lib/PublicInbox/FakeImport.pm
index dea25cbe..bccc3321 100644
--- a/lib/PublicInbox/FakeImport.pm
+++ b/lib/PublicInbox/FakeImport.pm
@@ -4,12 +4,15 @@
# pretend to do PublicInbox::Import::add for "lei index"
package PublicInbox::FakeImport;
use strict;
+use v5.10.1;
use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub new { bless { bytes_added => 0 }, __PACKAGE__ }
sub add {
my ($self, $eml, $check_cb, $smsg) = @_;
+ PublicInbox::Import::drop_unwanted_headers($eml);
$smsg->populate($eml);
my $raw = $eml->as_string;
$smsg->{blob} = git_sha(1, \$raw)->hexdigest;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7d0f63dc..347dd280 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1420,4 +1420,9 @@ sub refresh_watches {
}
}
+sub git_blob_id {
+ my ($lei, $eml) = @_;
+ ($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
+}
+
1;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 37b660f9..962ad49e 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -73,7 +73,7 @@ sub lxs_prepare {
my @only = @{$opt->{only} // []};
# --local is enabled by default unless --only is used
# we'll allow "--only $LOCATION --local"
- my $sto = $self->_lei_store(1); # FIXME: should not create
+ my $sto = $self->_lei_store(1);
$self->{lse} = $sto->search;
if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
$lxs->prepare_external($self->{lse});
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index e7deecb8..580787c0 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -32,7 +32,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
$smsg = $res if ref($res) eq ref($smsg);
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : git_sha(1, $eml)->hexdigest;
+ : $lei->git_blob_id($eml);
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
push @{$self->{smsg}}, $smsg;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index ce66014f..3f33d114 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -20,7 +20,7 @@ use PublicInbox::Eml;
use PublicInbox::Import;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::ContentHash qw(content_hash git_sha);
use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
@@ -508,4 +508,11 @@ sub write_prepare {
$lei->{sto} = $self;
}
+# TODO: support SHA-256
+sub git_blob_id { # called via LEI->git_blob_id
+ my ($self, $eml) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+ git_sha(1, $eml)->hexdigest;
+}
+
1;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 971f3a06..5e34d864 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -274,8 +274,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
$smsg->{kw} = []; # short-circuit xsmsg_vmd
}
}
- $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : git_sha(1, $eml)->hexdigest;
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
_smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
^ permalink raw reply related [relevance 58%]
* [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors
@ 2021-08-14 0:29 71% Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
2/3 is probably a fix for a long-standing problem, 3/3 was
noticed while working on it. If 2/3 doesn't fix it, then maybe
1/3 will help us narrow it down.
Eric Wong (3):
lei: diagnostics for /Document \d+ not found/ errors
lei <q|up>: wait on remote mboxrd imports synchronously
lei: hexdigest mocks account for unwanted headers
lib/PublicInbox/FakeImport.pm | 3 +++
lib/PublicInbox/IPC.pm | 2 +-
lib/PublicInbox/LEI.pm | 5 +++++
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 9 +++++----
lib/PublicInbox/LeiSearch.pm | 17 ++++++++++-------
lib/PublicInbox/LeiStore.pm | 10 +++++++++-
lib/PublicInbox/LeiXSearch.pm | 11 +++++++----
8 files changed, 41 insertions(+), 18 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei: diagnostics for /Document \d+ not found/ errors
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
@ 2021-08-14 0:29 68% ` Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
This may help diagnose "Exception: Document \d+ not found"
errors I'm seeing from "lei up" with HTTPS endpoints.
---
lib/PublicInbox/IPC.pm | 2 +-
lib/PublicInbox/LeiSearch.pm | 17 ++++++++++-------
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 497a6035..d909dc1c 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -236,7 +236,7 @@ sub recv_and_run {
undef $buf;
my $sub = shift @$args;
eval { $self->$sub(@$args) };
- warn "$$ wq_worker: $@" if $@;
+ warn "$$ $0 wq_worker: $@" if $@;
delete @$self{0..($nfd-1)};
$n;
}
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 79b2fd7d..f9e5c8e9 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -55,13 +55,16 @@ sub _xsmsg_vmd { # retry_reopen
$kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged});
my @num = $self->over->blob_exists($smsg->{blob});
for my $num (@num) { # there should only be one...
- $doc = $xdb->get_document(num2docid($self, $num));
- $x = xap_terms('K', $doc);
- %kw = (%kw, %$x);
- if ($want_label) { # JSON/JMAP only
- $x = xap_terms('L', $doc);
- %L = (%L, %$x);
- }
+ eval {
+ $doc = $xdb->get_document(num2docid($self, $num));
+ $x = xap_terms('K', $doc);
+ %kw = (%kw, %$x);
+ if ($want_label) { # JSON/JMAP only
+ $x = xap_terms('L', $doc);
+ %L = (%L, %$x);
+ }
+ };
+ warn "$$ $0 #$num (nshard=$self->{nshard}) $smsg->{blob}: $@";
}
$smsg->{kw} = [ sort keys %kw ] if scalar(keys(%kw));
$smsg->{L} = [ sort keys %L ] if scalar(keys(%L));
^ permalink raw reply related [relevance 68%]
* [PATCH 0/2] "lei up" improvements
@ 2021-08-12 23:40 71% Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
Some obvious things for now, some maybe not-so-obvious things on
the way...
Eric Wong (2):
lei up: support multiple output folders w/o --all=local
lei up: note errors if one output destination fails
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 23 +++++++++++++++++++----
2 files changed, 20 insertions(+), 5 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei up: note errors if one output destination fails
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
@ 2021-08-12 23:40 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
We can keep going if one (out of multiple) output destinations
fail, but the error needs to be communicated to the caller as an
exit code.
---
lib/PublicInbox/LeiUp.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 49b558fd..c5a01527 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -57,7 +57,7 @@ sub up1_redispatch {
up1($l, $out);
$l->qerr("# $out done");
};
- $l->err($@) if $@;
+ $l->child_error(1 << 8, $@) if $@;
}
sub lei_up {
^ permalink raw reply related [relevance 71%]
* [PATCH 1/2] lei up: support multiple output folders w/o --all=local
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
@ 2021-08-12 23:40 63% ` Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
Being able to update 1 folder, or all (local) folders is
sometimes too limiting, so just allow updating any subset
of local folders.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 21 ++++++++++++++++++---
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 54fac7b4..7d0f63dc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -178,7 +178,7 @@ our %CMD = ( # sorted in order of importance/use:
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
-'up' => [ 'OUTPUT|--all', 'update saved search',
+'up' => [ 'OUTPUT...|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 3356d11e..49b558fd 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -61,11 +61,13 @@ sub up1_redispatch {
}
sub lei_up {
- my ($lei, $out) = @_;
+ my ($lei, @outs) = @_;
$lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
$opt->{save} = -1;
+ my @local;
if (defined $opt->{all}) {
+ return $lei->fail("--all and @outs incompatible") if @outs;
length($opt->{mua}//'') and return
$lei->fail('--all and --mua= are incompatible');
@@ -74,7 +76,20 @@ sub lei_up {
$opt->{all} eq 'local' or return
$lei->fail('only --all=local works at the moment');
my @all = PublicInbox::LeiSavedSearch::list($lei);
- my @local = grep(!m!\Aimaps?://!i, @all);
+ @local = grep(!m!\Aimaps?://!i, @all);
+ } else {
+ @local = @outs;
+ }
+ if (scalar(@outs) > 1) {
+ length($opt->{mua}//'') and return $lei->fail(<<EOM);
+multiple outputs and --mua= are incompatible
+EOM
+ # TODO:
+ return $lei->fail(<<EOM) if grep(m!\Aimaps?://!i, @outs);
+multiple destinations only supported for local outputs (FIXME)
+EOM
+ }
+ if (scalar(@local) > 1) {
$lei->_lei_store->write_prepare($lei); # share early
# daemon mode, re-dispatch into our event loop w/o
# creating an extra fork-level
@@ -89,7 +104,7 @@ sub lei_up {
$lei->event_step_init;
$op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
} else {
- up1($lei, $out);
+ up1($lei, $local[0]);
}
}
^ permalink raw reply related [relevance 63%]
* [PATCH 0/3] lei pathname canonicalization fixes
@ 2021-08-11 11:26 71% Eric Wong
2021-08-11 11:26 42% ` [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-08-11 11:26 UTC (permalink / raw)
To: meta
Pathnames with "x/../y" components weren't being canonicalized
properly after creation, and some of it was being put into the
lei_mail_sync.sqlite3 DB and configs for saved searches.
This will become more important as we start using inotify more
to track keyword changes on messages.
Eric Wong (3):
treewide: use *nix-specific dirname regexps
lei_saved_search: canonicalized relative save paths
lei: attempt to canonicalize away "/../" pathnames
lib/PublicInbox/IMAPTracker.pm | 4 ++--
lib/PublicInbox/LEI.pm | 16 +++++++++++-----
lib/PublicInbox/LeiALE.pm | 8 ++++----
lib/PublicInbox/LeiBlob.pm | 4 ++--
lib/PublicInbox/LeiInit.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 9 ++++++++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/OverIdx.pm | 4 ++--
lib/PublicInbox/Xapcmd.pm | 5 ++---
script/public-inbox-init | 3 +--
t/init.t | 1 -
t/lei-q-save.t | 9 +++++++++
t/lei_xsearch.t | 8 +++++---
17 files changed, 52 insertions(+), 32 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames
2021-08-11 11:26 71% [PATCH 0/3] lei pathname canonicalization fixes Eric Wong
@ 2021-08-11 11:26 42% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-11 11:26 UTC (permalink / raw)
To: meta
As documented, File::Spec->canonpath does not canonicalize
"/../". While we want to do our best to preserve symlinks in
pathnames, leaving "/../" can mislead our inotify|kqueue usage.
---
lib/PublicInbox/LEI.pm | 14 ++++++++++----
lib/PublicInbox/LeiALE.pm | 8 ++++----
lib/PublicInbox/LeiBlob.pm | 4 ++--
lib/PublicInbox/LeiInit.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 +-
t/lei_xsearch.t | 8 +++++---
10 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index be4754df..54fac7b4 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -94,6 +94,12 @@ sub rel2abs {
# abs_path resolves symlinks in parent iff all parents exist
sub abs_path { Cwd::abs_path($_[1]) // rel2abs(@_) }
+sub canonpath_harder {
+ my $p = $_[-1]; # $_[0] may be self
+ $p = File::Spec->canonpath($p);
+ $p =~ m!(?:/*|\A)\.\.(?:/*|\z)! && -e $p ? Cwd::abs_path($p) : $p;
+}
+
sub share_path ($) { # $HOME/.local/share/lei/$FOO
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_DATA_HOME} //
@@ -808,8 +814,8 @@ sub _lei_cfg ($;$) {
my $cfg = PublicInbox::Config->git_config_dump($f);
$cfg->{-st} = $cur_st;
$cfg->{'-f'} = $f;
- if ($sto && File::Spec->canonpath($sto_dir // store_path($self))
- eq File::Spec->canonpath($cfg->{'leistore.dir'} //
+ if ($sto && canonpath_harder($sto_dir // store_path($self))
+ eq canonpath_harder($cfg->{'leistore.dir'} //
store_path($self))) {
$cfg->{-lei_store} = $sto;
$cfg->{-lei_note_event} = $lne;
@@ -1382,7 +1388,7 @@ sub refresh_watches {
next;
}
if ($url =~ /\Amaildir:(.+)/i) {
- my $d = File::Spec->canonpath($1);
+ my $d = canonpath_harder($1);
if ($state eq 'pause') {
cancel_maildir_watch($d, $cfg_f);
} elsif (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
@@ -1400,7 +1406,7 @@ sub refresh_watches {
next if exists $seen{$url};
delete $old->{$url};
if ($url =~ /\Amaildir:(.+)/i) {
- my $d = File::Spec->canonpath($1);
+ my $d = canonpath_harder($1);
cancel_maildir_watch($d, $cfg_f);
} else { # TODO: imap/nntp/jmap
$lei->child_error(1, "E: watch $url TODO");
diff --git a/lib/PublicInbox/LeiALE.pm b/lib/PublicInbox/LeiALE.pm
index cb570ab4..cc9a2095 100644
--- a/lib/PublicInbox/LeiALE.pm
+++ b/lib/PublicInbox/LeiALE.pm
@@ -33,7 +33,7 @@ sub new {
for my $loc ($lei->externals_each) { # locals only
$lxs->prepare_external($loc) if -d $loc;
}
- $self->refresh_externals($lxs);
+ $self->refresh_externals($lxs, $lei);
$self;
}
@@ -50,7 +50,7 @@ sub overs_all { # for xoids_for (called only in lei workers?)
}
sub refresh_externals {
- my ($self, $lxs) = @_;
+ my ($self, $lxs, $lei) = @_;
$self->git->cleanup;
my $lk = $self->lock_for_scope;
my $cur_lxs = ref($lxs)->new;
@@ -72,7 +72,7 @@ sub refresh_externals {
}
my @ibxish = $cur_lxs->locals;
for my $x ($lxs->locals) {
- my $d = File::Spec->canonpath($x->{inboxdir} // $x->{topdir});
+ my $d = $lei->canonpath_harder($x->{inboxdir} // $x->{topdir});
$seen_ibxish{$d} //= do {
$new .= "$d\n";
push @ibxish, $x;
@@ -95,7 +95,7 @@ sub refresh_externals {
$old = <$fh> // die "readline($f): $!";
}
for my $x (@ibxish) {
- $new .= File::Spec->canonpath($x->git->{git_dir})."/objects\n";
+ $new .= $lei->canonpath_harder($x->git->{git_dir})."/objects\n";
}
$self->{ibxish} = \@ibxish;
return if $old eq $new;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 09217964..3158ca3b 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -112,7 +112,7 @@ sub lei_blob {
if ($opt->{mail} // ($has_hints ? 0 : 1)) {
if (grep(defined, @$opt{qw(include only)})) {
$lxs = $lei->lxs_prepare;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
}
my $rdr = {};
if ($opt->{mail}) {
@@ -155,7 +155,7 @@ sub lei_blob {
return $lei->fail('no --git-dir to try') unless @$git_dirs;
unless ($lxs) {
$lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
}
if ($lxs->remotes) {
require PublicInbox::LeiRemote;
diff --git a/lib/PublicInbox/LeiInit.pm b/lib/PublicInbox/LeiInit.pm
index c6c0c01b..6558ac0a 100644
--- a/lib/PublicInbox/LeiInit.pm
+++ b/lib/PublicInbox/LeiInit.pm
@@ -4,7 +4,6 @@
# for the "lei init" command, not sure if it's even needed...
package PublicInbox::LeiInit;
use v5.10.1;
-use File::Spec;
sub lei_init {
my ($self, $dir) = @_;
@@ -13,7 +12,7 @@ sub lei_init {
$dir //= $self->store_path;
$dir = $self->rel2abs($dir);
my @cur = stat($cur) if defined($cur);
- $cur = File::Spec->canonpath($cur // $dir);
+ $cur = $self->canonpath_harder($cur // $dir);
my @dir = stat($dir);
my $exists = "# leistore.dir=$cur already initialized" if @dir;
if (@cur) {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index cb5eb5b4..4a0c24a9 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -128,7 +128,7 @@ sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
sub lei_lcat {
my ($lei, @argv) = @_;
my $lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
my $sto = $lei->_lei_store(1);
$lei->{lse} = $sto->search;
my $opt = $lei->{opt};
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index e4242d9b..223db222 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -76,7 +76,7 @@ sub new {
$fmt //= $devfd >= 0 ? 'json' : (detect_fmt($lei, $dst) or return);
if (index($dst, '://') < 0) { # not a URL, so assume path
- $dst = File::Spec->canonpath($dst);
+ $dst = $lei->canonpath_harder($dst);
} # else URL
my $self = bless { fmt => $fmt, dst => $dst }, $class;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index eb7b98d4..37b660f9 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -116,7 +116,7 @@ sub lei_q {
my ($self, @argv) = @_;
PublicInbox::Config->json; # preload before forking
my $lxs = lxs_prepare($self) or return;
- $self->ale->refresh_externals($lxs);
+ $self->ale->refresh_externals($lxs, $self);
my $opt = $self->{opt};
my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 7607b44f..0ba5897c 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -215,7 +215,7 @@ sub lei_rediff {
$lei->{curl} //= which('curl') or return
$lei->fail('curl needed for', $lxs->remotes);
}
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
my $self = bless {
-force_eml => 1, # for LeiInput->input_fh
lxs => $lxs,
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 9069232b..3356d11e 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -42,7 +42,7 @@ sub up1 ($$) {
}
$lei->{lss} = $lss; # for LeiOverview->new
my $lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
$lei->_start_query;
}
diff --git a/t/lei_xsearch.t b/t/lei_xsearch.t
index 3eb44233..d9ddb297 100644
--- a/t/lei_xsearch.t
+++ b/t/lei_xsearch.t
@@ -11,6 +11,7 @@ require PublicInbox::ExtSearchIdx;
require_git 2.6;
require_ok 'PublicInbox::LeiXSearch';
require_ok 'PublicInbox::LeiALE';
+require_ok 'PublicInbox::LEI';
my ($home, $for_destroy) = tmpdir();
my @ibx;
for my $V (1..2) {
@@ -88,18 +89,19 @@ is($lxs->over, undef, '->over fails');
my $smsg = $lxs->smsg_for($mitem) or BAIL_OUT 'smsg_for broken';
my $ale = PublicInbox::LeiALE::_new("$home/ale");
- $ale->refresh_externals($lxs);
+ my $lei = bless {}, 'PublicInbox::LEI';
+ $ale->refresh_externals($lxs, $lei);
my $exp = [ $smsg->{blob}, 'blob', -s 't/utf8.eml' ];
is_deeply([ $ale->git->check($smsg->{blob}) ], $exp, 'ale->git->check');
$lxs = PublicInbox::LeiXSearch->new;
$lxs->prepare_external($v2ibx);
- $ale->refresh_externals($lxs);
+ $ale->refresh_externals($lxs, $lei);
is_deeply([ $ale->git->check($smsg->{blob}) ], $exp,
'ale->git->check remembered inactive external');
rename("$home/v1tmp", "$home/v1moved") or BAIL_OUT "rename: $!";
- $ale->refresh_externals($lxs);
+ $ale->refresh_externals($lxs, $lei);
is($ale->git->check($smsg->{blob}), undef,
'missing after directory gone');
}
^ permalink raw reply related [relevance 42%]
* [PATCH] lei export-kw: workaround race in updating Maildir locations
@ 2021-08-05 2:33 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-05 2:33 UTC (permalink / raw)
To: meta
Inotify updates may simultaneously remove or update the location
of a message, so ensure we at least have knowledge of the new
location if the old one cannot be updated.
---
lib/PublicInbox/LeiMailSync.pm | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 82740d59..6dfa03be 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -158,7 +158,13 @@ sub mv_src {
my $sth = $self->{dbh}->prepare_cached(<<'');
UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
- $sth->execute($newbn, $fid, $oidbin, $$id);
+ my $nr = $sth->execute($newbn, $fid, $oidbin, $$id);
+ if ($nr == 0) { # may race with a clear_src, ensure new value exists
+ $sth = $self->{dbh}->prepare_cached(<<'');
+INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?)
+
+ $sth->execute($oidbin, $fid, $newbn);
+ }
}
# read-only, iterates every oidbin + UID or name for a given folder
^ permalink raw reply related [relevance 71%]
* Re: [RFC v2] lei: close inotify FD in forked child
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
@ 2021-08-04 10:40 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-04 10:40 UTC (permalink / raw)
To: meta
Pushed with present tense commit message now that L::I2 2.3+ is out:
https://public-inbox.org/meta/7fc6e30aeab9925bece4bb00f88bb91af5646aa2/s/
^ permalink raw reply [relevance 71%]
* [RFC v2] lei: close inotify FD in forked child
2021-07-28 12:34 58% [RFC] lei: address lifetime problems from Linux::Inotify2 Eric Wong
@ 2021-07-29 10:01 62% ` Eric Wong
2021-08-04 10:40 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-07-29 10:01 UTC (permalink / raw)
To: meta
It looks like Linux::Inotify2 2.3+ will include an ->fh method
to give us the ability to safely close an FD without hitting
EBADF (and automatically use FD_CLOEXEC).
We'll still need a new wrapper class (LI2Wrap) to handle it for
users of old versions, though.
http://lists.schmorp.de/pipermail/perl/2021q3/thread.html
---
MANIFEST | 1 +
lib/PublicInbox/DirIdle.pm | 11 +++++++++++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LI2Wrap.pm | 20 ++++++++++++++++++++
4 files changed, 33 insertions(+), 1 deletion(-)
create mode 100644 lib/PublicInbox/LI2Wrap.pm
diff --git a/MANIFEST b/MANIFEST
index a3913501..fb9f16bf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -197,6 +197,7 @@ lib/PublicInbox/InputPipe.pm
lib/PublicInbox/Isearch.pm
lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
+lib/PublicInbox/LI2Wrap.pm
lib/PublicInbox/LeiALE.pm
lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 65896f95..d572c274 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -84,4 +84,15 @@ sub event_step {
warn "$self->{inot}->read err: $@\n" if $@;
}
+sub force_close {
+ my ($self) = @_;
+ my $inot = delete $self->{inot} // return;
+ if ($inot->can('fh')) { # Linux::Inotify2 2.3+
+ close($inot->fh) or warn "CLOSE ERROR: $!";
+ } elsif ($inot->isa('Linux::Inotify2')) {
+ require PublicInbox::LI2Wrap;
+ PublicInbox::LI2Wrap::wrapclose($inot);
+ }
+}
+
1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d9fd40fd..e6f763e1 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -556,7 +556,7 @@ sub _lei_atfork_child {
}
close $listener if $listener;
undef $listener;
- undef $dir_idle;
+ $dir_idle->force_close if $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
diff --git a/lib/PublicInbox/LI2Wrap.pm b/lib/PublicInbox/LI2Wrap.pm
new file mode 100644
index 00000000..b0f4f8b8
--- /dev/null
+++ b/lib/PublicInbox/LI2Wrap.pm
@@ -0,0 +1,20 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Wrapper for Linux::Inotify2 < 2.3 which lacked ->fh and auto-close
+# Remove this when supported LTS/enterprise distros are all
+# Linux::Inotify2 >= 2.3
+package PublicInbox::LI2Wrap;
+use v5.10.1;
+our @ISA = qw(Linux::Inotify2);
+
+sub wrapclose {
+ my ($inot) = @_;
+ my $fd = $inot->fileno;
+ open my $fh, '<&=', $fd or die "open <&= $fd $!";
+
+}
+
+sub DESTROY {} # no-op
+
+1
^ permalink raw reply related [relevance 62%]
* [RFC] lei: address lifetime problems from Linux::Inotify2
@ 2021-07-28 12:34 58% Eric Wong
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-07-28 12:34 UTC (permalink / raw)
To: meta
Linux::Inotify2 doesn't supply an explicit close() wrapper,
and Linux::Inotify2::Event objects maintain a reference to the
parent Linux::Inotify2 object, so we were leaking inotify FDs
into the "lei note-event" worker processes.
This delays the fork()-ing code paths into the next tick of the
event loop to ensure the inotify FD is closed when $dir_idle
is undef'ed in ->_lei_atfork_child.
Note: I'm not sure if I want to keep this, since
The correct fix would be to have a way to close inotify FDs,
but the current Linux::Inotify2 API doesn't make it easy.
And leaking an inotify FD isn't the worst thing in the world,
since it won't hang other processes. So maybe the cure
is worse than the disease, in this case.
---
lib/PublicInbox/LEI.pm | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d9fd40fd..b81f95c9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -28,8 +28,8 @@ use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
our $quit = \&CORE::exit;
-our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
- $recv_cmd, $send_cmd);
+our ($current_lei, $errors_log, $listener, $oldset,
+ $dir_idle, $lne_nxt, $lne_q, $recv_cmd, $send_cmd);
my $GLP = Getopt::Long::Parser->new;
$GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
my $GLP_PASS = Getopt::Long::Parser->new;
@@ -557,6 +557,8 @@ sub _lei_atfork_child {
close $listener if $listener;
undef $listener;
undef $dir_idle;
+ undef $lne_nxt;
+ undef $lne_q;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
@@ -1153,6 +1155,21 @@ sub cfg2lei ($) {
$lei;
}
+sub lne_task { # requeue, runs without dir_idle stuff on stack
+ undef $lne_nxt;
+ my $q = $lne_q // return;
+ $lne_q = undef;
+
+ while (my ($loc, $nc, $bn, $fn, $cfg) = splice(@$q, 0, 5)) {
+ eval {
+ local %ENV = %{$cfg->{-env}};
+ my $lei = cfg2lei($cfg);
+ $lei->dispatch('note-event', $loc, $nc, $bn, $fn);
+ };
+ warn "E note-event $cfg->{-f}: $@\n" if $@;
+ }
+}
+
sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
my ($ev) = @_; # Linux::Inotify2::Event or duck type
my $fn = $ev->fullname;
@@ -1161,14 +1178,9 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
$nc = '' if $ev->IN_DELETE;
for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
my $cfg = $PATH2CFG{$f} // next;
- eval {
- local %ENV = %{$cfg->{-env}};
- my $lei = cfg2lei($cfg);
- $lei->dispatch('note-event',
- "maildir:$mdir", $nc, $bn, $fn);
- };
- warn "E note-event $f: $@\n" if $@;
+ push @$lne_q, "maildir:$mdir", $nc, $bn, $fn, $cfg;
}
+ $lne_nxt //= PublicInbox::DS::requeue(\&lne_task) if $lne_q;
}
if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
$ev->cancel;
@@ -1261,6 +1273,7 @@ sub lazy_start {
undef $sig;
local $SIG{PIPE} = 'IGNORE';
require PublicInbox::DirIdle;
+ local ($lne_q, $lne_nxt);
local $dir_idle = PublicInbox::DirIdle->new([$sock_dir], sub {
# just rely on wakeup to hit PostLoopCallback set below
dir_idle_handler($_[0]) if $_[0]->fullname ne $path;
^ permalink raw reply related [relevance 58%]
* [PATCH 1/2] lei: die on ECONNRESET
@ 2021-07-28 0:37 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-28 0:37 UTC (permalink / raw)
To: meta
ECONNRESET should be rare on a private local socket, and if
we hit it, it's because we're hitting the listen() limit.
---
script/lei | 1 -
1 file changed, 1 deletion(-)
diff --git a/script/lei b/script/lei
index fce8124a..99d94b4e 100755
--- a/script/lei
+++ b/script/lei
@@ -115,7 +115,6 @@ while (1) {
my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
if (scalar(@fds) == 1 && !defined($fds[0])) {
next if $!{EINTR};
- last if $!{ECONNRESET};
die "recvmsg: $!";
}
last if $buf eq '';
^ permalink raw reply related [relevance 71%]
* lei rediff: add --dequote and --requote?
@ 2021-07-25 12:29 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 12:29 UTC (permalink / raw)
To: meta
I find "lei rediff" useful when writing replies to patch emails.
Like any good *nix tool, my $EDITOR allows piping either the
full or partial editor buffer into arbitrary commands.
Besides typing text, the most important feature of my editor is
the ability to run/pipe-to arbitrary commands (e.g. git-am, lei
blob, git-show); and "lei rediff" is a new one.
I barely customize my editor; however, I do have some editor
macros for quickly stripping and adding quote prefixes (at least
the standard "> " mail quoting style, along with '#' for sh/Perl).
But maybe others do not have these macros, so I wonder if
"rediff" should support --dequote and --requote switches to ease
this usage pattern.
Or/and if "lei dequote" and "lei quote" should exist as
(slower) replacements for `sed 's/^[> ]\+//'` and `sed 's/^/> /'`,
respectively.
^ permalink raw reply [relevance 71%]
* [PATCH] doc: lei-{p2q,rediff}: note implicit --stdin
@ 2021-07-25 12:03 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 12:03 UTC (permalink / raw)
To: meta
lei actually uses implicit --stdin everywhere, but I thing
these patch-related commands are the most common use of them.
---
Documentation/lei-p2q.pod | 3 ++-
Documentation/lei-rediff.pod | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-p2q.pod b/Documentation/lei-p2q.pod
index f404ede0..44798ac3 100644
--- a/Documentation/lei-p2q.pod
+++ b/Documentation/lei-p2q.pod
@@ -48,7 +48,8 @@ Default: C<dfpost7>
=item --stdin
-Read patch from stdin.
+Read message from stdin. This is implicit if no arguments are given
+and stdin is a pipe or regular file.
=item --debug
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index 5fdde230..e968fb20 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -24,7 +24,8 @@ supported.
=item --stdin
-Read message from stdin.
+Read message from stdin. This is implicit if no arguments are given
+and stdin is a pipe or regular file.
=item --git-dir=DIR
^ permalink raw reply related [relevance 71%]
* [PATCH] t/lei-watch.t: improve test reliability
@ 2021-07-25 11:15 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 11:15 UTC (permalink / raw)
To: meta
On single CPU (and overloaded SMP) systems, we can't rely on
inotify in lei-daemon firing before a "lei note-event done"
client hits it. So force in a single tick() to ensure the
scheduler can yield to lei-daemon and see the inotify wakeup
before "lei note-event done" to commit the write.
---
t/lei-watch.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 9a3bfd80..86fa6649 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -52,7 +52,7 @@ test_lei(sub {
my @f = glob("$md/cur/*:2,");
is(scalar(@f), 1, 'got populated maildir with one result');
rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een
- $have_fast_inotify or tick(2);
+ tick($have_fast_inotify ? 0.1 : 2.1); # always needed for 1 CPU systems
lei_ok qw(note-event done); # flushes immediately (instead of 5s)
lei_ok qw(q mid:testmessage@example.com -o), $md2, '-I', "$ro_home/t1";
^ permalink raw reply related [relevance 71%]
Results 401-600 of ~1312 next (older) | prev (newer) | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-07-25 11:15 71% [PATCH] t/lei-watch.t: improve test reliability Eric Wong
2021-07-25 12:03 71% [PATCH] doc: lei-{p2q,rediff}: note implicit --stdin Eric Wong
2021-07-25 12:29 71% lei rediff: add --dequote and --requote? Eric Wong
2021-07-28 0:37 [PATCH 0/2] fix "make check-run" reliability Eric Wong
2021-07-28 0:37 71% ` [PATCH 1/2] lei: die on ECONNRESET Eric Wong
2021-07-28 12:34 58% [RFC] lei: address lifetime problems from Linux::Inotify2 Eric Wong
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
2021-08-04 10:40 71% ` Eric Wong
2021-08-05 2:33 71% [PATCH] lei export-kw: workaround race in updating Maildir locations Eric Wong
2021-08-11 11:26 71% [PATCH 0/3] lei pathname canonicalization fixes Eric Wong
2021-08-11 11:26 42% ` [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames Eric Wong
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
2021-08-24 20:14 71% ` [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors Eric Wong
2021-08-17 8:52 71% [PATCH 0/3] lei: some mail sync stuff Eric Wong
2021-08-17 8:52 54% ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
2021-08-17 8:52 88% ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
2021-08-18 11:56 71% "lei q --save" - should it be the default? Eric Wong
2021-08-19 1:36 31% ` [PATCH] lei q: make --save the default Eric Wong
2021-08-19 9:49 54% [PATCH] lei: implicitly watch all Maildirs it knows about Eric Wong
2021-08-24 13:04 71% [PATCH] lei: add missing LeiWatch lazy-load Eric Wong
2021-08-24 13:06 70% [PATCH] lei: non-blocking lei/store->done in lei-daemon Eric Wong
2021-08-25 8:40 71% [PATCH 0/2] minor lei usability tweaks Eric Wong
2021-08-25 8:40 59% ` [PATCH 1/2] lei up: improve --all=local stderr output Eric Wong
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
2021-08-31 11:21 71% ` [PATCH 05/10] t/lei-watch: avoid race between glob + readlink Eric Wong
2021-08-31 11:21 71% ` [PATCH 06/10] lei note-event: always flush changes on daemon exit Eric Wong
2021-08-31 11:21 71% ` [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir Eric Wong
2021-08-31 11:21 94% ` [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon Eric Wong
2021-08-31 11:21 71% ` [PATCH 10/10] lei/store: correctly delete entries from over Eric Wong
2021-08-31 19:38 90% [PATCH] lei up: only show finmsg in top-level lei-daemon Eric Wong
2021-09-02 10:17 71% [PATCH 0/3] lei: auto keyword propagation to Maildirs Eric Wong
2021-09-02 10:17 51% ` [PATCH 3/3] lei: propagate keyword changes from lei/store Eric Wong
2021-09-02 10:25 67% ` [SQUASH 4/3] t/lei-auto-watch: workaround for FreeBSD kevent Eric Wong
2021-09-02 21:12 61% Showcasing lei at Linux Plumbers Konstantin Ryabitsev
2021-09-02 21:58 65% ` Eric Wong
2021-09-03 15:15 71% ` Konstantin Ryabitsev
2021-09-07 21:33 71% ` Konstantin Ryabitsev
2021-09-07 22:14 71% ` Eric Wong
2021-09-08 13:36 65% ` Konstantin Ryabitsev
2021-09-08 14:49 66% ` Eric Wong
2021-09-08 17:17 67% ` Konstantin Ryabitsev
2021-09-08 17:32 71% ` Eric Wong
2021-09-02 22:36 [PATCH 0/2] test reliability fixes Eric Wong
2021-09-02 22:36 68% ` [PATCH 1/2] t/lei-auto-watch: improve test reliability Eric Wong
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
2021-09-03 8:54 71% ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
2021-09-03 8:54 52% ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
2021-09-03 8:54 83% ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
2021-09-03 8:54 58% ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
2021-09-03 8:54 61% ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
2021-09-03 8:54 63% ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
2021-09-07 11:32 71% ` [PATCH 2/4] lei: dump and clear log at exit Eric Wong
2021-09-07 11:32 38% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
2021-09-07 11:32 52% ` [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address Eric Wong
2021-09-07 22:41 71% [PATCH] lei q|up: fix write counter for v2 Eric Wong
2021-09-08 18:48 54% [PATCH] lei-rm: add man page, support LeiInput args Eric Wong
2021-09-08 19:04 71% [PATCH] lei prune-mail-sync: ignore missing locations Eric Wong
2021-09-09 5:34 70% [PATCH] lei up: print messages before disconnecting Eric Wong
2021-09-09 5:47 71% lei refresh-mail-sync [vs. prune-mail-sync] Eric Wong
2021-09-09 15:19 71% Tracking one-off threads with lei Konstantin Ryabitsev
2021-09-09 20:06 71% ` Eric Wong
2021-09-09 20:56 71% ` Konstantin Ryabitsev
2021-09-09 21:06 71% ` Eric Wong
2021-09-09 21:39 64% Using lei with podman + toolbox Konstantin Ryabitsev
2021-09-09 23:36 68% ` Eric Wong
2021-09-09 23:51 71% ` native C++ Xapian wrapper [was: Using lei with podman + toolbox] Eric Wong
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
2021-09-10 13:56 71% ` Eric Wong
2021-09-10 14:48 71% ` Konstantin Ryabitsev
2021-09-10 5:51 51% [PATCH] lei add-external --mirror: deduce paths for PSGI mount prefixes Eric Wong
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
2021-09-10 9:08 65% ` [PATCH 3/4] lei: do not read ~/.netrc by default Eric Wong
2021-09-10 9:08 57% ` [PATCH 4/4] doc: lei-index manpage Eric Wong
2021-09-10 9:15 68% [PATCH] lei add-external --mirror: quiet unlink error on ENOENT Eric Wong
2021-09-10 11:46 71% [PATCH] lei up: only delay non-zero "# $NR written to ..." Eric Wong
2021-09-10 14:11 65% RFC: normalize whitespace in lei queries Konstantin Ryabitsev
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
2021-09-11 0:19 53% ` [PATCH 1/3] lei: fix handling of broken lei.saved-search config files Eric Wong
2021-09-11 0:19 63% ` [PATCH 2/3] lei: pass client stderr to git-config in more places Eric Wong
2021-09-11 0:19 71% ` [PATCH 3/3] lei: normalize whitespace in remote queries Eric Wong
2021-09-11 8:33 71% [PATCH 0/1] lei q -f reply ... perhaps my new favorite feature Eric Wong
2021-09-11 8:33 40% ` [PATCH 1/1] lei q|lcat: support "-f reply" output format Eric Wong
2021-09-12 8:42 [PATCH 0/3] test dependency fixes for minimal installs Eric Wong
2021-09-12 8:42 67% ` [PATCH 1/3] t/lei-*.t: guard setup_public_inboxes with test_lei Eric Wong
2021-09-12 8:42 65% ` [PATCH 2/3] lei sucks: allow it to work without SQLite Eric Wong
2021-09-13 20:53 [PATCH 0/6] a batch of misc fixes Eric Wong
2021-09-13 20:53 67% ` [PATCH 2/6] lei: stop_pager: restore stdout when done Eric Wong
2021-09-13 20:53 71% ` [PATCH 4/6] lei up: localize %ENV in redispatch Eric Wong
2021-09-13 20:53 62% ` [PATCH 6/6] lei up: fix --mua with single output Eric Wong
2021-09-14 2:39 71% [PATCH 0/5] lei: TEST_LEI_DAEMON_PERSIST bugfixes Eric Wong
2021-09-14 2:39 71% ` [PATCH 1/5] lei: warn on event loop errors Eric Wong
2021-09-14 2:39 71% ` [PATCH 2/5] lei: sto_done_request: add eval guard Eric Wong
2021-09-14 2:39 56% ` [PATCH 5/5] lei up: fix env/cwd mismatches with multiple folders Eric Wong
2021-09-14 20:42 71% RFC: lei-daemon and auto-up Konstantin Ryabitsev
2021-09-14 22:32 71% ` Eric Wong
2021-09-15 21:34 52% make menuconfig interface for lei / grok-pull Luis Chamberlain
2021-09-15 21:44 71% ` Eric Wong
2021-09-15 21:57 71% ` Luis Chamberlain
2021-09-15 23:12 71% ` Eric Wong
2021-09-15 22:36 71% ` Kyle Meyer
2021-09-15 23:31 71% ` Eric Wong
2021-09-16 2:59 71% ` Eric Wong
2021-09-15 23:06 57% ` Eric Wong
2021-09-16 17:43 71% ` Konstantin Ryabitsev
2021-09-16 20:04 70% ` IMAP vs NNTP [was: make menuconfig interface for lei / grok-pull] Eric Wong
2021-09-19 21:21 71% ` make menuconfig interface for lei / grok-pull Eric Wong
2021-09-16 17:38 71% ` Konstantin Ryabitsev
2021-09-16 23:54 71% ` Luis Chamberlain
2021-09-16 21:09 90% ` lei import on epochs [was: make menuconfig interface for lei / grok-pull] Eric Wong
2021-09-16 23:53 90% ` Luis Chamberlain
2021-09-17 0:22 89% ` Eric Wong
2021-09-17 0:48 90% ` Luis Chamberlain
2021-09-16 2:19 [PATCH 0/3] IMAP and Tor .onion niceities Eric Wong
2021-09-16 2:19 71% ` [PATCH 2/3] lei ls-mail-source: sort IMAP folder names Eric Wong
2021-09-16 7:45 63% [PATCH] doc: lei-mail-formats: add "eml" and expand on git things Eric Wong
2021-09-16 9:41 71% [PATCH 0/3] lei refresh-mail-sync Eric Wong
2021-09-16 9:41 59% ` [PATCH 1/3] lei: git_oid: replace git_blob_id Eric Wong
2021-09-16 9:41 39% ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-16 11:04 66% [PATCH] lei: sqlite: do not forcibly enable WAL Eric Wong
2021-09-17 2:16 71% ` Eric Wong
2021-09-17 1:56 71% [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
2021-09-17 1:56 36% ` [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices Eric Wong
2021-09-17 1:56 71% ` [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path Eric Wong
2021-09-17 1:56 61% ` [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders Eric Wong
2021-09-17 1:56 56% ` [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info Eric Wong
2021-09-17 12:12 71% [PATCH] script/lei: umask(077) before execve Eric Wong
2021-09-17 12:38 38% [PATCH] doc: add lei-security(7) manpage Eric Wong
2021-09-18 9:33 67% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
2021-09-18 9:33 51% ` [PATCH 1/9] lei: lock worker counts Eric Wong
2021-09-18 9:33 39% ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
2021-09-18 14:31 71% ` Kyle Meyer
2021-09-18 20:30 71% ` Eric Wong
2021-09-18 22:38 66% [PATCH] t/lei-refresh-mail-sync: improve test reliability Eric Wong
2021-09-19 12:50 61% [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
2021-09-19 12:50 34% ` [PATCH 03/16] lei/store: use SOCK_SEQPACKET rather than pipe Eric Wong
2021-09-19 12:50 58% ` [PATCH 04/16] lei: simplify sto_done_request Eric Wong
2021-09-19 12:50 68% ` [PATCH 07/16] lei: clamp internal worker processes to 4 Eric Wong
2021-09-19 12:50 71% ` [PATCH 08/16] lei ls-mail-source: use "high"/"low" for NNTP Eric Wong
2021-09-19 12:50 62% ` [PATCH 09/16] lei ls-mail-source: pretty JSON support Eric Wong
2021-09-19 12:50 36% ` [PATCH 14/16] lei config --edit: use controlling terminal Eric Wong
2021-09-19 12:50 57% ` [PATCH 16/16] doc: lei-config: document various knobs Eric Wong
2021-09-19 16:14 71% ` Kyle Meyer
2021-09-19 20:00 65% ` Eric Wong
2021-09-21 7:41 68% [PATCH 00/12] lei: fix various annoyances Eric Wong
2021-09-21 7:41 64% ` [PATCH 01/12] lei inspect: convert to WQ worker Eric Wong
2021-09-21 7:41 42% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
2021-09-21 7:41 45% ` [PATCH 04/12] lei: simplify internal arg2folder usage Eric Wong
2021-09-21 7:41 61% ` [PATCH 05/12] lei lcat: use single queue for ordering Eric Wong
2021-09-21 7:41 71% ` [PATCH 06/12] doc: lei-security: section for WIP auth methods Eric Wong
2021-09-21 7:41 44% ` [PATCH 07/12] lei lcat: support NNTP URLs Eric Wong
2021-09-21 7:41 39% ` [PATCH 08/12] lei: various completion improvements Eric Wong
2021-09-21 7:41 63% ` [PATCH 09/12] lei q: show progress on >1s preparation phase Eric Wong
2021-09-21 7:41 61% ` [PATCH 11/12] lei q: update messages to reflect --save default Eric Wong
2021-09-21 7:41 39% ` [PATCH 12/12] lei q: improve --limit behavior and progress Eric Wong
2021-09-21 9:29 71% ` [PATCH 0/3] lei: a few more annoyances fixed Eric Wong
2021-09-21 9:29 71% ` [PATCH 1/3] t/lei-up: use '-q' to silence non-redirected test Eric Wong
2021-09-21 9:29 62% ` [PATCH 2/3] script/lei: handle SIGTSTP and SIGCONT Eric Wong
2021-09-21 9:29 71% ` [PATCH 3/3] lei: umask(077) before opening errors.log Eric Wong
2021-09-22 2:24 70% [PATCH 0/7] lei bugfixes and other fixes Eric Wong
2021-09-22 2:24 71% ` [PATCH 3/7] script/lei: describe purpose of sleep loop Eric Wong
2021-09-22 2:24 71% ` [PATCH 4/7] lei: dclose: do not close unnecessarily Eric Wong
2021-09-22 2:24 49% ` [PATCH 6/7] lei up: avoid excessively parallel --all Eric Wong
2021-09-22 2:24 58% ` [PATCH 7/7] lei: drop redundant WQ EOF callbacks Eric Wong
2021-09-23 10:36 71% [PATCH] lei: common --all[=remote|local] help message Eric Wong
2021-09-24 12:51 71% [PATCH 0/4] doc: lei: some manpage updaets Eric Wong
2021-09-24 12:51 53% ` [PATCH 1/4] doc: lei blob+rediff+p2q: add notes about git directory Eric Wong
2021-09-24 12:51 56% ` [PATCH 2/4] doc: lei-overview: implicit stdin, correct Inline::C notes Eric Wong
2021-09-24 12:51 68% ` [PATCH 3/4] doc: lei-index: remove --stdin, reword -F Eric Wong
2021-09-24 12:51 49% ` [PATCH 4/4] doc: lei: manpages for export-kw and refresh-mail-sync Eric Wong
2021-09-25 5:49 71% [PATCH 0/3] lei: some robustness fixes Eric Wong
2021-09-25 5:49 71% ` [PATCH 1/3] lei up: show timezone offset with localtime Eric Wong
2021-09-25 5:49 71% ` [PATCH 2/3] lei: restore old sigmask before daemon exit Eric Wong
2021-09-25 6:17 55% [PATCH] lei: make pkt_op easier-to-use and understand Eric Wong
2021-09-25 7:08 71% [PATCH] doc: lei-rm: remove unnecessary -F values Eric Wong
2021-09-25 8:49 71% [PATCH 0/3] lei *-external: split off into separate files Eric Wong
2021-09-25 8:49 50% ` [PATCH 1/3] lei forget-external: split into separate file Eric Wong
2021-09-25 8:49 48% ` [PATCH 2/3] lei add-external: " Eric Wong
2021-09-25 8:49 58% ` [PATCH 3/3] lei ls-external: " Eric Wong
2021-09-26 0:02 62% [PATCH] lei note-event: ignore kw_changed exceptions Eric Wong
2021-09-26 1:42 71% [PATCH] lei -f reply: fix Cc: header combining Eric Wong
2021-09-26 5:38 71% [PATCH] lei: ensure refresh_watches isn't called from workers Eric Wong
2021-09-27 4:59 71% [PATCH 0/2] lei rediff: --drq and --dequote-only Eric Wong
2021-09-27 4:59 69% ` [PATCH 1/2] lei rediff: quiet warnings from Import and Eml Eric Wong
2021-09-27 4:59 42% ` [PATCH 2/2] lei rediff: add --drq and --dequote-only Eric Wong
2021-09-27 21:05 [PATCH 0/3] fixes for odd/old/missing dependencies Eric Wong
2021-09-27 21:05 63% ` [PATCH 2/3] t/lei-index: IMAP and NNTP dependencies are optional Eric Wong
2021-09-27 21:05 58% ` [PATCH 3/3] lei completion: workaround old Perl bug Eric Wong
2021-09-27 23:53 71% [PATCH] doc: lei-rediff: grammar fixes for --drq and --dequote-only Kyle Meyer
2021-09-28 1:05 71% ` Eric Wong
2021-10-01 9:54 [PATCH 0/9] daemon-related things Eric Wong
2021-10-01 9:54 69% ` [PATCH 1/9] doc: lei-security: some more updates Eric Wong
2021-10-01 9:54 49% ` [PATCH 9/9] doc: lei-daemon: new manpage Eric Wong
2021-10-02 0:19 71% ` Kyle Meyer
2021-10-02 8:08 71% ` Eric Wong
2021-10-02 8:16 [PATCH 0/2] lei inspect stuffs Eric Wong
2021-10-02 8:16 53% ` [PATCH 2/2] lei inspect: fix "mid:" prefix, expand to Xapian Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).