* [PATCH 3/3] lei rediff: do not automatically store patches/mails
2021-05-05 17:49 55% ` [PATCH 2/3] lei rediff: capture and regenerate file modes Eric Wong
@ 2021-05-05 17:49 65% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-05 17:49 UTC (permalink / raw)
To: meta
We can use a temporary lei/store to avoid cluttering up
future search results. This is especially useful since
we expect "lei rediff" to be useful for non-email diffs
and individual attachments, too.
---
lib/PublicInbox/LeiRediff.pm | 9 +++++----
t/solver_git.t | 3 +++
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index d73e3e28..deb6c3ae 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -70,7 +70,6 @@ EOM
my $tb = $ta;
$tb =~ tr!A!B!;
my $lei = $self->{lei};
- my $wait = delete($self->{-do_done}) ? $lei->{sto}->ipc_do('done') : 0;
while (my ($oid_a, $oid_b, $pa, $pb, $ma, $mb) = splice(@$ctxq, 0, 6)) {
my $xa = $blob->{$oid_a} //= solve_1($self, $oid_a,
{ path_b => $pa });
@@ -193,8 +192,8 @@ sub extract_oids { # Eml each_part callback
sub input_eml_cb { # callback for all emails
my ($self, $eml) = @_;
- $self->{lei}->{sto}->ipc_do('add_eml', $eml);
- $self->{-do_done} = 1;
+ $self->{tmp_sto}->add_eml($eml);
+ $self->{tmp_sto}->done;
$eml->each_part(\&extract_oids, $self, 1);
}
@@ -239,7 +238,9 @@ sub ipc_atfork_child {
binmode $lei->{1}, ':utf8';
$self->{blobs} = {}; # oidhex => filename
$self->{rdtmp} = File::Temp->newdir('lei-rediff-XXXX', TMPDIR => 1);
- $self->{rmt} = [ map {
+ $self->{tmp_sto} = PublicInbox::LeiStore->new(
+ "$self->{rdtmp}/tmp.store", { creat => 1 });
+ $self->{rmt} = [ $self->{tmp_sto}->search, map {
PublicInbox::LeiRemote->new($lei, $_)
} $self->{lxs}->remotes ];
$self->{gits} = [ map {
diff --git a/t/solver_git.t b/t/solver_git.t
index ad0c295b..13e478b3 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -86,6 +86,9 @@ index 15ac20eb..771486c4
EOM
ok(index($lei_out, $exp) >= 0,
'preserve mode, regen header + context from -U0 patch');
+ my $e = { GIT_DIR => "$ENV{HOME}/.local/share/lei/store/ALL.git" };
+ my @x = xqx([qw(git cat-file --batch-all-objects --batch-check)], $e);
+ is_deeply(\@x, [], 'no objects stored') or diag explain(\@x);
});
test_lei({tmpdir => "$tmpdir/index-eml-only"}, sub {
^ permalink raw reply related [relevance 65%]
* [PATCH 2/3] lei rediff: capture and regenerate file modes
@ 2021-05-05 17:49 55% ` Eric Wong
2021-05-05 17:49 65% ` [PATCH 3/3] lei rediff: do not automatically store patches/mails Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-05 17:49 UTC (permalink / raw)
To: meta
Don't lose file mode information when regenerating a diff.
---
MANIFEST | 1 +
lib/PublicInbox/LeiRediff.pm | 19 +++++++++++++++----
t/solve/bare.patch | 8 ++++++++
t/solver_git.t | 13 +++++++++++++
4 files changed, 37 insertions(+), 4 deletions(-)
create mode 100644 t/solve/bare.patch
diff --git a/MANIFEST b/MANIFEST
index 7be07aa5..a6d94c77 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -480,6 +480,7 @@ t/shared_kv.t
t/sigfd.t
t/solve/0001-simple-mod.patch
t/solve/0002-rename-with-modifications.patch
+t/solve/bare.patch
t/solver_git.t
t/spamcheck_spamc.t
t/spawn.t
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 6c734bef..d73e3e28 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -16,6 +16,8 @@ use PublicInbox::Import;
use PublicInbox::LEI;
use PublicInbox::SolverGit;
+my $MODE = '(100644|120000|100755|160000)';
+
sub rediff_user_cb { # called by solver when done
my ($res, $self) = @_;
my $lei = $self->{lei};
@@ -69,7 +71,7 @@ EOM
$tb =~ tr!A!B!;
my $lei = $self->{lei};
my $wait = delete($self->{-do_done}) ? $lei->{sto}->ipc_do('done') : 0;
- while (my ($oid_a, $oid_b, $pa, $pb) = splice(@$ctxq, 0, 4)) {
+ while (my ($oid_a, $oid_b, $pa, $pb, $ma, $mb) = splice(@$ctxq, 0, 6)) {
my $xa = $blob->{$oid_a} //= solve_1($self, $oid_a,
{ path_b => $pa });
my $xb = $blob->{$oid_b} //= solve_1($self, $oid_b, {
@@ -77,8 +79,8 @@ EOM
path_a => $pa,
path_b => $pb
});
- $ta .= "M 100644 $xa ".git_quote($pa)."\n" if $xa;
- $tb .= "M 100644 $xb ".git_quote($pb)."\n" if $xb;
+ $ta .= "M $ma $xa ".git_quote($pa)."\n" if $xa;
+ $tb .= "M $mb $xb ".git_quote($pb)."\n" if $xb;
}
my $rw = $self->{gits}->[-1]; # has all known alternates
if (!$rw->{-tmp}) {
@@ -148,6 +150,15 @@ sub extract_oids { # Eml each_part callback
if (scalar(@top) >= 4 &&
$top[1] =~ $PublicInbox::ViewDiff::IS_OID &&
$top[0] =~ $PublicInbox::ViewDiff::IS_OID) {
+ my ($ma, $mb);
+ $x =~ /^old mode $MODE/sm and $ma = $1;
+ $x =~ /^new mode $MODE/sm and $mb = $1;
+ if (!defined($ma) && $x =~
+ /^index [a-z0-9]+\.\.[a-z0-9]+ $MODE/sm) {
+ $ma = $mb = $1;
+ }
+ $ma //= '100644';
+ $mb //= $ma;
my ($oid_a, $oid_b, $pa, $pb) = splice(@top, 0, 4);
$pa eq '/dev/null' or
$pa = (split(m'/', git_unquote($pa), 2))[1];
@@ -155,7 +166,7 @@ sub extract_oids { # Eml each_part callback
$pb = (split(m'/', git_unquote($pb), 2))[1];
$blobs->{$oid_a} //= undef;
$blobs->{$oid_b} //= undef;
- push @$ctxq, $oid_a, $oid_b, $pa, $pb;
+ push @$ctxq, $oid_a, $oid_b, $pa, $pb, $ma, $mb;
} elsif ($ctxq) {
my @out;
for (split(/^/sm, $x)) {
diff --git a/t/solve/bare.patch b/t/solve/bare.patch
new file mode 100644
index 00000000..25d7f344
--- /dev/null
+++ b/t/solve/bare.patch
@@ -0,0 +1,8 @@
+diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
+old mode 100644
+new mode 100755
+index 15ac20eb..771486c4
+--- a/script/public-inbox-extindex
++++ b/script/public-inbox-extindex
+@@ -4 +3,0 @@
+-# Basic tool to create a Xapian search index for a public-inbox.
diff --git a/t/solver_git.t b/t/solver_git.t
index 44cbbfdb..ad0c295b 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -73,6 +73,19 @@ test_lei({tmpdir => "$tmpdir/rediff"}, sub {
lei_ok(qw(rediff -q -U9 t/solve/0001-simple-mod.patch));
like($lei_out, qr!^\Q+++\E b/TODO\n@@ -103,9 \+103,11 @@!sm,
'got more context with -U9');
+ lei_ok(qw(rediff -q -U9 t/solve/bare.patch));
+ my $exp = <<'EOM';
+diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
+old mode 100644
+new mode 100755
+index 15ac20eb..771486c4
+--- a/script/public-inbox-extindex
++++ b/script/public-inbox-extindex
+@@ -1,13 +1,12 @@
+ #!perl -w
+EOM
+ ok(index($lei_out, $exp) >= 0,
+ 'preserve mode, regen header + context from -U0 patch');
});
test_lei({tmpdir => "$tmpdir/index-eml-only"}, sub {
^ permalink raw reply related [relevance 55%]
* lei timestamp resolution for mail synchronization
@ 2021-05-07 0:13 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-07 0:13 UTC (permalink / raw)
To: meta
Is millisecond resolution "good enough" for mail synchronization?
Or maybe even seconds is sufficient...
When syncing mail across machines/devices/reboots, using the
system clock seems required (anybody syncing mail also uses
NTP, right?).
While Time::HiRes allows access to .tv_nsec for some APIs;
Time::HiRes::stat converts it to a double-precision float
(NV/Numeric Value in perlguts-speak). That's not enough for
microsecond accuracy, even; and of course I don't think most
kernels or HW is actually accurate down to the nanosecond level.
^ permalink raw reply [relevance 71%]
* [PATCH] lei rediff: mark --color-moved value as optional
@ 2021-05-16 2:41 71% Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-16 2:41 UTC (permalink / raw)
To: meta
`git-diff --color-moved' can be called without a mode, in which case
it defaults to "zebra".
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5f178418..3e17a614 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -141,7 +141,7 @@ my @lxs_opt = (qw(remote! local! external! include|I=s@ exclude=s@ only=s@
our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
output-indicator-context=s indent-heuristic!
minimal patience histogram anchored=s@ diff-algorithm=s
- color-moved=s color-moved-ws=s no-color-moved no-color-moved-ws
+ color-moved:s color-moved-ws=s no-color-moved no-color-moved-ws
word-diff:s word-diff-regex=s color-words:s no-renames
rename-empty! check ws-error-highlight=s full-index binary
abbrev:i break-rewrites|B:s find-renames|M:s find-copies:s
base-commit: 79e274b293a71fb41dd8bf6e9598a8e8a24fed4f
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH] lei rediff: handle stdin like other commands
@ 2021-05-16 2:42 71% Kyle Meyer
2021-05-16 15:23 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-05-16 2:42 UTC (permalink / raw)
To: meta
`lei rediff' reads from stdin when no argument is specified, but this
is likely unintentional given that other lei commands instead have a
--stdin|- option and that `lei rediff --help' includes --stdin.
Align rediff's handling with the other commands'.
---
lib/PublicInbox/LEI.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5f178418..2144e390 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -180,8 +180,9 @@ our %CMD = ( # sorted in order of importance/use:
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@lxs_opt, @c_opt ],
-'rediff' => [ '[--stdin|LOCATION...]',
+'rediff' => [ '--stdin|LOCATION...',
'regenerate a diff with different options',
+ 'stdin|', # /|\z/ must be first for lone dash
qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
@diff_opt, @lxs_opt, @c_opt ],
base-commit: 79e274b293a71fb41dd8bf6e9598a8e8a24fed4f
--
2.31.1
^ permalink raw reply related [relevance 71%]
* Re: [PATCH] lei rediff: handle stdin like other commands
2021-05-16 2:42 71% [PATCH] lei rediff: handle stdin like other commands Kyle Meyer
@ 2021-05-16 15:23 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-16 15:23 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed your 3 patches as:
8cc23ac6 lei rediff: handle stdin like other commands
2c1cbdc7 lei rediff: mark --color-moved value as optional
1110da3e lei_input: drop misplaced word from error message
^ permalink raw reply [relevance 71%]
* [PATCH 0/9] doc: lei manpages, round 5
@ 2021-05-17 3:35 61% Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
` (7 more replies)
0 siblings, 8 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
This series updates the lei manpages, continuing from
<20210329031117.28516-1-kyle@kyleam.com>. It covers changes up to the
current tip of master (236831da32b1240d..8cc23ac6f7a38479).
The second patch touches outside of lei manpages as a followup to
e226f18934eb7291 (doc: lei q: split =item aliases onto separate lines,
2021-04-28).
[1/9] doc lei blob: avoid combined description of separate options
[2/9] doc: split option variants into separate items
[3/9] doc lei blob: point to lei-q for shared options
[4/9] doc lei: resort lei-tag entries
[5/9] doc lei q: fix a typo
[6/9] doc lei q: add missing value for --lock
[7/9] doc lei: add manpage for convert
[8/9] doc lei: add manpages for new commands
[9/9] doc lei: update manpages with new options
Documentation/lei-add-external.pod | 12 ++-
Documentation/lei-blob.pod | 62 +++++++--------
.../{lei-import.pod => lei-convert.pod} | 40 ++++++----
Documentation/lei-edit-search.pod | 28 +++++++
Documentation/lei-forget-external.pod | 4 +-
Documentation/lei-forget-search.pod | 28 +++++++
Documentation/lei-import.pod | 20 ++++-
Documentation/lei-init.pod | 4 +-
Documentation/lei-lcat.pod | 79 +++++++++++++++++++
Documentation/lei-ls-external.pod | 8 +-
Documentation/lei-ls-label.pod | 8 +-
...i-ls-external.pod => lei-ls-mail-sync.pod} | 22 +++---
Documentation/lei-ls-search.pod | 65 +++++++++++++++
Documentation/lei-overview.pod | 12 +++
Documentation/lei-p2q.pod | 8 +-
Documentation/lei-q.pod | 16 +++-
Documentation/lei-rediff.pod | 79 +++++++++++++++++++
Documentation/lei-tag.pod | 8 +-
Documentation/lei-up.pod | 48 +++++++++++
Documentation/lei.pod | 18 +++++
Documentation/public-inbox-compact.pod | 6 +-
Documentation/public-inbox-convert.pod | 4 +-
Documentation/public-inbox-daemon.pod | 16 +++-
Documentation/public-inbox-extindex.pod | 4 +-
Documentation/public-inbox-imapd.pod | 4 +-
Documentation/public-inbox-index.pod | 12 ++-
Documentation/public-inbox-init.pod | 20 +++--
Documentation/public-inbox-nntpd.pod | 4 +-
Documentation/public-inbox-xcpdb.pod | 14 +++-
Documentation/txt2pre | 11 ++-
MANIFEST | 8 ++
Makefile.PL | 8 +-
32 files changed, 580 insertions(+), 100 deletions(-)
copy Documentation/{lei-import.pod => lei-convert.pod} (54%)
create mode 100644 Documentation/lei-edit-search.pod
create mode 100644 Documentation/lei-forget-search.pod
create mode 100644 Documentation/lei-lcat.pod
copy Documentation/{lei-ls-external.pod => lei-ls-mail-sync.pod} (64%)
create mode 100644 Documentation/lei-ls-search.pod
create mode 100644 Documentation/lei-rediff.pod
create mode 100644 Documentation/lei-up.pod
base-commit: 8cc23ac6f7a3847977ec57c2a3e9e391fdb94da6
--
2.31.1
^ permalink raw reply [relevance 61%]
* [PATCH 1/9] doc lei blob: avoid combined description of separate options
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
@ 2021-05-17 3:35 68% ` Kyle Meyer
2021-05-17 3:35 82% ` [PATCH 3/9] doc lei blob: point to lei-q for shared options Kyle Meyer
` (6 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
The next commit will update the manpages to split each option's
variants into separate items. This change won't mix well with
--oid-a, --path-a, and --path-b. These different options all share a
single description, and, if each form is on its own line, the link
between the variants of each option would no longer be clear.
Use a dedicated description for each option to avoid confusion.
---
Documentation/lei-blob.pod | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index bb316e71..341b5505 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -33,12 +33,15 @@ C<--oid-a>, C<--path-a>, and C<--path-b>.
=item -A OID-A, --oid-a=OID-A
+Provide pre-image object ID as a hint for reconstructing C<OID>.
+
=item -a PATH-A, --path-a=PATH-A
+Provide pre-image pathname as a hint for reconstructing C<OID>.
+
=item -b PATH-B, --path-b=PATH-B
-Provide pre-image object ID, pre-image pathname, or post-image
-pathname as a hint for reconstructing C<OID>.
+Provide post-image pathname as a hint for reconstructing C<OID>.
=for comment
TODO: The below options are shared with lei-q. Any good approaches to
--
2.31.1
^ permalink raw reply related [relevance 68%]
* [PATCH 4/9] doc lei: resort lei-tag entries
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
2021-05-17 3:35 82% ` [PATCH 3/9] doc lei blob: point to lei-q for shared options Kyle Meyer
@ 2021-05-17 3:35 90% ` Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 5/9] doc lei q: fix a typo Kyle Meyer
` (4 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
The command was renamed in 54da988cfb049ea2 (lei tag: rename from "lei
mark", 2021-03-30). Relocate its entries in txt2pre and Makefile.PL
to restore alphabetical sorting.
---
Documentation/txt2pre | 2 +-
Makefile.PL | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 7b9d7853..25d81cf9 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -21,10 +21,10 @@ for (qw[lei(1)
lei-init(1)
lei-ls-external(1)
lei-ls-label(1)
- lei-tag(1)
lei-overview(7)
lei-p2q(1)
lei-q(1)
+ lei-tag(1)
public-inbox.cgi(1)
public-inbox-compact(1)
public-inbox-config(5)
diff --git a/Makefile.PL b/Makefile.PL
index 401c033e..b06b6ab8 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -47,7 +47,7 @@ $v->{-m1} = [ map {
qw(
lei-add-external lei-blob lei-config lei-daemon-kill lei-daemon-pid
lei-forget-external lei-import lei-init lei-ls-external lei-ls-label
- lei-tag lei-p2q lei-q)];
+ lei-p2q lei-q lei-tag)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
--
2.31.1
^ permalink raw reply related [relevance 90%]
* [PATCH 3/9] doc lei blob: point to lei-q for shared options
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
@ 2021-05-17 3:35 82% ` Kyle Meyer
2021-05-17 3:35 90% ` [PATCH 4/9] doc lei: resort lei-tag entries Kyle Meyer
` (5 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
lei-blob shares a good number of options with lei-q. Refer to lei-q's
manpage rather than repeating the descriptions.
Also, add lei-q to lei-blob's "see also" section.
Link: https://public-inbox.org/meta/YGFImEcWX1mCJJwv@dcvr/
---
Documentation/lei-blob.pod | 47 ++++++++++----------------------------
1 file changed, 12 insertions(+), 35 deletions(-)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index 76918324..36c75d53 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -49,60 +49,37 @@ Provide pre-image pathname as a hint for reconstructing C<OID>.
Provide post-image pathname as a hint for reconstructing C<OID>.
-=for comment
-TODO: The below options are shared with lei-q. Any good approaches to
-not repeating the text?
+=item -v
-=item --[no-]remote
+=item --verbose
-Whether to include results requiring network access. When local
-externals are configured, C<--remote> must be explicitly passed to
-enable reporting of results from remote externals.
+Provide more feedback on stderr.
-=item --no-local
+=back
-Limit operations to those requiring network access.
+The following options are also supported and are described in
+L<lei-q(1)>.
-=item --no-external
+=over
-Don't include results from externals.
+=item --[no-]remote
-=item -I LOCATION
+=item --no-local
-=item --include=LOCATION
+=item --no-external
-Include specified external in search. This option may be given
-multiple times.
+=item -I LOCATION, --include=LOCATION
=item --exclude=LOCATION
-Exclude specified external from search. This option may be given
-multiple times.
-
=item --only=LOCATION
-Use only the specified external for search. This option may be given
-multiple times, in which case the search uses only the specified set.
-
=item --no-import-remote
-Disable the default behavior of memoizing remote messages into the
-local store.
-
-=item -v
-
-=item --verbose
-
-Provide more feedback on stderr.
-
=item --torsocks=auto|no|yes
=item --no-torsocks
-Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
-
-Default: C<auto>
-
=back
=head1 CONTACT
@@ -121,4 +98,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-add-external(1)>
+L<lei-add-external(1)>, L<lei-q(1)>
--
2.31.1
^ permalink raw reply related [relevance 82%]
* [PATCH 5/9] doc lei q: fix a typo
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (2 preceding siblings ...)
2021-05-17 3:35 90% ` [PATCH 4/9] doc lei: resort lei-tag entries Kyle Meyer
@ 2021-05-17 3:35 71% ` Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 6/9] doc lei q: add missing value for --lock Kyle Meyer
` (3 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-q.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 46884aa5..e1c20a0d 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -99,7 +99,7 @@ Augment output destination instead of clobbering it.
=item --no-import-before
-Do not importing keywords before writing to an existing output
+Do not import keywords before writing to an existing output
destination.
=item --threads
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 6/9] doc lei q: add missing value for --lock
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (3 preceding siblings ...)
2021-05-17 3:35 71% ` [PATCH 5/9] doc lei q: fix a typo Kyle Meyer
@ 2021-05-17 3:35 71% ` Kyle Meyer
2021-05-17 3:35 54% ` [PATCH 7/9] doc lei: add manpage for convert Kyle Meyer
` (2 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-q.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index e1c20a0d..39199849 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -171,7 +171,7 @@ option applies to C<--include>, C<--exclude>, and C<--only>.
Disable the default behavior of memoizing remote messages into the
local store.
-=item --lock
+=item --lock=METHOD
L<mbox(5)> locking method(s) to use: C<dotlock>, C<fcntl>, C<flock> or
C<none>.
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 7/9] doc lei: add manpage for convert
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (4 preceding siblings ...)
2021-05-17 3:35 71% ` [PATCH 6/9] doc lei q: add missing value for --lock Kyle Meyer
@ 2021-05-17 3:35 54% ` Kyle Meyer
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
2021-05-17 3:35 66% ` [PATCH 9/9] doc lei: update manpages with new options Kyle Meyer
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
Link: https://public-inbox.org/meta/20210429015738.GA30172@dcvr/
---
Documentation/lei-convert.pod | 73 +++++++++++++++++++++++++++++++++++
Documentation/lei.pod | 2 +
Documentation/txt2pre | 1 +
MANIFEST | 1 +
Makefile.PL | 6 +--
5 files changed, 80 insertions(+), 3 deletions(-)
create mode 100644 Documentation/lei-convert.pod
diff --git a/Documentation/lei-convert.pod b/Documentation/lei-convert.pod
new file mode 100644
index 00000000..e8a71393
--- /dev/null
+++ b/Documentation/lei-convert.pod
@@ -0,0 +1,73 @@
+=head1 NAME
+
+lei-convert - one-time conversion from URL or filesystem to another format
+
+=head1 SYNOPSIS
+
+lei convert [OPTIONS] LOCATION
+
+lei convert [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+Convert messages to another format. C<LOCATION> is a source of
+messages: a directory (Maildir), a file, or a URL (C<imap://>,
+C<imaps://>, C<nntp://>, or C<nntps://>). URLs requiring
+authentication must use L<netrc(5)> and/or L<git-credential(1)> to
+fill in the username and password.
+
+For a regular file, the location must have a C<E<lt>formatE<gt>:>
+prefix specifying one of the following formats: C<eml>, C<mboxrd>,
+C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+
+=head1 OPTIONS
+
+=over
+
+=item -F MAIL_FORMAT
+
+=item --in-format=MAIL_FORMAT
+
+Message input format. Unless messages are given on stdin, using a
+format prefix with C<LOCATION> is preferred.
+
+=back
+
+The following options are also supported and are described in
+L<lei-q(1)>.
+
+=over
+
+=item -o MFOLDER, --output=MFOLDER
+
+=item --lock METHOD
+
+=item --no-kw
+
+=item --no-import-remote
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+=item --proxy=PROTO://HOST[:PORT]
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-add-external(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 58646577..78e1fb1e 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -84,6 +84,8 @@ Other subcommands include
=item * L<lei-config(1)>
+=item * L<lei-convert(1)>
+
=item * L<lei-daemon-kill(1)>
=item * L<lei-daemon-pid(1)>
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 25d81cf9..201566ee 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -14,6 +14,7 @@ for (qw[lei(1)
lei-add-external(1)
lei-blob(1)
lei-config(1)
+ lei-convert(1)
lei-daemon-kill(1)
lei-daemon-pid(1)
lei-forget-external(1)
diff --git a/MANIFEST b/MANIFEST
index a6d94c77..29e62880 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -24,6 +24,7 @@ Documentation/include.mk
Documentation/lei-add-external.pod
Documentation/lei-blob.pod
Documentation/lei-config.pod
+Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
Documentation/lei-forget-external.pod
diff --git a/Makefile.PL b/Makefile.PL
index b06b6ab8..d149a164 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -45,9 +45,9 @@ $v->{-m1} = [ map {
}
} @EXE_FILES,
qw(
- lei-add-external lei-blob lei-config lei-daemon-kill lei-daemon-pid
- lei-forget-external lei-import lei-init lei-ls-external lei-ls-label
- lei-p2q lei-q lei-tag)];
+ lei-add-external lei-blob lei-config lei-convert lei-daemon-kill
+ lei-daemon-pid lei-forget-external lei-import lei-init lei-ls-external
+ lei-ls-label lei-p2q lei-q lei-tag)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
--
2.31.1
^ permalink raw reply related [relevance 54%]
* [PATCH 8/9] doc lei: add manpages for new commands
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (5 preceding siblings ...)
2021-05-17 3:35 54% ` [PATCH 7/9] doc lei: add manpage for convert Kyle Meyer
@ 2021-05-17 3:35 27% ` Kyle Meyer
2021-05-17 8:09 71% ` Eric Wong
2021-05-17 3:35 66% ` [PATCH 9/9] doc lei: update manpages with new options Kyle Meyer
7 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-edit-search.pod | 28 ++++++++++
Documentation/lei-forget-search.pod | 28 ++++++++++
Documentation/lei-lcat.pod | 79 +++++++++++++++++++++++++++++
Documentation/lei-ls-mail-sync.pod | 55 ++++++++++++++++++++
Documentation/lei-ls-search.pod | 65 ++++++++++++++++++++++++
Documentation/lei-overview.pod | 12 +++++
Documentation/lei-q.pod | 2 +-
Documentation/lei-rediff.pod | 79 +++++++++++++++++++++++++++++
Documentation/lei-up.pod | 48 ++++++++++++++++++
Documentation/lei.pod | 16 ++++++
Documentation/txt2pre | 8 +++
MANIFEST | 7 +++
Makefile.PL | 8 +--
13 files changed, 431 insertions(+), 4 deletions(-)
create mode 100644 Documentation/lei-edit-search.pod
create mode 100644 Documentation/lei-forget-search.pod
create mode 100644 Documentation/lei-lcat.pod
create mode 100644 Documentation/lei-ls-mail-sync.pod
create mode 100644 Documentation/lei-ls-search.pod
create mode 100644 Documentation/lei-rediff.pod
create mode 100644 Documentation/lei-up.pod
diff --git a/Documentation/lei-edit-search.pod b/Documentation/lei-edit-search.pod
new file mode 100644
index 00000000..7908b5a2
--- /dev/null
+++ b/Documentation/lei-edit-search.pod
@@ -0,0 +1,28 @@
+=head1 NAME
+
+lei-edit-search - edit saved search
+
+=head1 SYNOPSIS
+
+lei edit-search [OPTIONS] OUTPUT
+
+=head1 DESCRIPTION
+
+Invoke C<git config --edit> to edit the saved search at C<OUTPUT>.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-ls-search(1)>, L<lei-forget-search(1)>, L<lei-up(1)>, L<lei-q(1)>
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
new file mode 100644
index 00000000..49bc1d68
--- /dev/null
+++ b/Documentation/lei-forget-search.pod
@@ -0,0 +1,28 @@
+=head1 NAME
+
+lei-forget-search - forget saved search
+
+=head1 SYNOPSIS
+
+lei forget-search [OPTIONS] OUTPUT
+
+=head1 DESCRIPTION
+
+Forget a saved search at C<OUTPUT>.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-edit-search(1)>, L<lei-ls-search(1)>, L<lei-up(1)>, L<lei-q(1)>
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
new file mode 100644
index 00000000..5a2bdb5a
--- /dev/null
+++ b/Documentation/lei-lcat.pod
@@ -0,0 +1,79 @@
+=head1 NAME
+
+lei-lcat - display local copy of messages(s)
+
+=head1 SYNOPSIS
+
+lei lcat [OPTIONS] MSGID_OR_URL [MSGID_OR_URL...]
+
+lei lcat [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+lcat (local cat) is a wrapper around L<lei-q(1)> that displays local
+messages by Message-ID. It is able to extract Message-IDs from URLs
+as well as from common formats such as C<E<lt>$MSGIDE<gt>> and
+C<id:$MSGID>. When reading from stdin, input that isn't understood is
+discarded, so the caller doesn't have to bother extracting the
+Message-ID or link from surrounding text (e.g., a "Link: $URL" line).
+
+=head1 OPTIONS
+
+The following options, described in L<lei-q(1)>, are supported.
+
+=over
+
+=item --[no-]remote
+
+=item --no-local
+
+=item --no-external
+
+=item --no-import-remote
+
+=item --torsocks=auto|no|yes, --no-torsocks
+
+=item --proxy=PROTO://HOST[:PORT]
+
+=item -o MFOLDER, --output=MFOLDER
+
+=item -d STRATEGY, --dedupe=STRATEGY
+
+=item -t, --threads
+
+=item -s KEY, --sort=KEY
+
+=item -r, --reverse
+
+=item --offset=NUMBER
+
+=item -g, --globoff
+
+=item -a, --augment
+
+=item --lock=METHOD
+
+=item --alert=COMMAND
+
+=item --mua=COMMAND
+
+=item --no-color
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-blob(1)>
diff --git a/Documentation/lei-ls-mail-sync.pod b/Documentation/lei-ls-mail-sync.pod
new file mode 100644
index 00000000..37aa910f
--- /dev/null
+++ b/Documentation/lei-ls-mail-sync.pod
@@ -0,0 +1,55 @@
+=head1 NAME
+
+lei-ls-mail-sync - list mail sync folders
+
+=head1 SYNOPSIS
+
+lei mail-sync [OPTIONS] [FILTER]
+
+=head1 DESCRIPTION
+
+List mail sync folders. If C<FILTER> is given, restrict the output to
+matching entries.
+
+=head1 OPTIONS
+
+=over
+
+=item -g
+
+=item --globoff
+
+Do not match C<FILTER> using C<*?> wildcards and C<[]> ranges.
+
+=item --local
+
+Limit operations to the local filesystem.
+
+=item --remote
+
+Limit operations to those requiring network access.
+
+=item -z
+
+=item -0
+
+Use C<\0> (NUL) instead of newline (CR) to delimit lines.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-up(1)>
diff --git a/Documentation/lei-ls-search.pod b/Documentation/lei-ls-search.pod
new file mode 100644
index 00000000..138dbbff
--- /dev/null
+++ b/Documentation/lei-ls-search.pod
@@ -0,0 +1,65 @@
+=head1 NAME
+
+lei-ls-search - list saved search queries
+
+=head1 SYNOPSIS
+
+lei ls-search [OPTIONS] [PREFIX]
+
+=head1 DESCRIPTION
+
+List saved search queries. If C<PREFIX> is given, restrict the output
+to entries that start with the specified value.
+
+=head1 OPTIONS
+
+=over
+
+=item -f FORMAT
+
+=item --format=FORMAT
+
+Display JSON output rather than default short output that includes
+only the saved search location. Possible values are C<json>,
+C<jsonl>, or C<concatjson>.
+
+=item --pretty
+
+Pretty print C<json> or C<concatjson> output. If stdout is opened to
+a tty and used as the C<--output> destination, C<--pretty> is enabled
+by default.
+
+=item -l
+
+Long listing format (shortcut for C<--format=json>).
+
+=item --ascii
+
+Escape non-ASCII characters.
+
+=item -z
+
+=item -0
+
+Use C<\0> (NUL) instead of newline (CR) to delimit lines. This option
+is incompatible with C<--format>.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-up(1)>, L<lei-edit-search(1)>,
+L<lei-forget-search(1)>
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index 4a34bc16..e80cb094 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -98,6 +98,18 @@ Search for messages that have post-image git blob IDs that match those
of the current repository's HEAD commit, writing them to the Maildir
directory "mdir" and flagging the messages that were an exact match.
+=item $ git show -s HEAD | lei lcat -
+
+Display a local message for the public-inbox link contained in a
+commit message.
+
+=item $ lei q -f text m:MID | lei rediff -U5 -
+
+Feed a message containing a diff to L<lei-rediff(1)> to regenerate its
+diff with five context lines. Unless C<--git-dir> is specified, this
+requires the current working directory to be within the associated
+code repository.
+
=back
=head1 PERFORMANCE NOTES
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 39199849..c4a7eea3 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -246,5 +246,5 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-add-external(1)>,
+L<lei-add-external(1)>, L<lei-lcat(1)>, L<lei-up(1)>,
L<Xapian::QueryParser Syntax|https://xapian.org/docs/queryparser.html>
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
new file mode 100644
index 00000000..5fdde230
--- /dev/null
+++ b/Documentation/lei-rediff.pod
@@ -0,0 +1,79 @@
+=head1 NAME
+
+lei-rediff - regenerate a diff with different options
+
+=head1 SYNOPSIS
+
+lei rediff [OPTIONS] LOCATION [LOCATION...]
+
+lei rediff [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+Read a message from C<LOCATION> or stdin and regenerate its diff with
+the specified L<git-diff(1)> options. This is useful if you want to
+change the display of the original patch (e.g., increasing context,
+coloring moved lines differently, or using an external diff viewer).
+
+=head1 OPTIONS
+
+In addition to many L<git-diff(1)>, the following options are
+supported.
+
+=over
+
+=item --stdin
+
+Read message from stdin.
+
+=item --git-dir=DIR
+
+Specify an additional .git/ directory to scan. This option may be
+given multiple times.
+
+=item --no-cwd
+
+Do not look in the git repository of the current working directory.
+
+=item -v
+
+=item --verbose
+
+Provide more feedback on stderr.
+
+=back
+
+The options below, described in L<lei-q(1)>, are also supported.
+
+=over
+
+=item --[no-]remote
+
+=item --no-local
+
+=item --no-external
+
+=item --no-import-remote
+
+=item --torsocks=auto|no|yes, --no-torsocks
+
+=item --proxy=PROTO://HOST[:PORT]
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-blob(1)>, L<lei-p2q(1)>
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
new file mode 100644
index 00000000..cea0f619
--- /dev/null
+++ b/Documentation/lei-up.pod
@@ -0,0 +1,48 @@
+=head1 NAME
+
+lei-up - update a saved search
+
+=head1 SYNOPSIS
+
+lei up [OPTIONS] OUTPUT
+
+lei up [OPTIONS] --all=TYPE
+
+=head1 DESCRIPTION
+
+Update the saved search at C<OUTPUT> or all saved searches of C<TYPE>
+(currently C<local> is the only supported value).
+
+=head1 OPTIONS
+
+The following options, described in L<lei-q(1)>, are supported.
+
+=over
+
+=item --lock=METHOD
+
+=item --alert=CMD
+
+=item --mua=CMD
+
+This option is incompatible with C<--all>.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-ls-search(1)>, L<lei-edit-search(1)>,
+L<lei-forget-search(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 78e1fb1e..3af9e2ee 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -72,10 +72,24 @@ store and configured externals are
=item * L<lei-blob(1)>
+=item * L<lei-config(1)>
+
+=item * L<lei-edit-search(1)>
+
+=item * L<lei-forget-search(1)>
+
+=item * L<lei-lcat(1)>
+
+=item * L<lei-ls-search(1)>
+
=item * L<lei-p2q(1)>
=item * L<lei-q(1)>
+=item * L<lei-rediff(1)>
+
+=item * L<lei-up(1)>
+
=back
Other subcommands include
@@ -92,6 +106,8 @@ Other subcommands include
=item * L<lei-ls-label(1)>
+=item * L<lei-ls-mail-sync(1)>
+
=back
=head1 FILES
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 201566ee..d98dfdd3 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -17,15 +17,22 @@ for (qw[lei(1)
lei-convert(1)
lei-daemon-kill(1)
lei-daemon-pid(1)
+ lei-edit-search(1)
lei-forget-external(1)
+ lei-forget-search(1)
lei-import(1)
lei-init(1)
+ lei-lcat(1)
lei-ls-external(1)
lei-ls-label(1)
+ lei-ls-mail-sync(1)
+ lei-ls-search(1)
lei-overview(7)
lei-p2q(1)
lei-q(1)
+ lei-rediff(1)
lei-tag(1)
+ lei-up(1)
public-inbox.cgi(1)
public-inbox-compact(1)
public-inbox-config(5)
@@ -68,6 +75,7 @@ for (qw[git(1)
git-config(1)
git-credential(1)
git-daemon(1)
+ git-diff(1)
git-fast-import(1)
git-fetch(1)
git-filter-branch(1)
diff --git a/MANIFEST b/MANIFEST
index 29e62880..c7084671 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -23,21 +23,28 @@ Documentation/hosted.txt
Documentation/include.mk
Documentation/lei-add-external.pod
Documentation/lei-blob.pod
+Documentation/lei-cat.pod
Documentation/lei-config.pod
Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
+Documentation/lei-edit-search.pod
Documentation/lei-forget-external.pod
+Documentation/lei-forget-search.pod
Documentation/lei-import.pod
Documentation/lei-init.pod
Documentation/lei-ls-external.pod
Documentation/lei-ls-label.pod
+Documentation/lei-ls-mail-sync.pod
+Documentation/lei-ls-search.pod
Documentation/lei-mail-formats.pod
Documentation/lei-overview.pod
Documentation/lei-p2q.pod
Documentation/lei-q.pod
+Documentation/lei-rediff.pod
Documentation/lei-store-format.pod
Documentation/lei-tag.pod
+Documentation/lei-up.pod
Documentation/lei.pod
Documentation/lei_design_notes.txt
Documentation/marketing.txt
diff --git a/Makefile.PL b/Makefile.PL
index d149a164..2af8c2f1 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -45,9 +45,11 @@ $v->{-m1} = [ map {
}
} @EXE_FILES,
qw(
- lei-add-external lei-blob lei-config lei-convert lei-daemon-kill
- lei-daemon-pid lei-forget-external lei-import lei-init lei-ls-external
- lei-ls-label lei-p2q lei-q lei-tag)];
+ lei-add-external lei-blob lei-config lei-convert lei-edit-search
+ lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
+ lei-import lei-init lei-lcat lei-ls-external lei-ls-label
+ lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-tag
+ lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
--
2.31.1
^ permalink raw reply related [relevance 27%]
* [PATCH 9/9] doc lei: update manpages with new options
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (6 preceding siblings ...)
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
@ 2021-05-17 3:35 66% ` Kyle Meyer
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-blob.pod | 2 ++
Documentation/lei-import.pod | 16 ++++++++++++++++
Documentation/lei-q.pod | 10 ++++++++++
3 files changed, 28 insertions(+)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index 36c75d53..5b611d11 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -80,6 +80,8 @@ L<lei-q(1)>.
=item --no-torsocks
+=item --proxy=PROTO://HOST[:PORT]
+
=back
=head1 CONTACT
diff --git a/Documentation/lei-import.pod b/Documentation/lei-import.pod
index 1fe2dac3..c29a085b 100644
--- a/Documentation/lei-import.pod
+++ b/Documentation/lei-import.pod
@@ -46,6 +46,22 @@ Default: fcntl,dotlock
Don't import message keywords (or "flags" in IMAP terminology).
+=item --no-incremental
+
+Import already seen IMAP and NNTP articles.
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
+
+Default: C<auto>
+
+=item --proxy=PROTO://HOST[:PORT]
+
+Use the specified proxy (e.g., C<socks5h://0:9050>).
+
=back
=head1 CONTACT
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index c4a7eea3..fbe61920 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -26,6 +26,10 @@ TODO: mention curl options?
Read search terms from stdin.
+=item --save
+
+Save a search for L<lei-up(1)>.
+
=item --output=MFOLDER
=item -o MFOLDER
@@ -66,6 +70,10 @@ default destination (stdout).
Using a C<format:> prefix with the C<--output> destination is
preferred when not writing to stdout.
+=item --no-color
+
+Disable color (for C<--format=text>).
+
=item --pretty
Pretty print C<json> or C<concatjson> output. If stdout is opened to
@@ -229,6 +237,8 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
Default: C<auto>
+=item --proxy=PROTO://HOST[:PORT]
+
=back
=head1 CONTACT
--
2.31.1
^ permalink raw reply related [relevance 66%]
* [PATCH] lei lcat: fix handling of multiple MSGID_OR_URL arguments
@ 2021-05-17 3:37 71% Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:37 UTC (permalink / raw)
To: meta
`lei lcat' is documented as being able to display multiple messages,
but this works only with --stdin because the positional argument
MSGID_OR_URL is missing a period.
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b82fb003..98e79a76 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -168,7 +168,7 @@ our %CMD = ( # sorted in order of importance/use:
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
-'lcat' => [ '--stdin|MSGID_OR_URL..', 'display local copy of message(s)',
+'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
# some of these options are ridiculous for lcat
@lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
base-commit: 8cc23ac6f7a3847977ec57c2a3e9e391fdb94da6
--
2.31.1
^ permalink raw reply related [relevance 71%]
* Re: [PATCH 8/9] doc lei: add manpages for new commands
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
@ 2021-05-17 8:09 71% ` Eric Wong
2021-05-17 23:24 71% ` Kyle Meyer
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-17 8:09 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
Thanks, pushed this series out and your lcat fix as
commit 94be0536ce5eaf1d5a67f217dce112607624389c
("lei lcat: fix handling of multiple MSGID_OR_URL arguments")
> diff --git a/MANIFEST b/MANIFEST
> index 29e62880..c7084671 100644
> --- a/MANIFEST
> +++ b/MANIFEST
> @@ -23,21 +23,28 @@ Documentation/hosted.txt
> Documentation/include.mk
> Documentation/lei-add-external.pod
> Documentation/lei-blob.pod
> +Documentation/lei-cat.pod
I squashed this fix in, though ('make check' revealed it):
diff -u b/MANIFEST b/MANIFEST
--- b/MANIFEST
+++ b/MANIFEST
@@ -23,7 +23,6 @@
Documentation/include.mk
Documentation/lei-add-external.pod
Documentation/lei-blob.pod
-Documentation/lei-cat.pod
Documentation/lei-config.pod
Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
@@ -33,6 +32,7 @@
Documentation/lei-forget-search.pod
Documentation/lei-import.pod
Documentation/lei-init.pod
+Documentation/lei-lcat.pod
Documentation/lei-ls-external.pod
Documentation/lei-ls-label.pod
Documentation/lei-ls-mail-sync.pod
^ permalink raw reply [relevance 71%]
* Re: [PATCH 8/9] doc lei: add manpages for new commands
2021-05-17 8:09 71% ` Eric Wong
@ 2021-05-17 23:24 71% ` Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 23:24 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> I squashed this fix in, though ('make check' revealed it):
Oy, sorry about that. Thanks for the fixup.
^ permalink raw reply [relevance 71%]
* [PATCH] lei: relax rules for "new" in Maildir
@ 2021-05-19 8:54 57% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-19 8:54 UTC (permalink / raw)
To: meta
mbsync and offlineimap both use ":2," suffixes for filenames in
"new/", however my interpretation of the Maildir spec at
<https://cr.yp.to/proto/maildir.html> is that ":2," is only for
files in "cur/". My interpretation also matches that of
doveecot, but we'll allow what mbsync and offlineimap do given
their popularity.
---
lib/PublicInbox/LeiMailSync.pm | 13 ++++++++++---
lib/PublicInbox/MdirReader.pm | 5 ++++-
t/lei-index.t | 10 +++++++++-
3 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 803de48c..3bada42d 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -233,9 +233,16 @@ WHERE b.oidbin = ?
$b2n->execute(pack('H*', $oidhex));
while (my ($d, $n) = $b2n->fetchrow_array) {
substr($d, 0, length('maildir:')) = '';
- my $f = "$d/" . ($n =~ /:2,[a-zA-Z]*\z/ ? "cur/$n" : "new/$n");
- open my $fh, '<', $f or next;
- if (-s $fh) {
+ # n.b. both mbsync and offlineimap use ":2," as a suffix
+ # in "new/", despite (from what I understand of reading
+ # <https://cr.yp.to/proto/maildir.html>), the ":2," only
+ # applies to files in "cur/".
+ my @try = $n =~ /:2,[a-zA-Z]+\z/ ? qw(cur new) : qw(new cur);
+ for my $x (@try) {
+ my $f = "$d/$x/$n";
+ open my $fh, '<', $f or next;
+ # some (buggy) Maildir writers are non-atomic:
+ next unless -s $fh;
local $/;
my $raw = <$fh>;
if ($vrfy && git_sha(1, \$raw)->hexdigest ne $oidhex) {
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index b49c8ceb..7a0641fb 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -61,7 +61,10 @@ sub maildir_each_eml {
while (defined(my $bn = readdir($dh))) {
next if substr($bn, 0, 1) eq '.';
my @f = split(/:/, $bn, -1);
- next if scalar(@f) != 1;
+
+ # mbsync and offlineimap both use "2," in "new/"
+ next if ($f[1] // '2,') ne '2,' || defined($f[2]);
+
next if defined($mod) && !shard_ok($bn, $mod, $shard);
my $f = $pfx.$bn;
my $eml = eml_from_path($f) or next;
diff --git a/t/lei-index.t b/t/lei-index.t
index 3382d42b..b7dafb71 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -27,6 +27,11 @@ my $expect = do {
local $/;
<$fh>;
};
+
+# mbsync and offlineimap both put ":2," in "new/" files:
+symlink(File::Spec->rel2abs('t/utf8.eml'), "$tmpdir/md/new/u:2,") or
+ xbail "symlink $!";
+
test_lei({ tmpdir => $tmpdir }, sub {
my $store_path = "$ENV{HOME}/.local/share/lei/store/";
@@ -40,13 +45,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok(qw(q mid:qp@example.com -f text));
like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
+ lei_ok(qw(q mid:testmessage@example.com -f text));
+ lei_ok(qw(-C / blob --mail 9bf1002c49eb075df47247b74d69bcd555e23422));
+
my $all_obj = ['git', "--git-dir=$store_path/ALL.git",
qw(cat-file --batch-check --batch-all-objects)];
is_deeply([xqx($all_obj)], [], 'no git objects');
lei_ok('import', 't/plack-qp.eml');
ok(grep(/\A$blob blob /, my @objs = xqx($all_obj)),
'imported blob');
- lei_ok(qw(q z:0.. --dedupe=none));
+ lei_ok(qw(q m:qp@example.com --dedupe=none));
my $res_b = json_utf8->decode($lei_out);
is_deeply($res_b, $res_a, 'no extra DB entries');
^ permalink raw reply related [relevance 57%]
* [PATCH] lei rediff: fix construction of git-diff options
@ 2021-05-21 4:38 70% Kyle Meyer
2021-05-21 9:52 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-05-21 4:38 UTC (permalink / raw)
To: meta
When generating git-diff options, lei-rediff extracts the single
character option from the lei option spec. However, there's no check
that the regular expression actually matches, leading to an
unintentional git-diff option when there isn't a short option (e.g.,
--inter-hunk-context=1 maps to the invalid `git diff --color -w1').
Check for a match before trying to extract the single character
option.
Fixes: cf0c7ce3ce81b5c3 (lei rediff: regenerate diffs from stdin)
---
lib/PublicInbox/LeiRediff.pm | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 3c8ebe41..2e793df5 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -108,8 +108,9 @@ EOM
push @cmd, '--'.($opt->{color} && !$opt->{'no-color'} ? '' : 'no-').
'color';
for my $o (@PublicInbox::LEI::diff_opt) {
- $o =~ s/\|([a-z0-9])\b//i; # remove single char short option
- my $c = $1;
+ my $c = '';
+ # remove single char short option
+ $o =~ s/\|([a-z0-9])\b//i and $c = $1;
if ($o =~ s/=[is]@\z//) {
my $v = $opt->{$o} or next;
push @cmd, map { $c ? "-$c$_" : "--$o=$_" } @$v;
base-commit: 2f720902ed702b64d918165ba21a96dabbeeca26
--
2.31.1
^ permalink raw reply related [relevance 70%]
* Re: [PATCH] lei rediff: fix construction of git-diff options
2021-05-21 4:38 70% [PATCH] lei rediff: fix construction of git-diff options Kyle Meyer
@ 2021-05-21 9:52 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 9:52 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> When generating git-diff options, lei-rediff extracts the single
> character option from the lei option spec. However, there's no check
> that the regular expression actually matches, leading to an
> unintentional git-diff option when there isn't a short option (e.g.,
> --inter-hunk-context=1 maps to the invalid `git diff --color -w1').
>
> Check for a match before trying to extract the single character
> option.
>
> Fixes: cf0c7ce3ce81b5c3 (lei rediff: regenerate diffs from stdin)
Oops :x Spending years in another language corrupted my
mind to believe $1 gets clobbered on failed matches;
but that's not the case for Perl.
An unrelated patch I'm working on had the same problem :x
Anyways, pushed as commit 3c1d0da4d0608b5a87371e602a911964d7c1498c
Thanks.
^ permalink raw reply [relevance 71%]
* [PATCH 0/8] lei: export-kw, IMAP import incompatibility
@ 2021-05-21 10:28 64% Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
"lei export-kw" is a new command. I'm not sure exactly how
it'll be used but it's probably more of a plumbing command,
for now. My brain hurts thinking about synchronization
and merge/conflict resolution when it comes to propagating
keywords assignments/clearing.
(I frequently mark messages as Unread in my MUA so I know to
reread them in the future, and I suspect it's a common thing).
mail_sync.sqlite3 now tracks AUTH=ANONYMOUS or username in the
folder name to account for lei(Unix) users having multiple IMAP
accounts on the same host with the same folders+UIDVALIDITY.
"lei import imap(s)://" users will waste a bit of bandwidth
resyncing as a result.
Eric Wong (8):
treewide: favor open(..., '+<&=', $fd)
lei: drop EOFpipe in favor of PktOp
lei tag: support tagging index-only messages
lei_input: fix canonicalization of Maildirs for sync
lei index: support command-line options
lei export-kw: new command to export keywords to Maildirs
uri_imap: support uid/auth/user as full accessors
lei import: store IMAP user+auth in mail_sync folder URI
MANIFEST | 2 +
examples/unsubscribe.milter | 3 +-
lib/PublicInbox/DS.pm | 3 +-
lib/PublicInbox/Daemon.pm | 2 +-
lib/PublicInbox/LEI.pm | 18 +++-
lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiInput.pm | 3 +-
lib/PublicInbox/LeiMailSync.pm | 10 ++
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiSearch.pm | 22 +++-
lib/PublicInbox/LeiTag.pm | 10 +-
lib/PublicInbox/LeiToMail.pm | 12 ++-
lib/PublicInbox/MdirReader.pm | 14 +++
lib/PublicInbox/NetReader.pm | 42 +++++---
lib/PublicInbox/Sigfd.pm | 3 +-
lib/PublicInbox/URIimap.pm | 82 +++++++++++----
t/epoll.t | 7 +-
t/lei-export-kw.t | 35 +++++++
t/lei-import-imap.t | 9 +-
t/lei-index.t | 12 ++-
t/mdir_reader.t | 5 +
t/uri_imap.t | 60 ++++++++---
22 files changed, 461 insertions(+), 75 deletions(-)
create mode 100644 lib/PublicInbox/LeiExportKw.pm
create mode 100644 t/lei-export-kw.t
^ permalink raw reply [relevance 64%]
* [PATCH 5/8] lei index: support command-line options
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
2021-05-21 10:28 53% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
@ 2021-05-21 10:28 71% ` Eric Wong
2021-05-21 10:28 32% ` [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs Eric Wong
2021-05-21 10:28 46% ` [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
This mostly takes after "lei import", and at least --quiet needs
to be supported.
---
lib/PublicInbox/LEI.pm | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d7768426..15680fe3 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -233,6 +233,11 @@ our %CMD = ( # sorted in order of importance/use:
'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
qw(prune), @c_opt ],
+'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
+ qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
+ verbose|v+ incremental!),
+ PublicInbox::LeiQuery::curl_opt(), # mainly for --proxy=
+ @c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s
^ permalink raw reply related [relevance 71%]
* [PATCH 2/8] lei: drop EOFpipe in favor of PktOp
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
@ 2021-05-21 10:28 66% ` Eric Wong
2021-05-21 10:28 53% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
lei already uses PktOp and SOCK_SEQPACKET throughout; whereas
EOFpipe had one single use in lei. Since PktOp is a strict
superset of EOFpipe functionality, we may be able to get rid of
EOFpipe entirely.
However, lei is considered a portability canary and I'm not sure
if the stable public-inbox-* code can drop EOFpipe just yet.
---
lib/PublicInbox/LEI.pm | 9 ++++-----
lib/PublicInbox/LeiOverview.pm | 2 +-
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 98e79a76..d7768426 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1151,7 +1151,7 @@ sub lazy_start {
(Socket::MsgHdr || Inline::C) missing/unconfigured (narg=$narg);
require PublicInbox::Listener;
- require PublicInbox::EOFpipe;
+ require PublicInbox::PktOp;
(-p STDOUT) or die "E: stdout must be a pipe\n";
open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
STDIN->autoflush(1);
@@ -1165,13 +1165,12 @@ sub lazy_start {
my $exit_code;
my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
local $quit = do {
- pipe(my ($eof_r, $eof_w)) or die "pipe: $!";
- PublicInbox::EOFpipe->new($eof_r, \&noop, undef);
+ my (undef, $eof_p) = PublicInbox::PktOp->pair;
sub {
$exit_code //= shift;
my $lis = $pil or exit($exit_code);
- # closing eof_w triggers \&noop wakeup
- $listener = $eof_w = $pil = $path = undef;
+ # closing eof_p triggers \&noop wakeup
+ $listener = $eof_p = $pil = $path = undef;
$lis->close; # DS::close
PublicInbox::DS->SetLoopTimeout(1000);
};
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index bfb8b143..28891460 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -119,7 +119,7 @@ sub ovv_begin {
} # TODO HTML/Atom/...
}
-# called once by parent (via PublicInbox::EOFpipe)
+# called once by parent (via PublicInbox::PktOp '' => query_done)
sub ovv_end {
my ($self, $lei) = @_;
if ($self->{fmt} eq 'json') {
^ permalink raw reply related [relevance 66%]
* [PATCH 3/8] lei tag: support tagging index-only messages
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
@ 2021-05-21 10:28 53% ` Eric Wong
2021-05-21 10:28 71% ` [PATCH 5/8] lei index: support command-line options Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
This will make some of our tests faster and allow users to try
more features of lei without high storage requirements.
---
lib/PublicInbox/LeiSearch.pm | 8 ++++++--
lib/PublicInbox/LeiTag.pm | 10 ++++++++--
lib/PublicInbox/LeiToMail.pm | 4 +++-
t/lei-index.t | 12 +++++++++++-
4 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index c2b12146..fb19229f 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -63,7 +63,9 @@ sub content_key ($) {
}
sub _cmp_1st { # git->cat_async callback
- my ($bref, $oid, $type, $size, $cmp) = @_; # cmp: [chash, xoids, smsg]
+ my ($bref, $oid, $type, $size, $cmp) = @_;
+ # cmp: [chash, xoids, smsg, lms]
+ $bref //= $cmp->[3] ? $cmp->[3]->local_blob($oid, 1) : undef;
if ($bref && content_hash(PublicInbox::Eml->new($bref)) eq $cmp->[0]) {
$cmp->[1]->{$oid} = $cmp->[2]->{num};
}
@@ -78,6 +80,8 @@ sub xoids_for {
my @overs = ($self->over // $self->overs_all);
my $git = $self->git;
my $xoids = {};
+ # no lms when used via {ale}:
+ my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir});
for my $mid (@$mids) {
for my $o (@overs) {
my ($id, $prev);
@@ -85,7 +89,7 @@ sub xoids_for {
next if $cur->{bytes} == 0 ||
$xoids->{$cur->{blob}};
$git->cat_async($cur->{blob}, \&_cmp_1st,
- [ $chash, $xoids, $cur ]);
+ [$chash, $xoids, $cur, $lms]);
if ($min && scalar(keys %$xoids) >= $min) {
$git->cat_async_wait;
return $xoids;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index c650e886..b6abd533 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -9,7 +9,8 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
- if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) {
+ if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
+ $self->{lei}->{ale}->xoids_for($eml)) {
$self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
$self->{vmd_mod});
} else {
@@ -17,7 +18,11 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
}
}
-sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
+sub input_mbox_cb {
+ my ($eml, $self) = @_;
+ $eml->header_set($_) for (qw(X-Status Status));
+ input_eml_cb($self, $eml);
+}
sub input_maildir_cb { # maildir_each_eml cb
my ($f, $kw, $eml, $self) = @_;
@@ -60,6 +65,7 @@ sub note_missing {
sub ipc_atfork_child {
my ($self) = @_;
PublicInbox::LeiInput::input_only_atfork_child($self);
+ $self->{lse} = $self->{lei}->{sto}->search;
# this goes out-of-scope at worker process exit:
PublicInbox::OnDestroy->new($$, \¬e_missing, $self);
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index da3a95d2..0cbdff8b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -650,7 +650,9 @@ sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
$lei->_lei_atfork_child;
- $self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse};
+ if (my $lse = $lei->{lse}) {
+ $self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
+ }
$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
diff --git a/t/lei-index.t b/t/lei-index.t
index b7dafb71..9a45d885 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -40,7 +40,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $res_a = json_utf8->decode($lei_out);
my $blob = $res_a->[0]->{'blob'};
like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example');
- lei_ok('blob', $blob);
+ lei_ok(qw(-C / blob), $blob);
is($lei_out, $expect, 'got expected blob via Maildir');
lei_ok(qw(q mid:qp@example.com -f text));
like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
@@ -58,6 +58,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $res_b = json_utf8->decode($lei_out);
is_deeply($res_b, $res_a, 'no extra DB entries');
+ # ensure tag works on index-only messages:
+ lei_ok(qw(tag +kw:seen t/utf8.eml));
+ lei_ok(qw(q mid:testmessage@example.com));
+ is_deeply(json_utf8->decode($lei_out)->[0]->{kw},
+ ['seen'], 'seen kw can be set on index-only message');
+
+ lei_ok(qw(q z:0.. -o), "$tmpdir/all-results") for (1..2);
+ is_deeply([xqx($all_obj)], \@objs,
+ 'no new objects after 2x q to trigger implicit import');
+
lei_ok('index', "nntp://$nntp_host_port/t.v2");
lei_ok('index', "imap://$imap_host_port/t.v2.0");
is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
^ permalink raw reply related [relevance 53%]
* [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
` (3 preceding siblings ...)
2021-05-21 10:28 32% ` [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs Eric Wong
@ 2021-05-21 10:28 46% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
Just having UIDVALIDITY in the URI isn't enough, since a single
lei user may have multiple IMAP logins on the same server.
This leads to compatibility problems and forces a reimport for
the few users already using this lei functionality, but it's not
stable nor released, yet.
---
lib/PublicInbox/NetReader.pm | 42 ++++++++++++++++++++++--------------
t/lei-import-imap.t | 9 +++++---
2 files changed, 32 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index fd0d1682..a532b218 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -58,12 +58,10 @@ sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
# mic_for may prompt the user and store auth info, prepares mic_get
sub mic_for ($$$$) { # mic = Mail::IMAPClient
- my ($self, $url, $mic_args, $lei) = @_;
- require PublicInbox::URIimap;
- my $uri = PublicInbox::URIimap->new($url);
+ my ($self, $uri, $mic_args, $lei) = @_;
require PublicInbox::GitCredential;
my $cred = bless {
- url => $url,
+ url => "$uri",
protocol => $uri->scheme,
host => $uri->host,
username => $uri->user,
@@ -83,13 +81,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
};
require PublicInbox::IMAPClient;
my $mic = mic_new($self, $mic_arg, $sec, $uri) or
- die "E: <$url> new: $@\n";
+ die "E: <$uri> new: $@\n";
# default to using STARTTLS if it's available, but allow
# it to be disabled since I usually connect to localhost
if (!$mic_arg->{Ssl} && !defined($mic_arg->{Starttls}) &&
$mic->has_capability('STARTTLS') &&
$mic->can('starttls')) {
- $mic->starttls or die "E: <$url> STARTTLS: $@\n";
+ $mic->starttls or die "E: <$uri> STARTTLS: $@\n";
}
# do we even need credentials?
@@ -111,8 +109,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
if ($mic->login && $mic->IsAuthenticated) {
# success! keep IMAPClient->new arg in case we get disconnected
$self->{mic_arg}->{$sec} = $mic_arg;
+ if ($cred) {
+ $uri->user($cred->{username}) if !defined($uri->user);
+ } elsif ($mic_arg->{Authmechanism} eq 'ANONYMOUS') {
+ $uri->auth('ANONYMOUS') if !defined($uri->auth);
+ }
} else {
- $err = "E: <$url> LOGIN: $@\n";
+ $err = "E: <$uri> LOGIN: $@\n";
if ($cred && defined($cred->{password})) {
$err =~ s/\Q$cred->{password}\E/*******/g;
}
@@ -304,15 +307,16 @@ sub imap_common_init ($;$) {
# make sure we can connect and cache the credentials in memory
$self->{mic_arg} = {}; # schema://authority => IMAPClient->new args
my $mics = {}; # schema://authority => IMAPClient obj
- for my $uri (@{$self->{imap_order}}) {
- my $sec = uri_section($uri);
+ for my $orig_uri (@{$self->{imap_order}}) {
+ my $sec = uri_section($orig_uri);
+ my $uri = PublicInbox::URIimap->new("$sec/");
my $mic = $mics->{$sec} //=
- mic_for($self, "$sec/", $mic_args, $lei) //
+ mic_for($self, $uri, $mic_args, $lei) //
die "Unable to continue\n";
next unless $self->isa('PublicInbox::NetWriter');
- my $dst = $uri->mailbox // next;
+ my $dst = $orig_uri->mailbox // next;
next if $mic->exists($dst); # already exists
- $mic->create($dst) or die "CREATE $dst failed <$uri>: $@";
+ $mic->create($dst) or die "CREATE $dst failed <$orig_uri>: $@";
}
$mics;
}
@@ -419,12 +423,18 @@ sub run_commit_cb ($) {
$cb->(@args);
}
-sub _itrk_last ($$;$) {
- my ($self, $uri, $r_uidval) = @_;
+sub itrk_last ($$;$$) {
+ my ($self, $uri, $r_uidval, $mic) = @_;
return (undef, undef, $r_uidval) unless $self->{incremental};
my ($itrk, $l_uid, $l_uidval);
if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0
$uri->uidvalidity($r_uidval) if defined $r_uidval;
+ if ($mic) {
+ my $auth = $mic->Authmechanism // '';
+ $uri->auth($auth) if $auth eq 'ANONYMOUS';
+ my $user = $mic->User;
+ $uri->user($user) if defined($user);
+ }
my $x;
$l_uid = ($lms && ($x = $lms->location_stats($$uri))) ?
$x->{'uid.max'} : undef;
@@ -459,7 +469,7 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
EOF
my $uri = $orig_uri->clone;
- my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval);
+ my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
@@ -612,7 +622,7 @@ sub _nntp_fetch_all ($$$) {
# IMAPTracker is also used for tracking NNTP, UID == article number
# LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
# expensive. So we assume newsgroups don't change:
- my ($itrk, $l_art) = _itrk_last($self, $uri);
+ my ($itrk, $l_art) = itrk_last($self, $uri);
# allow users to specify articles to refetch
# cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index fd15ef4f..d424ebb1 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -23,9 +23,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('import', $url);
lei_ok 'ls-mail-sync';
- like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-mail-sync');
+ like($lei_out, qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E
+ /t\.v2\.0;UIDVALIDITY=\d+\n\z!x, 'ls-mail-sync');
chomp(my $u = $lei_out);
lei_ok('import', $u, \'UIDVALIDITY match in URL');
+ $url = $u;
$u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s;
ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected');
@@ -33,7 +35,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $inspect = json_utf8->decode($lei_out);
my @k = keys %$inspect;
is(scalar(@k), 1, 'one URL resolved');
- like($k[0], qr!\A\Q$url\E;UIDVALIDITY=\d+\z!, 'inspect URL matches');
+ is($k[0], $url, 'inspect URL matches');
my $stats = $inspect->{$k[0]};
is_deeply([ sort keys %$stats ],
[ qw(uid.count uid.max uid.min) ], 'keys match');
@@ -55,7 +57,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $x = json_utf8->decode($lei_out);
is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect');
is(ref($x->{'mail-sync'}), 'HASH', 'sync in inspect');
- is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect');
+ is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect')
+ or diag explain($x);
my $psgi_attach = 'cfa3622cbeffc9bd6b0fc66c4d60d420ba74f60d';
lei_ok('blob', $psgi_attach);
^ permalink raw reply related [relevance 46%]
* [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
` (2 preceding siblings ...)
2021-05-21 10:28 71% ` [PATCH 5/8] lei index: support command-line options Eric Wong
@ 2021-05-21 10:28 32% ` Eric Wong
2021-05-21 10:28 46% ` [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
IMAP will eventually be supported.
---
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 4 +
lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 10 ++
lib/PublicInbox/LeiSearch.pm | 14 +++
lib/PublicInbox/LeiToMail.pm | 8 +-
lib/PublicInbox/MdirReader.pm | 14 +++
t/lei-export-kw.t | 35 +++++++
t/mdir_reader.t | 5 +
9 files changed, 270 insertions(+), 2 deletions(-)
create mode 100644 lib/PublicInbox/LeiExportKw.pm
create mode 100644 t/lei-export-kw.t
diff --git a/MANIFEST b/MANIFEST
index 684128aa..2d1ad5c3 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -202,6 +202,7 @@ lib/PublicInbox/LeiConvert.pm
lib/PublicInbox/LeiCurl.pm
lib/PublicInbox/LeiDedupe.pm
lib/PublicInbox/LeiEditSearch.pm
+lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
@@ -408,6 +409,7 @@ t/iso-2202-jp.eml
t/kqnotify.t
t/lei-convert.t
t/lei-daemon.t
+t/lei-export-kw.t
t/lei-externals.t
t/lei-import-http.t
t/lei-import-imap.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 15680fe3..628908b5 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -243,6 +243,10 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| offset=i recursive|r exclude=s include|I=s
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+
+'export-kw' => [ 'LOCATION...|--all',
+ 'one-time export of keywords of sync sources',
+ qw(all:s mode=s), @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
new file mode 100644
index 00000000..db4f7441
--- /dev/null
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -0,0 +1,180 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# front-end for the "lei export-kw" sub-command
+package PublicInbox::LeiExportKw;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use Errno qw(EEXIST ENOENT);
+
+sub export_kw_md { # LeiMailSync->each_src callback
+ my ($oidbin, $id, $self, $mdir) = @_;
+ my $oidhex = unpack('H*', $oidbin);
+ my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return;
+ my $bn = $$id;
+ my ($md_kw, $unknown, @try);
+ if ($bn =~ s/:2,([a-zA-Z]*)\z//) {
+ ($md_kw, $unknown) = PublicInbox::MdirReader::flags2kw($1);
+ @try = qw(cur new);
+ } else {
+ $unknown = [];
+ @try = qw(new cur);
+ }
+ if ($self->{-merge_kw} && $md_kw) { # merging keywords is the default
+ @$sto_kw{keys %$md_kw} = values(%$md_kw);
+ }
+ $bn .= ':2,'.
+ PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown);
+ my $dst = "$mdir/cur/$bn";
+ my @fail;
+ for my $d (@try) {
+ my $src = "$mdir/$d/$$id";
+ next if $src eq $dst;
+
+ # we use link(2) + unlink(2) since rename(2) may
+ # inadvertently clobber if the "uniquefilename" part wasn't
+ # actually unique.
+ if (link($src, $dst)) { # success
+ # unlink(2) may ENOENT from parallel invocation,
+ # ignore it, but not other serious errors
+ if (!unlink($src) and $! != ENOENT) {
+ $self->{lei}->child_error(1,
+ "E: unlink($src): $!");
+ }
+ $self->{lms}->mv_src("maildir:$mdir",
+ $oidbin, $id, $bn) or die;
+ return; # success anyways if link(2) worked
+ }
+ if ($! == ENOENT && !-e $src) { # some other process moved it
+ $self->{lms}->clear_src("maildir:$mdir", $id);
+ next;
+ }
+ push @fail, $src if $! != EEXIST;
+ }
+ return unless @fail;
+ # both tries failed
+ my $e = $!;
+ my $orig = '['.join('|', @fail).']';
+ $self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
+}
+
+# overrides PublicInbox::LeiInput::input_path_url
+sub input_path_url {
+ my ($self, $input, @args) = @_;
+ my $lms = $self->{lms} //= $self->{lse}->lms;
+ $lms->lms_begin;
+ if ($input =~ s/\Amaildir://i) {
+ require PublicInbox::LeiToMail; # kw2suffix
+ $lms->each_src("maildir:$input", \&export_kw_md, $self, $input);
+ }
+ $lms->lms_commit;
+}
+
+sub lei_export_kw {
+ my ($lei, @folders) = @_;
+ my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
+lei/store uninitialized, see lei-import(1)
+EOM
+ my $lse = $sto->search;
+ my $lms = $lse->lms or return $lei->fail(<<EOM);
+lei mail_sync uninitialized, see lei-import(1)
+EOM
+ my $opt = $lei->{opt};
+ my $all = $opt->{all};
+ my @all = $lms->folders;
+ if (defined $all) { # --all=<local|remote>
+ my %x = map { $_ => $_ } split(/,/, $all);
+ my @ok = grep(defined, delete(@x{qw(local remote), ''}));
+ my @no = keys %x;
+ if (@no) {
+ @no = (join(',', @no));
+ return $lei->fail(<<EOM);
+--all=@no not accepted (must be `local' and/or `remote')
+EOM
+ }
+ my (%seen, @inc);
+ for my $ok (@ok) {
+ if ($ok eq 'local') {
+ @inc = grep(!m!\A[a-z0-9\+]+://!i, @all);
+ } elsif ($ok eq 'remote') {
+ @inc = grep(m!\A[a-z0-9\+]+://!i, @all);
+ } elsif ($ok ne '') {
+ return $lei->fail("--all=$all not understood");
+ } else {
+ @inc = @all;
+ }
+ for (@inc) {
+ push(@folders, $_) unless $seen{$_}++;
+ }
+ }
+ return $lei->fail(<<EOM) if !@folders;
+no --mail-sync folders known to lei
+EOM
+ } else {
+ my %all = map { $_ => 1 } @all;
+ my @no;
+ for (@folders) {
+ next if $all{$_}; # ok
+ if (-d "$_/new" && -d "$_/cur") {
+ my $d = 'maildir:'.$lei->rel2abs($_);
+ push(@no, $_) unless $all{$d};
+ $_ = $d;
+ } else {
+ push @no, $_;
+ }
+ }
+ my $no = join("\n\t", @no);
+ return $lei->fail(<<EOF) if @no;
+No sync information for: $no
+Run `lei ls-mail-sync' to display valid choices
+EOF
+ }
+ my $self = bless { lse => $lse }, __PACKAGE__;
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ $self->prepare_inputs($lei, \@folders) or return;
+ my $j = $opt->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
+ return $lei->fail("cannot export to read-only folders: @ro");
+ }
+ if (my $net = $lei->{net}) {
+ require PublicInbox::NetWriter;
+ bless $net, 'PublicInbox::NetWriter';
+ }
+ undef $lms;
+ my $m = $opt->{mode} // 'merge';
+ if ($m eq 'merge') { # default
+ $self->{-merge_kw} = 1;
+ } elsif ($m eq 'set') {
+ } else {
+ return $lei->fail(<<EOM);
+--mode=$m not supported (`set' or `merge')
+EOM
+ }
+ my $ops = {};
+ $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $self->{-wq_nr_workers} = $j // 1; # locked
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $op_c->op_wait_event($ops); # calls net_merge_all_done if $lei->{auth}
+}
+
+sub _complete_export_kw {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ map { $match_cb->($_) } $lms->folders;
+}
+
+no warnings 'once';
+
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
+# the following works even when LeiAuth is lazy-loaded
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 3bada42d..32e17c65 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -138,6 +138,16 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ?
$sth->execute($fid, $id);
}
+# Maildir-only
+sub mv_src {
+ my ($self, $folder, $oidbin, $id, $newbn) = @_;
+ my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $sth = $self->{dbh}->prepare_cached(<<'');
+UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
+
+ $sth->execute($newbn, $fid, $oidbin, $$id);
+}
+
# read-only, iterates every oidbin + UID or name for a given folder
sub each_src {
my ($self, $folder, $cb, @args) = @_;
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index fb19229f..9297d060 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -27,6 +27,20 @@ sub msg_keywords {
wantarray ? sort(keys(%$kw)) : $kw;
}
+# returns undef if blob is unknown
+sub oid_keywords {
+ my ($self, $oidhex) = @_;
+ my @num = $self->over->blob_exists($oidhex) or return;
+ my $xdb = $self->xdb; # set {nshard};
+ my %kw;
+ for my $num (@num) { # there should only be one...
+ my $doc = $xdb->get_document(num2docid($self, $num));
+ my $x = xap_terms('K', $doc);
+ %kw = (%kw, %$x);
+ }
+ \%kw;
+}
+
# lookup keywords+labels for external messages
sub xsmsg_vmd {
my ($self, $smsg, $want_label) = @_;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 0cbdff8b..96a1f881 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -243,10 +243,14 @@ sub _rand () {
sprintf('%x,%x,%x,%x', rand(0xffffffff), time, $$, ++$seq);
}
+sub kw2suffix ($;@) {
+ my $kw = shift;
+ join('', sort(map { $kw2char{$_} // () } @$kw, @_));
+}
+
sub _buf2maildir {
my ($dst, $buf, $smsg) = @_;
my $kw = $smsg->{kw} // [];
- my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw));
my $rand = ''; # chosen by die roll :P
my ($tmp, $fh, $base, $ok);
my $common = $smsg->{blob} // _rand;
@@ -263,7 +267,7 @@ sub _buf2maildir {
$dst .= 'cur/';
$rand = '';
do {
- $base = $rand.$common.':2,'.$sfx
+ $base = $rand.$common.':2,'.kw2suffix($kw);
} while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} &&
($rand = _rand.','));
die "link($tmp, $dst$base): $!" unless $ok;
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index 7a0641fb..304be63d 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -86,4 +86,18 @@ sub maildir_each_eml {
sub new { bless {}, __PACKAGE__ }
+sub flags2kw ($) {
+ my @unknown;
+ my %kw;
+ for (split(//, $_[0])) {
+ my $k = $c2kw{$_};
+ if (defined($k)) {
+ $kw{$k} = 1;
+ } else {
+ push @unknown, $_;
+ }
+ }
+ (\%kw, \@unknown);
+}
+
1;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
new file mode 100644
index 00000000..9531949a
--- /dev/null
+++ b/t/lei-export-kw.t
@@ -0,0 +1,35 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Copy qw(cp);
+use File::Path qw(make_path);
+require_mods(qw(lei -imapd Mail::IMAPClient));
+my ($tmpdir, $for_destroy) = tmpdir;
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my $expect = eml_load('t/data/0001.patch');
+test_lei({ tmpdir => $tmpdir }, sub {
+ my $home = $ENV{HOME};
+ my $md = "$home/md";
+ make_path("$md/new", "$md/cur", "$md/tmp");
+ cp('t/data/0001.patch', "$md/new/y") or xbail "cp $md $!";
+ cp('t/data/message_embed.eml', "$md/cur/x:2,S") or xbail "cp $md $!";
+ lei_ok qw(index -q), $md;
+ lei_ok qw(tag t/data/0001.patch +kw:seen);
+ lei_ok qw(export-kw --all=local);
+ ok(!-e "$md/new/y", 'original gone');
+ is_deeply(eml_load("$md/cur/y:2,S"), $expect,
+ "`seen' kw exported");
+
+ lei_ok qw(tag t/data/0001.patch +kw:answered);
+ lei_ok qw(export-kw --all=local);
+ ok(!-e "$md/cur/y:2,S", 'seen-only file gone');
+ is_deeply(eml_load("$md/cur/y:2,RS"), $expect, "`R' added");
+
+ lei_ok qw(tag t/data/0001.patch -kw:answered -kw:seen);
+ lei_ok qw(export-kw --mode=set --all=local);
+ ok(!-e "$md/cur/y:2,RS", 'seen+answered file gone');
+ is_deeply(eml_load("$md/cur/y:2,"), $expect, 'no keywords left');
+});
+
+done_testing;
diff --git a/t/mdir_reader.t b/t/mdir_reader.t
index 51b38af4..c927e1a7 100644
--- a/t/mdir_reader.t
+++ b/t/mdir_reader.t
@@ -19,4 +19,9 @@ is(maildir_path_flags('/path/to/foo:2,'), '', 'no flags in path');
use_ok 'PublicInbox::InboxWritable', qw(eml_from_path);
is(eml_from_path('.'), undef, 'eml_from_path fails on directory');
+is_deeply([PublicInbox::MdirReader::flags2kw('S')], [{ 'seen' => 1 }, []],
+ "`seen' kw set from flag");
+is_deeply([PublicInbox::MdirReader::flags2kw('Su')], [{ 'seen' => 1 }, ['u']],
+ 'unknown flag ignored');
+
done_testing;
^ permalink raw reply related [relevance 32%]
* [PATCH 0/3] lei export-kw: IMAP support
@ 2021-05-23 1:38 71% Eric Wong
2021-05-23 1:38 49% ` [PATCH 2/3] lei export-kw: support exporting keywords to IMAP Eric Wong
2021-05-23 1:38 47% ` [PATCH 3/3] lei export-kw: relax IMAP URL matching Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-23 1:38 UTC (permalink / raw)
To: meta
Dealing with high-latency connections to be addressed at some
point, but probably not right away...
Eric Wong (3):
net_reader|net_writer: pass URI refs deeper into callbacks
lei export-kw: support exporting keywords to IMAP
lei export-kw: relax IMAP URL matching
lib/PublicInbox/LeiExportKw.pm | 72 ++++++++++++++++++++++++++++++----
lib/PublicInbox/LeiImport.pm | 4 +-
lib/PublicInbox/LeiToMail.pm | 13 +++---
lib/PublicInbox/NetReader.pm | 11 +++---
lib/PublicInbox/NetWriter.pm | 22 +++++++----
xt/net_writer-imap.t | 38 ++++++++++++------
6 files changed, 117 insertions(+), 43 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei export-kw: support exporting keywords to IMAP
2021-05-23 1:38 71% [PATCH 0/3] lei export-kw: IMAP support Eric Wong
@ 2021-05-23 1:38 49% ` Eric Wong
2021-05-23 1:38 47% ` [PATCH 3/3] lei export-kw: relax IMAP URL matching Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 1:38 UTC (permalink / raw)
To: meta
We support writing to IMAP stores in other places (just like
Maildir), and it's actually less complex for us to write to
IMAP. Neither usability nor performance is ideal, but usability
will be addressed in the next commit to relax CLI argument
checking.
Performance is poor due to the synchronous Mail::IMAPClient
API and will need to be addressed with pipelining sometime
further in the future.
---
lib/PublicInbox/LeiExportKw.pm | 31 +++++++++++++++++++++++--------
lib/PublicInbox/LeiToMail.pm | 9 +++++----
lib/PublicInbox/NetWriter.pm | 10 ++++++++++
xt/net_writer-imap.t | 18 +++++++++++++-----
4 files changed, 51 insertions(+), 17 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index db4f7441..5ad33959 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -59,15 +59,28 @@ sub export_kw_md { # LeiMailSync->each_src callback
$self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
}
+sub export_kw_imap { # LeiMailSync->each_src callback
+ my ($oidbin, $id, $self, $mic) = @_;
+ my $oidhex = unpack('H*', $oidbin);
+ my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return;
+ $self->{imap_mod_kw}->($self->{nwr}, $mic, $id, [ keys %$sto_kw ]);
+}
+
# overrides PublicInbox::LeiInput::input_path_url
sub input_path_url {
my ($self, $input, @args) = @_;
my $lms = $self->{lms} //= $self->{lse}->lms;
$lms->lms_begin;
- if ($input =~ s/\Amaildir://i) {
+ if ($input =~ /\Amaildir:(.+)/i) {
+ my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
- $lms->each_src("maildir:$input", \&export_kw_md, $self, $input);
- }
+ $lms->each_src($input, \&export_kw_md, $self, $mdir);
+ } elsif ($input =~ m!\Aimaps?://!) {
+ my $uri = PublicInbox::URIimap->new($input);
+ my $mic = $self->{nwr}->mic_for_folder($uri);
+ $lms->each_src($$uri, \&export_kw_imap, $self, $mic);
+ $mic->expunge;
+ } else { die "BUG: $input not supported" }
$lms->lms_commit;
}
@@ -137,11 +150,6 @@ EOF
if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
return $lei->fail("cannot export to read-only folders: @ro");
}
- if (my $net = $lei->{net}) {
- require PublicInbox::NetWriter;
- bless $net, 'PublicInbox::NetWriter';
- }
- undef $lms;
my $m = $opt->{mode} // 'merge';
if ($m eq 'merge') { # default
$self->{-merge_kw} = 1;
@@ -151,6 +159,13 @@ EOF
--mode=$m not supported (`set' or `merge')
EOM
}
+ if (my $net = $lei->{net}) {
+ require PublicInbox::NetWriter;
+ $self->{nwr} = bless $net, 'PublicInbox::NetWriter';
+ $self->{imap_mod_kw} = $net->can($self->{-merge_kw} ?
+ 'imap_add_kw' : 'imap_set_kw');
+ }
+ undef $lms;
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9d4c856..f3c03969 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -307,11 +307,12 @@ sub _imap_write_cb ($$) {
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
my $append = $lei->{net}->can('imap_append');
- my $mic = $lei->{net}->mic_get($self->{uri});
- my $folder = $self->{uri}->mailbox;
+ my $uri = $self->{uri};
+ my $mic = $lei->{net}->mic_get($uri);
+ my $folder = $uri->mailbox;
+ $uri->uidvalidity($mic->uidvalidity($folder));
my $lse = $lei->{lse}; # may be undef
my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
- my $out = $lei->{ovv}->{dst};
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
@@ -327,7 +328,7 @@ sub _imap_write_cb ($$) {
# imap_append returns UID if IMAP server has UIDPLUS extension
($sto && $uid =~ /\A[0-9]+\z/) and
$sto->ipc_do('set_sync_info',
- $smsg->{blob}, $out, $uid + 0);
+ $smsg->{blob}, $$uri, $uid + 0);
++$lei->{-nr_write};
}
}
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 2032a1fd..8ec7f85c 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,10 +26,20 @@ sub imap_append {
die "APPEND $folder: $@";
}
+# updates $uri with UIDVALIDITY
sub mic_for_folder {
my ($self, $uri) = @_;
my $mic = $self->mic_get($uri) or die "E: not connected: $@";
$mic->select($uri->mailbox) or return;
+ my $uidval;
+ for ($mic->Results) {
+ /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
+ $uidval = $1;
+ last;
+ }
+ $uidval //= $mic->uidvalidity($uri->mailbox) or
+ die "E: failed to get uidvalidity from <$uri>: $@";
+ $uri->uidvalidity($uidval);
$mic;
}
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 1298b958..0e6d4831 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -157,12 +157,20 @@ test_lei(sub {
lei_ok qw(import -F eml), $f, \'import local copy w/o keywords';
+ lei_ok 'ls-mail-sync'; diag $lei_out;
+ lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3
+ lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f;
+ lei_ok 'ls-mail-sync'; diag $lei_out;
+ chomp(my $uri_val = $lei_out);
+ lei_ok 'export-kw', $uri_val;
$mic = $nwr->mic_for_folder($folder_uri);
- # dummy set to ensure second set_kw clobbers
- $nwr->imap_set_kw($mic, $uid[0], [ qw(seen answered flagged) ]
- )->expunge or BAIL_OUT "expunge $@";
- $nwr->imap_set_kw($mic, $uid[0], [ 'seen' ]
- )->expunge or BAIL_OUT "expunge $@";
+ my $flags = $mic->flags($uid[0]);
+ is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ],
+ 'IMAP flags set by export-kw') or diag explain($flags);
+
+ # ensure this imap_set_kw clobbers
+ $nwr->imap_set_kw($mic, $uid[0], [ 'seen' ])->expunge or
+ BAIL_OUT "expunge $@";
$mic = undef;
@res = ();
$nwr->imap_each($folder_uri, $imap_slurp_all, \@res);
^ permalink raw reply related [relevance 49%]
* [PATCH 3/3] lei export-kw: relax IMAP URL matching
2021-05-23 1:38 71% [PATCH 0/3] lei export-kw: IMAP support Eric Wong
2021-05-23 1:38 49% ` [PATCH 2/3] lei export-kw: support exporting keywords to IMAP Eric Wong
@ 2021-05-23 1:38 47% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 1:38 UTC (permalink / raw)
To: meta
It's unreasonable to expect UIDVALIDITY= to be specified in
command-line arguments. We'll also check for cases without
"$USER@" or ";AUTH=", since we accept those forms on the
command-line.
---
lib/PublicInbox/LeiExportKw.pm | 41 ++++++++++++++++++++++++++++++++++
xt/net_writer-imap.t | 30 ++++++++++++++-----------
2 files changed, 58 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 5ad33959..82a4db04 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -84,6 +84,37 @@ sub input_path_url {
$lms->lms_commit;
}
+sub match_imap_url ($$) {
+ my ($all, $url) = @_; # $all = [ $lms->folders ];
+ require PublicInbox::URIimap;
+ my $cli = PublicInbox::URIimap->new($url)->canonical;
+ my ($s, $h, $mb) = ($cli->scheme, $cli->host, $cli->mailbox);
+ my @uri = map { PublicInbox::URIimap->new($_)->canonical }
+ grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$mb\E\b!, @$all);
+ my @match;
+ for my $x (@uri) {
+ next if $x->mailbox ne $cli->mailbox;
+ next if $x->host ne $cli->host;
+ next if $x->port != $cli->port;
+ my $x_uidval = $x->uidvalidity;
+ next if ($cli->uidvalidity // $x_uidval) != $x_uidval;
+
+ # allow nothing in CLI to possibly match ";AUTH=ANONYMOUS"
+ if (defined($x->auth) && !defined($cli->auth) &&
+ !defined($cli->user)) {
+ push @match, $x;
+ # or maybe user was forgotten on CLI:
+ } elsif (defined($x->user) && !defined($cli->user)) {
+ push @match, $x;
+ } elsif (($x->user//"\0") eq ($cli->user//"\0")) {
+ push @match, $x;
+ }
+ }
+ return $match[0] if scalar(@match) <= 1;
+ warn "E: `$url' is ambiguous:\n\t", join("\n\t", @match), "\n";
+ undef;
+}
+
sub lei_export_kw {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
@@ -133,6 +164,16 @@ EOM
my $d = 'maildir:'.$lei->rel2abs($_);
push(@no, $_) unless $all{$d};
$_ = $d;
+ } elsif (m!\Aimaps?://!i) {
+ my $orig = $_;
+ if (my $canon = match_imap_url(\@all, $orig)) {
+ $lei->qerr(<<EOM);
+# using `$canon' instead of `$orig'
+EOM
+ $_ = $canon;
+ } else {
+ push @no, $orig;
+ }
} else {
push @no, $_;
}
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 0e6d4831..ec8f80d1 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -23,7 +23,8 @@ my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)).
"-$$-".sprintf('%x', int(rand(0xffffffff)));
my $nwr = PublicInbox::NetWriter->new;
chop($imap_url) if substr($imap_url, -1) eq '/';
-my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder");
+my $folder_url = "$imap_url/$folder";
+my $folder_uri = PublicInbox::URIimap->new($folder_url);
is($folder_uri->mailbox, $folder, 'folder correct') or
BAIL_OUT "BUG: bad $$uri";
$nwr->add_url($$folder_uri);
@@ -120,13 +121,13 @@ test_lei(sub {
}
$set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) if $cred_set;
- lei_ok qw(q f:qp@example.com -o), $$folder_uri;
+ lei_ok qw(q f:qp@example.com -o), $folder_url;
$nwr->imap_each($folder_uri, $imap_slurp_all, my $res = []);
is(scalar(@$res), 1, 'got one deduped result') or diag explain($res);
is_deeply($res->[0]->[1], $plack_qp_eml,
'lei q wrote expected result');
- lei_ok qw(q f:matz -a -o), $$folder_uri;
+ lei_ok qw(q f:matz -a -o), $folder_url;
$nwr->imap_each($folder_uri, $imap_slurp_all, my $aug = []);
is(scalar(@$aug), 2, '2 results after augment') or diag explain($aug);
my $exp = $res->[0]->[1]->as_string;
@@ -136,13 +137,13 @@ test_lei(sub {
is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1,
'new result shown after augment');
- lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string;
+ lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_url;
$nwr->imap_each($folder_uri, $imap_slurp_all, my $empty = []);
is(scalar(@$empty), 0, 'no results w/o augment');
my $f = 't/utf8.eml'; # <testmessage@example.com>
$exp = eml_load($f);
- lei_ok qw(convert -F eml -o), $$folder_uri, $f;
+ lei_ok qw(convert -F eml -o), $folder_url, $f;
my (@uid, @res);
$nwr->imap_each($folder_uri, sub {
my ($u, $uid, $kw, $eml) = @_;
@@ -157,12 +158,15 @@ test_lei(sub {
lei_ok qw(import -F eml), $f, \'import local copy w/o keywords';
- lei_ok 'ls-mail-sync'; diag $lei_out;
- lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3
+ lei_ok 'import', $folder_url; # populate mail_sync.sqlite3
lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f;
- lei_ok 'ls-mail-sync'; diag $lei_out;
- chomp(my $uri_val = $lei_out);
- lei_ok 'export-kw', $uri_val;
+ lei_ok 'ls-mail-sync';
+ my @ls = split(/\n/, $lei_out);
+ is(scalar(@ls), 1, 'only one folder in ls-mail-sync') or xbail(\@ls);
+ for my $l (@ls) {
+ like($l, qr/;UIDVALIDITY=\d+\z/, 'UIDVALIDITY');
+ }
+ lei_ok 'export-kw', $folder_url;
$mic = $nwr->mic_for_folder($folder_uri);
my $flags = $mic->flags($uid[0]);
is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ],
@@ -177,7 +181,7 @@ test_lei(sub {
is_deeply(\@res, [ [ ['seen'], $exp ] ], 'seen flag set') or
diag explain(\@res);
- lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string,
+ lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_url,
\'clobber folder but import flag';
$nwr->imap_each($folder_uri, $imap_slurp_all, $empty = []);
is_deeply($empty, [], 'clobbered folder');
@@ -206,7 +210,7 @@ EOM
run_script(\@cmd) or BAIL_OUT "init wtest";
xsys(qw(git config), "--file=$ENV{HOME}/.public-inbox/config",
'publicinbox.wtest.watch',
- $$folder_uri) == 0 or BAIL_OUT "git config $?";
+ $folder_url) == 0 or BAIL_OUT "git config $?";
my $watcherr = "$ENV{HOME}/watch.err";
open my $err_wr, '>>', $watcherr or BAIL_OUT $!;
my $pub_cfg = PublicInbox::Config->new;
@@ -231,7 +235,7 @@ EOM
ok(defined($mm->num_for('forwarded@test.example.com')),
'-watch takes forwarded message');
undef $w; # done with watch
- lei_ok qw(import), $$folder_uri;
+ lei_ok qw(import), $folder_url;
lei_ok qw(q m:forwarded@test.example.com);
is_deeply(json_utf8->decode($lei_out)->[0]->{kw}, ['forwarded'],
'forwarded kw imported from IMAP');
^ permalink raw reply related [relevance 47%]
* [PATCH] lei <q|up>: set \Recent on non-empty mbox and Maildir
@ 2021-05-23 8:01 35% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 8:01 UTC (permalink / raw)
To: meta
Despite JMAP not supporting the equivalent of the IMAP \Recent
flag, it is useful for "lei q --augment", and "lei up" users to
be able to distinguish new results from old-but-unread messages
in an mbox or Maildir.
For mbox family messages, we'll drop the "O" status flag when
appending to mboxes, and we'll write to the "new" subdirectory
of Maildirs.
Behavior when writing to initially empty Maildirs and mboxes
remains unchanged since there's no need to distinguish between
new and old results in the initial case. Having users wait
for a rename(2) storm or complete mbox rewrite hurts UX.
With IMAP mailboxes, \Recent is already enforced by the IMAP
server and IMAP clients have no way of changing it(*)
(*) mutt uses the "Old" IMAP flag which isn't part of RFC 3501,
other MUAs may do similar things.
---
lib/PublicInbox/LeiDedupe.pm | 6 +++++
lib/PublicInbox/LeiSavedSearch.pm | 7 ++++++
lib/PublicInbox/LeiToMail.pm | 42 +++++++++++++++++++++----------
lib/PublicInbox/LeiXSearch.pm | 15 +++++++++--
t/lei-q-kw.t | 13 ++++++++--
t/lei-q-save.t | 4 +--
t/lei_to_mail.t | 16 ++++++------
7 files changed, 75 insertions(+), 28 deletions(-)
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index 378f748e..ed52e417 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -127,4 +127,10 @@ sub pause_dedupe {
delete($skv->{dbh}) if $skv;
}
+sub dedupe_nr {
+ my $skv = $_[0]->[0] or return undef;
+ my @n = $skv->count;
+ $n[0];
+}
+
1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 01b987d1..48d252f1 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -309,6 +309,13 @@ E: rename($dir_old, $dir_new) error: $!
EOM
}
+# cf. LeiDedupe->dedupe_nr
+sub dedupe_nr {
+ my $oidx = $_[0]->{oidx} // die 'BUG: no {oidx}';
+ my @n = $oidx->{dbh}->selectrow_array('SELECT COUNT(*) FROM over');
+ $n[0];
+}
+
no warnings 'once';
*nntp_url = \&cloneurl;
*base_url = \&PublicInbox::Inbox::base_url;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index f3c03969..ad6b9439 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -41,11 +41,14 @@ sub _mbox_hdr_buf ($$$) {
warn "# keyword `$k' not supported for mbox\n";
}
}
- # Messages are always 'O' (non-\Recent in IMAP), it saves
- # MUAs the trouble of rewriting the mbox if no other
- # changes are made. We put 'O' at the end (e.g. "Status: RO")
- # to match mutt(1) output.
- $eml->header_set('Status', join('', sort(@{$hdr{Status}})). 'O');
+ # When writing to empty mboxes, messages are always 'O'
+ # (not-\Recent in IMAP), it saves MUAs the trouble of
+ # rewriting the mbox if no other changes are made.
+ # We put 'O' at the end (e.g. "Status: RO") to match mutt(1) output.
+ # We only set smsg->{-recent} if augmenting existing stores.
+ my $status = join('', sort(@{$hdr{Status}}));
+ $status .= 'O' unless $smsg->{-recent};
+ $eml->header_set('Status', $status) if $status;
if (my $chars = delete $hdr{'X-Status'}) {
$eml->header_set('X-Status', join('', sort(@$chars)));
}
@@ -196,11 +199,13 @@ sub _mbox_write_cb ($$) {
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe;
my $lse = $lei->{lse}; # may be undef
+ my $set_recent = $dedupe->dedupe_nr;
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($buf);
return if $dedupe->is_dup($eml, $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
+ $smsg->{-recent} = 1 if $set_recent;
$buf = $eml2mbox->($eml, $smsg);
if ($atomic_append) {
atomic_append($lei, $buf);
@@ -248,8 +253,8 @@ sub kw2suffix ($;@) {
join('', sort(map { $kw2char{$_} // () } @$kw, @_));
}
-sub _buf2maildir {
- my ($dst, $buf, $smsg) = @_;
+sub _buf2maildir ($$$$) {
+ my ($dst, $buf, $smsg, $dir) = @_;
my $kw = $smsg->{kw} // [];
my $rand = ''; # chosen by die roll :P
my ($tmp, $fh, $base, $ok);
@@ -260,11 +265,7 @@ sub _buf2maildir {
} while (!($ok = sysopen($fh, $tmp, O_CREAT|O_EXCL|O_WRONLY)) &&
$!{EEXIST} && ($rand = _rand.','));
if ($ok && print $fh $$buf and close($fh)) {
- # ignore new/ and write only to cur/, otherwise MUAs
- # with R/W access to the Maildir will end up doing
- # a mass rename which can take a while with thousands
- # of messages.
- $dst .= 'cur/';
+ $dst .= $dir; # 'new/' or 'cur/'
$rand = '';
do {
$base = $rand.$common.':2,'.kw2suffix($kw);
@@ -289,6 +290,11 @@ sub _maildir_write_cb ($$) {
my $lse = $lei->{lse}; # may be undef
my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
my $out = $sto ? 'maildir:'.$lei->rel2abs($dst) : undef;
+
+ # Favor cur/ and only write to new/ when augmenting. This
+ # saves MUAs from having to do a mass rename when the initial
+ # search result set is huge.
+ my $dir = $dedupe && $dedupe->dedupe_nr ? 'new/' : 'cur/';
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
@@ -296,7 +302,8 @@ sub _maildir_write_cb ($$) {
PublicInbox::Eml->new($$bref),
$smsg);
$lse->xsmsg_vmd($smsg) if $lse;
- my $n = _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+ my $n = _buf2maildir($dst, $bref // \($eml->as_string),
+ $smsg, $dir);
$sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
++$lei->{-nr_write};
}
@@ -648,7 +655,16 @@ sub do_post_auth {
$lei->{1} = $zpipe->[1];
close $zpipe->[0];
}
+ my $au_peers = delete $self->{au_peers};
+ if ($au_peers) { # wait for peer l2m to finish augmenting:
+ $au_peers->[1] = undef;
+ sysread($au_peers->[0], my $barrier1, 1);
+ }
$self->{wcb} = $self->write_cb($lei);
+ if ($au_peers) { # wait for peer l2m to set write_cb
+ $au_peers->[3] = undef;
+ sysread($au_peers->[2], my $barrier2, 1);
+ }
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index e69a4edd..3482082d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -482,11 +482,22 @@ sub do_query {
if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {
$lei->start_mua;
}
+ my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
+ if ($l2m->{-wq_nr_workers} > 1 &&
+ $l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
+ # setup two barriers to coordinate dedupe_nr
+ # between l2m workers
+ pipe(my ($a_r, $a_w)) or die "pipe: $!";
+ fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
+ pipe(my ($b_r, $b_w)) or die "pipe: $!";
+ fcntl($b_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
+ $l2m->{au_peers} = [ $a_r, $a_w, $b_r, $b_w ];
+ }
$l2m->wq_workers_start('lei2mail', undef,
$lei->oldset, { lei => $lei });
pipe($lei->{startq}, $lei->{au_done}) or die "pipe: $!";
- # 1031: F_SETPIPE_SZ
- fcntl($lei->{startq}, 1031, 4096) if $^O eq 'linux';
+ fcntl($lei->{startq}, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
+ delete $l2m->{au_peers};
}
$self->wq_workers_start('lei_xsearch', undef,
$lei->oldset, { lei => $lei });
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index c00a0a43..074c573d 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -14,7 +14,6 @@ my $exp = {
'<testmessage@example.com>' => eml_load('t/utf8.eml'),
};
$exp->{'<qp@example.com>'}->header_set('Status', 'RO');
-$exp->{'<testmessage@example.com>'}->header_set('Status', 'O');
test_lei(sub {
lei_ok(qw(import -F eml t/plack-qp.eml));
@@ -105,7 +104,17 @@ for my $sfx ('', '.gz') {
my %res;
PublicInbox::MboxReader->mboxrd($fh, sub {
my ($eml) = @_;
- $res{$eml->header_raw('Message-ID')} = $eml;
+ my $mid = $eml->header_raw('Message-ID');
+ if ($mid eq '<testmessage@example.com>') {
+ is_deeply([$eml->header('Status')], [],
+ "no status $sfx");
+ $eml->header_set('Status');
+ } elsif ($mid eq '<qp@example.com>') {
+ is($eml->header('Status'), 'RO', 'status preserved');
+ } else {
+ fail("unknown mid $mid");
+ }
+ $res{$mid} = $eml;
});
is_deeply(\%res, $exp, '--augment worked');
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 753d5b20..aed38a51 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -42,7 +42,7 @@ test_lei(sub {
lei_ok qw(up -q md -C), $home;
lei_ok qw(up -q . -C), "$home/md";
lei_ok qw(up -q), "/$home/md";
- my %after = map { $_ => 1 } glob("$home/md/cur/*");
+ my %after = map { $_ => 1 } glob("$home/md/{new,cur}/*");
is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
is(scalar(keys %after), 1, 'one new message added');
is_deeply(eml_load((keys %after)[0]), $doc2, 'doc2 matches');
@@ -155,7 +155,7 @@ test_lei(sub {
$im->add(PublicInbox::Eml->new($diff));
$im->done;
lei_ok('up', $o);
- @m = glob("$o/cur/*");
+ @m = glob("$o/{new,cur}/*");
is(scalar(@m), 2, 'got 2nd result due to different OID');
SKIP: {
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 32532a98..35904706 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -90,7 +90,7 @@ my $fn = "$tmpdir/x.mbox";
my ($mbox) = shuffle(@MBOX); # pick one, shouldn't matter
my $wcb_get = sub {
my ($fmt, $dst) = @_;
- delete $lei->{dedupe};
+ delete $lei->{dedupe}; # to be recreated
$lei->{ovv} = bless {
fmt => $fmt,
dst => $dst
@@ -119,13 +119,12 @@ my $orig = do {
like($raw, qr/^blah\n/sm, 'wrote content');
unlink $fn or BAIL_OUT $!;
- local $lei->{opt} = { jobs => 2 };
$wcb = $wcb_get->($mbox, $fn);
ok(-f $fn && !-s _, 'truncated mbox destination');
$wcb->(\($dup = $buf), $deadbeef);
$commit->($wcb);
open $fh, '<', $fn or BAIL_OUT $!;
- is(do { local $/; <$fh> }, $raw, 'jobs > 1');
+ is(do { local $/; <$fh> }, $raw, 'wrote identical content');
$raw;
};
@@ -158,21 +157,20 @@ for my $zsfx (qw(gz bz2 xz)) {
ok($dc_cmd, "decompressor for .$zsfx");
my $f = "$fn.$zsfx";
my $wcb = $wcb_get->($mbox, $f);
- $wcb->(\(my $dup = $buf), $deadbeef);
+ $wcb->(\(my $dup = $buf), { %$deadbeef });
$commit->($wcb);
my $uncompressed = xqx([@$dc_cmd, $f]);
is($uncompressed, $orig, "$zsfx works unlocked");
- local $lei->{opt} = { jobs => 2 }; # for atomic writes
unlink $f or BAIL_OUT "unlink $!";
$wcb = $wcb_get->($mbox, $f);
- $wcb->(\($dup = $buf), $deadbeef);
+ $wcb->(\($dup = $buf), { %$deadbeef });
$commit->($wcb);
is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock");
local $lei->{opt} = { augment => 1 };
$wcb = $wcb_get->($mbox, $f);
- $wcb->(\($dup = $buf . "\nx\n"), $deadbeef);
+ $wcb->(\($dup = $buf . "\nx\n"), { %$deadbeef });
$commit->($wcb);
my $cat = popen_rd([@$dc_cmd, $f]);
@@ -182,9 +180,9 @@ for my $zsfx (qw(gz bz2 xz)) {
like($raw[1], qr/\nblah\n\nx\n\z/s, "augmented $zsfx");
like($raw[0], qr/\nblah\n\z/s, "original preserved $zsfx");
- local $lei->{opt} = { augment => 1, jobs => 2 };
+ local $lei->{opt} = { augment => 1 };
$wcb = $wcb_get->($mbox, $f);
- $wcb->(\($dup = $buf . "\ny\n"), $deadbeef);
+ $wcb->(\($dup = $buf . "\ny\n"), { %$deadbeef });
$commit->($wcb);
my @raw3;
^ permalink raw reply related [relevance 35%]
* [PATCH 0/2] lei IMAP usability stuff
@ 2021-05-23 21:36 71% Eric Wong
2021-05-23 21:36 46% ` [PATCH 1/2] lei inspect: use LeiMailSync->match_imap_url Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-23 21:36 UTC (permalink / raw)
To: meta
Slowly making IMAP usability less bad.
Eric Wong (2):
lei inspect: use LeiMailSync->match_imap_url
lei_mail_sync: reject IMAP URLs w/o UIDVALIDITY
lib/PublicInbox/LeiExportKw.pm | 43 ++++++----------------------------
lib/PublicInbox/LeiInspect.pm | 13 +---------
lib/PublicInbox/LeiMailSync.pm | 37 +++++++++++++++++++++++++++++
t/lei-import-imap.t | 10 ++++++--
4 files changed, 53 insertions(+), 50 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei inspect: use LeiMailSync->match_imap_url
2021-05-23 21:36 71% [PATCH 0/2] lei IMAP usability stuff Eric Wong
@ 2021-05-23 21:36 46% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 21:36 UTC (permalink / raw)
To: meta
Move match_imap_url into LeiMailSync so it can be used in more
places, such as "lei inspect". Upcoming commands such as
"lei forget-mail-sync" and {add,forget,pause,resume}-watch will
also support relaxed IMAP matching rules since there's
no reasonable way to expect users use ";UIDVALIDITY=" on the
command-line.
---
lib/PublicInbox/LeiExportKw.pm | 43 ++++++----------------------------
lib/PublicInbox/LeiInspect.pm | 13 +---------
lib/PublicInbox/LeiMailSync.pm | 32 +++++++++++++++++++++++++
t/lei-import-imap.t | 10 ++++++--
4 files changed, 48 insertions(+), 50 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 82a4db04..404570c5 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -75,7 +75,7 @@ sub input_path_url {
my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
$lms->each_src($input, \&export_kw_md, $self, $mdir);
- } elsif ($input =~ m!\Aimaps?://!) {
+ } elsif ($input =~ m!\Aimaps?://i!) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{nwr}->mic_for_folder($uri);
$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
@@ -84,37 +84,6 @@ sub input_path_url {
$lms->lms_commit;
}
-sub match_imap_url ($$) {
- my ($all, $url) = @_; # $all = [ $lms->folders ];
- require PublicInbox::URIimap;
- my $cli = PublicInbox::URIimap->new($url)->canonical;
- my ($s, $h, $mb) = ($cli->scheme, $cli->host, $cli->mailbox);
- my @uri = map { PublicInbox::URIimap->new($_)->canonical }
- grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$mb\E\b!, @$all);
- my @match;
- for my $x (@uri) {
- next if $x->mailbox ne $cli->mailbox;
- next if $x->host ne $cli->host;
- next if $x->port != $cli->port;
- my $x_uidval = $x->uidvalidity;
- next if ($cli->uidvalidity // $x_uidval) != $x_uidval;
-
- # allow nothing in CLI to possibly match ";AUTH=ANONYMOUS"
- if (defined($x->auth) && !defined($cli->auth) &&
- !defined($cli->user)) {
- push @match, $x;
- # or maybe user was forgotten on CLI:
- } elsif (defined($x->user) && !defined($cli->user)) {
- push @match, $x;
- } elsif (($x->user//"\0") eq ($cli->user//"\0")) {
- push @match, $x;
- }
- }
- return $match[0] if scalar(@match) <= 1;
- warn "E: `$url' is ambiguous:\n\t", join("\n\t", @match), "\n";
- undef;
-}
-
sub lei_export_kw {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
@@ -166,12 +135,14 @@ EOM
$_ = $d;
} elsif (m!\Aimaps?://!i) {
my $orig = $_;
- if (my $canon = match_imap_url(\@all, $orig)) {
+ my $res = $lms->match_imap_url($orig, $all);
+ if (ref $res) {
+ $_ = $$res;
$lei->qerr(<<EOM);
-# using `$canon' instead of `$orig'
+# using `$res' instead of `$orig'
EOM
- $_ = $canon;
} else {
+ $lei->err($res) if defined $res;
push @no, $orig;
}
} else {
@@ -188,7 +159,7 @@ EOF
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $opt->{jobs} // scalar(@{$self->{inputs}}) || 1;
- if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
+ if (my @ro = grep(!/\A(?:maildir|imaps?):/i, @folders)) {
return $lei->fail("cannot export to read-only folders: @ro");
}
my $m = $opt->{mode} // 'merge';
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index f79ebc9a..7fd33289 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -31,18 +31,7 @@ sub inspect_sync_folder ($$) {
my $lms = $lse->lms or return $ent;
my @folders;
if ($folder =~ m!\Aimaps?://!i) {
- require PublicInbox::URIimap;
- my $uri = PublicInbox::URIimap->new($folder)->canonical;
- if (defined($uri->uidvalidity)) {
- $folders[0] = $$uri;
- } else {
- my @maybe = $lms->folders($$uri);
- @folders = grep {
- my $u = PublicInbox::URIimap->new($_);
- $uri->uidvalidity($u->uidvalidity);
- $$uri eq $$u;
- } @maybe;
- }
+ @folders = map { $_->as_string } $lms->match_imap_url($folder);
} elsif ($folder =~ m!\A(maildir|mh):(.+)!i) {
my $type = lc $1;
$folders[0] = "$type:".$lei->abs_path($2);
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 32e17c65..b2986686 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -265,4 +265,36 @@ WHERE b.oidbin = ?
undef;
}
+sub match_imap_url {
+ my ($self, $url, $all) = @_; # $all = [ $lms->folders ];
+ $all //= [ $self->folders ];
+ require PublicInbox::URIimap;
+ my $want = PublicInbox::URIimap->new($url)->canonical;
+ my ($s, $h, $mb) = ($want->scheme, $want->host, $want->mailbox);
+ my @uri = map { PublicInbox::URIimap->new($_)->canonical }
+ grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$mb\E\b!, @$all);
+ my @match;
+ for my $x (@uri) {
+ next if $x->mailbox ne $want->mailbox;
+ next if $x->host ne $want->host;
+ next if $x->port != $want->port;
+ my $x_uidval = $x->uidvalidity;
+ next if ($want->uidvalidity // $x_uidval) != $x_uidval;
+
+ # allow nothing in want to possibly match ";AUTH=ANONYMOUS"
+ if (defined($x->auth) && !defined($want->auth) &&
+ !defined($want->user)) {
+ push @match, $x;
+ # or maybe user was forgotten on CLI:
+ } elsif (defined($x->user) && !defined($want->user)) {
+ push @match, $x;
+ } elsif (($x->user//"\0") eq ($want->user//"\0")) {
+ push @match, $x;
+ }
+ }
+ return @match if wantarray;
+ scalar(@match) <= 1 ? $match[0] :
+ "E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
+}
+
1;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index d424ebb1..d3935c82 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -22,9 +22,15 @@ test_lei({ tmpdir => $tmpdir }, sub {
is_deeply(json_utf8->decode($lei_out), {}, 'no inspect stats, yet');
lei_ok('import', $url);
+ lei_ok('inspect', $url);
+ my $res = json_utf8->decode($lei_out);
+ is(scalar keys %$res, 1, 'got one key in inspect URL');
+ my $re = qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E
+ /t\.v2\.0;UIDVALIDITY=\d+!x;
+ like((keys %$res)[0], qr/$re\z/, 'got expanded key');
+
lei_ok 'ls-mail-sync';
- like($lei_out, qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E
- /t\.v2\.0;UIDVALIDITY=\d+\n\z!x, 'ls-mail-sync');
+ like($lei_out, qr!$re\n\z!, 'ls-mail-sync');
chomp(my $u = $lei_out);
lei_ok('import', $u, \'UIDVALIDITY match in URL');
$url = $u;
^ permalink raw reply related [relevance 46%]
* [PATCH] lei export-kw: fix case-insensitivity typo for IMAP URLs
@ 2021-05-24 22:48 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-24 22:48 UTC (permalink / raw)
To: meta
Fixes: a9bfba8eb31dbb4c ("lei inspect: use LeiMailSync->match_imap_url")
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 404570c5..d8ba8fd5 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -75,7 +75,7 @@ sub input_path_url {
my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
$lms->each_src($input, \&export_kw_md, $self, $mdir);
- } elsif ($input =~ m!\Aimaps?://i!) {
+ } elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{nwr}->mic_for_folder($uri);
$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
^ permalink raw reply related [relevance 71%]
* [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs
2021-05-25 11:01 71% [PATCH 0/3] lei forget-mail-sync: drop sync information Eric Wong
@ 2021-05-25 11:01 90% ` Eric Wong
2021-05-25 11:01 55% ` [PATCH 3/3] lei forget-mail-sync: new command to drop sync information Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-25 11:01 UTC (permalink / raw)
To: meta
This is to match LeiInput behavior when mail-sync is
enabled, since rel2abs won't resolve symlinks.
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d8ba8fd5..fabc01f8 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -130,7 +130,7 @@ EOM
for (@folders) {
next if $all{$_}; # ok
if (-d "$_/new" && -d "$_/cur") {
- my $d = 'maildir:'.$lei->rel2abs($_);
+ my $d = 'maildir:'.$lei->abs_path($_);
push(@no, $_) unless $all{$d};
$_ = $d;
} elsif (m!\Aimaps?://!i) {
^ permalink raw reply related [relevance 90%]
* [PATCH 0/3] lei forget-mail-sync: drop sync information
@ 2021-05-25 11:01 71% Eric Wong
2021-05-25 11:01 90% ` [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs Eric Wong
2021-05-25 11:01 55% ` [PATCH 3/3] lei forget-mail-sync: new command to drop sync information Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-25 11:01 UTC (permalink / raw)
To: meta
Sometimes users lose interest in a Maildir or IMAP folder, etc;
this drops sync information to not clutter up disk space,
"ls-mail-sync" output, etc.
Eric Wong (3):
lei export-kw: use lei->abs_path instead of rel2abs
lei_mail_sync: args2folder: common folder lookup sub
lei forget-mail-sync: new command to drop sync information
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 3 +-
lib/PublicInbox/LeiExportKw.pm | 34 +++---------------
lib/PublicInbox/LeiForgetMailSync.pm | 30 ++++++++++++++++
lib/PublicInbox/LeiInspect.pm | 21 +++++------
lib/PublicInbox/LeiMailSync.pm | 54 ++++++++++++++++++++++++++++
t/lei-import-imap.t | 4 +++
7 files changed, 104 insertions(+), 43 deletions(-)
create mode 100644 lib/PublicInbox/LeiForgetMailSync.pm
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei forget-mail-sync: new command to drop sync information
2021-05-25 11:01 71% [PATCH 0/3] lei forget-mail-sync: drop sync information Eric Wong
2021-05-25 11:01 90% ` [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs Eric Wong
@ 2021-05-25 11:01 55% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-25 11:01 UTC (permalink / raw)
To: meta
Sometimes a user stops caring to sync an IMAP or Maildir
folder, or wants to force a resync. Let them run this
command to have lei forget all the sync information about
the mail folder.
This won't delete any stored messages in git, but will
leave "lei index" users with dangling references.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 3 ++-
lib/PublicInbox/LeiForgetMailSync.pm | 30 ++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 11 ++++++++++
t/lei-import-imap.t | 4 ++++
5 files changed, 48 insertions(+), 1 deletion(-)
create mode 100644 lib/PublicInbox/LeiForgetMailSync.pm
diff --git a/MANIFEST b/MANIFEST
index 2d1ad5c3..23423e0b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -204,6 +204,7 @@ lib/PublicInbox/LeiDedupe.pm
lib/PublicInbox/LeiEditSearch.pm
lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
+lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 628908b5..c8d2f315 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -243,7 +243,8 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| offset=i recursive|r exclude=s include|I=s
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
-
+'forget-mail-sync' => [ 'LOCATION...',
+ 'forget sync information for a mail folder', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
qw(all:s mode=s), @c_opt ],
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
new file mode 100644
index 00000000..46dde1a7
--- /dev/null
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -0,0 +1,30 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei forget-mail-sync" drop synchronization information
+# TODO: figure out what to do about "lei index" users having
+# dangling references. Perhaps just documenting "lei index"
+# use being incompatible with "forget-mail-sync" use is
+# sufficient.
+
+package PublicInbox::LeiForgetMailSync;
+use strict;
+use v5.10.1;
+use PublicInbox::LeiExportKw;
+
+sub lei_forget_mail_sync {
+ my ($lei, @folders) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
+ delete $lms->{dbh};
+ $lms->lms_begin;
+ $lms->forget_folder($_) for @folders;
+ $lms->lms_commit;
+}
+
+*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 094cf1fd..d9c30580 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -345,4 +345,15 @@ EOF
$err;
}
+sub forget_folder {
+ my ($self, $folder) = @_;
+ my ($fid, $sth);
+ $fid = delete($self->{fmap}->{$folder}) //
+ _fid_for($self, $folder) // return;
+ my $dbh = $self->{dbh};
+ $dbh->do('DELETE FROM blob2name WHERE fid = ?', undef, $fid);
+ $dbh->do('DELETE FROM blob2num WHERE fid = ?', undef, $fid);
+ $dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
+}
+
1;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index d3935c82..5283cc23 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -71,5 +71,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
like($lei_out, qr!^Content-Type: multipart/mixed;!sm, 'got full blob');
lei_ok('blob', "$psgi_attach:2");
is($lei_out, "b64\xde\xad\xbe\xef\n", 'got attachment');
+
+ lei_ok 'forget-mail-sync', $url;
+ lei_ok 'ls-mail-sync';
+ unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
});
done_testing;
^ permalink raw reply related [relevance 55%]
* [PATCH 0/2] ipc: fix "lei q" w/ HTTP(S) externals
@ 2021-05-25 22:19 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-25 22:19 UTC (permalink / raw)
To: meta
1/2 was found while making 2/2
Eric Wong (2):
ipc: avoid potential stack-not-refcounted bug
ipc: wq: handle >MAX_ARG_STRLEN && <EMSGSIZE case
lib/PublicInbox/IPC.pm | 47 +++++++++++++++++++++++--------------
lib/PublicInbox/WQWorker.pm | 2 +-
t/ipc.t | 11 ++++++---
3 files changed, 38 insertions(+), 22 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH] lei: require Socket::MsgHdr or Inline::C, drop oneshot
@ 2021-05-26 18:08 28% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-26 18:08 UTC (permalink / raw)
To: meta
The cost of supporting separate code paths between oneshot and
daemon isn't worth the trouble; especially if there are more
users to support. The test suite time nearly doubles with
oneshot, so that's hurting developer productivity.
FD passing is currently required to work efficiently with
remote HTTP(S) queries which return large messages, as seen in
commit 708b182a57373172f5523f3dc297659d58e03b58
("ipc: wq: handle >MAX_ARG_STRLEN && <EMSGSIZE case").
Additionally, upcoming support for IMAP IDLE and inotify-based
monitoring of Maildirs cannot work properly without a background
daemon.
---
lib/PublicInbox/GitCredential.pm | 2 +-
lib/PublicInbox/LEI.pm | 58 +------------
lib/PublicInbox/LeiEditSearch.pm | 19 ++---
lib/PublicInbox/LeiStore.pm | 13 +--
lib/PublicInbox/LeiSucks.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 26 +++---
lib/PublicInbox/TestCommon.pm | 15 +---
script/lei | 137 +++++++++++++++----------------
t/lei-daemon.t | 9 --
9 files changed, 97 insertions(+), 185 deletions(-)
diff --git a/lib/PublicInbox/GitCredential.pm b/lib/PublicInbox/GitCredential.pm
index 2d81817c..b29780d6 100644
--- a/lib/PublicInbox/GitCredential.pm
+++ b/lib/PublicInbox/GitCredential.pm
@@ -9,7 +9,7 @@ sub run ($$;$) {
my ($in_r, $in_w, $out_r);
my $cmd = [ qw(git credential), $op ];
pipe($in_r, $in_w) or die "pipe: $!";
- if ($lei && !$lei->{oneshot}) { # we'll die if disconnected:
+ if ($lei) { # we'll die if disconnected:
pipe($out_r, my $out_w) or die "pipe: $!";
$lei->send_exec_cmd([ $in_r, $out_w ], $cmd, {});
} else {
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index c8d2f315..6ff249d0 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -444,19 +444,6 @@ sub x_it ($$) {
dump_and_clear_log();
if (my $s = $self->{pkt_op_p} // $self->{sock}) {
send($s, "x_it $code", MSG_EOR);
- } elsif ($self->{oneshot}) {
- # don't want to end up using $? from child processes
- _drop_wq($self);
- # cleanup anything that has tempfiles or open file handles
- %PATH2CFG = ();
- delete @$self{qw(ovv dedupe sto cfg)};
- if (my $signum = ($code & 127)) { # usually SIGPIPE (13)
- $SIG{PIPE} = 'DEFAULT'; # $SIG{$signum} doesn't work
- kill $signum, $$;
- sleep(1) while 1; # wait for signal
- } else {
- $quit->($code >> 8);
- }
} # else ignore if client disconnected
}
@@ -921,17 +908,6 @@ sub start_mua {
my $io = [];
$io->[0] = $self->{1} if $self->{opt}->{stdin} && -t $self->{1};
send_exec_cmd($self, $io, \@cmd, {});
- } elsif ($self->{oneshot}) {
- my $pid = fork // die "fork: $!";
- if ($pid > 0) { # original process
- if ($self->{opt}->{stdin} && -t STDOUT) {
- open STDIN, '+<&', \*STDOUT or die "dup2: $!";
- }
- exec(@cmd);
- warn "exec @cmd: $!\n";
- POSIX::_exit(1);
- }
- POSIX::setsid() > 0 or die "setsid: $!";
}
if ($self->{lxs} && $self->{au_done}) { # kick wait_startq
syswrite($self->{au_done}, 'q' x ($self->{lxs}->{jobs} // 0));
@@ -952,14 +928,11 @@ sub send_exec_cmd { # tell script/lei to execute a command
sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
my ($self) = @_;
my $alerts = $self->{opt}->{alert} // return;
+ my $sock = $self->{sock};
while (my $op = shift(@$alerts)) {
if ($op eq ':WINCH') {
# hit the process group that started the MUA
- if ($self->{sock}) {
- send($self->{sock}, '-WINCH', MSG_EOR);
- } elsif ($self->{oneshot}) {
- kill('-WINCH', $$);
- }
+ send($sock, '-WINCH', MSG_EOR) if $sock;
} elsif ($op eq ':bell') {
out($self, "\a");
} elsif ($op =~ /(?<!\\),/) { # bare ',' (not ',,')
@@ -968,11 +941,7 @@ sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
my $cmd = $1; # run an arbitrary command
require Text::ParseWords;
$cmd = [ Text::ParseWords::shellwords($cmd) ];
- if (my $s = $self->{sock}) {
- send($s, exec_buf($cmd, {}), MSG_EOR);
- } elsif ($self->{oneshot}) {
- $self->{"pid.$self.$$"}->{spawn($cmd)} = $cmd;
- }
+ send($sock, exec_buf($cmd, {}), MSG_EOR) if $sock;
} else {
err($self, "W: unsupported --alert=$op"); # non-fatal
}
@@ -1009,9 +978,6 @@ sub start_pager {
if ($self->{sock}) { # lei(1) process runs it
delete @$new_env{keys %$env}; # only set iff unset
send_exec_cmd($self, [ @$rdr{0..2} ], [$pager], $new_env);
- } elsif ($self->{oneshot}) {
- my $cmd = [$pager];
- $self->{"pid.$self.$$"}->{spawn($cmd, $new_env, $rdr)} = $cmd;
} else {
die 'BUG: start_pager w/o socket';
}
@@ -1253,29 +1219,13 @@ sub lazy_start {
sub busy { 1 } # prevent daemon-shutdown if client is connected
-# for users w/o Socket::Msghdr installed or Inline::C enabled
-sub oneshot {
- my ($main_pkg) = @_;
- my $exit = $main_pkg->can('exit'); # caller may override exit()
- local $quit = $exit if $exit;
- local %PATH2CFG;
- umask(077) // die("umask(077): $!");
- my $self = bless { oneshot => 1, env => \%ENV }, __PACKAGE__;
- for (0..2) { open($self->{$_}, '+<&=', $_) or die "open fd=$_: $!" }
- dispatch($self, @ARGV);
- x_it($self, $self->{child_error}) if $self->{child_error};
-}
-
# ensures stdout hits the FS before sock disconnects so a client
# can immediately reread it
sub DESTROY {
my ($self) = @_;
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
- my $err = $?;
- my $oneshot_pids = delete $self->{"pid.$self.$$"} or return;
- waitpid($_, 0) for keys %$oneshot_pids;
- $? = $err if $err; # preserve ->fail or ->x_it code
+ # preserve $? for ->fail or ->x_it code
}
sub wq_done_wait { # dwaitpid callback
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 30ac65bd..13713d24 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -14,19 +14,12 @@ sub lei_edit_search {
my @cmd = (qw(git config --edit -f), $lss->{'-f'});
$lei->qerr("# spawning @cmd");
$lss->edit_begin($lei);
- if ($lei->{oneshot}) {
- require PublicInbox::Spawn;
- waitpid(PublicInbox::Spawn::spawn(\@cmd), 0);
- # non-fatal, editor could fail after successful write
- $lei->child_error($?) if $?;
- $lss->edit_done($lei);
- } else { # run in script/lei foreground
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- # $op_p will EOF when $EDITOR is done
- $op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
- $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p ], \@cmd, {});
- }
+ # run in script/lei foreground
+ require PublicInbox::PktOp;
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ # $op_p will EOF when $EDITOR is done
+ $op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
+ $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p ], \@cmd, {});
}
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index a7a0ebef..af5edbc2 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -431,20 +431,15 @@ sub write_prepare {
my $d = $lei->store_path;
$self->ipc_lock_init("$d/ipc.lock");
substr($d, -length('/lei/store'), 10, '');
- my $err_pipe;
- unless ($lei->{oneshot}) {
- pipe(my ($r, $w)) or die "pipe: $!";
- $err_pipe = [ $r, $w ];
- }
+ pipe(my ($r, $w)) or die "pipe: $!";
+ my $err_pipe = [ $r, $w ];
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
$self->ipc_worker_spawn("lei/store $d", $lei->oldset,
{ lei => $lei, err_pipe => $err_pipe });
- if ($err_pipe) {
- require PublicInbox::LeiStoreErr;
- PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
- }
+ require PublicInbox::LeiStoreErr;
+ PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
}
$lei->{sto} = $self;
}
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index 2ce64d62..a71158f3 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -23,8 +23,7 @@ sub lei_sucks {
}
eval { require PublicInbox };
my $pi_ver = eval('$PublicInbox::VERSION') // '(???)';
- my $daemon = $lei->{oneshot} ? 'oneshot' : 'daemon';
- my @out = ("lei $pi_ver mode=$daemon\n",
+ my @out = ("lei $pi_ver\n",
"perl $Config{version} / $os $rel / $mac ".
"ptrsize=$Config{ptrsize}\n");
chomp(my $gv = `git --version` || "git missing");
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 4399c4fb..9069232b 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -76,22 +76,18 @@ sub lei_up {
my @all = PublicInbox::LeiSavedSearch::list($lei);
my @local = grep(!m!\Aimaps?://!i, @all);
$lei->_lei_store->write_prepare($lei); # share early
- if ($lei->{oneshot}) { # synchronous
- up1_redispatch($lei, $_) for @local;
- } else {
- # daemon mode, re-dispatch into our event loop w/o
- # creating an extra fork-level
- require PublicInbox::DS;
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@local) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
- }
- $lei->event_step_init;
- $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
+ # daemon mode, re-dispatch into our event loop w/o
+ # creating an extra fork-level
+ require PublicInbox::DS;
+ require PublicInbox::PktOp;
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ for my $o (@local) {
+ PublicInbox::DS::requeue(sub {
+ up1_redispatch($lei, $o, $op_p);
+ });
}
+ $lei->event_step_init;
+ $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
} else {
up1($lei, $out);
}
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 460c9da0..83dcf650 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -544,7 +544,8 @@ EOM
($tmpdir, $for_destroy) = tmpdir unless $tmpdir;
state $persist_xrd = $ENV{TEST_LEI_DAEMON_PERSIST_DIR};
SKIP: {
- skip 'TEST_LEI_ONESHOT set', 1 if $ENV{TEST_LEI_ONESHOT};
+ $ENV{TEST_LEI_ONESHOT} and
+ xbail 'TEST_LEI_ONESHOT no longer supported';
my $home = "$tmpdir/lei-daemon";
mkdir($home, 0700) or BAIL_OUT "mkdir: $!";
local $ENV{HOME} = $home;
@@ -568,22 +569,12 @@ EOM
lei_ok(qw(daemon-kill), \"daemon-kill after $t");
}
}; # SKIP for lei_daemon
- unless ($test_opt->{daemon_only}) {
- $ENV{TEST_LEI_DAEMON_ONLY} and
- skip 'TEST_LEI_DAEMON_ONLY set', 1;
- require_ok 'PublicInbox::LEI';
- my $home = "$tmpdir/lei-oneshot";
- mkdir($home, 0700) or BAIL_OUT "mkdir: $!";
- local $ENV{HOME} = $home;
- local $ENV{XDG_RUNTIME_DIR} = '/dev/null';
- $cb->();
- }
if ($daemon_pid) {
for (0..10) {
kill(0, $daemon_pid) or last;
tick;
}
- ok(!kill(0, $daemon_pid), "$t daemon stopped after oneshot");
+ ok(!kill(0, $daemon_pid), "$t daemon stopped");
my $f = "$daemon_xrd/lei/errors.log";
open my $fh, '<', $f or BAIL_OUT "$f: $!";
my @l = <$fh>;
diff --git a/script/lei b/script/lei
index bec6b001..4d752fd8 100755
--- a/script/lei
+++ b/script/lei
@@ -9,10 +9,18 @@ my $narg = 5;
my $sock;
my $recv_cmd = PublicInbox::CmdIPC4->can('recv_cmd4');
my $send_cmd = PublicInbox::CmdIPC4->can('send_cmd4') // do {
+ my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //= (
+ $ENV{XDG_CACHE_HOME} //
+ ( ($ENV{HOME} // '/nonexistent').'/.cache' )
+ ).'/public-inbox/inline-c';
+ if (!-d $inline_dir) {
+ require File::Path;
+ File::Path::make_path($inline_dir);
+ }
require PublicInbox::Spawn; # takes ~50ms even if built *sigh*
$recv_cmd = PublicInbox::Spawn->can('recv_cmd4');
PublicInbox::Spawn->can('send_cmd4');
-};
+} // die 'please install Inline::C or Socket::MsgHdr';
my %pids;
my $sigchld = sub {
@@ -66,80 +74,69 @@ my $exec_cmd = sub {
}
};
-if ($send_cmd && eval {
- my $path = do {
- my $runtime_dir = ($ENV{XDG_RUNTIME_DIR} // '') . '/lei';
- die \0 if $runtime_dir eq '/dev/null/lei'; # oneshot forced
- if ($runtime_dir eq '/lei') {
- require File::Spec;
- $runtime_dir = File::Spec->tmpdir."/lei-$<";
- }
- unless (-d $runtime_dir) {
- require File::Path;
- File::Path::mkpath($runtime_dir, 0, 0700);
- }
- "$runtime_dir/$narg.seq.sock";
- };
- my $addr = pack_sockaddr_un($path);
- socket($sock, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
- unless (connect($sock, $addr)) { # start the daemon if not started
- local $ENV{PERL5LIB} = join(':', @INC);
- open(my $daemon, '-|', $^X, qw[-MPublicInbox::LEI
- -E PublicInbox::LEI::lazy_start(@ARGV)],
- $path, $! + 0, $narg) or die "popen: $!";
- while (<$daemon>) { warn $_ } # EOF when STDERR is redirected
- close($daemon) or warn <<"";
+my $runtime_dir = ($ENV{XDG_RUNTIME_DIR} // '') . '/lei';
+if ($runtime_dir eq '/lei') {
+ require File::Spec;
+ $runtime_dir = File::Spec->tmpdir."/lei-$<";
+}
+unless (-d $runtime_dir) {
+ require File::Path;
+ File::Path::make_path($runtime_dir, { mode => 0700 });
+}
+my $path = "$runtime_dir/$narg.seq.sock";
+my $addr = pack_sockaddr_un($path);
+socket($sock, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
+unless (connect($sock, $addr)) { # start the daemon if not started
+ local $ENV{PERL5LIB} = join(':', @INC);
+ open(my $daemon, '-|', $^X, qw[-MPublicInbox::LEI
+ -E PublicInbox::LEI::lazy_start(@ARGV)],
+ $path, $! + 0, $narg) or die "popen: $!";
+ while (<$daemon>) { warn $_ } # EOF when STDERR is redirected
+ close($daemon) or warn <<"";
lei-daemon could not start, exited with \$?=$?
- # try connecting again anyways, unlink+bind may be racy
- connect($sock, $addr) or die <<"";
+ # try connecting again anyways, unlink+bind may be racy
+ connect($sock, $addr) or die <<"";
connect($path): $! (after attempted daemon start)
Falling back to (slow) one-shot mode
+}
+# (Socket::MsgHdr|Inline::C), $sock are all available:
+open my $dh, '<', '.' or die "open(.) $!";
+my $buf = join("\0", scalar(@ARGV), @ARGV);
+while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
+$buf .= "\0\0";
+my $n = $send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR);
+if (!$n) {
+ die "sendmsg: $! (check RLIMIT_NOFILE)\n" if $!{ETOOMANYREFS};
+ die "sendmsg: $!\n";
+}
+my $x_it_code = 0;
+while (1) {
+ my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
+ if (scalar(@fds) == 1 && !defined($fds[0])) {
+ next if $!{EINTR};
+ last if $!{ECONNRESET};
+ die "recvmsg: $!";
}
- # (Socket::MsgHdr|Inline::C), $sock are all available:
- open my $dh, '<', '.' or die "open(.) $!";
- my $buf = join("\0", scalar(@ARGV), @ARGV);
- while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
- $buf .= "\0\0";
- my $n = $send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR);
- if (!$n) {
- die "sendmsg: $! (check RLIMIT_NOFILE)\n" if $!{ETOOMANYREFS};
- die "sendmsg: $!\n";
- }
- 1;
-}) { # connected and request sent to lei-daemon, wait for responses or EOF
- my $x_it_code = 0;
- while (1) {
- my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
- if (scalar(@fds) == 1 && !defined($fds[0])) {
- next if $!{EINTR};
- last if $!{ECONNRESET};
- die "recvmsg: $!";
- }
- last if $buf eq '';
- if ($buf =~ /\Aexec (.+)\z/) {
- $exec_cmd->(\@fds, split(/\0/, $1));
- } elsif ($buf eq '-WINCH') {
- kill($buf, @parent); # for MUA
- } elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
- $x_it_code ||= $1 + 0;
- last;
- } elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
- $x_it_code ||= $1 + 0;
- } else {
- $sigchld->();
- die $buf;
- }
- }
- $sigchld->();
- if (my $sig = ($x_it_code & 127)) {
- kill $sig, $$;
- sleep(1) while 1;
+ last if $buf eq '';
+ if ($buf =~ /\Aexec (.+)\z/) {
+ $exec_cmd->(\@fds, split(/\0/, $1));
+ } elsif ($buf eq '-WINCH') {
+ kill($buf, @parent); # for MUA
+ } elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
+ $x_it_code ||= $1 + 0;
+ last;
+ } elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
+ $x_it_code ||= $1 + 0;
+ } else {
+ $sigchld->();
+ die $buf;
}
- exit($x_it_code >> 8);
-} else { # for systems lacking Socket::MsgHdr or Inline::C
- warn $@ if $@ && !ref($@);
- require PublicInbox::LEI;
- PublicInbox::LEI::oneshot(__PACKAGE__);
}
+$sigchld->();
+if (my $sig = ($x_it_code & 127)) {
+ kill $sig, $$;
+ sleep(1) while 1;
+}
+exit($x_it_code >> 8);
diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index 84e2791d..a7c4b799 100644
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -73,15 +73,6 @@ test_lei({ daemon_only => 1 }, sub {
lei_ok('daemon-pid');
chomp $lei_out;
is($lei_out, $new_pid, 'PID unchanged after -0/-CHLD');
-
- SKIP: { # socket inaccessible
- skip "cannot test connect EPERM as root", 3 if $> == 0;
- chmod 0000, $sock or BAIL_OUT "chmod 0000: $!";
- lei_ok('help', \'connect fail, one-shot fallback works');
- like($lei_err, qr/\bconnect\(/, 'connect error noted');
- like($lei_out, qr/^usage: /, 'help output works');
- chmod 0700, $sock or BAIL_OUT "chmod 0700: $!";
- }
unlink $sock or BAIL_OUT "unlink($sock) $!";
for (0..100) {
kill('CHLD', $new_pid) or last;
^ permalink raw reply related [relevance 28%]
* [PATCH] lei rm: new command to remove messages from index
@ 2021-05-26 23:50 50% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-26 23:50 UTC (permalink / raw)
To: meta
This is similar to "public-inbox-learn rm", but it's
possible to point an entire Maildir/IMAP/mbox*/newsgroup
at it.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 5 +++-
lib/PublicInbox/LeiRm.pm | 50 +++++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiStore.pm | 29 ++++++++++++++++++++-
t/lei-import-maildir.t | 7 ++++++
5 files changed, 90 insertions(+), 2 deletions(-)
create mode 100644 lib/PublicInbox/LeiRm.pm
diff --git a/MANIFEST b/MANIFEST
index 23423e0b..0b4bb380 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -223,6 +223,7 @@ lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
+lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiSavedSearch.pm
lib/PublicInbox/LeiSearch.pm
lib/PublicInbox/LeiStore.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6ff249d0..7acc05bf 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -206,7 +206,10 @@ our %CMD = ( # sorted in order of importance/use:
qw(verbose|v+), @c_opt ],
'edit-search' => [ 'OUTPUT', "edit saved search via `git config --edit'",
@c_opt ],
-
+'rm' => [ '--stdin|LOCATION...',
+ 'remove a message from the index and prevent reindexing',
+ 'stdin|', # /|\z/ must be first for lone dash
+ @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
new file mode 100644
index 00000000..185b6a15
--- /dev/null
+++ b/lib/PublicInbox/LeiRm.pm
@@ -0,0 +1,50 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# implements the "lei rm" command, you can point this at
+# an entire spam mailbox or read a message from stdin
+package PublicInbox::LeiRm;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+
+sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
+ my ($self, $eml) = @_;
+ $self->{lei}->{sto}->ipc_do('remove_eml', $eml);
+}
+
+sub input_mbox_cb { # MboxReader callback
+ my ($eml, $self) = @_;
+ input_eml_cb($self, $eml);
+}
+
+sub input_net_cb { # callback for ->imap_each, ->nntp_each
+ my (undef, undef, $kw, $eml, $self) = @_; # @_[0,1]: url + uid ignored
+ input_eml_cb($self, $eml);
+}
+
+sub input_maildir_cb {
+ my (undef, $kw, $eml, $self) = @_; # $_[0] $filename ignored
+ input_eml_cb($self, $eml);
+}
+
+sub lei_rm {
+ my ($lei, @inputs) = @_;
+ $lei->_lei_store(1)->write_prepare($lei);
+ $lei->{opt}->{stdin} = 1 if !@inputs;
+ $lei->{opt}->{'in-format'} //= 'eml';
+ my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
+ $self->prepare_inputs($lei, \@inputs) or return;
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $op_c->op_wait_event($ops);
+}
+
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+
+1;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index af5edbc2..6888afb4 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -183,7 +183,7 @@ sub add_eml_vmd {
\@docids;
}
-sub remove_eml_vmd {
+sub remove_eml_vmd { # remove just the VMD
my ($self, $eml, $vmd) = @_;
my ($eidx, $tl) = eidx_init($self);
my @docids = _docids_for($self, $eml);
@@ -204,6 +204,33 @@ sub set_sync_info {
})->set_src($oidhex, $folder, $id);
}
+sub _remove_if_local { # git->cat_async arg
+ my ($bref, $oidhex, $type, $size, $self) = @_;
+ $self->{im}->remove($bref) if $bref;
+}
+
+# remove the entire message from the index, does not touch mail_sync.sqlite3
+sub remove_eml {
+ my ($self, $eml) = @_;
+ my $im = $self->importer; # may create new epoch
+ my ($eidx, $tl) = eidx_init($self);
+ my $oidx = $eidx->{oidx};
+ my @docids = _docids_for($self, $eml);
+ my $git = $eidx->git;
+ for my $docid (@docids) {
+ my $xr3 = $oidx->get_xref3($docid, 1);
+ for my $row (@$xr3) {
+ my (undef, undef, $oidbin) = @$row;
+ my $oidhex = unpack('H*', $oidbin);
+ $git->cat_async($oidhex, \&_remove_if_local, $self);
+ }
+ $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
+ $oidx->delete_by_num($docid);
+ }
+ $git->cat_async_wait;
+ \@docids;
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index f813440a..688b10ce 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -68,5 +68,12 @@ test_lei(sub {
$res = json_utf8->decode($lei_out);
is_deeply($res, [ undef ], 'trashed message not imported')
or diag explain($imp_err, $res);
+
+ lei_ok qw(rm t/data/0001.patch);
+ lei_ok(qw(q s:boolean));
+ is($lei_out, "[null]\n", 'removed message gone from results');
+ my $g0 = "$ENV{HOME}/.local/share/lei/store/local/0.git";
+ my $x = xqx(['git', "--git-dir=$g0", qw(cat-file blob HEAD:d)]);
+ is($?, 0, "git cat-file shows file is `d'");
});
done_testing;
^ permalink raw reply related [relevance 50%]
* [PATCH] lei: handle a single IMAP message in most places
@ 2021-05-27 10:49 44% Eric Wong
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-27 10:49 UTC (permalink / raw)
To: meta
"lei import" can now import a single IMAP message via
<imaps://example.com/MAILBOX/;UID=$UID>
Likewise, "lei inspect" can show the blob information for UID
URLs and "lei lcat" can display the blob without network access
if imported.
"lei lcat" also gets rid of some unused code and supports
"blob:$OIDHEX" syntax as described in the comments (and used by
our "text" output format).
---
lib/PublicInbox/LeiInspect.pm | 24 +++++++++++++++++++++--
lib/PublicInbox/LeiLcat.pm | 35 +++++++++++++++++++++-------------
lib/PublicInbox/LeiMailSync.pm | 23 ++++++++++++++++++++++
lib/PublicInbox/LeiToMail.pm | 8 +++++++-
lib/PublicInbox/NetReader.pm | 9 +++++++--
t/lei-import-imap.t | 14 ++++++++++++++
6 files changed, 95 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 46b9197f..7205979e 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -24,6 +24,19 @@ sub inspect_blob ($$) {
$ent;
}
+sub inspect_imap_uid ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $ent = {};
+ my $lse = $lei->{lse} or return $ent;
+ my $lms = $lse->lms or return $ent;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ $ent->{$$uid_uri} = $oidhex;
+ $ent;
+}
+
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
@@ -49,8 +62,15 @@ sub inspect1 ($$$) {
my $ent;
if ($item =~ /\Ablob:(.+)/) {
$ent = inspect_blob($lei, $1);
- } elsif ($item =~ m!\Aimaps?://!i ||
- $item =~ m!\A(?:maildir|mh):!i || -d $item) {
+ } elsif ($item =~ m!\Aimaps?://!i) {
+ require PublicInbox::URIimap;
+ my $uri = PublicInbox::URIimap->new($item);
+ if (defined($uri->uid)) {
+ $ent = inspect_imap_uid($lei, $uri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ } elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
} else { # TODO: more things
return $lei->fail("$item not understood");
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 87729acf..e5d8e805 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -9,27 +9,33 @@ use strict;
use v5.10.1;
use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
-use URI;
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_redispatch {
- my ($lei, $out, $op_p) = @_;
- my $l = bless { %$lei }, ref($lei);
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
- eval {
- $l->qerr("# updating $out");
- up1($l, $out);
- $l->qerr("# $out done");
- };
- $l->err($@) if $@;
+sub lcat_imap_uid_uri ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $lms = $lei->{lse}->lms or return;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
+ '""'; # blank query
}
sub extract_1 ($$) {
my ($lei, $x) = @_;
- if ($x =~ m!\b([a-z]+?://\S+)!i) {
+ if ($x =~ m!\b(imaps?://[^>]+)!i) {
+ my $u = $1;
+ require PublicInbox::URIimap;
+ $u = PublicInbox::URIimap->new($u);
+ if (!defined($u->uid)) {
+ $lei->qerr("# no UID= in $u");
+ }
+ lcat_imap_uid_uri($lei, $u);
+ } elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
+ require URI;
$u = URI->new($u);
my $p = $u->path;
my $term;
@@ -57,6 +63,9 @@ sub extract_1 ($$) {
$1;
} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
"mid:$1";
+ } elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
+ push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+ '""'; # blank query
} else {
undef;
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index b3e5e035..eb2b0c0f 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -356,6 +356,29 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
+sub imap_oid {
+ my ($self, $lei, $uid_uri) = @_;
+ my $mailbox_uri = $uid_uri->clone;
+ $mailbox_uri->uid(undef);
+ warn "$uid_uri $mailbox_uri";
+ my $folders = [ $$mailbox_uri ];
+ if (my $err = $self->arg2folder($lei, $folders)) {
+ if ($err->{fail}) {
+ $lei->qerr("# no sync information for $mailbox_uri");
+ return;
+ }
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ my $fid = $self->{fmap}->{$folders->[0]} //=
+ _fid_for($self, $folders->[0]) // return;
+ my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
+EOM
+ $sth->execute($fid, $uid_uri->uid);
+ my ($oidbin) = $sth->fetchrow_array;
+ $oidbin ? unpack('H*', $oidbin) : undef;
+}
+
sub DESTROY {
my ($self) = @_;
my $dbh = $self->{dbh} or return;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ad6b9439..b3aec50b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -702,8 +702,14 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
- delete $self->{wcb};
my $lei = $self->{lei};
+ if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
+ for my $oid (@{$lei->{lcat_blob}}) {
+ my $smsg = { blob => $oid, pct => 100 };
+ write_mail($self, $smsg);
+ }
+ }
+ delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
my $nr = delete($lei->{-nr_write}) or return;
return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 73b8b1cd..76d2fe62 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -469,7 +469,10 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
EOF
my $uri = $orig_uri->clone;
+ my $single_uid = $uri->uid;
my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
+ $itrk = $l_uid = undef if defined($single_uid);
+
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
@@ -499,7 +502,9 @@ EOF
# I wish "UID FETCH $START:*" could work, but:
# 1) servers do not need to return results in any order
# 2) Mail::IMAPClient doesn't offer a streaming API
- unless ($uids = $mic->search("UID $l_uid:*")) {
+ if (defined $single_uid) {
+ $uids = [ $single_uid ];
+ } elsif (!($uids = $mic->search("UID $l_uid:*"))) {
return if $!{EINTR} && $self->{quit};
return "E: $uri UID SEARCH $l_uid:* error: $!";
}
@@ -541,7 +546,7 @@ EOF
}
run_commit_cb($self);
$itrk->update_last($r_uidval, $last_uid) if $itrk;
- } until ($err || $self->{quit});
+ } until ($err || $self->{quit} || defined($single_uid));
$err;
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 5283cc23..2f928a2e 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -75,5 +75,19 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok 'forget-mail-sync', $url;
lei_ok 'ls-mail-sync';
unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
+ my $uid_url = "$url/;UID=".$stats->{'uid.max'};
+ lei_ok 'import', $uid_url;
+ lei_ok 'inspect', $uid_url;
+ $lei_out =~ /([a-f0-9]{40,})/ or
+ xbail 'inspect missed blob with UID URL';
+ my $blob = $1;
+ lei_ok 'lcat', $uid_url;
+ like $lei_out, qr/^Subject: /sm,
+ 'lcat shows mail text with UID URL';
+ like $lei_out, qr/\bblob:$blob\b/, 'lcat showed blob';
+ my $orig = $lei_out;
+ lei_ok 'lcat', "blob:$blob";
+ is($lei_out, $orig, 'lcat understands blob:...');
});
+
done_testing;
^ permalink raw reply related [relevance 44%]
* [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
@ 2021-05-28 0:07 71% ` Eric Wong
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
2021-05-28 0:07 71% ` [PATCH 6/6] lei: add TODO item for FUSE mount Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
This is low priority, for now.
---
lib/PublicInbox/LEI.pm | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7acc05bf..ad5f06be 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -274,13 +274,11 @@ our %CMD = ( # sorted in order of importance/use:
'daemon-pid' => [ '', 'show the PID of the lei-daemon' ],
'help' => [ '[SUBCOMMAND]', 'show help' ],
-# XXX do we need this?
-# 'git' => [ '[ANYTHING...]', 'git(1) wrapper', pass_through('git') ],
-
-'reorder-local-store-and-break-history' => [ '[REFNAME]',
- 'rewrite git history in an attempt to improve compression',
- qw(gc!), @c_opt ],
-
+# TODO
+#'reorder-local-store-and-break-history' => [ '[REFNAME]',
+# 'rewrite git history in an attempt to improve compression',
+# qw(gc!), @c_opt ],
+#
# internal commands are prefixed with '_'
'_complete' => [ '[...]', 'internal shell completion helper',
pass_through('everything') ],
^ permalink raw reply related [relevance 71%]
* [PATCH 6/6] lei: add TODO item for FUSE mount
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
@ 2021-05-28 0:07 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
It seems possible and natural to allow browsing lei/store as
a Maildir (as well as read-write JMAP/IMAP store).
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ad5f06be..07378ca7 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -278,6 +278,7 @@ our %CMD = ( # sorted in order of importance/use:
#'reorder-local-store-and-break-history' => [ '[REFNAME]',
# 'rewrite git history in an attempt to improve compression',
# qw(gc!), @c_opt ],
+#'fuse-mount' => [ 'PATHNAME', 'expose lei/store as Maildir(s)', @c_opt ],
#
# internal commands are prefixed with '_'
'_complete' => [ '[...]', 'internal shell completion helper',
^ permalink raw reply related [relevance 71%]
* [PATCH 0/6] lei: odds and ends and a resend
2021-05-27 10:49 44% [PATCH] lei: handle a single IMAP message in most places Eric Wong
@ 2021-05-28 0:07 70% ` Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
"lei: handle a single IMAP message in most places" didn't actually
apply, since I was still running 3/6 in this series. I've been
having trouble reproducing the uncommitted transaction error
messages I've been seeing from lms, but hopefully 3/6 can
eventually find it.
1/6 and 2/6 were somethings I noticed randomly. Been looking
into FUSE support for exposing lei/store as a Maildir, too.
Eric Wong (6):
viewdiff: make $UNSAFE a variable
viewdiff: escape '{' and '}' for regexp
lei_mail_sync: debug code for uncommitted txn
lei: mark reorder-and-rewrite-local-history as a TODO item
lei: handle a single IMAP message in most places
lei: add TODO item for FUSE mount
lib/PublicInbox/LEI.pm | 13 ++++++-------
lib/PublicInbox/LeiInspect.pm | 24 ++++++++++++++++++++++--
lib/PublicInbox/LeiLcat.pm | 33 ++++++++++++++++++++-------------
lib/PublicInbox/LeiMailSync.pm | 33 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiToMail.pm | 8 +++++++-
lib/PublicInbox/LeiXSearch.pm | 3 +++
lib/PublicInbox/NetReader.pm | 9 +++++++--
lib/PublicInbox/ViewDiff.pm | 11 +++++------
t/lei-import-imap.t | 15 +++++++++++++++
9 files changed, 118 insertions(+), 31 deletions(-)
^ permalink raw reply [relevance 70%]
* [PATCH 5/6] lei: handle a single IMAP message in most places
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
@ 2021-05-28 0:07 46% ` Eric Wong
2021-05-28 9:37 71% ` Eric Wong
2021-05-28 0:07 71% ` [PATCH 6/6] lei: add TODO item for FUSE mount Eric Wong
2 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
"lei import" can now import a single IMAP message via
<imaps://example.com/MAILBOX/;UID=$UID>
Likewise, "lei inspect" can show the blob information for UID
URLs and "lei lcat" can display the blob without network access
if imported.
"lei lcat" also gets rid of some unused code and supports
"blob:$OIDHEX" syntax as described in the comments (and used by
our "text" output format).
v2: enforce UID in URL, fail without
---
lib/PublicInbox/LeiInspect.pm | 24 ++++++++++++++++++++++--
lib/PublicInbox/LeiLcat.pm | 33 ++++++++++++++++++++-------------
lib/PublicInbox/LeiToMail.pm | 8 +++++++-
lib/PublicInbox/NetReader.pm | 9 +++++++--
t/lei-import-imap.t | 15 +++++++++++++++
5 files changed, 71 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 46b9197f..7205979e 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -24,6 +24,19 @@ sub inspect_blob ($$) {
$ent;
}
+sub inspect_imap_uid ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $ent = {};
+ my $lse = $lei->{lse} or return $ent;
+ my $lms = $lse->lms or return $ent;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ $ent->{$$uid_uri} = $oidhex;
+ $ent;
+}
+
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
@@ -49,8 +62,15 @@ sub inspect1 ($$$) {
my $ent;
if ($item =~ /\Ablob:(.+)/) {
$ent = inspect_blob($lei, $1);
- } elsif ($item =~ m!\Aimaps?://!i ||
- $item =~ m!\A(?:maildir|mh):!i || -d $item) {
+ } elsif ($item =~ m!\Aimaps?://!i) {
+ require PublicInbox::URIimap;
+ my $uri = PublicInbox::URIimap->new($item);
+ if (defined($uri->uid)) {
+ $ent = inspect_imap_uid($lei, $uri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ } elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
} else { # TODO: more things
return $lei->fail("$item not understood");
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 87729acf..c4712662 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -9,27 +9,31 @@ use strict;
use v5.10.1;
use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
-use URI;
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_redispatch {
- my ($lei, $out, $op_p) = @_;
- my $l = bless { %$lei }, ref($lei);
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
- eval {
- $l->qerr("# updating $out");
- up1($l, $out);
- $l->qerr("# $out done");
- };
- $l->err($@) if $@;
+sub lcat_imap_uid_uri ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $lms = $lei->{lse}->lms or return;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
}
sub extract_1 ($$) {
my ($lei, $x) = @_;
- if ($x =~ m!\b([a-z]+?://\S+)!i) {
+ if ($x =~ m!\b(imaps?://[^>]+)!i) {
+ my $u = $1;
+ require PublicInbox::URIimap;
+ $u = PublicInbox::URIimap->new($u);
+ defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
+ $lei->fail("# no UID= in $u");
+ '""'; # blank query, using {lcat_blob}
+ } elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
+ require URI;
$u = URI->new($u);
my $p = $u->path;
my $term;
@@ -57,6 +61,9 @@ sub extract_1 ($$) {
$1;
} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
"mid:$1";
+ } elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
+ push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+ '""'; # blank query
} else {
undef;
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ad6b9439..b3aec50b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -702,8 +702,14 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
- delete $self->{wcb};
my $lei = $self->{lei};
+ if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
+ for my $oid (@{$lei->{lcat_blob}}) {
+ my $smsg = { blob => $oid, pct => 100 };
+ write_mail($self, $smsg);
+ }
+ }
+ delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
my $nr = delete($lei->{-nr_write}) or return;
return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 73b8b1cd..76d2fe62 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -469,7 +469,10 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
EOF
my $uri = $orig_uri->clone;
+ my $single_uid = $uri->uid;
my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
+ $itrk = $l_uid = undef if defined($single_uid);
+
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
@@ -499,7 +502,9 @@ EOF
# I wish "UID FETCH $START:*" could work, but:
# 1) servers do not need to return results in any order
# 2) Mail::IMAPClient doesn't offer a streaming API
- unless ($uids = $mic->search("UID $l_uid:*")) {
+ if (defined $single_uid) {
+ $uids = [ $single_uid ];
+ } elsif (!($uids = $mic->search("UID $l_uid:*"))) {
return if $!{EINTR} && $self->{quit};
return "E: $uri UID SEARCH $l_uid:* error: $!";
}
@@ -541,7 +546,7 @@ EOF
}
run_commit_cb($self);
$itrk->update_last($r_uidval, $last_uid) if $itrk;
- } until ($err || $self->{quit});
+ } until ($err || $self->{quit} || defined($single_uid));
$err;
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 5283cc23..59d481d5 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -75,5 +75,20 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok 'forget-mail-sync', $url;
lei_ok 'ls-mail-sync';
unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
+ my $uid_url = "$url/;UID=".$stats->{'uid.max'};
+ lei_ok 'import', $uid_url;
+ lei_ok 'inspect', $uid_url;
+ $lei_out =~ /([a-f0-9]{40,})/ or
+ xbail 'inspect missed blob with UID URL';
+ my $blob = $1;
+ lei_ok 'lcat', $uid_url;
+ like $lei_out, qr/^Subject: /sm,
+ 'lcat shows mail text with UID URL';
+ like $lei_out, qr/\bblob:$blob\b/, 'lcat showed blob';
+ my $orig = $lei_out;
+ lei_ok 'lcat', "blob:$blob";
+ is($lei_out, $orig, 'lcat understands blob:...');
+ ok(!lei('lcat', $url), "lcat doesn't work on IMAP URL w/o UID");
});
+
done_testing;
^ permalink raw reply related [relevance 46%]
* Re: [PATCH 5/6] lei: handle a single IMAP message in most places
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
@ 2021-05-28 9:37 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:37 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> v2: enforce UID in URL, fail without
Pushed v3 as commit 9b3cd5e254fafa08c774a24f85c2b2eac12a9de5
v3: fix error reporting (s/fail/child_error/)
Using ->fail was causing errors to cascade elsewhere, not 100%
sure why...
interdiff:
diff -u b/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
--- b/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -28,7 +28,7 @@
require PublicInbox::URIimap;
$u = PublicInbox::URIimap->new($u);
defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
- $lei->fail("# no UID= in $u");
+ $lei->child_error(1 << 8, "# no UID= in $u");
'""'; # blank query, using {lcat_blob}
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
^ permalink raw reply [relevance 71%]
* [PATCH 0/3] lei: diagnostic and reliability fix
@ 2021-05-28 9:45 71% Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
1/3 may no longer be needed, but doesn't hurt, either.
It was made while tracking down occasional failures for
v3 of "lei: handle a single IMAP message in most places":
https://public-inbox.org/meta/20210528093740.GB18768@dcvr/
2/3 is necessary for parallel invocations from different dirs,
and in multi-tenant situations; and I just noticed 3/3 because
I had a typo somewhere else :x
Eric Wong (3):
t/lei-*: better diagnostics for occasional failures
lei: restore working directory in more places
script/lei: drop leftover message about fallback
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiLcat.pm | 4 +---
lib/PublicInbox/LeiQuery.pm | 4 +---
lib/PublicInbox/LeiXSearch.pm | 1 +
script/lei | 1 -
t/lei-q-kw.t | 2 +-
t/lei-q-remote-import.t | 6 +++---
t/lei-q-save.t | 6 ++++--
8 files changed, 17 insertions(+), 13 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] script/lei: drop leftover message about fallback
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
2021-05-28 9:45 62% ` [PATCH 2/3] lei: restore working directory in more places Eric Wong
@ 2021-05-28 9:45 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
Non-daemon lei isn't implemented, anymore.
---
script/lei | 1 -
1 file changed, 1 deletion(-)
diff --git a/script/lei b/script/lei
index 4d752fd8..fce8124a 100755
--- a/script/lei
+++ b/script/lei
@@ -98,7 +98,6 @@ lei-daemon could not start, exited with \$?=$?
# try connecting again anyways, unlink+bind may be racy
connect($sock, $addr) or die <<"";
connect($path): $! (after attempted daemon start)
-Falling back to (slow) one-shot mode
}
# (Socket::MsgHdr|Inline::C), $sock are all available:
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] lei: restore working directory in more places
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
@ 2021-05-28 9:45 62% ` Eric Wong
2021-05-28 9:45 71% ` [PATCH 3/3] script/lei: drop leftover message about fallback Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
Every tick of the event loop can change the working directory,
so we need to restore it for every client if they operate
in different directories.
This would be easier if we had openat(2) and friends in Perl;
but Inline::C is practically required for lei, now.
---
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiLcat.pm | 4 +---
lib/PublicInbox/LeiQuery.pm | 4 +---
lib/PublicInbox/LeiXSearch.pm | 1 +
4 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 07378ca7..e5ff9e5d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1239,6 +1239,12 @@ sub wq_done_wait { # dwaitpid callback
$lei->dclose;
}
+sub fchdir {
+ my ($lei) = @_;
+ my $dh = $lei->{3} // die 'BUG: lei->{3} (CWD) gone';
+ chdir($dh) || $lei->fail("fchdir: $!");
+}
+
sub wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 0f585ff5..f9d9633a 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -89,9 +89,7 @@ sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
my ($lei) = @_; # $_[1] = $rbuf
if (defined($_[1])) {
$_[1] eq '' and return eval {
- if (my $dfd = $lei->{3}) {
- chdir($dfd) or return $lei->fail("fchdir: $!");
- }
+ $lei->fchdir or return;
my @argv = split(/\s+/, $lei->{mset_opt}->{qstr});
$lei->{mset_opt}->{qstr} = extract_all($lei, @argv)
or return;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 1999a534..0435a516 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -51,9 +51,7 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
my ($self) = @_; # $_[1] = $rbuf
if (defined($_[1])) {
$_[1] eq '' and return eval {
- if (my $dfd = $self->{3}) {
- chdir($dfd) or return $self->fail("fchdir: $!");
- }
+ $self->fchdir or return;
$self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr};
$self->{lse}->query_approxidate($self->{lse}->git,
$self->{mset_opt}->{qstr});
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index e2a8e8e3..760f9718 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -394,6 +394,7 @@ Error closing $lei->{ovv}->{dst}: $!
sub do_post_augment {
my ($lei) = @_;
my $l2m = $lei->{l2m} or return; # client disconnected
+ $lei->fchdir or return;
my $err;
eval { $l2m->post_augment($lei) };
$err = $@;
^ permalink raw reply related [relevance 62%]
* [PATCH 1/3] t/lei-*: better diagnostics for occasional failures
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
@ 2021-05-28 9:45 61% ` Eric Wong
2021-05-28 9:45 62% ` [PATCH 2/3] lei: restore working directory in more places Eric Wong
2021-05-28 9:45 71% ` [PATCH 3/3] script/lei: drop leftover message about fallback Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
Some of these have been failing occasionally, not sure
how, yet...
---
t/lei-q-kw.t | 2 +-
t/lei-q-remote-import.t | 6 +++---
t/lei-q-save.t | 6 ++++--
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 074c573d..528751b4 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -116,7 +116,7 @@ for my $sfx ('', '.gz') {
}
$res{$mid} = $eml;
});
- is_deeply(\%res, $exp, '--augment worked');
+ is_deeply(\%res, $exp, '--augment worked') or diag $lei_err;
lei_ok(qw(q -o), "mboxrd:/dev/stdout", qw(m:qp@example.com)) or
diag $lei_err;
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 80067061..7db684d9 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -49,10 +49,10 @@ test_lei({ tmpdir => $tmpdir }, sub {
open my $fh, '>', "$o.lock";
$cmd[-1] = 'm:qp@example.com';
- unlink $o or BAIL_OUT $!;
+ unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
lei_ok(@cmd, '--lock=none');
- ok(-f $o && -s _, '--lock=none respected');
- unlink $o or BAIL_OUT $!;
+ ok(-f $o && -s _, '--lock=none respected') or diag $lei_err;
+ unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
ok(!lei(@cmd, '--lock=dotlock,timeout=0.000001'), 'dotlock fails');
ok(-f $o && !-s _, 'nothing output on lock failure');
unlink "$o.lock" or BAIL_OUT $!;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index aed38a51..bea65133 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -25,7 +25,8 @@ test_lei(sub {
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
lei_ok qw(q -q --save z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
my %before = map { $_ => 1 } glob("$home/md/cur/*");
- is_deeply(eml_load((keys %before)[0]), $doc1, 'doc1 matches');
+ my $f = (keys %before)[0] or xbail({before => \%before});
+ is_deeply(eml_load($f), $doc1, 'doc1 matches');
lei_ok qw(ls-mail-sync);
is($lei_out, "maildir:$home/md\n", 'canonicalized mail sync name');
@@ -45,7 +46,8 @@ test_lei(sub {
my %after = map { $_ => 1 } glob("$home/md/{new,cur}/*");
is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
is(scalar(keys %after), 1, 'one new message added');
- is_deeply(eml_load((keys %after)[0]), $doc2, 'doc2 matches');
+ $f = (keys %after)[0] or xbail({after => \%after});
+ is_deeply(eml_load($f), $doc2, 'doc2 matches');
# check stdin
lei_ok [qw(q --save - -o), "mboxcl2:mbcl2" ],
^ permalink raw reply related [relevance 61%]
* [PATCH] lei: retry_reopen on read-only Xapian access
@ 2021-05-28 19:47 49% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 19:47 UTC (permalink / raw)
To: meta
Xapian DBs may be modified by a parallel process while we're
reading it, and Xapian's MVCC model places the burden on readers
to retry operations.
We'll also have retry_reopen croak instead of die on errors,
which ought to help us track down some "Document not found"
errors I've occasionally seen when using "lei <q|up>".
---
lib/PublicInbox/LeiSearch.pm | 33 ++++++++++++++++++++++++---------
lib/PublicInbox/LeiXSearch.pm | 21 ++++++++++++++-------
lib/PublicInbox/Search.pm | 7 ++++---
3 files changed, 42 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 9297d060..b09d1e45 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -18,7 +18,7 @@ sub num2docid ($$) {
($num - 1) * $nshard + $num % $nshard + 1;
}
-sub msg_keywords {
+sub _msg_kw { # retry_reopen callback
my ($self, $num) = @_; # num_or_mitem
my $xdb = $self->xdb; # set {nshard};
my $docid = ref($num) ? $num->get_docid : num2docid($self, $num);
@@ -27,13 +27,16 @@ sub msg_keywords {
wantarray ? sort(keys(%$kw)) : $kw;
}
-# returns undef if blob is unknown
-sub oid_keywords {
- my ($self, $oidhex) = @_;
- my @num = $self->over->blob_exists($oidhex) or return;
+sub msg_keywords {
+ my ($self, $num) = @_; # num_or_mitem
+ $self->retry_reopen(\&_msg_kw, $num);
+}
+
+sub _oid_kw { # retry_reopen callback
+ my ($self, $nums) = @_;
my $xdb = $self->xdb; # set {nshard};
my %kw;
- for my $num (@num) { # there should only be one...
+ for my $num (@$nums) { # there should only be one...
my $doc = $xdb->get_document(num2docid($self, $num));
my $x = xap_terms('K', $doc);
%kw = (%kw, %$x);
@@ -41,10 +44,15 @@ sub oid_keywords {
\%kw;
}
-# lookup keywords+labels for external messages
-sub xsmsg_vmd {
+# returns undef if blob is unknown
+sub oid_keywords {
+ my ($self, $oidhex) = @_;
+ my @num = $self->over->blob_exists($oidhex) or return;
+ $self->retry_reopen(\&_oid_kw, \@num);
+}
+
+sub _xsmsg_vmd { # retry_reopen
my ($self, $smsg, $want_label) = @_;
- return if $smsg->{kw};
my $xdb = $self->xdb; # set {nshard};
my (%kw, %L, $doc, $x);
$kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged});
@@ -62,6 +70,13 @@ sub xsmsg_vmd {
$smsg->{L} = [ sort keys %L ] if scalar(keys(%L));
}
+# lookup keywords+labels for external messages
+sub xsmsg_vmd {
+ my ($self, $smsg, $want_label) = @_;
+ return if $smsg->{kw}; # already set by LeiXSearch->mitem_kw
+ $self->retry_reopen(\&_xsmsg_vmd, $smsg, $want_label);
+}
+
# when a message has no Message-IDs at all, this is needed for
# unsent Draft messages, at least
sub content_key ($) {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 760f9718..2e548a7a 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -67,17 +67,23 @@ sub remotes { @{$_[0]->{remotes} // []} }
# called by PublicInbox::Search::xdb (usually via ->mset)
sub xdb_shards_flat { @{$_[0]->{shards_flat} // []} }
-sub mitem_kw ($$;$) {
- my ($smsg, $mitem, $flagged) = @_;
- my $kw = xap_terms('K', my $doc = $mitem->get_document);
+sub _mitem_kw { # retry_reopen callback
+ my ($srch, $smsg, $mitem, $flagged) = @_;
+ my $doc = $mitem->get_document;
+ my $kw = xap_terms('K', $doc);
$kw->{flagged} = 1 if $flagged;
+ my $L = xap_terms('L', $doc);
# we keep the empty {kw} array here to prevent expensive work in
# ->xsmsg_vmd, _unbless_smsg will clobber it iff it's empty
$smsg->{kw} = [ sort keys %$kw ];
- my $L = xap_terms('L', $doc);
$smsg->{L} = [ sort keys %$L ] if scalar(keys %$L);
}
+sub mitem_kw ($$$;$) {
+ my ($srch, $smsg, $mitem, $flagged) = @_;
+ $srch->retry_reopen(\&_mitem_kw, $smsg, $mitem, $flagged);
+}
+
# like over->get_art
sub smsg_for {
my ($self, $mitem) = @_;
@@ -90,7 +96,7 @@ sub smsg_for {
my $smsg = $ibx->over->get_art($num);
return if $smsg->{bytes} == 0; # external message
if ($ibx->can('msg_keywords')) {
- mitem_kw($smsg, $mitem);
+ mitem_kw($self, $smsg, $mitem);
}
$smsg;
}
@@ -194,7 +200,8 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $mitem = delete $n2item{$n};
next if $smsg->{bytes} == 0;
if ($mitem && $can_kw) {
- mitem_kw($smsg, $mitem, $fl);
+ mitem_kw($srch, $smsg, $mitem,
+ $fl);
} elsif ($mitem && $fl) {
# call ->xsmsg_vmd, later
$smsg->{lei_q_tt_flagged} = 1;
@@ -210,7 +217,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $mitem = $items[$i++];
my $smsg = $over->get_art($n) or next;
next if $smsg->{bytes} == 0;
- mitem_kw($smsg, $mitem, $fl) if $can_kw;
+ mitem_kw($srch, $smsg, $mitem, $fl) if $can_kw;
$each_smsg->($smsg, $mitem);
}
}
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index fbcff2c3..59a5a3b0 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -9,6 +9,7 @@ use parent qw(Exporter);
our @EXPORT_OK = qw(retry_reopen int_val get_pct xap_terms);
use List::Util qw(max);
use POSIX qw(strftime);
+use Carp ();
# values for searching, changing the numeric value breaks
# compatibility with old indices (so don't change them it)
@@ -405,16 +406,16 @@ sub retry_reopen {
# Exception: The revision being read has been discarded -
# you should call Xapian::Database::reopen()
if (ref($@) =~ /\bDatabaseModifiedError\b/) {
- warn "reopen try #$i on $@\n";
+ warn "# reopen try #$i on $@\n";
reopen($self);
} else {
# let caller decide how to spew, because ExtMsg queries
# get wonky and trigger:
# "something terrible happened at .../Xapian/Enquire.pm"
- die;
+ Carp::croak($@);
}
}
- die "Too many Xapian database modifications in progress\n";
+ Carp::croak("Too many Xapian database modifications in progress\n");
}
sub _do_enquire {
^ permalink raw reply related [relevance 49%]
* [PATCH] lei q|up: support v2:/path/to/inboxdir destination
@ 2021-05-28 22:39 39% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 22:39 UTC (permalink / raw)
To: meta
This allows "lei-managed pseudo mailing lists" as described
by Konstantin.
Alternates use is optional and can be enables via --shared.
This doesn't manage or edit ~/.public-inbox/config; presumably
there'll need to be some tweaking of search parameters before
finalizing and making the inbox publicly accessible via HTTP/NNTP.
Link: https://public-inbox.org/meta/20210426164454.5zd5kgugfhfwfkpo@nitro.local/T/
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 ++
lib/PublicInbox/LeiSavedSearch.pm | 8 ++++-
lib/PublicInbox/LeiToMail.pm | 59 +++++++++++++++++++++++++++++--
lib/PublicInbox/LeiXSearch.pm | 1 +
lib/PublicInbox/V2Writable.pm | 11 +++++-
t/lei-q-save.t | 30 ++++++++++++++++
7 files changed, 107 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e5ff9e5d..f2dfc320 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -163,7 +163,7 @@ our %CMD = ( # sorted in order of importance/use:
qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
- color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
+ shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 28891460..e4242d9b 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -108,6 +108,8 @@ sub new {
$opt->{alert} //= [ ':WINCH,:bell' ] if -t $lei->{1};
}
}
+ return $lei->fail('--shared is only for v2 inbox output') if
+ $self->{fmt} ne 'v2' && $lei->{opt}->{shared};
$self;
}
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 48d252f1..929380ed 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -14,7 +14,7 @@ use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::MID qw(mids_for_index);
use Digest::SHA qw(sha256_hex);
-my $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf):!i; # TODO: put in LeiToMail?
+my $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
# move this to PublicInbox::Config if other things use it:
my %cquote = ("\n" => '\\n', "\t" => '\\t', "\b" => '\\b');
@@ -290,6 +290,12 @@ EOM
my $dir_old = lss_dir_for($lei, \$old_path, 1);
my $dir_new = lss_dir_for($lei, \$new_path);
return if $dir_new eq $dir_old; # no change, likely
+
+ ($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
+ return $lei->fail(<<EOM);
+conversions from/to v2 inboxes not supported at this time
+EOM
+
return $lei->fail(<<EOM) if -e $dir_new;
lei.q.output changed from `$old_out' to `$new_out'
However, $dir_new exists
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b3aec50b..a7382169 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -353,16 +353,28 @@ sub _text_write_cb ($$) {
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$lse->xsmsg_vmd($smsg) if $lse;
- $eml //= PublicInbox::Eml->new($bref); # copy bref
+ $eml //= PublicInbox::Eml->new($bref);
return if $dedupe && $dedupe->is_dup($eml, $smsg);
my $lk = $ovv->lock_for_scope;
$lei->out(${$lvt->eml_to_text($smsg, $eml)}, "\n");
}
}
+sub _v2_write_cb ($$) {
+ my ($self, $lei) = @_;
+ my $dedupe = $lei->{dedupe};
+ $dedupe->prepare_dedupe if $dedupe;
+ sub { # for git_to_mail
+ my ($bref, $smsg, $eml) = @_;
+ $eml //= PublicInbox::Eml->new($bref);
+ return if $dedupe && $dedupe->is_dup($eml, $smsg);
+ $lei->{v2w}->ipc_do('add', $eml); # V2Writable->add
+ }
+}
+
sub write_cb { # returns a callback for git_to_mail
my ($self, $lei) = @_;
- # _mbox_write_cb, _maildir_write_cb or _imap_write_cb
+ # _mbox_write_cb, _maildir_write_cb, _imap_write_cb, _v2_write_cb
my $m = "_$self->{base_type}_write_cb";
$self->$m($lei);
}
@@ -400,6 +412,13 @@ sub new {
require PublicInbox::LeiViewText;
$lei->{lvt} = PublicInbox::LeiViewText->new($lei);
$self->{base_type} = 'text';
+ } elsif ($fmt eq 'v2') {
+ die "--dedupe=oid and v2 are incompatible\n" if
+ ($lei->{opt}->{dedupe}//'') eq 'oid';
+ $self->{base_type} = 'v2';
+ $lei->{opt}->{save} = \1;
+ die "--mua incompatible with v2\n" if $lei->{opt}->{mua};
+ $dst = $lei->{ovv}->{dst} = $lei->abs_path($dst);
} else {
die "bad mail --format=$fmt\n";
}
@@ -599,9 +618,43 @@ sub _do_augment_mbox {
$dedupe->pause_dedupe if $dedupe;
}
+sub _pre_augment_v2 {
+ my ($self, $lei) = @_;
+ my $dir = $self->{dst};
+ require PublicInbox::InboxWritable;
+ my ($ibx, @creat);
+ if (-d $dir) {
+ my $opt = { -min_inbox_version => 2 };
+ require PublicInbox::Admin;
+ my @ibx = PublicInbox::Admin::resolve_inboxes([ $dir ], $opt);
+ $ibx = $ibx[0] or die "$dir is not a v2 inbox\n";
+ } else {
+ $creat[0] = {};
+ $ibx = PublicInbox::Inbox->new({
+ name => 'lei-result', # XXX configurable
+ inboxdir => $dir,
+ version => 2,
+ address => [ 'lei@example.com' ],
+ });
+ }
+ PublicInbox::InboxWritable->new($ibx, @creat);
+ $ibx->init_inbox if @creat;
+ my $v2w = $lei->{v2w} = $ibx->importer;
+ $v2w->ipc_lock_init("$dir/ipc.lock");
+ $v2w->ipc_worker_spawn("lei/v2w $dir", $lei->oldset, { lei => $lei });
+ return if !$lei->{opt}->{shared};
+ my $d = "$lei->{ale}->{git}->{git_dir}/objects";
+ my $al = "$dir/git/0.git/objects/info/alternates";
+ open my $fh, '+>>', $al or die "open($al): $!";
+ seek($fh, 0, SEEK_SET) or die "seek($al): $!";
+ grep(/\A\Q$d\E\n/, <$fh>) and return;
+ print $fh "$d\n" or die "print($al): $!";
+ close $fh or die "close($al): $!";
+}
+
sub pre_augment { # fast (1 disk seek), runs in same process as post_augment
my ($self, $lei) = @_;
- # _pre_augment_maildir, _pre_augment_mbox
+ # _pre_augment_maildir, _pre_augment_mbox, _pre_augment_v2
my $m = $self->can("_pre_augment_$self->{base_type}") or return;
$m->($self, $lei);
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 2e548a7a..d6d42a01 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -368,6 +368,7 @@ sub query_done { # EOF callback for main daemon
warn "BUG: {sto} missing with --mail-sync";
}
my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
+ $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 0461257f..573d9c6f 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -6,7 +6,7 @@
package PublicInbox::V2Writable;
use strict;
use v5.10.1;
-use parent qw(PublicInbox::Lock);
+use parent qw(PublicInbox::Lock PublicInbox::IPC);
use PublicInbox::SearchIdxShard;
use PublicInbox::IPC;
use PublicInbox::Eml;
@@ -1431,4 +1431,13 @@ W: interrupted, --xapian-only --reindex required upon restart
EOF
}
+sub ipc_atfork_child {
+ my ($self) = @_;
+ if (my $lei = delete $self->{lei}) {
+ $lei->_lei_atfork_child;
+ close(delete $lei->{pkt_op_p});
+ }
+ $self->SUPER::ipc_atfork_child;
+}
+
1;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index bea65133..694b33b2 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -3,6 +3,8 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
use PublicInbox::Smsg;
+use List::Util qw(sum);
+
my $doc1 = eml_load('t/plack-qp.eml');
$doc1->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 5)}));
my $doc2 = eml_load('t/utf8.eml');
@@ -165,5 +167,33 @@ test_lei(sub {
skip "symlinks not supported in $home?: $!", 1;
lei_ok('up', "$home/ln -s");
};
+
+ my $v2 = "$home/v2"; # v2: as an output destination
+ my (@before, @after);
+ require PublicInbox::MboxReader;
+ lei_ok(qw(q z:0.. -o), "v2:$v2");
+ lei_ok(qw(q z:0.. -o), "mboxrd:$home/before", '--only', $v2, '-j1,1');
+ open my $fh, '<', "$home/before";
+ PublicInbox::MboxReader->mboxrd($fh, sub { push @before, $_[0] });
+ isnt(scalar(@before), 0, 'initial v2 written');
+ my $orig = sum(map { -f $_ ? -s _ : () } (
+ glob("$v2/git/0.git/objects/*/*")));
+ lei_ok(qw(import t/data/0001.patch));
+ lei_ok 'up', $v2;
+ lei_ok(qw(q z:0.. -o), "mboxrd:$home/after", '--only', $v2, '-j1,1');
+ open $fh, '<', "$home/after";
+ PublicInbox::MboxReader->mboxrd($fh, sub { push @after, $_[0] });
+
+ my $last = shift @after;
+ $last->header_set('Status');
+ is_deeply($last, eml_load('t/data/0001.patch'), 'lei up worked on v2');
+ is_deeply(\@before, \@after, 'got same results');
+
+ my $v2s = "$home/v2s";
+ lei_ok(qw(q --shared z:0.. -o), "v2:$v2s");
+ my $shared = sum(map { -f $_ ? -s _ : () } (
+ glob("$v2s/git/0.git/objects/*/*")));
+ ok($shared < $orig, 'fewer bytes stored with --shared') or
+ diag "shared=$shared orig=$orig";
});
done_testing;
^ permalink raw reply related [relevance 39%]
* [PATCH 0/3] lei: v2 and lcat/import/IMAP things
@ 2021-05-29 20:20 71% Eric Wong
2021-05-29 20:20 36% ` [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support Eric Wong
2021-05-29 20:20 61% ` [PATCH 3/3] lei q: --sort and --save|v2 are incompatible Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-29 20:20 UTC (permalink / raw)
To: meta
Been using lei more and more for IMAP things.
Eric Wong (3):
TODO: add pipelining note
lei import|lcat: improve+fix single message IMAP support
lei q: --sort and --save|v2 are incompatible
TODO | 2 ++
lib/PublicInbox/LeiLcat.pm | 25 ++++++++------
lib/PublicInbox/LeiMailSync.pm | 23 ++++++++-----
lib/PublicInbox/LeiQuery.pm | 5 ++-
lib/PublicInbox/LeiToMail.pm | 18 +++-------
lib/PublicInbox/LeiXSearch.pm | 60 +++++++++++++++++++++++++++++++---
lib/PublicInbox/NetReader.pm | 6 ++--
t/lei-import-imap.t | 13 +++++++-
8 files changed, 112 insertions(+), 40 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei q: --sort and --save|v2 are incompatible
2021-05-29 20:20 71% [PATCH 0/3] lei: v2 and lcat/import/IMAP things Eric Wong
2021-05-29 20:20 36% ` [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support Eric Wong
@ 2021-05-29 20:20 61% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-29 20:20 UTC (permalink / raw)
To: meta
Saved searches rely on (reverse) docid ordering for efficient
incremental results, and sorting any other way prevents that.
Update comment description in LeiQuery while we're at it:
"ls-query" and "rm-query" are "ls-search" and "forget-search",
respectively, and "mv-query" is implicit with "edit-search"
---
lib/PublicInbox/LeiQuery.pm | 5 ++++-
lib/PublicInbox/LeiToMail.pm | 12 +++++-------
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 0435a516..eb7b98d4 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -1,7 +1,8 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# handles lei <q|ls-query|rm-query|mv-query> commands
+# handles "lei q" command and provides internals for
+# several other sub-commands (up, lcat, ...)
package PublicInbox::LeiQuery;
use strict;
use v5.10.1;
@@ -130,6 +131,8 @@ sub lei_q {
} else {
die "unrecognized --sort=$sort\n";
}
+ $opt->{save} and return
+ $self->fail('--save and --sort are incompatible');
}
$self->{mset_opt} = \%mset_opt;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 997ce599..5b5caee7 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -384,6 +384,7 @@ sub new {
my $fmt = $lei->{ovv}->{fmt};
my $dst = $lei->{ovv}->{dst};
my $self = bless {}, $cls;
+ my @conflict;
if ($fmt eq 'maildir') {
require PublicInbox::MdirReader;
$self->{base_type} = 'maildir';
@@ -412,13 +413,14 @@ sub new {
require PublicInbox::LeiViewText;
$lei->{lvt} = PublicInbox::LeiViewText->new($lei);
$self->{base_type} = 'text';
+ @conflict = qw(mua save);
} elsif ($fmt eq 'v2') {
die "--dedupe=oid and v2 are incompatible\n" if
($lei->{opt}->{dedupe}//'') eq 'oid';
$self->{base_type} = 'v2';
$lei->{opt}->{save} = \1;
- die "--mua incompatible with v2\n" if $lei->{opt}->{mua};
$dst = $lei->{ovv}->{dst} = $lei->abs_path($dst);
+ @conflict = qw(mua sort);
} else {
die "bad mail --format=$fmt\n";
}
@@ -426,12 +428,8 @@ sub new {
(-d $dst || (-e _ && !-w _)) and die
"$dst exists and is not a writable file\n";
}
- if ($self->{base_type} eq 'text') {
- my @err = map {
- defined($lei->{opt}->{$_}) ? "--$_" : ();
- } (qw(mua save));
- die "@err incompatible with $fmt\n" if @err;
- }
+ my @err = map { defined($lei->{opt}->{$_}) ? "--$_" : () } @conflict;
+ die "@err incompatible with $fmt\n" if @err;
$self->{dst} = $dst;
$lei->{dedupe} = $lei->{lss} // do {
my $dd_cls = 'PublicInbox::'.
^ permalink raw reply related [relevance 61%]
* [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support
2021-05-29 20:20 71% [PATCH 0/3] lei: v2 and lcat/import/IMAP things Eric Wong
@ 2021-05-29 20:20 36% ` Eric Wong
2021-05-29 20:20 61% ` [PATCH 3/3] lei q: --sort and --save|v2 are incompatible Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-29 20:20 UTC (permalink / raw)
To: meta
lcat can now dump the memoized contents of entire IMAP folders,
not just a single UID. It's now parallelized and pipelined for
multiple lei2mail workers.
Furthemore, various forms of JSON output work consistently
with blob-only output, now.
While working on this, I noticed NetReader was passing UID URLs
to imap_each callbacks, which was causing mail_sync.sqlite3 to
store UIDs in `folders' and clearly wrong so it's now fixed.
---
lib/PublicInbox/LeiLcat.pm | 25 ++++++++------
lib/PublicInbox/LeiMailSync.pm | 23 ++++++++-----
lib/PublicInbox/LeiToMail.pm | 6 ----
lib/PublicInbox/LeiXSearch.pm | 60 +++++++++++++++++++++++++++++++---
lib/PublicInbox/NetReader.pm | 6 ++--
t/lei-import-imap.t | 13 +++++++-
6 files changed, 101 insertions(+), 32 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index f9d9633a..effc3682 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -11,14 +11,21 @@ use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_imap_uid_uri ($$) {
- my ($lei, $uid_uri) = @_;
+sub lcat_imap_uri ($$) {
+ my ($lei, $uri) = @_;
my $lms = $lei->{lse}->lms or return;
- my $oidhex = $lms->imap_oid($lei, $uid_uri);
- if (ref(my $err = $oidhex)) { # art2folder error
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ # cf. LeiToMail->wq_atexit_child
+ if (defined $uri->uid) {
+ my $oidhex = $lms->imap_oid($lei, $uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ push @{$lei->{lcat_blob}}, $oidhex;
+ } elsif (defined(my $fid = $lms->fid_for($$uri))) {
+ push @{$lei->{lcat_fid}}, $fid;
+ } else {
+ $lei->child_error(1 << 8, "# unknown folder: $uri");
}
- push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
}
sub extract_1 ($$) {
@@ -26,10 +33,8 @@ sub extract_1 ($$) {
if ($x =~ m!\b(imaps?://[^>]+)!i) {
my $u = $1;
require PublicInbox::URIimap;
- $u = PublicInbox::URIimap->new($u);
- defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
- $lei->child_error(1 << 8, "# no UID= in $u");
- '""'; # blank query, using {lcat_blob}
+ lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
+ '""'; # blank query, using {lcat_blob} or {lcat
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 5c0988b5..c7f78239 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -64,9 +64,9 @@ CREATE TABLE IF NOT EXISTS blob2name (
}
-sub _fid_for {
+sub fid_for {
my ($self, $folder, $rw) = @_;
- my $dbh = $self->{dbh};
+ my $dbh = $self->{dbh} //= dbh_new($self, $rw);
my $sel = 'SELECT fid FROM folders WHERE loc = ? LIMIT 1';
my ($fid) = $dbh->selectrow_array($sel, undef, $folder);
return $fid if defined $fid;
@@ -111,7 +111,7 @@ EOM
sub set_src {
my ($self, $oidhex, $folder, $id) = @_;
- my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
my $sth;
if (ref($id)) { # scalar name
$id = $$id;
@@ -128,7 +128,7 @@ INSERT OR IGNORE INTO blob2num (oidbin, fid, uid) VALUES (?, ?, ?)
sub clear_src {
my ($self, $folder, $id) = @_;
- my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
my $sth;
if (ref($id)) { # scalar name
$id = $$id;
@@ -146,7 +146,7 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ?
# Maildir-only
sub mv_src {
my ($self, $folder, $oidbin, $id, $newbn) = @_;
- my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
my $sth = $self->{dbh}->prepare_cached(<<'');
UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
@@ -158,7 +158,12 @@ sub each_src {
my ($self, $folder, $cb, @args) = @_;
my $dbh = $self->{dbh} //= dbh_new($self);
my ($fid, $sth);
- $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder) // return;
+ if (ref($folder) eq 'HASH') {
+ $fid = $folder->{fid} // die "BUG: no `fid'";
+ } else {
+ $fid = $self->{fmap}->{$folder} //=
+ fid_for($self, $folder) // return;
+ }
$sth = $dbh->prepare('SELECT oidbin,uid FROM blob2num WHERE fid = ?');
$sth->execute($fid);
while (my ($oidbin, $id) = $sth->fetchrow_array) {
@@ -176,7 +181,7 @@ sub location_stats {
my $dbh = $self->{dbh} //= dbh_new($self);
my $fid;
my $ret = {};
- $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder) // return;
+ $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder) // return;
my ($row) = $dbh->selectrow_array(<<"", undef, $fid);
SELECT COUNT(name) FROM blob2name WHERE fid = ?
@@ -349,7 +354,7 @@ sub forget_folder {
my ($self, $folder) = @_;
my ($fid, $sth);
$fid = delete($self->{fmap}->{$folder}) //
- _fid_for($self, $folder) // return;
+ fid_for($self, $folder) // return;
my $dbh = $self->{dbh};
$dbh->do('DELETE FROM blob2name WHERE fid = ?', undef, $fid);
$dbh->do('DELETE FROM blob2num WHERE fid = ?', undef, $fid);
@@ -369,7 +374,7 @@ sub imap_oid {
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
my $fid = $self->{fmap}->{$folders->[0]} //=
- _fid_for($self, $folders->[0]) // return;
+ fid_for($self, $folders->[0]) // return;
my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
EOM
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 078f2551..997ce599 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -756,12 +756,6 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
my $lei = $self->{lei};
- if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
- for my $oid (@{$lei->{lcat_blob}}) {
- my $smsg = { blob => $oid, pct => 100 };
- write_mail($self, $smsg);
- }
- }
delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
my $nr = delete($lei->{-nr_write}) or return;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d6d42a01..f7c1e559 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -256,6 +256,14 @@ sub query_combined_mset { # non-parallel for non-"--threads" users
$lei->{ovv}->ovv_atexit_child($lei);
}
+sub _smsg_fill ($$) {
+ my ($smsg, $eml) = @_;
+ $smsg->populate($eml);
+ $smsg->parse_references($eml, mids($eml));
+ $smsg->{$_} //= '' for qw(from to cc ds subject references mid);
+ delete @$smsg{qw(From Subject -ds -ts)};
+}
+
sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
my $xoids = $lei->{ale}->xoids_for($eml, 1);
@@ -265,10 +273,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
my $smsg = bless {}, 'PublicInbox::Smsg';
$smsg->{blob} = $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
- $smsg->populate($eml);
- $smsg->parse_references($eml, mids($eml));
- $smsg->{$_} //= '' for qw(from to cc ds subject references mid);
- delete @$smsg{qw(From Subject -ds -ts)};
+ _smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
++$lei->{-nr_remote_eml};
@@ -453,6 +458,9 @@ sub start_query ($;$) { # always runs in main (lei-daemon) process
for my $uris (@$q) {
$self->wq_io_do('query_remote_mboxrd', [], $uris);
}
+ if ($self->{-do_lcat}) {
+ $self->wq_io_do('lcat_dump', []);
+ }
$self->wq_close(1); # lei_xsearch workers stop when done
}
@@ -518,6 +526,7 @@ sub do_query {
@$end = ();
$self->{opt_threads} = $lei->{opt}->{threads};
$self->{opt_sort} = $lei->{opt}->{'sort'};
+ $self->{-do_lcat} = $lei->{lcat_blob} // $lei->{lcat_fid};
if ($l2m) {
$l2m->net_merge_all_done unless $lei->{auth};
} else {
@@ -561,5 +570,48 @@ sub prepare_external {
push @{$self->{locals}}, $loc;
}
+sub _lcat_i { # LeiMailSync->each_src iterator callback
+ my ($oidbin, $id, $each_smsg) = @_;
+ $each_smsg->({blob => unpack('H*', $oidbin), pct => 100});
+}
+
+sub _lcat2smsg { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $smsg) = @_;
+ if ($bref) {
+ my $eml = PublicInbox::Eml->new($bref);
+ my $json_dump = delete $smsg->{-json_dump};
+ bless $smsg, 'PublicInbox::Smsg';
+ _smsg_fill($smsg, $eml);
+ $json_dump->($smsg, undef, $eml);
+ }
+}
+
+sub lcat_dump {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
+ my $git = $lei->{ale}->git;
+ if (!$lei->{l2m}) {
+ my $json_dump = $each_smsg;
+ $each_smsg = sub {
+ my ($smsg) = @_;
+ use Data::Dumper;
+ $smsg->{-json_dump} = $json_dump;
+ $git->cat_async($smsg->{blob}, \&_lcat2smsg, $smsg);
+ };
+ }
+ for my $oid (@{$lei->{lcat_blob} // []}) {
+ $each_smsg->({ blob => $oid, pct => 100 });
+ }
+ if (my $fids = delete $lei->{lcat_fid}) {
+ my $lms = $lei->{lse}->lms;
+ for my $fid (@$fids) {
+ $lms->each_src({fid => $fid}, \&_lcat_i, $each_smsg);
+ }
+ }
+ $git->async_wait_all;
+ undef $each_smsg; # may commit
+ $lei->{ovv}->ovv_atexit_child($lei);
+}
1;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 76d2fe62..54c6b082 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -471,8 +471,10 @@ EOF
my $uri = $orig_uri->clone;
my $single_uid = $uri->uid;
my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
- $itrk = $l_uid = undef if defined($single_uid);
-
+ if (defined($single_uid)) {
+ $itrk = $l_uid = undef;
+ $uri->uid(undef); # for eml_cb
+ }
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 59d481d5..895b19ff 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -48,6 +48,13 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok($stats->{'uid.min'} < $stats->{'uid.max'}, 'min < max');
ok($stats->{'uid.count'} > 0, 'count > 0');
+ lei_ok('lcat', $url);
+ is(scalar(grep(/^# blob:/, split(/\n/ms, $lei_out))),
+ $stats->{'uid.count'}, 'lcat on URL dumps folder');
+ lei_ok qw(lcat -f json), $url;
+ $out = json_utf8->decode($lei_out);
+ is(scalar(@$out) - 1, $stats->{'uid.count'}, 'lcat JSON dumps folder');
+
lei_ok(qw(q z:1..));
$out = json_utf8->decode($lei_out);
ok(scalar(@$out) > 1, 'got imported messages');
@@ -77,6 +84,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
my $uid_url = "$url/;UID=".$stats->{'uid.max'};
lei_ok 'import', $uid_url;
+ lei_ok 'ls-mail-sync';
+ is($lei_out, "$url\n", 'ls-mail-sync added URL w/o UID');
lei_ok 'inspect', $uid_url;
$lei_out =~ /([a-f0-9]{40,})/ or
xbail 'inspect missed blob with UID URL';
@@ -88,7 +97,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $orig = $lei_out;
lei_ok 'lcat', "blob:$blob";
is($lei_out, $orig, 'lcat understands blob:...');
- ok(!lei('lcat', $url), "lcat doesn't work on IMAP URL w/o UID");
+ lei_ok qw(lcat -f json), $uid_url;
+ $out = json_utf8->decode($lei_out);
+ is(scalar(@$out), 2, 'got JSON') or diag explain($out);
});
done_testing;
^ permalink raw reply related [relevance 36%]
* [PATCH 0/4] lei lcat usability things
@ 2021-05-30 6:33 71% Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
Nothing major, just some things to make lcat nicer for the
inevitable network outages to come
Eric Wong (4):
lei lcat+inspect: start wiring up completion
lei lcat: allow IMAP folder URLs w/o UIDVALIDITY
lei lcat: support maildir:... paths, too
lei: support implicit stdin by default
lib/PublicInbox/LEI.pm | 12 ++++++++++--
lib/PublicInbox/LeiInspect.pm | 8 ++++++++
lib/PublicInbox/LeiLcat.pm | 33 ++++++++++++++++++++++++++++++---
lib/PublicInbox/LeiRediff.pm | 1 -
lib/PublicInbox/LeiRm.pm | 1 -
t/lei-import-imap.t | 3 +++
t/lei-p2q.t | 4 ++++
7 files changed, 55 insertions(+), 7 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/4] lei lcat+inspect: start wiring up completion
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
@ 2021-05-30 6:33 71% ` Eric Wong
2021-05-30 6:33 68% ` [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
Colons and other delimiters still cause problems for our bash
completion, but some completion is better than no completion.
---
lib/PublicInbox/LeiInspect.pm | 8 ++++++++
lib/PublicInbox/LeiLcat.pm | 8 ++++++++
2 files changed, 16 insertions(+)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 7205979e..eb2634b4 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -99,4 +99,12 @@ sub lei_inspect {
$lei->out(']') if $multi;
}
+sub _complete_inspect {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ map { $match_cb->($_) } $lms->folders;
+}
+
1;
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index effc3682..81ab1e36 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -132,4 +132,12 @@ no args allowed on command-line with --stdin
$lei->_start_query;
}
+sub _complete_lcat {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ grep(m!\A[a-z]+://!, map { $match_cb->($_) } $lms->folders);
+}
+
1;
^ permalink raw reply related [relevance 71%]
* [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
@ 2021-05-30 6:33 68% ` Eric Wong
2021-05-30 6:33 63% ` [PATCH 3/4] lei lcat: support maildir: paths, too Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
Requiring UIDVALIDITY on the command-line is of course
unreasonable.
---
lib/PublicInbox/LeiLcat.pm | 12 +++++++++++-
t/lei-import-imap.t | 3 +++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 81ab1e36..5bd20c1f 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -24,7 +24,17 @@ sub lcat_imap_uri ($$) {
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
push @{$lei->{lcat_fid}}, $fid;
} else {
- $lei->child_error(1 << 8, "# unknown folder: $uri");
+ my $folders = [ $$uri ];
+ my $err = $lms->arg2folder($lei, $folders);
+ $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
+ if ($err && $err->{fail}) {
+ $lei->child_error(1 << 8, "# unknown folder: $uri");
+ } else {
+ for my $f (@$folders) {
+ my $fid = $lms->fid_for($f);
+ push @{$lei->{lcat_fid}}, $fid;
+ }
+ }
}
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 895b19ff..34fd6cf9 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -13,6 +13,7 @@ my $host_port = tcp_host_port($sock);
undef $sock;
test_lei({ tmpdir => $tmpdir }, sub {
my $url = "imap://$host_port/t.v2.0";
+ my $url_orig = $url;
lei_ok(qw(q z:1..));
my $out = json_utf8->decode($lei_out);
@@ -100,6 +101,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok qw(lcat -f json), $uid_url;
$out = json_utf8->decode($lei_out);
is(scalar(@$out), 2, 'got JSON') or diag explain($out);
+ lei_ok qw(lcat), $url_orig;
+ is($lei_out, $orig, 'lcat w/o UID works');
});
done_testing;
^ permalink raw reply related [relevance 68%]
* [PATCH 3/4] lei lcat: support maildir: paths, too
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
2021-05-30 6:33 68% ` [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY Eric Wong
@ 2021-05-30 6:33 63% ` Eric Wong
2021-05-30 6:33 61% ` [PATCH 4/4] lei: support implicit stdin by default Eric Wong
2021-05-30 11:52 71% ` [PATCH 0/4] lei lcat usability things Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
This could be helpful in case when a Maildir is on a slow
or unmounted filesystem and lei/store is on fast storage.
---
lib/PublicInbox/LeiLcat.pm | 37 +++++++++++++++++++++++--------------
1 file changed, 23 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 5bd20c1f..cb5eb5b4 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -11,10 +11,26 @@ use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
use PublicInbox::MID qw($MID_EXTRACT);
+sub lcat_folder ($$$) {
+ my ($lei, $lms, $folder) = @_;
+ $lms //= $lei->{lse}->lms // return;
+ my $folders = [ $folder];
+ my $err = $lms->arg2folder($lei, $folders);
+ $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
+ if ($err && $err->{fail}) {
+ $lei->child_error(1 << 8, "# unknown folder: $folder");
+ } else {
+ for my $f (@$folders) {
+ my $fid = $lms->fid_for($f);
+ push @{$lei->{lcat_fid}}, $fid;
+ }
+ }
+}
+
sub lcat_imap_uri ($$) {
my ($lei, $uri) = @_;
my $lms = $lei->{lse}->lms or return;
- # cf. LeiToMail->wq_atexit_child
+ # cf. LeiXsearch->lcat_dump
if (defined $uri->uid) {
my $oidhex = $lms->imap_oid($lei, $uri);
if (ref(my $err = $oidhex)) { # art2folder error
@@ -24,17 +40,7 @@ sub lcat_imap_uri ($$) {
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
push @{$lei->{lcat_fid}}, $fid;
} else {
- my $folders = [ $$uri ];
- my $err = $lms->arg2folder($lei, $folders);
- $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
- if ($err && $err->{fail}) {
- $lei->child_error(1 << 8, "# unknown folder: $uri");
- } else {
- for my $f (@$folders) {
- my $fid = $lms->fid_for($f);
- push @{$lei->{lcat_fid}}, $fid;
- }
- }
+ lcat_folder($lei, $lms, $$uri);
}
}
@@ -44,7 +50,10 @@ sub extract_1 ($$) {
my $u = $1;
require PublicInbox::URIimap;
lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
- '""'; # blank query, using {lcat_blob} or {lcat
+ '""'; # blank query, using {lcat_blob} or {lcat_fid}
+ } elsif ($x =~ m!\b(maildir:.+)!i) {
+ lcat_folder($lei, undef, $1);
+ '""'; # blank query, using {lcat_blob} or {lcat_fid}
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
@@ -147,7 +156,7 @@ sub _complete_lcat {
my $sto = $lei->_lei_store or return;
my $lms = $sto->search->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
- grep(m!\A[a-z]+://!, map { $match_cb->($_) } $lms->folders);
+ map { $match_cb->($_) } $lms->folders;
}
1;
^ permalink raw reply related [relevance 63%]
* [PATCH 4/4] lei: support implicit stdin by default
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
` (2 preceding siblings ...)
2021-05-30 6:33 63% ` [PATCH 3/4] lei lcat: support maildir: paths, too Eric Wong
@ 2021-05-30 6:33 61% ` Eric Wong
2021-05-30 11:52 71% ` [PATCH 0/4] lei lcat usability things Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
This adds implicit stdin suppport for p2q and lcat,
while rm and rediff no longer need explicit support
for it.
---
lib/PublicInbox/LEI.pm | 12 ++++++++++--
lib/PublicInbox/LeiRediff.pm | 1 -
lib/PublicInbox/LeiRm.pm | 1 -
t/lei-p2q.t | 4 ++++
4 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f2dfc320..3527cf09 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -638,7 +638,15 @@ sub optparse ($$$) {
my $ok;
for my $o (@or) {
if ($o =~ /\A--([a-z0-9\-]+)/) {
- $ok = defined($OPT->{$1});
+ my $sw = $1;
+ # assume pipe/regular file on stdin
+ # w/o args means stdin
+ if ($sw eq 'stdin' && !@$argv &&
+ (-p $self->{0} ||
+ -f _) && -r _) {
+ $OPT->{stdin} //= 1;
+ }
+ $ok = defined($OPT->{$sw});
last if $ok;
} elsif (defined($argv->[$i])) {
$ok = 1;
@@ -906,7 +914,7 @@ sub start_mua {
}
push @cmd, $mfolder unless defined($replaced);
if ($self->{sock}) { # lei(1) client process runs it
- # restore terminal: echo $query | lei q -stdin --mua=...
+ # restore terminal: echo $query | lei q --stdin --mua=...
my $io = [];
$io->[0] = $self->{1} if $self->{opt}->{stdin} && -t $self->{1};
send_exec_cmd($self, $io, \@cmd, {});
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 2e793df5..c8bd0dfb 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -201,7 +201,6 @@ sub input_eml_cb { # callback for all emails
sub lei_rediff {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
- $lei->{opt}->{stdin} = 1 if !@inputs;
$lei->{opt}->{'in-format'} //= 'eml';
# maybe it's a non-email (code) blob from a coderepo
my $git_dirs = $lei->{opt}->{'git-dir'} //= [];
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 185b6a15..c6d28045 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -31,7 +31,6 @@ sub input_maildir_cb {
sub lei_rm {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
- $lei->{opt}->{stdin} = 1 if !@inputs;
$lei->{opt}->{'in-format'} //= 'eml';
my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
$self->prepare_inputs($lei, \@inputs) or return;
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index f8b073cf..58506f94 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -14,6 +14,10 @@ test_lei(sub {
lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin') or diag $lei_err;
+ sysseek($fh, 0, 0) or xbail "lseek: $!";
+ lei_ok([qw(p2q -w dfpost)], undef, { %$lei_opt, 0 => $fh });
+ is($lei_out, "dfpost:6e006fd73b1d\n", 'implicit --stdin');
+
lei_ok(qw(p2q --uri t/data/0001.patch -w), 'dfpost,dfn');
is($lei_out, "dfpost%3A6e006fd73b1d+".
"dfn%3Alib%2FPublicInbox%2FSearch.pm\n",
^ permalink raw reply related [relevance 61%]
* [PATCH] lei import: import IMAP flag changes from old messages
@ 2021-05-30 11:45 42% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 11:45 UTC (permalink / raw)
To: meta
This makes "lei import" behavior with IMAP folders more
consistent with that with Maildir.
Opening IMAP folders read-write with "SELECT" (instead of
read-only with "EXAMINE") was necessary, since it lets an IMAP
server communicate to us as to whether or not it's worth
refetching IMAP flags of previously imported messages.
Fetching UID+FLAGS only is one of the fastest IMAP operations
with dovecot, our -imapd and presumably other common IMAP servers.
It is issued by common MUAs such as mutt after every SELECT.
Users may now rely on "lei import" exclusively to merge mail and
keywords into lei/store, and "lei export-kw" to propagate
keyword changes back to IMAP servers.
A sticks-and-stones workflow for personal mailboxes is currently:
lei import imaps://$MY_PERSONAL_INBOX
lei q --mua=$MUA -o /tmp/results SEARCH TERMS...
# do stuff from within $MUA to /tmp/results
lei import /tmp/results # read keyword changes from MUA
lei export-kw imaps://$MY_PERSONAL_INBOX
# repeat when new stuff shows up in personal inbox
The next goal is to automate repeated imports + export-kw
commands with with inotify and IMAP IDLE.
---
lib/PublicInbox/LeiImport.pm | 18 ++++++++-
lib/PublicInbox/LeiMailSync.pm | 21 +++++++----
lib/PublicInbox/NetReader.pm | 68 +++++++++++++++++++++++++++++-----
3 files changed, 88 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 01e6c93c..f9a46ec5 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -45,7 +45,16 @@ sub input_net_cb { # imap_each / nntp_each
my ($uri, $uid, $kw, $eml, $self) = @_;
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
$vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
- $self->input_eml_cb($eml, $vmd);
+ if (defined $eml) {
+ $self->input_eml_cb($eml, $vmd);
+ } elsif ($vmd) { # old message, kw only
+ my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
+ my @docids = $self->{lse}->over->blob_exists($oid) or return;
+ my $lei = $self->{lei};
+ $lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
+ $self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
+ $vmd, \@docids);
+ }
}
sub do_import_index ($$@) {
@@ -65,7 +74,12 @@ sub do_import_index ($$@) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
- $net->{-lms_ro} = $lei->_lei_store->search->lms // 0;
+ $net->{-lms_ro} = $sto->search->lms // 0;
+ if ($self->{-import_kw}) {
+ $net->{each_old} = 1;
+ $self->{-lms_ro} = $net->{-lms_ro};
+ $self->{lse} = $sto->search;
+ }
}
} else {
my $nproc = $self->detect_nproc;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index c7f78239..36cd564c 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -361,6 +361,17 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
+sub imap_oid2 ($$$) {
+ my ($self, $uri, $uid) = @_; # $uri MUST have UIDVALIDITY
+ my $fid = $self->{fmap}->{"$uri"} //= fid_for($self, "$uri") // return;
+ my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
+EOM
+ $sth->execute($fid, $uid);
+ my ($oidbin) = $sth->fetchrow_array;
+ $oidbin ? unpack('H*', $oidbin) : undef;
+}
+
sub imap_oid {
my ($self, $lei, $uid_uri) = @_;
my $mailbox_uri = $uid_uri->clone;
@@ -373,16 +384,10 @@ sub imap_oid {
}
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
- my $fid = $self->{fmap}->{$folders->[0]} //=
- fid_for($self, $folders->[0]) // return;
- my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
-SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
-EOM
- $sth->execute($fid, $uid_uri->uid);
- my ($oidbin) = $sth->fetchrow_array;
- $oidbin ? unpack('H*', $oidbin) : undef;
+ imap_oid2($self, $folders->[0], $uid_uri->uid);
}
+
# FIXME: something with "lei <up|q>" is causing uncommitted transaction
# warnings, not sure what...
sub DESTROY {
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 54c6b082..b97444fd 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -396,10 +396,8 @@ sub errors {
undef;
}
-sub _imap_do_msg ($$$$$) {
- my ($self, $uri, $uid, $raw, $flags) = @_;
- # our target audience expects LF-only, save storage
- $$raw =~ s/\r\n/\n/sg;
+sub flags2kw ($$$$) {
+ my ($self, $uri, $uid, $flags) = @_;
my $kw = [];
for my $f (split(/ /, $flags)) {
if (my $k = $IMAPflags2kw{$f}) {
@@ -412,6 +410,14 @@ sub _imap_do_msg ($$$$$) {
}
}
@$kw = sort @$kw; # for all UI/UX purposes
+ $kw;
+}
+
+sub _imap_do_msg ($$$$$) {
+ my ($self, $uri, $uid, $raw, $flags) = @_;
+ # our target audience expects LF-only, save storage
+ $$raw =~ s/\r\n/\n/sg;
+ my $kw = flags2kw($self, $uri, $uid, $flags) // return;
my ($eml_cb, @args) = @{$self->{eml_each}};
$eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args);
}
@@ -447,17 +453,56 @@ sub itrk_last ($$;$$) {
($itrk, $l_uid, $l_uidval //= $r_uidval);
}
+# import flags of already-seen messages
+sub each_old_flags ($$$$) {
+ my ($self, $mic, $uri, $l_uid) = @_;
+ $l_uid ||= 1;
+ my $sec = uri_section($uri);
+ my $bs = $self->{imap_opt}->{$sec}->{batch_size} // 10000;
+ my ($eml_cb, @args) = @{$self->{eml_each}};
+ for (my $n = 1; $n <= $l_uid; $n += $bs) {
+ my $end = $n + $bs;
+ $end = $l_uid if $end > $l_uid;
+ my $r = $mic->fetch_hash("$n:$end", 'FLAGS');
+ if (!$r) {
+ return if $!{EINTR} && $self->{quit};
+ return "E: $uri UID FETCH $n:$end error: $!";
+ }
+ while (my ($uid, $per_uid) = each %$r) {
+ my $kw = flags2kw($self, $uri, $uid, $per_uid->{FLAGS})
+ // next;
+ $eml_cb->($uri, $uid, $kw, undef, @args);
+ }
+ }
+}
+
+# returns true if PERMANENTFLAGS indicates FLAGS of already imported
+# messages are meaningful
+sub perm_fl_ok ($) {
+ my ($perm_fl) = @_;
+ return if !defined($perm_fl);
+ for my $f (split(/[ \t]+/, $perm_fl)) {
+ return 1 if $IMAPflags2kw{$f};
+ }
+ undef;
+}
+
sub _imap_fetch_all ($$$) {
my ($self, $mic, $orig_uri) = @_;
my $sec = uri_section($orig_uri);
my $mbx = $orig_uri->mailbox;
$mic->Clear(1); # trim results history
- $mic->examine($mbx) or return "E: EXAMINE $mbx ($sec) failed: $!";
- my ($r_uidval, $r_uidnext);
+
+ # we need to check for mailbox writability to see if we care about
+ # FLAGS from already-imported messages.
+ my $cmd = $self->{each_old} ? 'select' : 'examine';
+ $mic->$cmd($mbx) or return "E: \U$cmd\E $mbx ($sec) failed: $!";
+
+ my ($r_uidval, $r_uidnext, $perm_fl);
for ($mic->Results) {
+ /^\* OK \[PERMANENTFLAGS \(([^\)]*)\)\].*/ and $perm_fl = $1;
/^\* OK \[UIDVALIDITY ([0-9]+)\].*/ and $r_uidval = $1;
/^\* OK \[UIDNEXT ([0-9]+)\].*/ and $r_uidnext = $1;
- last if $r_uidval && $r_uidnext;
}
$r_uidval //= $mic->uidvalidity($mbx) //
return "E: $orig_uri cannot get UIDVALIDITY";
@@ -486,6 +531,13 @@ EOF
E: $uri local UID exceeds remote ($l_uid > $r_uid)
E: $uri strangely, UIDVALIDLITY matches ($l_uidval)
EOF
+ $mic->Uid(1); # the default, we hope
+ my $err;
+ if (!defined($single_uid) && $self->{each_old} &&
+ perm_fl_ok($perm_fl)) {
+ $err = each_old_flags($self, $mic, $uri, $l_uid);
+ return $err if $err;
+ }
return if $l_uid >= $r_uid; # nothing to do
$l_uid ||= 1;
my ($mod, $shard) = @{$self->{shard_info} // []};
@@ -493,13 +545,11 @@ EOF
my $m = $mod ? " [(UID % $mod) == $shard]" : '';
warn "# $uri fetching UID $l_uid:$r_uid$m\n";
}
- $mic->Uid(1); # the default, we hope
my $bs = $self->{imap_opt}->{$sec}->{batch_size} // 1;
my $req = $mic->imap4rev1 ? 'BODY.PEEK[]' : 'RFC822.PEEK';
my $key = $req;
$key =~ s/\.PEEK//;
my ($uids, $batch);
- my $err;
do {
# I wish "UID FETCH $START:*" could work, but:
# 1) servers do not need to return results in any order
^ permalink raw reply related [relevance 42%]
* Re: [PATCH 0/4] lei lcat usability things
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
` (3 preceding siblings ...)
2021-05-30 6:33 61% ` [PATCH 4/4] lei: support implicit stdin by default Eric Wong
@ 2021-05-30 11:52 71% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 11:52 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> Nothing major, just some things to make lcat nicer for the
> inevitable network outages to come
>
> Eric Wong (4):
> lei lcat+inspect: start wiring up completion
> lei lcat: allow IMAP folder URLs w/o UIDVALIDITY
Btw, "lei lcat $URL" is great in the case adding a label/keyword
with "lei import" was forgotten:
lei import $URL
# oops, I forgot to add +L:somelabel to the above import
# invocation
lei lcat $URL -f mboxrd | lei tag -f mboxrd - +L:somelabel
Running "lei tag $URL +L:somelabel" directly works, too,
but that's extra network traffic + latency.
^ permalink raw reply [relevance 71%]
* [PATCH] lei import: reduce writes to lei/store on IMAP sync
@ 2021-05-31 10:20 86% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-31 10:20 UTC (permalink / raw)
To: meta
We don't need to write VMD changes to lei/store if local
keywords are unchanged.
---
lib/PublicInbox/LeiImport.pm | 1 +
lib/PublicInbox/LeiSearch.pm | 8 +++++---
lib/PublicInbox/NetReader.pm | 2 +-
3 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index f9a46ec5..860a2c98 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -50,6 +50,7 @@ sub input_net_cb { # imap_each / nntp_each
} elsif ($vmd) { # old message, kw only
my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
my @docids = $self->{lse}->over->blob_exists($oid) or return;
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
my $lei = $self->{lei};
$lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
$self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index b09d1e45..d0963e92 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -133,9 +133,11 @@ sub xoids_for {
# returns true if $eml is indexed by lei/store and keywords don't match
sub kw_changed {
my ($self, $eml, $new_kw_sorted, $docids) = @_;
- my $xoids = xoids_for($self, $eml) // return;
- $docids //= [];
- @$docids = sort { $a <=> $b } values %$xoids;
+ if ($eml) {
+ my $xoids = xoids_for($self, $eml) // return;
+ $docids //= [];
+ @$docids = sort { $a <=> $b } values %$xoids;
+ }
my $cur_kw = msg_keywords($self, $docids->[0]);
# RFC 5550 sec 5.9 on the $Forwarded keyword states:
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index b97444fd..39a8f7fc 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -409,7 +409,7 @@ sub flags2kw ($$$$) {
warn "# unknown IMAP flag $f <$uri/;UID=$uid>\n";
}
}
- @$kw = sort @$kw; # for all UI/UX purposes
+ @$kw = sort @$kw; # for LeiSearch->kw_changed and UI/UX purposes
$kw;
}
^ permalink raw reply related [relevance 86%]
* [PATCH] lei: remove "forget" (old name for "rm")
@ 2021-06-01 19:20 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-01 19:20 UTC (permalink / raw)
To: meta
"rm" is probably the better name for it, since it matches
"public-inbox-learn rm"
---
lib/PublicInbox/LEI.pm | 3 ---
1 file changed, 3 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3527cf09..30f90798 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -218,9 +218,6 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt,
pass_through('-kw:foo for delete') ],
-'forget' => [ '[--stdin|--oid=OID|--by-mid=MID]',
- "exclude message(s) on stdin from `q' search results",
- qw(stdin| oid=s exact by-mid|mid:s), @c_opt ],
'purge-mailsource' => [ 'LOCATION|--all',
'remove imported messages from IMAP, Maildirs, and MH',
^ permalink raw reply related [relevance 71%]
* [PATCH] lei export-kw: do not write directly to mail_sync.sqlite3
@ 2021-06-02 10:03 58% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-02 10:03 UTC (permalink / raw)
To: meta
Only the lei/store process should be writing to files/DBs
in lei/store.
---
lib/PublicInbox/LeiExportKw.pm | 21 +++++++++++----------
lib/PublicInbox/LeiStore.pm | 23 +++++++++++++++++++----
2 files changed, 30 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 92c3aa43..b31b065f 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -28,6 +28,7 @@ sub export_kw_md { # LeiMailSync->each_src callback
PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown);
my $dst = "$mdir/cur/$bn";
my @fail;
+ my $lei = $self->{lei};
for my $d (@try) {
my $src = "$mdir/$d/$$id";
next if $src eq $dst;
@@ -39,15 +40,15 @@ sub export_kw_md { # LeiMailSync->each_src callback
# unlink(2) may ENOENT from parallel invocation,
# ignore it, but not other serious errors
if (!unlink($src) and $! != ENOENT) {
- $self->{lei}->child_error(1,
- "E: unlink($src): $!");
+ $lei->child_error(1, "E: unlink($src): $!");
}
- $self->{lms}->mv_src("maildir:$mdir",
- $oidbin, $id, $bn) or die;
+ $lei->{sto}->ipc_do('lms_mv_src', "maildir:$mdir",
+ $oidbin, $id, $bn);
return; # success anyways if link(2) worked
}
if ($! == ENOENT && !-e $src) { # some other process moved it
- $self->{lms}->clear_src("maildir:$mdir", $id);
+ $lei->{sto}->ipc_do('lms_clear_src',
+ "maildir:$mdir", $id);
next;
}
push @fail, $src if $! != EEXIST;
@@ -56,7 +57,7 @@ sub export_kw_md { # LeiMailSync->each_src callback
# both tries failed
my $e = $!;
my $orig = '['.join('|', @fail).']';
- $self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
+ $lei->child_error(1, "link($orig, $dst) ($oidhex): $e");
}
sub export_kw_imap { # LeiMailSync->each_src callback
@@ -69,8 +70,7 @@ sub export_kw_imap { # LeiMailSync->each_src callback
# overrides PublicInbox::LeiInput::input_path_url
sub input_path_url {
my ($self, $input, @args) = @_;
- my $lms = $self->{lms} //= $self->{lse}->lms;
- $lms->lms_begin;
+ my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
@@ -81,7 +81,7 @@ sub input_path_url {
$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
$mic->expunge;
} else { die "BUG: $input not supported" }
- $lms->lms_commit;
+ my $wait = $self->{lei}->{sto}->ipc_do('done');
}
sub lei_export_kw {
@@ -151,8 +151,9 @@ EOM
$self->{imap_mod_kw} = $net->can($self->{-merge_kw} ?
'imap_add_kw' : 'imap_set_kw');
}
- undef $lms;
+ undef $lms; # for fork
my $ops = {};
+ $sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 6888afb4..821782b9 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -193,15 +193,30 @@ sub remove_eml_vmd { # remove just the VMD
\@docids;
}
-sub set_sync_info {
- my ($self, $oidhex, $folder, $id) = @_;
- ($self->{lms} //= do {
+sub _lms_rw ($) {
+ my ($self) = @_;
+ $self->{lms} //= do {
require PublicInbox::LeiMailSync;
my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
my $lms = PublicInbox::LeiMailSync->new($f);
$lms->lms_begin;
$lms;
- })->set_src($oidhex, $folder, $id);
+ };
+}
+
+sub lms_clear_src {
+ my ($self, $folder, $id) = @_;
+ _lms_rw($self)->clear_src($folder, $id);
+}
+
+sub lms_mv_src {
+ my ($self, $folder, $oidbin, $id, $newbn) = @_;
+ _lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn);
+}
+
+sub set_sync_info {
+ my ($self, $oidhex, $folder, $id) = @_;
+ _lms_rw($self)->set_src($oidhex, $folder, $id);
}
sub _remove_if_local { # git->cat_async arg
^ permalink raw reply related [relevance 58%]
* [PATCH] lei import: speed up kw updates for old IMAP messages
@ 2021-06-03 0:17 40% Eric Wong
2021-06-03 1:05 38% ` [PATCH v2] " Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-06-03 0:17 UTC (permalink / raw)
To: meta
On a 4-core CPU, this speeds up "lei import" on a largish IMAP
inbox with 75K messages from ~21 minutes down to 40s.
Parallelizing with the new LeiImportKw WQ worker class gives a
near-linear speedup and brought the runtime down to ~5:40.
The new idx_fid_uid index on the "fid" and "uid" columns of
blob2num in mail_sync.sqlite3 brought us the final speedup.
An additional index on over.sqlite3#xref3(oidbin) did not help,
since idx_nntp already exists and speeds up the new ->oidbin_exists
internal API.
I initially experimented with a separate "lei import-kw" command
but decided against it since it's useless outside of IMAP+JMAP
and would require extra cognitive overhead for both users and
hackers. So LeiImportKw is just a WQ worker used by "lei import"
and not its own user-visible command.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 25 ++++++++--------
lib/PublicInbox/LeiImportKw.pm | 54 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 17 ++++++-----
lib/PublicInbox/NetReader.pm | 1 +
lib/PublicInbox/Over.pm | 10 ++++---
7 files changed, 85 insertions(+), 25 deletions(-)
create mode 100644 lib/PublicInbox/LeiImportKw.pm
diff --git a/MANIFEST b/MANIFEST
index 0b4bb380..5a70a144 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
+lib/PublicInbox/LeiImportKw.pm
lib/PublicInbox/LeiIndex.pm
lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 30f90798..7bda9408 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -421,7 +421,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1 ikw); # internal workers
sub _drop_wq {
my ($self) = @_;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 860a2c98..2efd4935 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -43,18 +43,14 @@ sub input_maildir_cb { # maildir_each_eml cb
sub input_net_cb { # imap_each / nntp_each
my ($uri, $uid, $kw, $eml, $self) = @_;
- my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
- $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
if (defined $eml) {
+ my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
+ $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
$self->input_eml_cb($eml, $vmd);
- } elsif ($vmd) { # old message, kw only
- my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
- my @docids = $self->{lse}->over->blob_exists($oid) or return;
- $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
- my $lei = $self->{lei};
- $lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
- $self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
- $vmd, \@docids);
+ } elsif (my $ikw = $self->{lei}->{ikw}) { # old message, kw only
+ # we send $uri as a bare SCALAR and not a URIimap ref to
+ # reduce socket traffic:
+ $ikw->wq_io_do('ck_update_kw', [], $$uri, $uid, $kw);
}
}
@@ -71,15 +67,17 @@ sub do_import_index ($$@) {
$lei->ale; # initialize for workers to read
my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ my $ikw;
if (my $net = $lei->{net}) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
- if ($self->{-import_kw}) {
+ if ($self->{-import_kw} && $net->{-lms_ro} &&
+ $net->{imap_order}) {
+ require PublicInbox::LeiImportKw;
+ $ikw = PublicInbox::LeiImportKw->new($lei);
$net->{each_old} = 1;
- $self->{-lms_ro} = $net->{-lms_ro};
- $self->{lse} = $sto->search;
}
}
} else {
@@ -93,6 +91,7 @@ sub do_import_index ($$@) {
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
+ $ikw->wq_close(1) if $ikw;
net_merge_all_done($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
new file mode 100644
index 00000000..e13dce07
--- /dev/null
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -0,0 +1,54 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# WQ worker for dealing with LeiImport IMAP flags on already-imported messages
+# WQ key: {ikw}
+package PublicInbox::LeiImportKw;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+sub new {
+ my ($cls, $lei) = @_;
+ my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
+ my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
+ $op_c->{ops} = $ops; # for PktOp->event_step
+ $lei->{ikw} = $self;
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->_lei_atfork_child;
+ my $net = delete $lei->{net} // die 'BUG: no lei->{net}';
+ $self->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
+ $self->{verbose} = $lei->{opt}->{verbose};
+ $self->{lse} = $self->{sto}->search;
+ $self->{over} = $self->{lse}->over;
+ $self->{-lms_ro} = $net->{-lms_ro} || die 'BUG: net->{-lms_ro} FALSE';
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub ck_update_kw { # via wq_io_do
+ my ($self, $url, $uid, $kw) = @_;
+ my $oidbin = $self->{-lms_ro}->imap_oidbin($url, $uid) // return;
+ my @docids = $self->{over}->oidbin_exists($oidbin) or return;
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ $self->{verbose} and
+ $self->{lei}->qerr('# '.unpack('H*', $oidbin)." => @$kw\n");
+ $self->{sto}->ipc_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
+}
+
+sub ikw_done_wait {
+ my ($lei) = @_;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lei->wq_done_wait;
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $ikw = delete $lei->{ikw} or return $lei->fail;
+ $ikw->wq_wait_old(\&ikw_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 22ee1109..75603d89 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS blob2num (
UNIQUE (oidbin, fid, uid)
)
+ # speeds up LeiImport->ck_update_kw (for "lei import") by 5-6x:
+ $dbh->do(<<'');
+CREATE INDEX IF NOT EXISTS idx_fid_uid ON blob2num(fid,uid)
+
$dbh->do(<<'');
CREATE TABLE IF NOT EXISTS blob2name (
oidbin VARBINARY NOT NULL,
@@ -361,15 +365,14 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
-sub imap_oid2 ($$$) {
- my ($self, $uri, $uid) = @_; # $uri MUST have UIDVALIDITY
- my $fid = $self->{fmap}->{"$uri"} //= fid_for($self, "$uri") // return;
+sub imap_oidbin ($$$) {
+ my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY
+ my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return;
my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
EOM
$sth->execute($fid, $uid);
- my ($oidbin) = $sth->fetchrow_array;
- $oidbin ? unpack('H*', $oidbin) : undef;
+ $sth->fetchrow_array;
}
sub imap_oid {
@@ -384,10 +387,10 @@ sub imap_oid {
}
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
- imap_oid2($self, $folders->[0], $uid_uri->uid);
+ my $oidbin = imap_oidbin($self, $folders->[0], $uid_uri->uid);
+ $oidbin ? unpack('H*', $oidbin) : undef;
}
-
# FIXME: something with "lei <up|q>" is causing uncommitted transaction
# warnings, not sure what...
sub DESTROY {
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 39a8f7fc..058f4313 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -471,6 +471,7 @@ sub each_old_flags ($$$$) {
while (my ($uid, $per_uid) = each %$r) {
my $kw = flags2kw($self, $uri, $uid, $per_uid->{FLAGS})
// next;
+ # LeiImport->input_net_cb
$eml_cb->($uri, $uid, $kw, undef, @args);
}
}
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 0e191c47..58fdea0e 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -349,13 +349,13 @@ sub check_inodes {
}
}
-sub blob_exists {
- my ($self, $oidhex) = @_;
+sub oidbin_exists {
+ my ($self, $oidbin) = @_;
if (wantarray) {
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
my $tmp = $sth->fetchall_arrayref;
map { $_->[0] } @$tmp;
@@ -363,10 +363,12 @@ SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT COUNT(*) FROM xref3 WHERE oidbin = ?
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
$sth->fetchrow_array;
}
}
+sub blob_exists { oidbin_exists($_[0], pack('H*', $_[1])) }
+
1;
^ permalink raw reply related [relevance 40%]
* [PATCH v2] lei import: speed up kw updates for old IMAP messages
2021-06-03 0:17 40% [PATCH] lei import: speed up kw updates for old IMAP messages Eric Wong
@ 2021-06-03 1:05 38% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-03 1:05 UTC (permalink / raw)
To: meta
On a 4-core CPU, this speeds up "lei import" on a largish IMAP
inbox with 75K messages from ~21 minutes down to 40s.
Parallelizing with the new LeiImportKw WQ worker class gives a
near-linear speedup and brought the runtime down to ~5:40.
The new idx_fid_uid index on the "fid" and "uid" columns of
blob2num in mail_sync.sqlite3 brought us the final speedup.
An additional index on over.sqlite3#xref3(oidbin) did not help,
since idx_nntp already exists and speeds up the new ->oidbin_exists
internal API.
I initially experimented with a separate "lei import-kw" command
but decided against it since it's useless outside of IMAP+JMAP
and would require extra cognitive overhead for both users and
hackers. So LeiImportKw is just a WQ worker used by "lei import"
and not its own user-visible command.
v2: fix ikw_done_wait arg handling (ugh, confusing API :x)
---
Interdiff against v1:
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index e13dce07..2878cbdf 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -40,9 +40,10 @@ sub ck_update_kw { # via wq_io_do
}
sub ikw_done_wait {
- my ($lei) = @_;
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
my $wait = $lei->{sto}->ipc_do('done');
- $lei->wq_done_wait;
+ $lei->can('wq_done_wait')->($arg, $pid);
}
sub _lei_wq_eof { # EOF callback for main lei daemon
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 25 ++++++++--------
lib/PublicInbox/LeiImportKw.pm | 55 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 17 ++++++-----
lib/PublicInbox/NetReader.pm | 1 +
lib/PublicInbox/Over.pm | 10 ++++---
7 files changed, 86 insertions(+), 25 deletions(-)
create mode 100644 lib/PublicInbox/LeiImportKw.pm
diff --git a/MANIFEST b/MANIFEST
index 0b4bb380..5a70a144 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
+lib/PublicInbox/LeiImportKw.pm
lib/PublicInbox/LeiIndex.pm
lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 30f90798..7bda9408 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -421,7 +421,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1 ikw); # internal workers
sub _drop_wq {
my ($self) = @_;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 860a2c98..2efd4935 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -43,18 +43,14 @@ sub input_maildir_cb { # maildir_each_eml cb
sub input_net_cb { # imap_each / nntp_each
my ($uri, $uid, $kw, $eml, $self) = @_;
- my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
- $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
if (defined $eml) {
+ my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
+ $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
$self->input_eml_cb($eml, $vmd);
- } elsif ($vmd) { # old message, kw only
- my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
- my @docids = $self->{lse}->over->blob_exists($oid) or return;
- $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
- my $lei = $self->{lei};
- $lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
- $self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
- $vmd, \@docids);
+ } elsif (my $ikw = $self->{lei}->{ikw}) { # old message, kw only
+ # we send $uri as a bare SCALAR and not a URIimap ref to
+ # reduce socket traffic:
+ $ikw->wq_io_do('ck_update_kw', [], $$uri, $uid, $kw);
}
}
@@ -71,15 +67,17 @@ sub do_import_index ($$@) {
$lei->ale; # initialize for workers to read
my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ my $ikw;
if (my $net = $lei->{net}) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
- if ($self->{-import_kw}) {
+ if ($self->{-import_kw} && $net->{-lms_ro} &&
+ $net->{imap_order}) {
+ require PublicInbox::LeiImportKw;
+ $ikw = PublicInbox::LeiImportKw->new($lei);
$net->{each_old} = 1;
- $self->{-lms_ro} = $net->{-lms_ro};
- $self->{lse} = $sto->search;
}
}
} else {
@@ -93,6 +91,7 @@ sub do_import_index ($$@) {
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
+ $ikw->wq_close(1) if $ikw;
net_merge_all_done($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
new file mode 100644
index 00000000..2878cbdf
--- /dev/null
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -0,0 +1,55 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# WQ worker for dealing with LeiImport IMAP flags on already-imported messages
+# WQ key: {ikw}
+package PublicInbox::LeiImportKw;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+sub new {
+ my ($cls, $lei) = @_;
+ my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
+ my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
+ $op_c->{ops} = $ops; # for PktOp->event_step
+ $lei->{ikw} = $self;
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->_lei_atfork_child;
+ my $net = delete $lei->{net} // die 'BUG: no lei->{net}';
+ $self->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
+ $self->{verbose} = $lei->{opt}->{verbose};
+ $self->{lse} = $self->{sto}->search;
+ $self->{over} = $self->{lse}->over;
+ $self->{-lms_ro} = $net->{-lms_ro} || die 'BUG: net->{-lms_ro} FALSE';
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub ck_update_kw { # via wq_io_do
+ my ($self, $url, $uid, $kw) = @_;
+ my $oidbin = $self->{-lms_ro}->imap_oidbin($url, $uid) // return;
+ my @docids = $self->{over}->oidbin_exists($oidbin) or return;
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ $self->{verbose} and
+ $self->{lei}->qerr('# '.unpack('H*', $oidbin)." => @$kw\n");
+ $self->{sto}->ipc_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
+}
+
+sub ikw_done_wait {
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lei->can('wq_done_wait')->($arg, $pid);
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $ikw = delete $lei->{ikw} or return $lei->fail;
+ $ikw->wq_wait_old(\&ikw_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 22ee1109..75603d89 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS blob2num (
UNIQUE (oidbin, fid, uid)
)
+ # speeds up LeiImport->ck_update_kw (for "lei import") by 5-6x:
+ $dbh->do(<<'');
+CREATE INDEX IF NOT EXISTS idx_fid_uid ON blob2num(fid,uid)
+
$dbh->do(<<'');
CREATE TABLE IF NOT EXISTS blob2name (
oidbin VARBINARY NOT NULL,
@@ -361,15 +365,14 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
-sub imap_oid2 ($$$) {
- my ($self, $uri, $uid) = @_; # $uri MUST have UIDVALIDITY
- my $fid = $self->{fmap}->{"$uri"} //= fid_for($self, "$uri") // return;
+sub imap_oidbin ($$$) {
+ my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY
+ my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return;
my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
EOM
$sth->execute($fid, $uid);
- my ($oidbin) = $sth->fetchrow_array;
- $oidbin ? unpack('H*', $oidbin) : undef;
+ $sth->fetchrow_array;
}
sub imap_oid {
@@ -384,10 +387,10 @@ sub imap_oid {
}
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
- imap_oid2($self, $folders->[0], $uid_uri->uid);
+ my $oidbin = imap_oidbin($self, $folders->[0], $uid_uri->uid);
+ $oidbin ? unpack('H*', $oidbin) : undef;
}
-
# FIXME: something with "lei <up|q>" is causing uncommitted transaction
# warnings, not sure what...
sub DESTROY {
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 39a8f7fc..058f4313 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -471,6 +471,7 @@ sub each_old_flags ($$$$) {
while (my ($uid, $per_uid) = each %$r) {
my $kw = flags2kw($self, $uri, $uid, $per_uid->{FLAGS})
// next;
+ # LeiImport->input_net_cb
$eml_cb->($uri, $uid, $kw, undef, @args);
}
}
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 0e191c47..58fdea0e 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -349,13 +349,13 @@ sub check_inodes {
}
}
-sub blob_exists {
- my ($self, $oidhex) = @_;
+sub oidbin_exists {
+ my ($self, $oidbin) = @_;
if (wantarray) {
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
my $tmp = $sth->fetchall_arrayref;
map { $_->[0] } @$tmp;
@@ -363,10 +363,12 @@ SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT COUNT(*) FROM xref3 WHERE oidbin = ?
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
$sth->fetchrow_array;
}
}
+sub blob_exists { oidbin_exists($_[0], pack('H*', $_[1])) }
+
1;
^ permalink raw reply related [relevance 38%]
* [RFC] lei import: support --new-only
@ 2021-06-03 5:00 67% Eric Wong
2021-06-09 22:39 62% ` [PATCH v2] lei import: support --new-only for IMAP Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-06-03 5:00 UTC (permalink / raw)
To: meta
Taking ~40s to synchronize a ~75K message IMAP folder is
still a lot of time, so support an option to only touch
new messages.
This is similar to "offlineimap -q" (quick) or "mbsync --new"
switches, but lei already accepts "-q" as a shortcut for
--quiet. "--new" could work, but "--new-only" might be more
descriptive (or "--only-new"?), since the default fetches
also fetches new messages.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7bda9408..9a2a323c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -240,7 +240,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s
+ qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 2efd4935..60f3241b 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -74,6 +74,7 @@ sub do_import_index ($$@) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
if ($self->{-import_kw} && $net->{-lms_ro} &&
+ !$lei->{opt}->{'new-only'} &&
$net->{imap_order}) {
require PublicInbox::LeiImportKw;
$ikw = PublicInbox::LeiImportKw->new($lei);
^ permalink raw reply related [relevance 67%]
* [PATCH] INSTALL: note about lei metadata storage
@ 2021-06-05 21:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-05 21:04 UTC (permalink / raw)
To: meta
Since lei is for personal mailboxes, I don't think lei needs to
keep keyword and label changes in history. And fix a minor
wording problem ("or" => "nor") while we're at it.
---
INSTALL | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/INSTALL b/INSTALL
index e8bfba97..e93731f1 100644
--- a/INSTALL
+++ b/INSTALL
@@ -174,9 +174,10 @@ Other installation notes
Debian 8.x (jessie) users, use Debian 8.5 or later if using Xapian:
https://bugs.debian.org/808610
-public-inbox will never store unregeneratable data in Xapian
-or any other search database we might use; Xapian corruption
-will not destroy critical data.
+public-inbox-* commands will never store unregeneratable data in
+Xapian nor any other search database we might use; Xapian
+corruption will not destroy critical data. Note: `lei' DOES store
+unregeneratable data in Xapian and SQLite.
See the public-inbox-overview(7) man page for the next steps once
the installation is complete.
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: don't drop WQ workers on normal exit
@ 2021-06-05 21:06 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-05 21:06 UTC (permalink / raw)
To: meta
This is dangerous and causes race conditions on commands
which utilize multiple workqueues.
---
lib/PublicInbox/LEI.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 2dd21fc6..8adf70fa 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -480,6 +480,7 @@ sub sigint_reap {
sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
+ $self->{failed}++;
err($self, $buf) if defined $buf;
# calls fail_handler:
$self->{pkt_op_p}->pkt_do('!') if $self->{pkt_op_p};
@@ -1043,7 +1044,7 @@ sub accept_dispatch { # Listener {post_accept} callback
sub dclose {
my ($self) = @_;
delete $self->{-progress};
- _drop_wq($self);
+ _drop_wq($self) if $self->{failed};
close(delete $self->{1}) if $self->{1}; # may reap_compress
$self->close if $self->{-event_init_done}; # PublicInbox::DS::close
}
^ permalink raw reply related [relevance 71%]
* [PATCH] lei/store: checkpoint commits mail_sync.sqlite3
@ 2021-06-07 19:06 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-07 19:06 UTC (permalink / raw)
To: meta
We mainly rely on ->done with lei/store, but moving to
->checkpoint probably makes sense. Note: over, msgmap, and
mail_sync all have slightly different transacation behavior;
perhaps they can be unified in the future.
---
lib/PublicInbox/LeiStore.pm | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 821782b9..0b033e3e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -419,6 +419,9 @@ sub checkpoint {
if (my $im = $self->{im}) {
$wait ? $im->barrier : $im->checkpoint;
}
+ if (my $lms = delete $self->{lms}) {
+ $lms->lms_commit;
+ }
$self->{priv_eidx}->checkpoint($wait);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/3] lei import: speedup repeated Maildir import
@ 2021-06-08 9:50 69% Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
More importantly, this series starts us down the road of
generalizing the use of auxiliary WQs like {ikw} (import
keywords for IMAP) and {pmd} (parallel Maildir).
It took me a bit to figure out some consistency problems
which I reordered and split out into 1/3. 2/3 should
make future work easier, and 3/3 gives the final speedup
along the lines of what I already did with IMAP.
Eric Wong (3):
lei: safety fix for multiple WQ classes
lei: generalize auxiliary WQ handling
lei import: speed up repeated Maildir imports
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 20 ++++++++--
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 39 +++++++++++++-------
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 31 +++++++++++-----
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMailSync.pm | 14 +++++++
lib/PublicInbox/LeiMirror.pm | 4 +-
lib/PublicInbox/LeiP2q.pm | 4 +-
lib/PublicInbox/LeiPmdir.pm | 67 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/MdirReader.pm | 22 ++++++-----
lib/PublicInbox/PktOp.pm | 6 ---
t/lei-import-maildir.t | 2 +-
20 files changed, 171 insertions(+), 57 deletions(-)
create mode 100644 lib/PublicInbox/LeiPmdir.pm
^ permalink raw reply [relevance 69%]
* [PATCH 1/3] lei: safety fix for multiple WQ classes
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
@ 2021-06-08 9:50 71% ` Eric Wong
2021-06-08 9:50 48% ` [PATCH 2/3] lei: generalize auxiliary WQ handling Eric Wong
2021-06-08 9:50 32% ` [PATCH 3/3] lei import: speed up repeated Maildir imports Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
For commands utilizing multiple workers, this simple change
generalizes the persistence mechanism and and prevents
lei->dclose from causing script/lei to exit if there are
still in-flight workers.
This ougth to prevent read-after-write consistency problems that
occasionally manifest in scripts (e.g. test cases) but usually
go unnoticed in normal use.
---
lib/PublicInbox/LEI.pm | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8adf70fa..0cf4d10b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -580,6 +580,8 @@ sub workers_start {
$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
+ # {-lei_sock} persists script/lei process until ops->{''} EOF callback
+ $op_c->{-lei_sock} = $lei->{sock};
@$end = ();
$lei->event_step_init;
($op_c, $ops);
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] lei: generalize auxiliary WQ handling
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
@ 2021-06-08 9:50 48% ` Eric Wong
2021-06-08 9:50 32% ` [PATCH 3/3] lei import: speed up repeated Maildir imports Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
op_wait_event is now more lei-specific since we no longer have
to care about oneshot and use a synchronous loop.
{ikw} (import-keywords) started a trend, but LeiPmdir (parallel
Maildir) is an upcoming WQ class that will follow this idea.
Eventually, {l2m} usage may be updated to follow this, too.
---
lib/PublicInbox/LEI.pm | 9 +++++++++
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 3 +--
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
lib/PublicInbox/LeiP2q.pm | 4 ++--
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/PktOp.pm | 6 ------
13 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0cf4d10b..ed01e8de 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -587,6 +587,15 @@ sub workers_start {
($op_c, $ops);
}
+# call this when we're ready to wait on events and yield to other clients
+sub wait_wq_events {
+ my ($lei, $op_c, $ops) = @_;
+ for my $wq (grep(defined, @$lei{qw(ikw)})) { # auxiliary WQs
+ $wq->wq_close(1);
+ }
+ $op_c->{ops} = $ops;
+}
+
sub _help {
require PublicInbox::LeiHelp;
PublicInbox::LeiHelp::call($_[0], $_[1], \%CMD, \%OPTDESC);
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 8de86565..09217964 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -169,7 +169,7 @@ sub lei_blob {
$lei->{wq1} = $self;
$self->wq_io_do('do_solve_blob', []);
$self->wq_close(1);
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 395a80f8..6550c242 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -59,7 +59,7 @@ sub lei_convert { # the main "lei convert" method
$lei->{wq1} = $self;
$self->wq_io_do('process_inputs', []);
$self->wq_close(1);
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index b31b065f..f8579221 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -160,7 +160,7 @@ EOM
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops); # calls net_merge_all_done if $lei->{auth}
+ $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
}
sub _complete_export_kw {
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 2efd4935..222f75c8 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -91,9 +91,8 @@ sub do_import_index ($$@) {
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
- $ikw->wq_close(1) if $ikw;
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub lei_import { # the main "lei import" method
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 6cea6ae8..70136135 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -76,7 +76,7 @@ sub bg_worker ($$$) {
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
$self->wq_close(1);
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub lei_ls_search {
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index a37e1d5c..39671f90 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -282,11 +282,11 @@ sub start {
require PublicInbox::Inbox;
require PublicInbox::Admin;
require PublicInbox::InboxWritable;
- my ($op, $ops) = $lei->workers_start($self, 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_mirror', []);
$self->wq_close(1);
- $op->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index f381a31c..c0c4563d 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -188,11 +188,11 @@ sub lei_p2q { # the "lei patch-to-query" entry point
} else {
$self->{input} = $input;
}
- my ($op, $ops) = $lei->workers_start($self, 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_p2q', []);
$self->wq_close(1);
- $op->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index c8bd0dfb..7607b44f 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -227,7 +227,7 @@ sub lei_rediff {
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index c6d28045..578e9811 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -38,7 +38,7 @@ sub lei_rm {
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index b6abd533..4b3ce7d8 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -53,7 +53,7 @@ sub lei_tag { # the "lei tag" method
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub note_missing {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 75e55d47..beb955bb 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -532,7 +532,7 @@ sub do_query {
start_query($self);
}
$lei->event_step_init; # wait for shutdowns
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub add_uri {
diff --git a/lib/PublicInbox/PktOp.pm b/lib/PublicInbox/PktOp.pm
index ca098d3c..92e150a4 100644
--- a/lib/PublicInbox/PktOp.pm
+++ b/lib/PublicInbox/PktOp.pm
@@ -63,10 +63,4 @@ sub event_step {
}
}
-# call this when we're ready to wait on events
-sub op_wait_event {
- my ($self, $ops) = @_;
- $self->{ops} = $ops;
-}
-
1;
^ permalink raw reply related [relevance 48%]
* [PATCH 3/3] lei import: speed up repeated Maildir imports
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
2021-06-08 9:50 48% ` [PATCH 2/3] lei: generalize auxiliary WQ handling Eric Wong
@ 2021-06-08 9:50 32% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
On a 4-core CPU, this speeds up "lei import" on a largish
Maildir inbox with 75K messages from ~8 minutes down to ~40s.
Parallelizing alone did not bring any improvement and may
even hurt performance slightly, depending on CPU availability.
However, creating the index on the "fid" and "name" columns in
blob2name yields us the same speedup we got.
Parallelizing IMAP makes more sense due to the fact most IMAP
stores are non-local and subject to network latency.
Followup-to: bdecd7ed8e0dcf0b45491b947cd737ba8cfe38a3 ("lei import: speed up kw updates for old IMAP messages")
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 11 +++---
lib/PublicInbox/LeiImport.pm | 36 ++++++++++++------
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 31 +++++++++++-----
lib/PublicInbox/LeiMailSync.pm | 14 +++++++
lib/PublicInbox/LeiPmdir.pm | 67 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/MdirReader.pm | 22 ++++++-----
t/lei-import-maildir.t | 2 +-
9 files changed, 148 insertions(+), 38 deletions(-)
create mode 100644 lib/PublicInbox/LeiPmdir.pm
diff --git a/MANIFEST b/MANIFEST
index 5a70a144..7bdbf252 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -221,6 +221,7 @@ lib/PublicInbox/LeiMailSync.pm
lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
+lib/PublicInbox/LeiPmdir.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ed01e8de..77fc5b8f 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -240,7 +240,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s
+ qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
@@ -421,7 +421,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m wq1 ikw); # internal workers
+my @WQ_KEYS = qw(lxs l2m ikw pmd wq1); # internal workers
sub _drop_wq {
my ($self) = @_;
@@ -566,7 +566,7 @@ sub pkt_op_pair {
}
sub workers_start {
- my ($lei, $wq, $jobs, $ops) = @_;
+ my ($lei, $wq, $jobs, $ops, $flds) = @_;
$ops = {
'!' => [ \&fail_handler, $lei ],
'|' => [ \&sigpipe_handler, $lei ],
@@ -577,7 +577,8 @@ sub workers_start {
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
- $wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
+ $flds->{lei} = $lei;
+ $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
# {-lei_sock} persists script/lei process until ops->{''} EOF callback
@@ -590,7 +591,7 @@ sub workers_start {
# call this when we're ready to wait on events and yield to other clients
sub wait_wq_events {
my ($lei, $op_c, $ops) = @_;
- for my $wq (grep(defined, @$lei{qw(ikw)})) { # auxiliary WQs
+ for my $wq (grep(defined, @$lei{qw(ikw pmd)})) { # auxiliary WQs
$wq->wq_close(1);
}
$op_c->{ops} = $ops;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 222f75c8..b0e7ba6b 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiImport;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::InboxWritable qw(eml_from_path);
# /^input_/ subs are used by (or override) PublicInbox::LeiInput superclass
@@ -28,17 +29,26 @@ sub input_mbox_cb { # MboxReader callback
input_eml_cb($self, $eml, $vmd);
}
-sub input_maildir_cb { # maildir_each_eml cb
- my ($f, $kw, $eml, $self) = @_;
+sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
+ my ($self, $f, @args) = @_;
+ my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+ die "BUG: $f was not from a Maildir?\n";
+ my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn);
+ return if index($fl, 'T') >= 0; # no Trashed messages
+ my $kw = PublicInbox::MdirReader::flags2kw($fl);
+ substr($folder, 0, 0) = 'maildir:'; # add prefix
+ my $lms = $self->{-lms_ro};
+ my $oidbin = $lms ? $lms->name_oidbin($folder, $bn) : undef;
+ my @docids = defined($oidbin) ?
+ $self->{over}->oidbin_exists($oidbin) : ();
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
- if ($self->{-mail_sync}) {
- if ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) { # ugh...
- $vmd->{sync_info} = [ "maildir:$1", \(my $n = $2) ];
- } else {
- warn "E: $f was not from a Maildir?\n";
- }
+ if (scalar @docids) {
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ }
+ if (my $eml = eml_from_path($f)) {
+ $vmd->{sync_info} = [ $folder, \$bn ] if $self->{-mail_sync};
+ $self->input_eml_cb($eml, $vmd);
}
- $self->input_eml_cb($eml, $vmd);
}
sub input_net_cb { # imap_each / nntp_each
@@ -62,11 +72,13 @@ sub do_import_index ($$@) {
my $vmd_mod = $self->vmd_mod_extract(\@inputs);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
$self->{all_vmd} = $vmd_mod if scalar keys %$vmd_mod;
- $self->prepare_inputs($lei, \@inputs) or return;
+ $lei->ale; # initialize for workers to read (before LeiPmdir->new)
$self->{-mail_sync} = $lei->{opt}->{'mail-sync'} // 1;
+ $self->prepare_inputs($lei, \@inputs) or return;
- $lei->ale; # initialize for workers to read
- my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ my $j = $lei->{opt}->{jobs} // 0;
+ $j =~ /\A([0-9]+),[0-9]+\z/ and $j = $1 + 0;
+ $j ||= scalar(@{$self->{inputs}}) || 1;
my $ikw;
if (my $net = $lei->{net}) {
# $j = $net->net_concurrency($j); TODO
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index cc3e83e7..4be0c649 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -35,7 +35,7 @@ sub lei_index {
no warnings 'once';
no strict 'refs';
-for my $m (qw(input_maildir_cb input_net_cb)) {
+for my $m (qw(pmdir_cb input_net_cb)) {
*$m = PublicInbox::LeiImport->can($m);
}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 4ff7a379..24211bf0 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -151,9 +151,16 @@ sub input_path_url {
return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
$input appears to be a maildir, not $ifmt
EOM
- PublicInbox::MdirReader->new->maildir_each_eml($input,
- $self->can('input_maildir_cb'),
- $self, @args);
+ my $mdr = PublicInbox::MdirReader->new;
+ if (my $pmd = $self->{pmd}) {
+ $mdr->maildir_each_file($input,
+ $pmd->can('each_mdir_fn'),
+ $pmd, @args);
+ } else {
+ $mdr->maildir_each_eml($input,
+ $self->can('input_maildir_cb'),
+ $self, @args);
+ }
} else {
$lei->fail("$input unsupported (TODO)");
}
@@ -215,7 +222,7 @@ sub prepare_inputs { # returns undef on error
push @{$sync->{no}}, '/dev/stdin' if $sync;
}
my $net = $lei->{net}; # NetWriter may be created by l2m
- my (@f, @d);
+ my (@f, @md);
# e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX
for my $input (@$inputs) {
my $input_path = $input;
@@ -247,11 +254,11 @@ sub prepare_inputs { # returns undef on error
PublicInbox::MboxReader->reads($ifmt) or return
$lei->fail("$ifmt not supported");
} elsif (-d $input_path) {
- require PublicInbox::MdirReader;
$ifmt eq 'maildir' or return
$lei->fail("$ifmt not supported");
$sync and $input = 'maildir:'.
$lei->abs_path($input_path);
+ push @md, $input;
} else {
return $lei->fail("Unable to handle $input");
}
@@ -266,21 +273,18 @@ $input is `eml', not --in-format=$in_fmt
if ($devfd >= 0 || -f $input || -p _) {
push @{$sync->{no}}, $input if $sync;
push @f, $input;
- } elsif (-d $input) {
+ } elsif (-d "$input/new" && -d "$input/cur") {
if ($sync) {
$input = $lei->abs_path($input);
push @{$sync->{ok}}, $input;
}
- push @d, $input;
+ push @md, $input;
} else {
return $lei->fail("Unable to handle $input")
}
}
}
if (@f) { check_input_format($lei, \@f) or return }
- if (@d) { # TODO: check for MH vs Maildir, here
- require PublicInbox::MdirReader;
- }
if ($sync && $sync->{no}) {
return $lei->fail(<<"") if !$sync->{ok};
--mail-sync specified but no inputs support it
@@ -299,6 +303,13 @@ $input is `eml', not --in-format=$in_fmt
$lei->{auth} //= PublicInbox::LeiAuth->new;
$lei->{net} //= $net;
}
+ if (scalar(@md)) {
+ require PublicInbox::MdirReader;
+ if ($self->can('pmdir_cb')) {
+ require PublicInbox::LeiPmdir;
+ $self->{pmd} = PublicInbox::LeiPmdir->new($lei, $self);
+ }
+ }
$self->{inputs} = $inputs;
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 75603d89..ec05404a 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -66,6 +66,10 @@ CREATE TABLE IF NOT EXISTS blob2name (
UNIQUE (oidbin, fid, name)
)
+ # speeds up LeiImport->pmdir_cb (for "lei import") by ~6x:
+ $dbh->do(<<'');
+CREATE INDEX IF NOT EXISTS idx_fid_name ON blob2name(fid,name)
+
}
sub fid_for {
@@ -375,6 +379,16 @@ EOM
$sth->fetchrow_array;
}
+sub name_oidbin ($$$) {
+ my ($self, $mdir, $nm) = @_;
+ my $fid = $self->{fmap}->{$mdir} //= fid_for($self, $mdir) // return;
+ my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2name WHERE fid = ? AND name = ?
+EOM
+ $sth->execute($fid, $nm);
+ $sth->fetchrow_array;
+}
+
sub imap_oid {
my ($self, $lei, $uid_uri) = @_;
my $mailbox_uri = $uid_uri->clone;
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
new file mode 100644
index 00000000..5efb012e
--- /dev/null
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -0,0 +1,67 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# WQ worker for dealing with parallel Maildir reads;
+# this does NOT use the {shard_info} field of LeiToMail
+# (and we may remove {shard_info})
+# WQ key: {pmd}
+package PublicInbox::LeiPmdir;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+sub new {
+ my ($cls, $lei, $ipt) = @_;
+ my $self = bless { -wq_ident => 'lei Maildir worker' }, $cls;
+ my $jobs = $lei->{opt}->{jobs};
+ $jobs =~ /\A[0-9]+,([0-9]+)\z/ and $jobs = $1;
+ my $nproc = $jobs // do {
+ # untested with >=4 CPUs, though I suspect I/O latency
+ # of SATA SSD storage will make >=4 processes unnecessary,
+ # here. NVMe users may wish to use '-j'
+ my $n = $self->detect_nproc;
+ $n = 4 if $n > 4;
+ };
+ my ($op_c, $ops) = $lei->workers_start($self, $nproc,
+ undef, { ipt => $ipt }); # LeiInput subclass
+ $op_c->{ops} = $ops; # for PktOp->event_step
+ $lei->{pmd} = $self;
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->_lei_atfork_child;
+ my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}';
+ $ipt->{lei} = $lei;
+ $ipt->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
+ $ipt->{lse} = $ipt->{sto}->search;
+ $ipt->{over} = $ipt->{lse}->over;
+ $ipt->{-lms_ro} //= $ipt->{lse}->lms; # may be undef or '0'
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub each_mdir_fn { # maildir_each_file callback
+ my ($f, $self, @args) = @_;
+ $self->wq_io_do('mdir_iter', [], $f, @args);
+}
+
+sub mdir_iter { # via wq_io_do
+ my ($self, $f, @args) = @_;
+ $self->{ipt}->pmdir_cb($f, @args);
+}
+
+sub pmd_done_wait {
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lei->can('wq_done_wait')->($arg, $pid);
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $pmd = delete $lei->{pmd} or return $lei->fail;
+ $pmd->wq_wait_old(\&pmd_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index 304be63d..484bf0a8 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -87,17 +87,21 @@ sub maildir_each_eml {
sub new { bless {}, __PACKAGE__ }
sub flags2kw ($) {
- my @unknown;
- my %kw;
- for (split(//, $_[0])) {
- my $k = $c2kw{$_};
- if (defined($k)) {
- $kw{$k} = 1;
- } else {
- push @unknown, $_;
+ if (wantarray) {
+ my @unknown;
+ my %kw;
+ for (split(//, $_[0])) {
+ my $k = $c2kw{$_};
+ if (defined($k)) {
+ $kw{$k} = 1;
+ } else {
+ push @unknown, $_;
+ }
}
+ (\%kw, \@unknown);
+ } else {
+ [ sort(map { $c2kw{$_} // () } split(//, $_[0])) ];
}
- (\%kw, \@unknown);
}
1;
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index 688b10ce..c81e7805 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -28,7 +28,7 @@ test_lei(sub {
is(scalar(keys %v), 1, 'inspect handles relative and absolute paths');
my $inspect = json_utf8->decode([ keys %v ]->[0]);
is_deeply($inspect, {"maildir:$md" => { 'name.count' => 1 }},
- 'inspect maildir: path had expected output');
+ 'inspect maildir: path had expected output') or xbail($inspect);
lei_ok(qw(q s:boolean));
my $res = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 32%]
* [PATCH] lei pmdir: fix nproc for <= 4 CPUs
@ 2021-06-08 23:56 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 23:56 UTC (permalink / raw)
To: meta
I forgot my FreeBSD VM has 8 cores, actually, and tweaked the
nproc detection on that machine before finalizing commit
10b523eb017162240b1ac3647f8dcbbf2be348a7
("lei import: speed up repeated Maildir imports")
Fixes: 10b523eb01716224 ("lei import: speed up repeated Maildir imports")
---
lib/PublicInbox/LeiPmdir.pm | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index 5efb012e..b71efe70 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -13,14 +13,14 @@ use parent qw(PublicInbox::IPC);
sub new {
my ($cls, $lei, $ipt) = @_;
my $self = bless { -wq_ident => 'lei Maildir worker' }, $cls;
- my $jobs = $lei->{opt}->{jobs};
+ my $jobs = $lei->{opt}->{jobs} // '';
$jobs =~ /\A[0-9]+,([0-9]+)\z/ and $jobs = $1;
- my $nproc = $jobs // do {
- # untested with >=4 CPUs, though I suspect I/O latency
+ my $nproc = $jobs || do {
+ # barely tested with >=4 CPUs, though I suspect I/O latency
# of SATA SSD storage will make >=4 processes unnecessary,
# here. NVMe users may wish to use '-j'
my $n = $self->detect_nproc;
- $n = 4 if $n > 4;
+ $n = $n > 4 ? 4 : $n;
};
my ($op_c, $ops) = $lei->workers_start($self, $nproc,
undef, { ipt => $ipt }); # LeiInput subclass
^ permalink raw reply related [relevance 70%]
* [PATCH] lei edit-search: fix and add a (weak) test
@ 2021-06-09 0:11 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 0:11 UTC (permalink / raw)
To: meta
This broke recently and lacked an automated test, so rely on
EDITOR=cat to ensure we have some coverage.
Fixes: d2670108f71b1eff ("pkt_op: make pkt_do an OO method")
---
lib/PublicInbox/LeiEditSearch.pm | 2 +-
t/lei-q-save.t | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 13713d24..82dfbf63 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -19,7 +19,7 @@ sub lei_edit_search {
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
# $op_p will EOF when $EDITOR is done
$op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
- $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p ], \@cmd, {});
+ $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], \@cmd, {});
}
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 694b33b2..6c592088 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -195,5 +195,8 @@ test_lei(sub {
glob("$v2s/git/0.git/objects/*/*")));
ok($shared < $orig, 'fewer bytes stored with --shared') or
diag "shared=$shared orig=$orig";
+
+ lei_ok([qw(edit-search), $v2s], { VISUAL => 'cat', EDITOR => 'cat' });
+ like($lei_out, qr/^\[lei/sm, 'edit-search can cat');
});
done_testing;
^ permalink raw reply related [relevance 70%]
* [PATCH 0/5] lei Maildir stuff
@ 2021-06-09 7:47 71% Eric Wong
2021-06-09 7:47 58% ` [PATCH 3/5] lei tag: parallelize Maildir access Eric Wong
2021-06-09 7:47 45% ` [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-06-09 7:47 UTC (permalink / raw)
To: meta
I'm not sure if "prune-mail-sync" needs to be exposed, but I
suppose it could be useful in some cases. It's certainly easier
to implement.
"lei tag" gets a nice speedup for Maildirs, IMAP speedup is
probably better off done after we get IMAP pipelining far into
the future.
Not sure if anybody is using InboxWritable->import_maildir...
Eric Wong (5):
inbox_writable: fix import_maildir
mdir_reader: maildir_each_file: pass flags, skip Trash
lei tag: parallelize Maildir access
lei_mail_sync: hoist out --all handling from export-kw
lei prune-mail-sync: new command to prune invalid sync data
MANIFEST | 1 +
lib/PublicInbox/InboxWritable.pm | 15 ++---
lib/PublicInbox/LEI.pm | 2 +
lib/PublicInbox/LeiExportKw.pm | 32 +---------
lib/PublicInbox/LeiImport.pm | 12 ++--
lib/PublicInbox/LeiMailSync.pm | 36 +++++++++++
lib/PublicInbox/LeiPmdir.pm | 18 ++----
lib/PublicInbox/LeiPruneMailSync.pm | 97 +++++++++++++++++++++++++++++
lib/PublicInbox/LeiTag.pm | 8 ++-
lib/PublicInbox/MdirReader.pm | 5 +-
lib/PublicInbox/NetReader.pm | 19 ++++++
lib/PublicInbox/NetWriter.pm | 21 +------
12 files changed, 184 insertions(+), 82 deletions(-)
create mode 100644 lib/PublicInbox/LeiPruneMailSync.pm
^ permalink raw reply [relevance 71%]
* [PATCH 3/5] lei tag: parallelize Maildir access
2021-06-09 7:47 71% [PATCH 0/5] lei Maildir stuff Eric Wong
@ 2021-06-09 7:47 58% ` Eric Wong
2021-06-09 7:47 45% ` [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 7:47 UTC (permalink / raw)
To: meta
Since Maildir isn't guaranteed to have any sort of order, we
can parallelize inputs, here. On a 4-core system, this reduced
one of my tag invocations from 5.5 to 1.4s.
---
lib/PublicInbox/LeiImport.pm | 8 ++++----
lib/PublicInbox/LeiPmdir.pm | 10 ++--------
lib/PublicInbox/LeiTag.pm | 8 +++++---
3 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index cddd5619..e3cb69ca 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -35,13 +35,13 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
die "BUG: $f was not from a Maildir?\n";
my $kw = PublicInbox::MdirReader::flags2kw($fl);
substr($folder, 0, 0) = 'maildir:'; # add prefix
- my $lms = $self->{-lms_ro};
+ my $lse = $self->{lse} //= $self->{lei}->{sto}->search;
+ my $lms = $self->{-lms_ro} //= $lse->lms; # may be 0 or undef
my $oidbin = $lms ? $lms->name_oidbin($folder, $bn) : undef;
- my @docids = defined($oidbin) ?
- $self->{over}->oidbin_exists($oidbin) : ();
+ my @docids = defined($oidbin) ? $lse->over->oidbin_exists($oidbin) : ();
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
if (scalar @docids) {
- $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ $lse->kw_changed(undef, $kw, \@docids) or return;
}
if (my $eml = eml_from_path($f)) {
$vmd->{sync_info} = [ $folder, \$bn ] if $self->{-mail_sync};
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index aa9ce713..760f276c 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -30,15 +30,9 @@ sub new {
sub ipc_atfork_child {
my ($self) = @_;
- my $lei = $self->{lei};
- $lei->_lei_atfork_child;
my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}';
- $ipt->{lei} = $lei;
- $ipt->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
- $ipt->{lse} = $ipt->{sto}->search;
- $ipt->{over} = $ipt->{lse}->over;
- $ipt->{-lms_ro} //= $ipt->{lse}->lms; # may be undef or '0'
- $self->SUPER::ipc_atfork_child;
+ $ipt->{lei} = $self->{lei};
+ $ipt->ipc_atfork_child;
}
sub each_mdir_fn { # maildir_each_file callback
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 4b3ce7d8..e0532653 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiTag;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::InboxWritable qw(eml_from_path);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
@@ -24,8 +25,9 @@ sub input_mbox_cb {
input_eml_cb($self, $eml);
}
-sub input_maildir_cb { # maildir_each_eml cb
- my ($f, $kw, $eml, $self) = @_;
+sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
+ my ($self, $f) = @_;
+ my $eml = eml_from_path($f) or return;
input_eml_cb($self, $eml);
}
@@ -42,12 +44,12 @@ sub lei_tag { # the "lei tag" method
$lei->ale; # refresh and prepare
my $vmd_mod = $self->vmd_mod_extract(\@argv);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
+ $self->{vmd_mod} = $vmd_mod; # before LeiPmdir->new in prepare_inputs
$self->prepare_inputs($lei, \@argv) or return;
grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or
return $lei->fail('no keywords or labels specified');
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{vmd_mod} = $vmd_mod;
my $j = $self->{-wq_nr_workers} = 1; # locked for now
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
^ permalink raw reply related [relevance 58%]
* [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data
2021-06-09 7:47 71% [PATCH 0/5] lei Maildir stuff Eric Wong
2021-06-09 7:47 58% ` [PATCH 3/5] lei tag: parallelize Maildir access Eric Wong
@ 2021-06-09 7:47 45% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 7:47 UTC (permalink / raw)
To: meta
This will be invoked automatically by "lei import" eventually,
but it may make sense to expose as a separate command.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 2 +
lib/PublicInbox/LeiPruneMailSync.pm | 97 +++++++++++++++++++++++++++++
lib/PublicInbox/NetReader.pm | 19 ++++++
lib/PublicInbox/NetWriter.pm | 21 +------
5 files changed, 121 insertions(+), 19 deletions(-)
create mode 100644 lib/PublicInbox/LeiPruneMailSync.pm
diff --git a/MANIFEST b/MANIFEST
index 7bdbf252..3d4c6cbd 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -222,6 +222,7 @@ lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiPmdir.pm
+lib/PublicInbox/LeiPruneMailSync.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 77fc5b8f..265b7047 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -245,6 +245,8 @@ our %CMD = ( # sorted in order of importance/use:
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
+'prune-mail-sync' => [ 'LOCATION...|--all',
+ 'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
qw(all:s mode=s), @c_opt ],
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
new file mode 100644
index 00000000..79f3325d
--- /dev/null
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -0,0 +1,97 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei prune-mail-sync" drops dangling sync information
+package PublicInbox::LeiPruneMailSync;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::LeiExportKw;
+use PublicInbox::InboxWritable qw(eml_from_path);
+
+sub eml_match ($$) {
+ my ($eml, $oidbin) = @_;
+ $oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
+}
+
+sub prune_mdir { # lms->each_src callback
+ my ($oidbin, $id, $self, $mdir) = @_;
+ my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur);
+ for my $d (@try) {
+ my $src = "$mdir/$d/$$id";
+ if ($self->{verify}) {
+ my $eml = eml_from_path($src) or next;
+ return if eml_match($eml, $oidbin);
+ } elsif (-f $src) {
+ return;
+ }
+ }
+ # both tries failed
+ $self->{lei}->qerr("# maildir:$mdir $$id gone");
+ $self->{lei}->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
+}
+
+sub prune_imap { # lms->each_src callback
+ my ($oidbin, $uid, $self, $uids, $url) = @_;
+ return if exists $uids->{$uid};
+ $self->{lei}->qerr("# $url $uid gone");
+ $self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
+}
+
+sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
+ my ($self, $input, @args) = @_;
+ my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
+ if ($input =~ /\Amaildir:(.+)/i) {
+ my $mdir = $1;
+ $lms->each_src($input, \&prune_mdir, $self, $mdir);
+ } elsif ($input =~ m!\Aimaps?://!i) {
+ my $uri = PublicInbox::URIimap->new($input);
+ my $mic = $self->{lei}->{net}->mic_for_folder($uri);
+ my $uids = $mic->search('UID 1:*');
+ $uids = +{ map { $_ => undef } @$uids };
+ $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
+ } else { die "BUG: $input not supported" }
+ my $wait = $self->{lei}->{sto}->ipc_do('done');
+}
+
+sub lei_prune_mail_sync {
+ my ($lei, @folders) = @_;
+ my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
+lei/store uninitialized, see lei-import(1)
+EOM
+ my $lse = $sto->search;
+ my $lms = $lse->lms or return $lei->fail(<<EOM);
+lei mail_sync uninitialized, see lei-import(1)
+EOM
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
+ }
+ delete $lms->{dbh};
+ $sto->write_prepare($lei);
+ my $self = bless { lse => $lse }, __PACKAGE__;
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ $self->prepare_inputs($lei, \@folders) or return;
+ my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
+ undef $lms; # for fork
+ my $ops = {};
+ $sto->write_prepare($lei);
+ $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $self->{-wq_nr_workers} = $j // 1; # locked
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
+}
+
+no warnings 'once';
+*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
+1;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 058f4313..2795a9d4 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -747,4 +747,23 @@ sub nntp_each {
sub new { bless {}, shift };
+# updates $uri with UIDVALIDITY
+sub mic_for_folder {
+ my ($self, $uri) = @_;
+ my $mic = $self->mic_get($uri) or die "E: not connected: $@";
+ my $m = $self->isa('PublicInbox::NetWriter') ? 'select' : 'examine';
+ $mic->$m($uri->mailbox) or return;
+ my $uidval;
+ for ($mic->Results) {
+ /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
+ $uidval = $1;
+ last;
+ }
+ $uidval //= $mic->uidvalidity($uri->mailbox) or
+ die "E: failed to get uidvalidity from <$uri>: $@";
+ $uri->uidvalidity($uidval);
+ $mic;
+}
+
+
1;
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 8ec7f85c..82288e6b 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,26 +26,9 @@ sub imap_append {
die "APPEND $folder: $@";
}
-# updates $uri with UIDVALIDITY
-sub mic_for_folder {
- my ($self, $uri) = @_;
- my $mic = $self->mic_get($uri) or die "E: not connected: $@";
- $mic->select($uri->mailbox) or return;
- my $uidval;
- for ($mic->Results) {
- /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
- $uidval = $1;
- last;
- }
- $uidval //= $mic->uidvalidity($uri->mailbox) or
- die "E: failed to get uidvalidity from <$uri>: $@";
- $uri->uidvalidity($uidval);
- $mic;
-}
-
sub imap_delete_all {
my ($self, $uri) = @_;
- my $mic = mic_for_folder($self, $uri) or return;
+ my $mic = $self->mic_for_folder($uri) or return;
my $sec = $self->can('uri_section')->($uri);
local $0 = $uri->mailbox." $sec";
if ($mic->delete_message('1:*')) {
@@ -55,7 +38,7 @@ sub imap_delete_all {
sub imap_delete_1 {
my ($self, $uri, $uid, $delete_mic) = @_;
- $$delete_mic //= mic_for_folder($self, $uri) or return;
+ $$delete_mic //= $self->mic_for_folder($uri) or return;
$$delete_mic->delete_message($uid);
}
^ permalink raw reply related [relevance 45%]
* [PATCH] lei/store: do eidx_init before creating R/W lms dbh
@ 2021-06-09 10:03 69% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 10:03 UTC (permalink / raw)
To: meta
Sharing lms->{dbh} with eidx shards appears to be the cause of
the "Issuing rollback() due to DESTROY without explicit
disconnect() of DBD::SQLite::db handle" messages I've been
seeing from "lei up".
---
lib/PublicInbox/LeiMailSync.pm | 5 ++---
lib/PublicInbox/LeiStore.pm | 1 +
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index ec05404a..558988f3 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -405,13 +405,12 @@ sub imap_oid {
$oidbin ? unpack('H*', $oidbin) : undef;
}
-# FIXME: something with "lei <up|q>" is causing uncommitted transaction
-# warnings, not sure what...
+# FIXED? something with "lei <up|q>" is causing uncommitted transaction
+# TODO: remove soon
sub DESTROY {
my ($self) = @_;
my $dbh = delete($self->{dbh}) or return;
return if $dbh->{ReadOnly};
- use Carp;
undef $dbh;
warn "BUG $$ $0 $self {dbh} OPEN ppid=".getppid.' '.Carp::longmess();
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 0b033e3e..5446873e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -195,6 +195,7 @@ sub remove_eml_vmd { # remove just the VMD
sub _lms_rw ($) {
my ($self) = @_;
+ my ($eidx, $tl) = eidx_init($self);
$self->{lms} //= do {
require PublicInbox::LeiMailSync;
my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
^ permalink raw reply related [relevance 69%]
* [PATCH v2] lei import: support --new-only for IMAP
2021-06-03 5:00 67% [RFC] lei import: support --new-only Eric Wong
@ 2021-06-09 22:39 62% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 22:39 UTC (permalink / raw)
To: meta
Taking ~40s to synchronize a ~75K message IMAP folder is
still a lot of time, so support an option to only touch
new messages.
This is similar to "offlineimap -q" (quick) or "mbsync --new"
switches, but lei already accepts "-q" as a shortcut for
--quiet. "--new" could work, but "--new-only" might be more
descriptive (or "--only-new"?), since the default fetches
also fetches new messages.
v2: warn for non-IMAP sources, I'm not sure it's worth it for
Maildir or other sources, yet. It will also make sense
for MH and JMAP once we support them.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 7 ++++++-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 265b7047..beeb8b48 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -240,7 +240,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s
+ qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index e3cb69ca..08794f71 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -78,12 +78,14 @@ sub do_import_index ($$@) {
$j =~ /\A([0-9]+),[0-9]+\z/ and $j = $1 + 0;
$j ||= scalar(@{$self->{inputs}}) || 1;
my $ikw;
- if (my $net = $lei->{net}) {
+ my $net = $lei->{net};
+ if ($net) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
if ($self->{-import_kw} && $net->{-lms_ro} &&
+ !$lei->{opt}->{'new-only'} &&
$net->{imap_order}) {
require PublicInbox::LeiImportKw;
$ikw = PublicInbox::LeiImportKw->new($lei);
@@ -94,6 +96,9 @@ sub do_import_index ($$@) {
my $nproc = $self->detect_nproc;
$j = $nproc if $j > $nproc;
}
+ if ($lei->{opt}->{'new-only'} && (!$net || !$net->{imap_order})) {
+ $lei->err('# --new-only is only for IMAP');
+ }
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
^ permalink raw reply related [relevance 62%]
* [PATCH] lei tag: less confusing warning about unimported messages
@ 2021-06-09 23:27 58% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 23:27 UTC (permalink / raw)
To: meta
"unimported" is more meaningful than "missing", here. And
instead of having every worker spew about unimported messages,
we'll accumulate and only print one warning line. This
necessitated alterating ->DESTROY behavior and persisting
the client socket within the $lei object itself, not just
the PktOp consumer object.
---
lib/PublicInbox/LEI.pm | 21 ++++++++++++++++-----
lib/PublicInbox/LeiTag.pm | 12 ++++++------
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index beeb8b48..d34997fd 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -567,6 +567,11 @@ sub pkt_op_pair {
$end;
}
+sub incr {
+ my ($self, $field, $nr) = @_;
+ $self->{counters}->{$field} += $nr;
+}
+
sub workers_start {
my ($lei, $wq, $jobs, $ops, $flds) = @_;
$ops = {
@@ -574,6 +579,7 @@ sub workers_start {
'|' => [ \&sigpipe_handler, $lei ],
'x_it' => [ \&x_it, $lei ],
'child_error' => [ \&child_error, $lei ],
+ 'incr' => [ \&incr, $lei ],
($ops ? %$ops : ()),
};
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
@@ -583,8 +589,6 @@ sub workers_start {
$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
- # {-lei_sock} persists script/lei process until ops->{''} EOF callback
- $op_c->{-lei_sock} = $lei->{sock};
@$end = ();
$lei->event_step_init;
($op_c, $ops);
@@ -1092,10 +1096,11 @@ sub event_step {
sub event_step_init {
my ($self) = @_;
- return if $self->{-event_init_done}++;
- if (my $sock = $self->{sock}) { # using DS->EventLoop
+ my $sock = $self->{sock} or return;
+ $self->{-event_init_done} //= do { # persist til $ops done
$self->SUPER::new($sock, EPOLLIN|EPOLLET);
- }
+ $sock;
+ };
}
sub noop {}
@@ -1246,6 +1251,12 @@ sub busy { 1 } # prevent daemon-shutdown if client is connected
# can immediately reread it
sub DESTROY {
my ($self) = @_;
+ if (my $counters = delete $self->{counters}) {
+ for my $k (sort keys %$counters) {
+ my $nr = $counters->{$k};
+ $self->child_error(1 << 8, "$nr $k messages");
+ }
+ }
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
# preserve $? for ->fail or ->x_it code
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index e0532653..463fb921 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -15,7 +15,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
$self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
$self->{vmd_mod});
} else {
- ++$self->{missing};
+ ++$self->{unimported};
}
}
@@ -40,7 +40,7 @@ sub lei_tag { # the "lei tag" method
my ($lei, @argv) = @_;
my $sto = $lei->_lei_store(1);
$sto->write_prepare($lei);
- my $self = bless { missing => 0 }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$lei->ale; # refresh and prepare
my $vmd_mod = $self->vmd_mod_extract(\@argv);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
@@ -58,10 +58,10 @@ sub lei_tag { # the "lei tag" method
$lei->wait_wq_events($op_c, $ops);
}
-sub note_missing {
+sub note_unimported {
my ($self) = @_;
- my $n = $self->{missing} or return;
- $self->{lei}->child_error(1 << 8, "$n missed messages");
+ my $n = $self->{unimported} or return;
+ $self->{lei}->{pkt_op_p}->pkt_do('incr', 'unimported', $n);
}
sub ipc_atfork_child {
@@ -69,7 +69,7 @@ sub ipc_atfork_child {
PublicInbox::LeiInput::input_only_atfork_child($self);
$self->{lse} = $self->{lei}->{sto}->search;
# this goes out-of-scope at worker process exit:
- PublicInbox::OnDestroy->new($$, \¬e_missing, $self);
+ PublicInbox::OnDestroy->new($$, \¬e_unimported, $self);
}
# Workaround bash word-splitting s to ['kw', ':', 'keyword' ...]
^ permalink raw reply related [relevance 58%]
* [PATCH] lei ls-mail-source: list IMAP folders and NNTP groups
@ 2021-06-11 9:42 41% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-11 9:42 UTC (permalink / raw)
To: meta
While other tools can provide the same functionality, having
integration with git-credential is convenient, here. Caching
and completion will be implemented separately.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 3 ++
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 82 ++++++++++++++++++++++++++++++
lib/PublicInbox/NetReader.pm | 15 +++---
t/lei-import-imap.t | 6 +++
t/lei-import-nntp.t | 7 +++
7 files changed, 108 insertions(+), 8 deletions(-)
create mode 100644 lib/PublicInbox/LeiLsMailSource.pm
diff --git a/MANIFEST b/MANIFEST
index 3d4c6cbd..d4b3e75d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -215,6 +215,7 @@ lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
lib/PublicInbox/LeiLcat.pm
lib/PublicInbox/LeiLsLabel.pm
+lib/PublicInbox/LeiLsMailSource.pm
lib/PublicInbox/LeiLsMailSync.pm
lib/PublicInbox/LeiLsSearch.pm
lib/PublicInbox/LeiMailSync.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d34997fd..833d9c4d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -196,6 +196,8 @@ our %CMD = ( # sorted in order of importance/use:
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
+'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
+ qw(z|0 ascii l), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
@@ -381,6 +383,7 @@ my %OPTDESC = (
'format|f=s ls-search' => ['OUT|json|jsonl|concatjson',
'listing output format' ],
'l ls-search' => 'long listing format',
+'l ls-mail-source' => 'long listing format',
'format|f=s ls-external' => $ls_format,
'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 24211bf0..92d67715 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -229,7 +229,7 @@ sub prepare_inputs { # returns undef on error
if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) {
require PublicInbox::NetReader;
$net //= PublicInbox::NetReader->new;
- $net->add_url($input);
+ $net->add_url($input, $self->{-ls_ok});
push @{$sync->{ok}}, $input if $sync;
} elsif ($input_path =~ m!\Ahttps?://!i) { # mboxrd.gz
# TODO: how would we detect r/w JMAP?
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
new file mode 100644
index 00000000..e95c0b34
--- /dev/null
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -0,0 +1,82 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# command for listing NNTP groups and IMAP folders,
+# handy for users with git-credential-helper configured
+# TODO: list JMAP labels
+package PublicInbox::LeiLsMailSource;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+
+sub input_path_url { # overrides LeiInput version
+ my ($self, $url) = @_;
+ # TODO: support ndjson and other JSONs we support elsewhere
+ my $json;
+ my $lei = $self->{lei};
+ my $ORS = "\n";
+ if ($self->{lei}->{opt}->{l}) {
+ $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
+ $json->ascii(1) if $lei->{opt}->{ascii};
+ } elsif ($self->{lei}->{opt}->{z}) {
+ $ORS = "\0";
+ }
+ if ($url =~ m!\Aimaps?://!i) {
+ my $uri = PublicInbox::URIimap->new($url);
+ my $mic = $lei->{net}->mic_get($uri);
+ my $l = $mic->folders_hash($uri->path); # server-side filter
+ if ($json) {
+ $lei->puts($json->encode($l));
+ } else {
+ $lei->out(join($ORS, (map { $_->{name} } @$l), ''));
+ }
+ } elsif ($url =~ m!\A(?:nntps?|s?news)://!i) {
+ my $uri = PublicInbox::URInntps->new($url);
+ my $nn = $lei->{net}->nn_get($uri);
+ my $l = $nn->newsgroups($uri->group); # name => description
+ if ($json) {
+ my $all = $nn->list;
+ my @x;
+ for my $ng (sort keys %$l) {
+ my $desc = $l->{$ng};
+
+# we need to drop CR ourselves iff using IO::Socket::SSL since
+# Net::Cmd::getline doesn't get used by Net::NNTP if TLS is in play, noted in:
+# <https://rt.cpan.org/Ticket/Display.html?id=129966>
+ $desc =~ s/\r\z//;
+
+ my ($hwm, $lwm, $status) = @{$all->{$ng}};
+ push @x, { name => $ng, lwm => $lwm + 0,
+ hwm => $hwm + 0, status => $status,
+ description => $desc };
+ }
+ $lei->puts($json->encode(\@x));
+ } else {
+ $lei->out(join($ORS, sort(keys %$l), ''));
+ }
+ } else { die "BUG: $url not supported" }
+}
+
+sub lei_ls_mail_source {
+ my ($lei, $url, $pfx) = @_;
+ $url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return
+ $lei->fail('only NNTP and IMAP URLs supported');
+ my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
+ $self->prepare_inputs($lei, [ $url ]) or return;
+ $lei->start_pager if -t $lei->{1};
+ my $ops = {};
+ $lei->{auth}->op_merge($ops, $self);
+ my $j = $self->{-wq_nr_workers} = 1; # locked
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
+}
+
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+
+1;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 2795a9d4..bcc6cbf0 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -236,18 +236,19 @@ W: see https://rt.cpan.org/Ticket/Display.html?id=129967 for updates
}
sub imap_uri {
- my ($url) = @_;
+ my ($url, $ls_ok) = @_;
require PublicInbox::URIimap;
my $uri = PublicInbox::URIimap->new($url);
- $uri ? $uri->canonical : undef;
+ $uri && ($ls_ok || $uri->mailbox) ? $uri->canonical : undef;
}
my %IS_NNTP = (news => 1, snews => 1, nntp => 1, nntps => 1);
sub nntp_uri {
- my ($url) = @_;
+ my ($url, $ls_ok) = @_;
require PublicInbox::URInntps;
my $uri = PublicInbox::URInntps->new($url);
- $uri && $IS_NNTP{$uri->scheme} && $uri->group ? $uri->canonical : undef;
+ $uri && $IS_NNTP{$uri->scheme} && ($ls_ok || $uri->group) ?
+ $uri->canonical : undef;
}
sub cfg_intvl ($$$) {
@@ -367,11 +368,11 @@ sub nntp_common_init ($;$) {
}
sub add_url {
- my ($self, $arg) = @_;
+ my ($self, $arg, $ls_ok) = @_;
my $uri;
- if ($uri = imap_uri($arg)) {
+ if ($uri = imap_uri($arg, $ls_ok)) {
push @{$self->{imap_order}}, $uri;
- } elsif ($uri = nntp_uri($arg)) {
+ } elsif ($uri = nntp_uri($arg, $ls_ok)) {
push @{$self->{nntp_order}}, $uri;
} else {
push @{$self->{unsupported_url}}, $arg;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 34fd6cf9..12f6fad0 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -14,6 +14,12 @@ undef $sock;
test_lei({ tmpdir => $tmpdir }, sub {
my $url = "imap://$host_port/t.v2.0";
my $url_orig = $url;
+ lei_ok(qw(ls-mail-source), "imap://$host_port/");
+ like($lei_out, qr/^t\.v2\.0$/ms, 'shows mailbox');
+ lei_ok(qw(ls-mail-source), $url);
+ is($lei_out, "t.v2.0\n", 'shows only mailbox with filter');
+ lei_ok(qw(ls-mail-source -l), "imap://$host_port/");
+ is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON');
lei_ok(qw(q z:1..));
my $out = json_utf8->decode($lei_out);
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index 662da309..f2c35406 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -17,6 +17,13 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $out = json_utf8->decode($lei_out);
is_deeply($out, [ undef ], 'nothing imported, yet');
my $url = "nntp://$host_port/t.v2";
+ lei_ok(qw(ls-mail-source), "nntp://$host_port/");
+ like($lei_out, qr/^t\.v2$/ms, 'shows newsgroup');
+ lei_ok(qw(ls-mail-source), $url);
+ is($lei_out, "t.v2\n", 'shows only newsgroup with filter');
+ lei_ok(qw(ls-mail-source -l), "nntp://$host_port/");
+ is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON');
+
lei_ok('import', $url);
lei_ok(qw(q z:1..));
$out = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 41%]
* [PATCH 1/5] lei: stop pager early on exit
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
@ 2021-06-12 0:10 71% ` Eric Wong
2021-06-12 0:10 48% ` [PATCH 2/5] lei ls-mail-source: write through to URL folder cache Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
This is necessary when using "ls-mail-source" on an unreachable
IMAP server.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 833d9c4d..122045ee 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -446,6 +446,7 @@ sub x_it ($$) {
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
dump_and_clear_log();
+ stop_pager($self);
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('x_it', $code);
} elsif ($self->{sock}) { # to lei(1) client
^ permalink raw reply related [relevance 71%]
* [PATCH 0/5] lei ls-mail-source-related things
@ 2021-06-12 0:10 71% Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
A couple of minor usability things that'll make "lei import"
easier for bash completion users.
Eric Wong (5):
lei: stop pager early on exit
lei ls-mail-source: write through to URL folder cache
t/lei-import-http: quiet unnecessary diag message
lei import: use url_folder_cache for completion
net_reader: canonicalize URL args on add_url
lib/PublicInbox/LEI.pm | 10 ++++++-
lib/PublicInbox/LeiImport.pm | 11 +++++---
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 42 ++++++++++++++++++++++++++++--
lib/PublicInbox/NetReader.pm | 2 ++
lib/PublicInbox/SharedKV.pm | 13 +++++++--
t/lei-import-http.t | 2 --
7 files changed, 71 insertions(+), 11 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/5] lei ls-mail-source: write through to URL folder cache
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
@ 2021-06-12 0:10 48% ` Eric Wong
2021-06-12 0:10 71% ` [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message Eric Wong
2021-06-12 0:10 71% ` [PATCH 4/5] lei import: use url_folder_cache for completion Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
We'll be able to use this for shell completion for
lei import, lcat, tag, etc..
This also adds --url support for scripting purposes.
---
lib/PublicInbox/LEI.pm | 9 ++++++-
lib/PublicInbox/LeiLsMailSource.pm | 42 ++++++++++++++++++++++++++++--
lib/PublicInbox/SharedKV.pm | 13 +++++++--
3 files changed, 59 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 122045ee..50a7fdad 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -116,6 +116,12 @@ sub cache_dir ($) {
.'/lei');
}
+sub url_folder_cache {
+ my ($self) = @_;
+ require PublicInbox::SharedKV; # URI => updated_at_sec_
+ PublicInbox::SharedKV->new(cache_dir($self).'/uri_folder');
+}
+
sub ale {
my ($self) = @_;
$self->{ale} //= do {
@@ -197,7 +203,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
- qw(z|0 ascii l), @c_opt ],
+ qw(z|0 ascii l url), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
@@ -384,6 +390,7 @@ my %OPTDESC = (
'listing output format' ],
'l ls-search' => 'long listing format',
'l ls-mail-source' => 'long listing format',
+'url ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
'format|f=s ls-external' => $ls_format,
'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index e95c0b34..cadc61ed 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -21,19 +21,27 @@ sub input_path_url { # overrides LeiInput version
} elsif ($self->{lei}->{opt}->{z}) {
$ORS = "\0";
}
+ my @f;
if ($url =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($url);
+ my $sec = $lei->{net}->can('uri_section')->($uri);
my $mic = $lei->{net}->mic_get($uri);
my $l = $mic->folders_hash($uri->path); # server-side filter
+ @f = map { "$sec/$_->{name}" } @$l;
if ($json) {
+ $_->{url} = "$sec/$_->{name}" for @$l;
$lei->puts($json->encode($l));
} else {
+ if ($self->{lei}->{opt}->{url}) {
+ $_->{name} = "$sec/$_->{name}" for @$l;
+ }
$lei->out(join($ORS, (map { $_->{name} } @$l), ''));
}
} elsif ($url =~ m!\A(?:nntps?|s?news)://!i) {
my $uri = PublicInbox::URInntps->new($url);
my $nn = $lei->{net}->nn_get($uri);
my $l = $nn->newsgroups($uri->group); # name => description
+ my $sec = $lei->{net}->can('uri_section')->($uri);
if ($json) {
my $all = $nn->list;
my @x;
@@ -46,15 +54,30 @@ sub input_path_url { # overrides LeiInput version
$desc =~ s/\r\z//;
my ($hwm, $lwm, $status) = @{$all->{$ng}};
- push @x, { name => $ng, lwm => $lwm + 0,
+ push @x, { name => $ng, url => "$sec/$ng",
+ lwm => $lwm + 0,
hwm => $hwm + 0, status => $status,
description => $desc };
}
+ @f = map { "$sec/$_" } keys %$all;
$lei->puts($json->encode(\@x));
} else {
- $lei->out(join($ORS, sort(keys %$l), ''));
+ @f = map { "$sec/$_" } keys %$l;
+ if ($self->{lei}->{opt}->{url}) {
+ $lei->out(join($ORS, sort(@f), ''));
+ } else {
+ $lei->out(join($ORS, sort(keys %$l), ''));
+ }
}
} else { die "BUG: $url not supported" }
+ if (@f) {
+ my $fc = $lei->url_folder_cache;
+ my $lk = $fc->lock_for_scope;
+ $fc->dbh->begin_work;
+ my $now = time;
+ $fc->set($_, $now) for @f;
+ $fc->dbh->commit;
+ }
}
sub lei_ls_mail_source {
@@ -62,6 +85,7 @@ sub lei_ls_mail_source {
$url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return
$lei->fail('only NNTP and IMAP URLs supported');
my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
+ $self->{cfg} = $lei->_lei_cfg; # may be undef
$self->prepare_inputs($lei, [ $url ]) or return;
$lei->start_pager if -t $lei->{1};
my $ops = {};
@@ -74,6 +98,20 @@ sub lei_ls_mail_source {
$lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
}
+sub _complete_ls_mail_source {
+ my ($lei, @argv) = @_;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
+ my %f = map { $_ => 1 } @m;
+ my $sto = $lei->_lei_store;
+ if (my $lms = $sto ? $sto->search->lms : undef) {
+ @m = map { $match_cb->($_) } grep(
+ m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
+ @f{@m} = @m;
+ }
+ keys %f;
+}
+
no warnings 'once';
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index d65c3158..8347b195 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -11,7 +11,7 @@ use parent qw(PublicInbox::Lock);
use File::Temp qw(tempdir);
use DBI ();
use PublicInbox::Spawn;
-use File::Path qw(rmtree);
+use File::Path qw(rmtree make_path);
sub dbh {
my ($self, $lock) = @_;
@@ -46,8 +46,8 @@ CREATE TABLE IF NOT EXISTS kv (
sub new {
my ($cls, $dir, $base, $opt) = @_;
my $self = bless { opt => $opt }, $cls;
+ make_path($dir) if defined($dir) && !-d $dir;
$dir //= $self->{"tmp$$.$self"} = tempdir("skv.$$-XXXX", TMPDIR => 1);
- -d $dir or mkdir($dir) or die "mkdir($dir): $!";
$base //= '';
my $f = $self->{filename} = "$dir/$base.sqlite3";
$self->{lock_path} = $opt->{lock_path} // "$dir/$base.flock";
@@ -83,6 +83,15 @@ SELECT k,v FROM kv
$sth
}
+sub keys {
+ my ($self) = @_;
+ my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
+SELECT k FROM kv
+
+ $sth->execute;
+ map { $_->[0] } @{$sth->fetchall_arrayref};
+}
+
sub delete_by_val {
my ($self, $val, $lock) = @_;
$lock //= $self->lock_for_scope_fast;
^ permalink raw reply related [relevance 48%]
* [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
2021-06-12 0:10 48% ` [PATCH 2/5] lei ls-mail-source: write through to URL folder cache Eric Wong
@ 2021-06-12 0:10 71% ` Eric Wong
2021-06-12 0:10 71% ` [PATCH 4/5] lei import: use url_folder_cache for completion Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
Leftover while writing the test.
---
t/lei-import-http.t | 2 --
1 file changed, 2 deletions(-)
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
index 6cb8a753..2104c778 100644
--- a/t/lei-import-http.t
+++ b/t/lei-import-http.t
@@ -42,7 +42,5 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(!lei(qw(import --mail-sync), "$url/x\@example.com/raw"),
'--mail-sync fails on HTTP');
- diag $lei_err;
-
});
done_testing;
^ permalink raw reply related [relevance 71%]
* [PATCH 4/5] lei import: use url_folder_cache for completion
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
` (2 preceding siblings ...)
2021-06-12 0:10 71% ` [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message Eric Wong
@ 2021-06-12 0:10 71% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
And fix "lei index" completion while we're at it.
---
lib/PublicInbox/LeiImport.pm | 11 ++++++++---
lib/PublicInbox/LeiIndex.pm | 2 +-
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 08794f71..7580e37e 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -118,10 +118,15 @@ sub lei_import { # the main "lei import" method
sub _complete_import {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
+ my %f = map { $_ => 1 } @m;
+ my $sto = $lei->_lei_store;
+ if (my $lms = $sto ? $sto->search->lms : undef) {
+ @m = map { $match_cb->($_) } $lms->folders;
+ @f{@m} = @m;
+ }
+ keys %f;
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 4be0c649..5b545998 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -39,7 +39,7 @@ for my $m (qw(pmdir_cb input_net_cb)) {
*$m = PublicInbox::LeiImport->can($m);
}
-*_complete_import = \&PublicInbox::LeiImport::_complete_import;
+*_complete_index = \&PublicInbox::LeiImport::_complete_import;
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
^ permalink raw reply related [relevance 71%]
* [PATCH 0/2] lei: support keywords off a single Maildir file
@ 2021-06-13 18:12 71% Eric Wong
2021-06-13 18:12 59% ` [PATCH 2/2] lei index+import: reject keywords from R/O IMAP Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-06-13 18:12 UTC (permalink / raw)
To: meta
I noticed the bug fixed by 2/2 while working on 1/2
Eric Wong (2):
lei_input: allow keywords when importing 1 file from Maildir
lei index+import: reject keywords from R/O IMAP
lib/PublicInbox/LeiInput.pm | 27 ++++++++++++++++++++++++++-
lib/PublicInbox/LeiStore.pm | 3 ++-
lib/PublicInbox/NetReader.pm | 11 ++++++-----
t/lei-index.t | 22 ++++++++++++++++++++++
4 files changed, 56 insertions(+), 7 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei index+import: reject keywords from R/O IMAP
2021-06-13 18:12 71% [PATCH 0/2] lei: support keywords off a single Maildir file Eric Wong
@ 2021-06-13 18:12 59% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-13 18:12 UTC (permalink / raw)
To: meta
Since users can't set IMAP flags in read-only IMAP folders,
we won't clobber local flags when importing from IMAP. This
also enables the local_blob fallback used for lei-index to
be used for index deduplication.
---
lib/PublicInbox/LeiStore.pm | 3 ++-
lib/PublicInbox/NetReader.pm | 11 ++++++-----
t/lei-index.t | 13 +++++++++++++
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 5446873e..f978288a 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -151,7 +151,8 @@ sub _docids_for ($$) {
my $oid = $cur->{blob};
my $docid = $cur->{num};
my $bref = $im ? $im->cat_blob($oid) : undef;
- $bref //= $eidx->git->cat_file($oid) // do {
+ $bref //= $eidx->git->cat_file($oid) //
+ _lms_rw($self)->local_blob($oid, 1) // do {
warn "W: $oid (#$docid) <$mid> not found\n";
next;
};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 30784199..0c2288d8 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -420,7 +420,8 @@ sub _imap_do_msg ($$$$$) {
my ($self, $uri, $uid, $raw, $flags) = @_;
# our target audience expects LF-only, save storage
$$raw =~ s/\r\n/\n/sg;
- my $kw = flags2kw($self, $uri, $uid, $flags) // return;
+ my $kw = defined($flags) ?
+ (flags2kw($self, $uri, $uid, $flags) // return) : undef;
my ($eml_cb, @args) = @{$self->{eml_each}};
$eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args);
}
@@ -537,8 +538,8 @@ E: $uri strangely, UIDVALIDLITY matches ($l_uidval)
EOF
$mic->Uid(1); # the default, we hope
my $err;
- if (!defined($single_uid) && $self->{each_old} &&
- perm_fl_ok($perm_fl)) {
+ my $use_fl = perm_fl_ok($perm_fl);
+ if (!defined($single_uid) && $self->{each_old} && $use_fl) {
$err = each_old_flags($self, $mic, $uri, $l_uid);
return $err if $err;
}
@@ -593,8 +594,8 @@ EOF
# messages get deleted, so holes appear
my $per_uid = delete $r->{$uid} // next;
my $raw = delete($per_uid->{$key}) // next;
- _imap_do_msg($self, $uri, $uid, \$raw,
- $per_uid->{FLAGS});
+ my $fl = $use_fl ? $per_uid->{FLAGS} : undef;
+ _imap_do_msg($self, $uri, $uid, \$raw, $fl);
$last_uid = $uid;
last if $self->{quit};
}
diff --git a/t/lei-index.t b/t/lei-index.t
index c142e79c..eeda5196 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -80,6 +80,19 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('index', "nntp://$nntp_host_port/t.v2");
lei_ok('index', "imap://$imap_host_port/t.v2.0");
is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
+
+ lei_ok qw(q m:multipart-html-sucks@11);
+ $res_a = json_utf8->decode($lei_out)->[0];
+ is_deeply($res_a->{'kw'}, ['seen'],
+ 'keywords still set after NNTP + IMAP import');
+
+ # ensure import works after lms->local_blob fallback in lei/store
+ lei_ok('import', 't/mda-mime.eml');
+ lei_ok qw(q m:multipart-html-sucks@11);
+ $res_b = json_utf8->decode($lei_out)->[0];
+ my $t = xqx(['git', "--git-dir=$store_path/ALL.git",
+ qw(cat-file -t), $res_b->{blob}]);
+ is($t, "blob\n", 'got blob');
});
done_testing;
^ permalink raw reply related [relevance 59%]
* [PATCH 0/3] lei: internal bug fixups
@ 2021-06-17 22:00 71% Eric Wong
2021-06-17 22:00 60% ` [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes Eric Wong
2021-06-17 22:00 55% ` [PATCH 3/3] lei/store: cull redundant docids based on blob OID Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-06-17 22:00 UTC (permalink / raw)
To: meta
Still chasing some oddness in day-to-day usage; but I think
3/3 is safe (1/3 helped me inspect things)
Eric Wong (3):
lei inspect: learn "num:" and "docid:" prefixes
lei_input: prefix bare Maildir paths w/ "maildir:"
lei/store: cull redundant docids based on blob OID
lib/PublicInbox/LeiInput.pm | 3 +-
lib/PublicInbox/LeiInspect.pm | 73 +++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiStore.pm | 54 +++++++++++++++++---------
lib/PublicInbox/SearchIdx.pm | 2 +-
4 files changed, 111 insertions(+), 21 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes
2021-06-17 22:00 71% [PATCH 0/3] lei: internal bug fixups Eric Wong
@ 2021-06-17 22:00 60% ` Eric Wong
2021-06-17 22:00 55% ` [PATCH 3/3] lei/store: cull redundant docids based on blob OID Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-17 22:00 UTC (permalink / raw)
To: meta
"num:" is useful for inspecting Inbox-ish directories, while
"docid:" can be used for any Xapian DB (not just stuff managed
by our code).
---
lib/PublicInbox/LeiInspect.pm | 73 +++++++++++++++++++++++++++++++++++
1 file changed, 73 insertions(+)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index eb2634b4..30714764 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -57,6 +57,75 @@ sub inspect_sync_folder ($$) {
$ent
}
+sub inspect_docid ($$;$) {
+ my ($lei, $docid, $ent) = @_;
+ require PublicInbox::Search;
+ $ent //= {};
+ my $xdb;
+ if ($xdb = delete $ent->{xdb}) { # from inspect_num
+ } elsif (defined(my $dir = $lei->{opt}->{dir})) {
+ no warnings 'once';
+ $xdb = $PublicInbox::Search::X{Database}->new($dir);
+ } else {
+ $xdb = $lei->{lse}->xdb;
+ }
+ $xdb or return $lei->fail('no Xapian DB');
+ my $doc = $xdb->get_document($docid); # raises
+ my $data = $doc->get_data;
+ $ent->{docid} = $docid;
+ $ent->{data_length} = length($data);
+ $ent->{description} => $doc->get_description;
+ $ent->{$_} = $doc->$_ for (qw(termlist_count values_count));
+ my $cur = $doc->termlist_begin;
+ my $end = $doc->termlist_end;
+ for (; $cur != $end; $cur++) {
+ my $tn = $cur->get_termname;
+ $tn =~ s/\A([A-Z]+)// or warn "$tn no prefix! (???)";
+ my $term = ($1 // '');
+ push @{$ent->{terms}->{$term}}, $tn;
+ }
+ @$_ = sort(@$_) for values %{$ent->{terms} // {}};
+ $cur = $doc->values_begin;
+ $end = $doc->values_end;
+ for (; $cur != $end; $cur++) {
+ my $n = $cur->get_valueno;
+ my $v = $cur->get_value;
+ my $iv = PublicInbox::Search::sortable_unserialise($v);
+ $v = $iv + 0 if defined $iv;
+ # not using ->[$n] since we may have large gaps in $n
+ $ent->{'values'}->{$n} = $v;
+ }
+ $ent;
+}
+
+sub inspect_num ($$) {
+ my ($lei, $num) = @_;
+ my ($docid, $ibx);
+ my $ent = { num => $num };
+ if (defined(my $dir = $lei->{opt}->{dir})) {
+ my $num2docid = $lei->{lse}->can('num2docid');
+ if (-f "$dir/ei.lock") {
+ require PublicInbox::ExtSearch;
+ $ibx = PublicInbox::ExtSearch->new($dir);
+ } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
+ require PublicInbox::Inbox; # v2, v1
+ $ibx = bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+ }
+ $ent->{xdb} = $ibx->xdb //
+ return $lei->fail("no Xapian DB for $dir");
+ $docid = $num2docid->($ibx, $num);
+ } else {
+ $ibx = $lei->{lse};
+ $lei->{lse}->xdb; # set {nshard} for num2docid
+ $docid = $lei->{lse}->num2docid($num);
+ }
+ if ($ibx && $ibx->over) {
+ my $smsg = $ibx->over->get_art($num);
+ $ent->{smsg} = { %$smsg } if $smsg;
+ }
+ inspect_docid($lei, $docid, $ent);
+}
+
sub inspect1 ($$$) {
my ($lei, $item, $more) = @_;
my $ent;
@@ -72,6 +141,10 @@ sub inspect1 ($$$) {
}
} elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
+ } elsif ($item =~ m!\Adocid:([0-9]+)\z!) {
+ $ent = inspect_docid($lei, $1 + 0);
+ } elsif ($item =~ m!\Anum:([0-9]+)\z!) {
+ $ent = inspect_num($lei, $1 + 0);
} else { # TODO: more things
return $lei->fail("$item not understood");
}
^ permalink raw reply related [relevance 60%]
* [PATCH 3/3] lei/store: cull redundant docids based on blob OID
2021-06-17 22:00 71% [PATCH 0/3] lei: internal bug fixups Eric Wong
2021-06-17 22:00 60% ` [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes Eric Wong
@ 2021-06-17 22:00 55% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-17 22:00 UTC (permalink / raw)
To: meta
I'm not sure how this happened (only once for me in March), but
it should not happen... In any case, we'll operate on the
lowest numbered docid and cull redundant index entries when
lei/store is open for read-write.
This also fixes the normal lei/store removal path to clean up
the xref3 table (since it's not done automatically for
public-facing -eidx due to the multi-list nature of it).
---
lib/PublicInbox/LeiStore.pm | 54 +++++++++++++++++++++++-------------
lib/PublicInbox/SearchIdx.pm | 2 +-
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index f978288a..4ba1e647 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -226,6 +226,18 @@ sub _remove_if_local { # git->cat_async arg
$self->{im}->remove($bref) if $bref;
}
+sub remove_docids ($;@) {
+ my ($self, @docids) = @_;
+ my $eidx = eidx_init($self);
+ for my $docid (@docids) {
+ $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
+ $self->{oidx}->delete_by_num($docid);
+ $self->{oidx}->{dbh}->do(<<EOF, undef, $docid);
+DELETE FROM xref3 WHERE docid = ?
+EOF
+ }
+}
+
# remove the entire message from the index, does not touch mail_sync.sqlite3
sub remove_eml {
my ($self, $eml) = @_;
@@ -241,13 +253,25 @@ sub remove_eml {
my $oidhex = unpack('H*', $oidbin);
$git->cat_async($oidhex, \&_remove_if_local, $self);
}
- $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
- $oidx->delete_by_num($docid);
}
$git->cat_async_wait;
+ remove_docids($self, @docids);
\@docids;
}
+sub oid2docid ($$) {
+ my ($self, $oid) = @_;
+ my $eidx = eidx_init($self);
+ my ($docid, @cull) = $eidx->{oidx}->blob_exists($oid);
+ if (@cull) { # fixup old bugs...
+ warn <<EOF;
+W: $oid indexed as multiple docids: $docid @cull, culling to fixup old bugs
+EOF
+ remove_docids($self, @cull);
+ }
+ wantarray ? ($docid) : $docid;
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
@@ -268,7 +292,7 @@ sub add_eml {
if (scalar keys %$xoids) {
my %docids = map { $_ => 1 } @$vivify_xvmd;
for my $oid (keys %$xoids) {
- my @id = $oidx->blob_exists($oid);
+ my @id = oid2docid($self, $oid);
@docids{@id} = @id;
}
@$vivify_xvmd = sort { $a <=> $b } keys(%docids);
@@ -356,15 +380,11 @@ sub update_xvmd {
my $oidx = $eidx->{oidx};
my %seen;
for my $oid (keys %$xoids) {
- my @docids = $oidx->blob_exists($oid) or next;
- scalar(@docids) > 1 and
- warn "W: $oid indexed as multiple docids: @docids\n";
- for my $docid (@docids) {
- next if $seen{$docid}++;
- my $idx = $eidx->idx_shard($docid);
- $idx->ipc_do('update_vmd', $docid, $vmd_mod);
- }
+ my $docid = oid2docid($self, $oid) // next;
delete $xoids->{$oid};
+ next if $seen{$docid}++;
+ my $idx = $eidx->idx_shard($docid);
+ $idx->ipc_do('update_vmd', $docid, $vmd_mod);
}
return unless scalar(keys(%$xoids));
@@ -395,15 +415,11 @@ sub set_xvmd {
# see if we can just update existing docs
for my $oid (keys %$xoids) {
- my @docids = $oidx->blob_exists($oid) or next;
- scalar(@docids) > 1 and
- warn "W: $oid indexed as multiple docids: @docids\n";
- for my $docid (@docids) {
- next if $seen{$docid}++;
- my $idx = $eidx->idx_shard($docid);
- $idx->ipc_do('set_vmd', $docid, $vmd);
- }
+ my $docid = oid2docid($self, $oid) // next;
delete $xoids->{$oid}; # all done with this oid
+ next if $seen{$docid}++;
+ my $idx = $eidx->idx_shard($docid);
+ $idx->ipc_do('set_vmd', $docid, $vmd);
}
return unless scalar(keys(%$xoids));
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f066cc92..f553eda6 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -572,7 +572,7 @@ sub apply_vmd_mod ($$) {
my $updated = 0;
my @x = @VMD_MAP;
while (my ($field, $pfx) = splice(@x, 0, 2)) {
- # field: "label" or "kw"
+ # field: "L" or "kw"
for my $val (@{$vmd_mod->{"-$field"} // []}) {
eval {
$doc->remove_term($pfx . $val);
^ permalink raw reply related [relevance 55%]
* [PATCH] lei/store: do not put NULL into over.num column
@ 2021-06-18 19:20 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-18 19:20 UTC (permalink / raw)
To: meta
Simplify oid2docid and filter out undefined docids in ->add_eml,
instead. This avoids SQLite "datatype mismatch" errors in
OverIdx->add_over
Fixes: d1052f03ea85d4af ("lei/store: cull redundant docids based on blob OID")
---
lib/PublicInbox/LeiStore.pm | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 4ba1e647..e26b622d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -269,7 +269,7 @@ W: $oid indexed as multiple docids: $docid @cull, culling to fixup old bugs
EOF
remove_docids($self, @cull);
}
- wantarray ? ($docid) : $docid;
+ $docid;
}
sub add_eml {
@@ -292,8 +292,8 @@ sub add_eml {
if (scalar keys %$xoids) {
my %docids = map { $_ => 1 } @$vivify_xvmd;
for my $oid (keys %$xoids) {
- my @id = oid2docid($self, $oid);
- @docids{@id} = @id;
+ my $docid = oid2docid($self, $oid);
+ $docids{$docid} = $docid if defined($docid);
}
@$vivify_xvmd = sort { $a <=> $b } keys(%docids);
}
^ permalink raw reply related [relevance 71%]
* [PATCH] lei sucks: don't warn or error out on missing dependencies
@ 2021-06-20 4:33 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-20 4:33 UTC (permalink / raw)
To: meta
%INC can hold undef. This can be hit on a Linux machine missing
Linux::Inotify2. Loading PublicInbox::KQNotify is attempted and
PublicInbox/KQNotify.pm always exists, causing the `undef' entry
in %INC when it fails to load IO::KQueue.
$ perl -MData::Dumper -I lib \
-E 'eval { require PublicInbox::KQNotify }; say Dumper(\%INC)'
---
lib/PublicInbox/LeiSucks.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index a71158f3..3e945d0b 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -55,7 +55,7 @@ sub lei_sucks {
my $dig = Digest::SHA->new(1);
push @out, "public-inbox blob OIDs of loaded features:\n";
for my $m (grep(m{^PublicInbox/}, sort keys %INC)) {
- my $f = $INC{$m};
+ my $f = $INC{$m} // next; # lazy require failed (missing dep)
$dig->add('blob '.(-s $f)."\0");
$dig->addfile($f);
push @out, ' '.$dig->hexdigest.' '.$m."\n";
^ permalink raw reply related [relevance 71%]
* [PATCH] lei import: help + completion for --new-only
@ 2021-06-20 8:39 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-20 8:39 UTC (permalink / raw)
To: meta
I've found it's very helpful for large IMAP folders.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 50a7fdad..546fa773 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -353,6 +353,7 @@ my %OPTDESC = (
"or\xa0`-'\x{a0}for\x{a0}stdout)" ],
'mua=s' => [ 'CMD',
"MUA to run on --output Maildir or mbox (e.g.\xa0`mutt\xa0-f\xa0%f')" ],
+'new-only import' => 'only import new messages from IMAP source',
'inbox-version=i' => [ 'NUM|1|2',
'force a public-inbox version with --mirror'],
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: use open() perlop for -C (chdir)
@ 2021-06-22 10:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-22 10:04 UTC (permalink / raw)
To: meta
This is for consistency with the open() at initial accept, in
case we hit a code path which expects Perl directory handles
rather than "file handles". Both work with the chdir() perlop
(fchdir(2), in our case).
---
lib/PublicInbox/LEI.pm | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 546fa773..cffe4dce 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -762,11 +762,7 @@ sub dispatch {
next if $d eq ''; # same as git(1)
chdir $d or return fail($self, "cd $d: $!");
}
- if (delete $self->{3}) { # update cwd for rel2abs
- opendir my $dh, '.' or
- return fail($self, "opendir . $!");
- $self->{3} = $dh;
- }
+ open $self->{3}, '.' or return fail($self, "open . $!");
}
$cb->($self, @argv);
} elsif (grep(/\A-/, $cmd, @argv)) { # --help or -h only
^ permalink raw reply related [relevance 71%]
* [PATCH 0/2] lei inspect: "mid:" prefix and pager
@ 2021-07-01 11:31 71% Eric Wong
2021-07-01 11:31 71% ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
2021-07-01 11:31 62% ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
To: meta
Some things that came up while chasing -extindex deduplication
weirdness (and my head hurts because of that...)
Eric Wong (2):
lei inspect: support automatic pager in output
lei inspect: support "mid:" (and "m:") prefix
lib/PublicInbox/LeiInspect.pm | 60 +++++++++++++++++++++++++++--------
1 file changed, 46 insertions(+), 14 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei inspect: support automatic pager in output
2021-07-01 11:31 71% [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
@ 2021-07-01 11:31 71% ` Eric Wong
2021-07-01 11:31 62% ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
To: meta
All commands which output non-trivial amounts of data to
the terminal should support this.
---
lib/PublicInbox/LeiInspect.pm | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 30714764..9a7900c7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -155,9 +155,6 @@ sub inspect1 ($$$) {
sub lei_inspect {
my ($lei, @argv) = @_;
- $lei->{1}->autoflush(0);
- my $multi = scalar(@argv) > 1;
- $lei->out('[') if $multi;
$lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
$lei->{lse} = ($lei->{opt}->{external} // 1) ? do {
my $sto = $lei->_lei_store;
@@ -166,6 +163,10 @@ sub lei_inspect {
if ($lei->{opt}->{pretty} || -t $lei->{1}) {
$lei->{json}->pretty(1)->indent(2);
}
+ $lei->start_pager if -t $lei->{1};
+ $lei->{1}->autoflush(0);
+ my $multi = scalar(@argv) > 1;
+ $lei->out('[') if $multi;
while (defined(my $x = shift @argv)) {
inspect1($lei, $x, scalar(@argv)) or return;
}
^ permalink raw reply related [relevance 71%]
* [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix
2021-07-01 11:31 71% [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
2021-07-01 11:31 71% ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
@ 2021-07-01 11:31 62% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
To: meta
Using this to track down deduplication failures in -extindex...
---
lib/PublicInbox/LeiInspect.pm | 53 +++++++++++++++++++++++++++--------
1 file changed, 42 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 9a7900c7..574da7a7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -98,22 +98,29 @@ sub inspect_docid ($$;$) {
$ent;
}
+sub dir2ibx ($$) {
+ my ($lei, $dir) = @_;
+ if (-f "$dir/ei.lock") {
+ require PublicInbox::ExtSearch;
+ PublicInbox::ExtSearch->new($dir);
+ } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
+ require PublicInbox::Inbox; # v2, v1
+ bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+ } else {
+ $lei->fail("no (indexed) inbox or extindex at $dir");
+ }
+}
+
sub inspect_num ($$) {
my ($lei, $num) = @_;
my ($docid, $ibx);
my $ent = { num => $num };
if (defined(my $dir = $lei->{opt}->{dir})) {
- my $num2docid = $lei->{lse}->can('num2docid');
- if (-f "$dir/ei.lock") {
- require PublicInbox::ExtSearch;
- $ibx = PublicInbox::ExtSearch->new($dir);
- } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
- require PublicInbox::Inbox; # v2, v1
- $ibx = bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+ $ibx = dir2ibx($lei, $dir) or return;
+ if ($ent->{xdb} = $ibx->xdb) {
+ my $num2docid = $lei->{lse}->can('num2docid');
+ $docid = $num2docid->($ibx, $num);
}
- $ent->{xdb} = $ibx->xdb //
- return $lei->fail("no Xapian DB for $dir");
- $docid = $num2docid->($ibx, $num);
} else {
$ibx = $lei->{lse};
$lei->{lse}->xdb; # set {nshard} for num2docid
@@ -123,7 +130,29 @@ sub inspect_num ($$) {
my $smsg = $ibx->over->get_art($num);
$ent->{smsg} = { %$smsg } if $smsg;
}
- inspect_docid($lei, $docid, $ent);
+ defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent;
+}
+
+sub inspect_mid ($$) {
+ my ($lei, $mid) = @_;
+ my ($ibx, $over);
+ my $ent = { mid => $mid };
+ if (defined(my $dir = $lei->{opt}->{dir})) {
+ my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid');
+ $ibx = dir2ibx($lei, $dir) or return;
+ # $ent->{xdb} = $ibx->xdb //
+ # return $lei->fail("no Xapian DB for $dir");
+ } else {
+ $ibx = $lei->{lse};
+ $lei->{lse}->xdb; # set {nshard} for num2docid
+ }
+ if ($ibx && $ibx->over) {
+ my ($id, $prev);
+ while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
+ push @{$ent->{smsg}}, { %$smsg }
+ }
+ }
+ $ent;
}
sub inspect1 ($$$) {
@@ -145,6 +174,8 @@ sub inspect1 ($$$) {
$ent = inspect_docid($lei, $1 + 0);
} elsif ($item =~ m!\Anum:([0-9]+)\z!) {
$ent = inspect_num($lei, $1 + 0);
+ } elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
+ $ent = inspect_mid($lei, $1);
} else { # TODO: more things
return $lei->fail("$item not understood");
}
^ permalink raw reply related [relevance 62%]
* [PATCH] lei inspect: help+completion for --dir option
@ 2021-07-02 20:42 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-02 20:42 UTC (permalink / raw)
To: meta
It's the most generic name I could find for it since it can
mean so many things...
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index cffe4dce..42e02efb 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -323,6 +323,7 @@ my %OPTDESC = (
'path-a|a=s' => 'pre-image pathname associated with OID',
'path-b|b=s' => 'post-image pathname associated with OID',
'git-dir=s@' => 'additional git repository to scan',
+'dir=s inspect' => 'specify a inboxdir, extindex topdir or Xapian shard',
'proxy=s' => [ 'PROTO://HOST[:PORT]', # shared with curl(1)
"proxy for (e.g. `socks5h://0:9050')" ],
'torsocks=s' => ['VAL|auto|no|yes',
^ permalink raw reply related [relevance 71%]
* [PATCH] lei import: increase flags search batch size, display progress
@ 2021-07-02 21:02 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-02 21:02 UTC (permalink / raw)
To: meta
IMAP flag-only synchronization doesn't fetch entire messages,
so we can safely bump the batch size iff a user specified one
for full messages to 10000 times that.
Since I sometimes wonder why nothing happens for several seconds
after starting "lei import $URL", we'll also show some progress
during the flag synchronization phase.
---
lib/PublicInbox/NetReader.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 0c2288d8..23445e7a 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -462,8 +462,9 @@ sub each_old_flags ($$$$) {
my ($self, $mic, $uri, $l_uid) = @_;
$l_uid ||= 1;
my $sec = uri_section($uri);
- my $bs = $self->{imap_opt}->{$sec}->{batch_size} // 10000;
+ my $bs = ($self->{imap_opt}->{$sec}->{batch_size} // 1) * 10000;
my ($eml_cb, @args) = @{$self->{eml_each}};
+ $self->{quiet} or warn "# $uri syncing flags 1:$l_uid\n";
for (my $n = 1; $n <= $l_uid; $n += $bs) {
my $end = $n + $bs;
$end = $l_uid if $end > $l_uid;
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: drop workers on EOF from clients
@ 2021-07-04 19:35 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-04 19:35 UTC (permalink / raw)
To: meta
Sometimes a user will be bored waiting for a command to finish,
so ensure we drop disconnect workers in this case.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 42e02efb..a9f5edae 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1095,6 +1095,7 @@ sub event_step {
}
die "unrecognized client signal: $buf";
}
+ _drop_wq($self); # EOF, client disconnected
dclose($self);
};
if (my $err = $@) {
^ permalink raw reply related [relevance 71%]
* deduplication doesn't always work with "lei up"
@ 2021-07-08 8:42 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-08 8:42 UTC (permalink / raw)
To: meta
It seems to only happen with HTTP(S) externals, and
not in a FreeBSD VM, only bare metal Debian.
I can't seem to reproduce it consistently, at all.
Been staring at the LeiSavedSearch and query_remote_mboxrd
off and on for a few days while testing dedupe and can't
seem to notice any problems.
Maybe the maxuid guard for locals is hiding the bug...
^ permalink raw reply [relevance 71%]
* [PATCH 0/2] lei Maildir inotify/kevent support
@ 2021-07-19 8:59 63% Eric Wong
2021-07-19 8:59 22% ` [PATCH 2/2] lei: start implementing inotify Maildir support Eric Wong
2021-07-21 14:07 47% ` [PATCH 3/2] lei: auto-refresh watches in config, cancel missing Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-07-19 8:59 UTC (permalink / raw)
To: meta
One of the most important features of lei that drove the
decision towards its daemonic architecture.
My brain still has some wires loose or crossed :<
So I've had a very tough time mapping out the internal design
for this that could deal with rename() storms and reduce the
need to handle IN_Q_OVERFLOW conditions. Eventually,
introducing a new concept in the form of LeiNoteEvent helped my
crippled brain make the necessary connections and cobble
together something that seems halfway working.
LeiNoteEvent represents a significant conceptual shift in lei
internals. Having long-lived workers is closer to my original
vision of how lei internals would've been. That old model had
some major weaknesses, however:
1) error handling/reporting back to the CLI required FD passing
(and large-scale FD passing got confusing pretty quick)
2) excessive FD passing overflowed default kernel buffers
and made parallelism difficult for unprivileged users
Since there's no CLI to report errors to with internal
notifications, we don't have FDs to pass. Of course, that
means we can lose errors right now, but it's probably not
a big deal for (mostly) innocuous keyword changes.
Eric Wong (2):
config: s/_one_val/get_1/ for public use
lei: start implementing inotify Maildir support
MANIFEST | 6 ++
lib/PublicInbox/Config.pm | 6 +-
lib/PublicInbox/DirIdle.pm | 17 +++++
lib/PublicInbox/LEI.pm | 116 ++++++++++++++++++++++++++---
lib/PublicInbox/LeiAddWatch.pm | 41 ++++++++++
lib/PublicInbox/LeiInput.pm | 1 +
lib/PublicInbox/LeiLsWatch.pm | 15 ++++
lib/PublicInbox/LeiNoteEvent.pm | 124 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSearch.pm | 2 +-
lib/PublicInbox/LeiSelfSocket.pm | 45 +++++++++++
lib/PublicInbox/LeiWatch.pm | 13 ++++
t/lei-watch.t | 49 ++++++++++++
12 files changed, 419 insertions(+), 16 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddWatch.pm
create mode 100644 lib/PublicInbox/LeiLsWatch.pm
create mode 100644 lib/PublicInbox/LeiNoteEvent.pm
create mode 100644 lib/PublicInbox/LeiSelfSocket.pm
create mode 100644 lib/PublicInbox/LeiWatch.pm
create mode 100644 t/lei-watch.t
^ permalink raw reply [relevance 63%]
* [PATCH 2/2] lei: start implementing inotify Maildir support
2021-07-19 8:59 63% [PATCH 0/2] lei Maildir inotify/kevent support Eric Wong
@ 2021-07-19 8:59 22% ` Eric Wong
2021-07-21 14:07 47% ` [PATCH 3/2] lei: auto-refresh watches in config, cancel missing Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-19 8:59 UTC (permalink / raw)
To: meta
This allows lei to automatically note keyword (message flag)
changes made to a Maildir and propagate it into lei/store:
lei add-watch --state=tag-ro /path/to/Maildir
This doesn't persist across restarts, yet. In the future,
it will be applied automatically to "lei q" output Maildirs
by default (with an option to disable it).
State values of tag-rw, index-<ro|rw>, import-<ro|rw> will all
be supported for Maildir.
This represents a fairly major internal change that's fairly
intrusive, but the whole daemon-oriented design was to
facilitate being able to automatically monitor (and propagate)
Maildir/IMAP flag changes.
---
MANIFEST | 6 ++
lib/PublicInbox/DirIdle.pm | 17 +++++
lib/PublicInbox/LEI.pm | 116 ++++++++++++++++++++++++++---
lib/PublicInbox/LeiAddWatch.pm | 41 ++++++++++
lib/PublicInbox/LeiInput.pm | 1 +
lib/PublicInbox/LeiLsWatch.pm | 15 ++++
lib/PublicInbox/LeiNoteEvent.pm | 124 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSearch.pm | 2 +-
lib/PublicInbox/LeiSelfSocket.pm | 45 +++++++++++
lib/PublicInbox/LeiWatch.pm | 13 ++++
t/lei-watch.t | 49 ++++++++++++
11 files changed, 416 insertions(+), 13 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddWatch.pm
create mode 100644 lib/PublicInbox/LeiLsWatch.pm
create mode 100644 lib/PublicInbox/LeiNoteEvent.pm
create mode 100644 lib/PublicInbox/LeiSelfSocket.pm
create mode 100644 lib/PublicInbox/LeiWatch.pm
create mode 100644 t/lei-watch.t
diff --git a/MANIFEST b/MANIFEST
index 146a32ab..1d79b7c9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -198,6 +198,7 @@ lib/PublicInbox/Isearch.pm
lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
lib/PublicInbox/LeiALE.pm
+lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
lib/PublicInbox/LeiBlob.pm
lib/PublicInbox/LeiConvert.pm
@@ -220,8 +221,10 @@ lib/PublicInbox/LeiLsLabel.pm
lib/PublicInbox/LeiLsMailSource.pm
lib/PublicInbox/LeiLsMailSync.pm
lib/PublicInbox/LeiLsSearch.pm
+lib/PublicInbox/LeiLsWatch.pm
lib/PublicInbox/LeiMailSync.pm
lib/PublicInbox/LeiMirror.pm
+lib/PublicInbox/LeiNoteEvent.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiPmdir.pm
@@ -232,6 +235,7 @@ lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiSavedSearch.pm
lib/PublicInbox/LeiSearch.pm
+lib/PublicInbox/LeiSelfSocket.pm
lib/PublicInbox/LeiStore.pm
lib/PublicInbox/LeiStoreErr.pm
lib/PublicInbox/LeiSucks.pm
@@ -239,6 +243,7 @@ lib/PublicInbox/LeiTag.pm
lib/PublicInbox/LeiToMail.pm
lib/PublicInbox/LeiUp.pm
lib/PublicInbox/LeiViewText.pm
+lib/PublicInbox/LeiWatch.pm
lib/PublicInbox/LeiXSearch.pm
lib/PublicInbox/Linkify.pm
lib/PublicInbox/Listener.pm
@@ -434,6 +439,7 @@ t/lei-q-save.t
t/lei-q-thread.t
t/lei-sigpipe.t
t/lei-tag.t
+t/lei-watch.t
t/lei.t
t/lei_dedupe.t
t/lei_external.t
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 5142d005..7031e5fd 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -53,6 +53,23 @@ sub new {
$self;
}
+sub add_watches {
+ my ($self, $dirs, $gone) = @_;
+ my $fl = $MAIL_IN | ($gone ? $MAIL_GONE : 0);
+ for my $d (@$dirs) {
+ $self->{inot}->watch($d, $fl);
+ }
+ PublicInbox::FakeInotify::poll_once($self) if !$ino_cls;
+}
+
+sub rm_watches {
+ my ($self, $dir) = @_;
+ my $inot = $self->{inot};
+ if (my $cb = $inot->can('rm_watches')) { # TODO for fake watchers
+ $cb->($inot, $dir);
+ }
+}
+
sub event_step {
my ($self) = @_;
my $cb = $self->{cb};
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a9f5edae..b92d7512 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -28,14 +28,15 @@ use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
our $quit = \&CORE::exit;
-our ($current_lei, $errors_log, $listener, $oldset, $dir_idle);
-my ($recv_cmd, $send_cmd);
+our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
+ $recv_cmd, $send_cmd);
my $GLP = Getopt::Long::Parser->new;
$GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
my $GLP_PASS = Getopt::Long::Parser->new;
$GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
+our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => undef }
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -232,10 +233,9 @@ our %CMD = ( # sorted in order of importance/use:
qw(exact! all jobs:i indexed), @c_opt ],
'add-watch' => [ 'LOCATION', 'watch for new messages and flag changes',
- qw(import! kw! interval=s recursive|r
- exclude=s include=s), @c_opt ],
+ qw(poll-interval=s state=s recursive|r), @c_opt ],
'ls-watch' => [ '[FILTER...]', 'list active watches with numbers and status',
- qw(format|f=s z), @c_opt ],
+ qw(l z|0), @c_opt ],
'pause-watch' => [ '[WATCH_NUMBER_OR_FILTER]', qw(all local remote), @c_opt ],
'resume-watch' => [ '[WATCH_NUMBER_OR_FILTER]', qw(all local remote), @c_opt ],
'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
@@ -391,6 +391,7 @@ my %OPTDESC = (
'format|f=s ls-search' => ['OUT|json|jsonl|concatjson',
'listing output format' ],
'l ls-search' => 'long listing format',
+'l ls-watch' => 'long listing format',
'l ls-mail-source' => 'long listing format',
'url ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
'format|f=s ls-external' => $ls_format,
@@ -435,7 +436,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m ikw pmd wq1); # internal workers
+my @WQ_KEYS = qw(lxs l2m ikw pmd wq1 lne); # internal workers
sub _drop_wq {
my ($self) = @_;
@@ -538,7 +539,7 @@ sub _lei_atfork_child {
chdir '/' or die "chdir(/): $!";
close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
if (my $cfg = $self->{cfg}) {
- delete $cfg->{-lei_store};
+ delete @$cfg{qw(-lei_store -watches -lei_note_event)};
}
} else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
@@ -555,6 +556,8 @@ sub _lei_atfork_child {
undef $listener;
undef $dir_idle;
%PATH2CFG = ();
+ $MDIR2CFGPATH = {};
+ eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
undef $errors_log;
$quit = \&CORE::exit;
$self->{-eml_noisy} or # only "lei import" sets this atm
@@ -781,10 +784,12 @@ sub _lei_cfg ($;$) {
my $f = _config_path($self);
my @st = stat($f);
my $cur_st = @st ? pack('dd', $st[10], $st[7]) : ''; # 10:ctime, 7:size
- my ($sto, $sto_dir);
+ my ($sto, $sto_dir, $watches, $lne);
if (my $cfg = $PATH2CFG{$f}) { # reuse existing object in common case
return ($self->{cfg} = $cfg) if $cur_st eq $cfg->{-st};
- ($sto, $sto_dir) = @$cfg{qw(-lei_store leistore.dir)};
+ ($sto, $sto_dir, $watches, $lne) =
+ @$cfg{qw(-lei_store leistore.dir -watches
+ -lei_note_event)};
}
if (!@st) {
unless ($creat) {
@@ -805,6 +810,8 @@ sub _lei_cfg ($;$) {
eq File::Spec->canonpath($cfg->{'leistore.dir'} //
store_path($self))) {
$cfg->{-lei_store} = $sto;
+ $cfg->{-lei_note_event} = $lne;
+ $cfg->{-watches} = $watches if $watches;
}
if (scalar(keys %PATH2CFG) > 5) {
# FIXME: use inotify/EVFILT_VNODE to detect unlinked configs
@@ -817,7 +824,7 @@ sub _lei_cfg ($;$) {
sub _lei_store ($;$) {
my ($self, $creat) = @_;
- my $cfg = _lei_cfg($self, $creat);
+ my $cfg = _lei_cfg($self, $creat) // return;
$cfg->{-lei_store} //= do {
require PublicInbox::LeiStore;
my $dir = $cfg->{'leistore.dir'} // store_path($self);
@@ -1126,6 +1133,53 @@ sub dump_and_clear_log {
}
}
+sub cfg2lei ($) {
+ my ($cfg) = @_;
+ my $lei = bless { env => { %{$cfg->{-env}} } }, __PACKAGE__;
+ open($lei->{0}, '<&', \*STDIN) or die "dup 0: $!";
+ open($lei->{1}, '>>&', \*STDOUT) or die "dup 1: $!";
+ open($lei->{2}, '>>&', \*STDERR) or die "dup 2: $!";
+ open($lei->{3}, '/') or die "open /: $!";
+ chdir($lei->{3}) or die "chdir /': $!";
+ my ($x, $y);
+ socketpair($x, $y, AF_UNIX, SOCK_SEQPACKET, 0) or die "socketpair: $!";
+ $lei->{sock} = $x;
+ require PublicInbox::LeiSelfSocket;
+ PublicInbox::LeiSelfSocket->new($y); # adds to event loop
+ $lei;
+}
+
+sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
+ my ($ev) = @_; # Linux::Inotify2::Event or duck type
+ my $fn = $ev->fullname;
+ if ($fn =~ m!\A(.+)/(new|cur)/([^/]+)\z!) { # Maildir file
+ my ($mdir, $nc, $bn) = ($1, $2, $3);
+ $nc = '' if $ev->IN_DELETE;
+ for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
+ my $cfg = $PATH2CFG{$f} // next;
+ eval {
+ local %ENV = %{$cfg->{-env}};
+ my $lei = cfg2lei($cfg);
+ $lei->dispatch('note-event',
+ "maildir:$mdir", $nc, $bn, $fn);
+ };
+ warn "E note-event $f: $@\n" if $@;
+ }
+ }
+ if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
+ $ev->cancel;
+ }
+ if ($fn =~ m!\A(.+)/(?:new|cur)\z! && !-e $fn) {
+ delete $MDIR2CFGPATH->{$1};
+ }
+ if (!-e $fn) { # config file or Maildir gone
+ for my $cfgpaths (values %$MDIR2CFGPATH) {
+ delete $cfgpaths->{$fn};
+ }
+ delete $PATH2CFG{$fn};
+ }
+}
+
# lei(1) calls this when it can't connect
sub lazy_start {
my ($path, $errno, $narg) = @_;
@@ -1175,6 +1229,7 @@ sub lazy_start {
return if $pid;
$0 = "lei-daemon $path";
local %PATH2CFG;
+ local $MDIR2CFGPATH;
$listener->blocking(0);
my $exit_code;
my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
@@ -1204,8 +1259,8 @@ sub lazy_start {
local $SIG{PIPE} = 'IGNORE';
require PublicInbox::DirIdle;
local $dir_idle = PublicInbox::DirIdle->new([$sock_dir], sub {
- # just rely on wakeup ot hit PostLoopCallback set below
- _dir_idle_handler(@_) if $_[0]->fullname ne $path;
+ # just rely on wakeup to hit PostLoopCallback set below
+ dir_idle_handler($_[0]) if $_[0]->fullname ne $path;
}, 1);
if ($sigfd) {
undef $sigfd; # unref, already in DS::DescriptorMap
@@ -1293,4 +1348,41 @@ sub wq_eof { # EOF callback for main daemon
$wq1->wq_wait_old(\&wq_done_wait, $lei);
}
+sub watch_state_ok ($) {
+ my ($state) = $_[-1]; # $_[0] may be $self
+ $state =~ /\Apause|(?:import|index|tag)-(?:ro|rw)\z/;
+}
+
+sub refresh_watches {
+ my ($lei) = @_;
+ my $cfg = _lei_cfg($lei) or return;
+ $cfg->{-env} //= { %{$lei->{env}}, PWD => '/' }; # for cfg2lei
+ my $watches = $cfg->{-watches} //= {};
+ require PublicInbox::LeiWatch;
+ for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
+ my $url = substr($w, length('watch.'), -length('.state'));
+ my $lw = $watches->{$w} //= PublicInbox::LeiWatch->new($url);
+ my $state = $cfg->get_1("watch.$url", 'state');
+ if (!watch_state_ok($state)) {
+ $lei->err("watch.$url.state=$state not supported");
+ next;
+ }
+ my $f = $cfg->{'-f'};
+ if ($url =~ /\Amaildir:(.+)/i) {
+ my $d = File::Spec->canonpath($1);
+ if ($state eq 'pause') {
+ delete $MDIR2CFGPATH->{$d}->{$f};
+ scalar(keys %{$MDIR2CFGPATH->{$d}}) or
+ delete $MDIR2CFGPATH->{$d};
+ } elsif (!exists($MDIR2CFGPATH->{$d}->{$f})) {
+ $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
+ $MDIR2CFGPATH->{$d}->{$f} = undef;
+ }
+ } else { # TODO: imap/nntp/jmap
+ $lei->child_error(1,
+ "E: watch $url not supported, yet");
+ }
+ }
+}
+
1;
diff --git a/lib/PublicInbox/LeiAddWatch.pm b/lib/PublicInbox/LeiAddWatch.pm
new file mode 100644
index 00000000..671d54f9
--- /dev/null
+++ b/lib/PublicInbox/LeiAddWatch.pm
@@ -0,0 +1,41 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei add-watch" command
+package PublicInbox::LeiAddWatch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::LeiInput);
+
+sub lei_add_watch {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg(1);
+ my $self = bless {}, __PACKAGE__;
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ my $state = $lei->{opt}->{'state'} // 'import-rw';
+ $lei->watch_state_ok($state) or
+ return $lei->fail("invalid state: $state");
+ my $vmd_mod = $self->vmd_mod_extract(\@argv);
+ return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
+ $self->prepare_inputs($lei, \@argv) or return;
+ my @vmd;
+ while (my ($type, $vals) = each %$vmd_mod) {
+ push @vmd, "$type:$_" for @$vals;
+ }
+ my $vmd0 = shift @vmd;
+ for my $w (@{$self->{inputs}}) {
+ # clobber existing, allow multiple
+ if (defined($vmd0)) {
+ $lei->_config("watch.$w.vmd", '--replace-all', $vmd0);
+ for my $v (@vmd) {
+ $lei->_config("watch.$w.vmd", $v);
+ }
+ }
+ next if defined $cfg->{"watch.$w.state"};
+ $lei->_config("watch.$w.state", $state);
+ }
+ delete $lei->{cfg}; # force reload
+ $lei->refresh_watches;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index de2a8ff1..fa330df5 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -372,6 +372,7 @@ sub input_only_net_merge_all_done {
# like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare
# for update_xvmd -> update_vmd
+# returns something like { "+L" => [ @Labels ], ... }
sub vmd_mod_extract {
my $argv = $_[-1];
my $vmd_mod = {};
diff --git a/lib/PublicInbox/LeiLsWatch.pm b/lib/PublicInbox/LeiLsWatch.pm
new file mode 100644
index 00000000..f96dc4ec
--- /dev/null
+++ b/lib/PublicInbox/LeiLsWatch.pm
@@ -0,0 +1,15 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+package PublicInbox::LeiLsWatch;
+use strict;
+use v5.10.1;
+
+sub lei_ls_watch {
+ my ($lei) = @_;
+ my $cfg = $lei->_lei_cfg or return;
+ my @w = (join("\n", keys %$cfg) =~ m/^watch\.(.+?)\.state$/sgm);
+ $lei->puts(join("\n", @w)) if @w;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
new file mode 100644
index 00000000..bf15cd26
--- /dev/null
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -0,0 +1,124 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# internal command for dealing with inotify, kqueue vnodes, etc
+package PublicInbox::LeiNoteEvent;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+my $flush_timer;
+our $to_flush; # { cfgpath => $lei }
+
+sub flush_lei ($) {
+ my ($lei) = @_;
+ if (my $lne = delete $lei->{cfg}->{-lei_note_event}) {
+ $lne->wq_close(1, undef, $lei); # runs _lei_wq_eof;
+ } else { # lms_clear_src calls only:
+ my $wait = $lei->{sto}->ipc_do('done');
+ }
+}
+
+# we batch up writes and flush every 5s (matching Linux default
+# writeback behavior) since MUAs can trigger a storm of inotify events
+sub flush_task { # PublicInbox::DS timer callback
+ undef $flush_timer;
+ my $todo = $to_flush // return;
+ $to_flush = undef;
+ for my $lei (values %$todo) { flush_lei($lei) }
+}
+
+# sets a timer to flush
+sub note_event_arm_done ($) {
+ my ($lei) = @_;
+ $flush_timer //= PublicInbox::DS::add_timer(5, \&flush_task);
+ $to_flush->{$lei->{cfg}->{'-f'}} //= $lei;
+}
+
+sub eml_event ($$$$) {
+ my ($self, $eml, $kw, $state) = @_;
+ my $sto = $self->{lei}->{sto};
+ my $lse = $self->{lse} //= $sto->search;
+ my $vmd = { kw => $kw };
+ if ($state =~ /\Aimport-(?:rw|ro)\z/) {
+ $sto->ipc_do('set_eml', $eml, $vmd);
+ } elsif ($state =~ /\Aindex-(?:rw|ro)\z/) {
+ my $xoids = $self->{lei}->ale->xoids_for($eml);
+ $sto->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+ } elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
+ my $c = $lse->kw_changed($eml, $kw, my $docids = []);
+ if (scalar @$docids) { # already in lei/store
+ $sto->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+ } elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
+ # it's in an external, only set kw, here
+ $sto->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+ } # else { totally unknown
+ } else {
+ warn "unknown state: $state (in $self->{lei}->{cfg}->{'-f'})\n";
+ }
+}
+
+sub maildir_event { # via wq_io_do
+ my ($self, $fn, $kw, $state) = @_;
+ my $eml = PublicInbox::InboxWritable::eml_from_path($fn) // return;
+ eml_event($self, $eml, $kw, $state);
+}
+
+sub lei_note_event {
+ my ($lei, $folder, $new_cur, $bn, $fn, @rest) = @_;
+ die "BUG: unexpected: @rest" if @rest;
+ my $cfg = $lei->_lei_cfg or return; # gone (race)
+ my $sto = $lei->_lei_store or return; # gone
+ return flush_lei($lei) if $folder eq 'done'; # special case
+ my $lms = $sto->search->lms or return;
+ my $err = $lms->arg2folder($lei, [ $folder ]);
+ return if $err->{fail};
+ undef $lms;
+ my $state = $cfg->get_1("watch.$folder", 'state') // 'pause';
+ return if $state eq 'pause';
+ $lei->ale; # prepare
+ $sto->write_prepare($lei);
+ if ($new_cur eq '') {
+ $sto->ipc_do('lms_clear_src', $folder, \$bn);
+ return note_event_arm_done($lei);
+ }
+ require PublicInbox::MdirReader;
+ my $self = $cfg->{-lei_note_event} //= do {
+ my $wq = bless {}, __PACKAGE__;
+ # MUAs such as mutt can trigger massive rename() storms so
+ # use all CPU power available:
+ my $jobs = $wq->detect_nproc // 1;
+ my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
+ $lei->wait_wq_events($op_c, $ops);
+ note_event_arm_done($lei);
+ $lei->{lne} = $wq;
+ };
+ if ($folder =~ /\Amaildir:/i) {
+ my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn)
+ // return;
+ return if index($fl, 'T') >= 0;
+ my $kw = PublicInbox::MdirReader::flags2kw($fl);
+ $self->wq_io_do('maildir_event', [], $fn, $kw, $state);
+ } # else: TODO: imap
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ $self->{lei}->_lei_atfork_child(1); # persistent, for a while
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub lne_done_wait {
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
+ $lei->can('wq_done_wait')->($arg, $pid);
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $lne = delete $lei->{lne} or return $lei->fail;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lne->wq_wait_old(\&lne_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 06ea6299..37bfc65e 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -5,7 +5,7 @@
package PublicInbox::LeiSearch;
use strict;
use v5.10.1;
-use parent qw(PublicInbox::ExtSearch);
+use parent qw(PublicInbox::ExtSearch); # PublicInbox::Search->reopen
use PublicInbox::Search qw(xap_terms);
use PublicInbox::ContentHash qw(content_digest content_hash);
use PublicInbox::MID qw(mids mids_for_index);
diff --git a/lib/PublicInbox/LeiSelfSocket.pm b/lib/PublicInbox/LeiSelfSocket.pm
new file mode 100644
index 00000000..3d847649
--- /dev/null
+++ b/lib/PublicInbox/LeiSelfSocket.pm
@@ -0,0 +1,45 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# dummy placeholder socket for internal lei commands.
+# This receives what script/lei receives, but isn't connected
+# to an interactive terminal so I'm not sure what to do with it...
+package PublicInbox::LeiSelfSocket;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::DS);
+use Data::Dumper;
+$Data::Dumper::Useqq = 1; # should've been the Perl default :P
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use PublicInbox::Spawn;
+my $recv_cmd;
+
+sub new {
+ my ($cls, $r) = @_;
+ my $self = bless { sock => $r }, $cls;
+ $r->blocking(0);
+ no warnings 'once';
+ $recv_cmd = $PublicInbox::LEI::recv_cmd;
+ $self->SUPER::new($r, EPOLLIN|EPOLLET);
+}
+
+sub event_step {
+ my ($self) = @_;
+ while (1) {
+ my (@fds) = $recv_cmd->($self->{sock}, my $buf, 4096 * 33);
+ if (scalar(@fds) == 1 && !defined($fds[0])) {
+ return if $!{EAGAIN};
+ next if $!{EINTR};
+ die "recvmsg: $!";
+ }
+ # open so perl can auto-close them:
+ for my $fd (@fds) {
+ open(my $newfh, '+<&=', $fd) or die "open +<&=$fd: $!";
+ }
+ return $self->close if $buf eq '';
+ warn Dumper({ 'unexpected self msg' => $buf, fds => \@fds });
+ # TODO: figure out what to do with these messages...
+ }
+}
+
+1;
diff --git a/lib/PublicInbox/LeiWatch.pm b/lib/PublicInbox/LeiWatch.pm
new file mode 100644
index 00000000..35267b58
--- /dev/null
+++ b/lib/PublicInbox/LeiWatch.pm
@@ -0,0 +1,13 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# represents a Maildir or IMAP "watch" item
+package PublicInbox::LeiWatch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+# "url" may be something like "maildir:/path/to/dir"
+sub new { bless { url => $_[1] }, $_[0] }
+
+1;
diff --git a/t/lei-watch.t b/t/lei-watch.t
new file mode 100644
index 00000000..3a2f9e64
--- /dev/null
+++ b/t/lei-watch.t
@@ -0,0 +1,49 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Path qw(make_path);
+require_mods('lei');
+my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+ eval { require IO::KQueue };
+
+$have_fast_inotify or
+ diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
+
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+test_lei(sub {
+ my $md = "$ENV{HOME}/md";
+ my $md2 = $md.'2';
+ lei_ok 'ls-watch';
+ is($lei_out, '', 'nothing in ls-watch, yet');
+ if (0) { # TODO
+ my $url = 'imaps://example.com/foo.bar.0';
+ lei_ok([qw(add-watch --state=pause), $url], undef, {});
+ lei_ok 'ls-watch';
+ is($lei_out, "$url\n", 'ls-watch shows added watch');
+ ok(!lei(qw(add-watch --state=pause), 'bogus'.$url),
+ 'bogus URL rejected');
+ }
+
+ # first, make sure tag-ro works
+ make_path("$md/new", "$md/cur", "$md/tmp");
+ lei_ok qw(add-watch --state=tag-ro), $md;
+ lei_ok 'ls-watch';
+ like($lei_out, qr/^\Qmaildir:$md\E$/sm, 'maildir shown');
+ lei_ok qw(q mid:testmessage@example.com -o), $md, '-I', "$ro_home/t1";
+ my @f = glob("$md/cur/*:2,");
+ is(scalar(@f), 1, 'got populated maildir with one result');
+ rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een
+ $have_fast_inotify or tick(2);
+ lei_ok qw(note-event done); # flushes immediately (instead of 5s)
+
+ lei_ok qw(q mid:testmessage@example.com -o), $md2, '-I', "$ro_home/t1";
+ my @f2 = glob("$md2/*/*");
+ is(scalar(@f2), 1, 'got one result');
+ like($f2[0], qr/S\z/, 'seen set from rename');
+ my $e2 = eml_load($f2[0]);
+ my $e1 = eml_load("$f[0]S");
+ is_deeply($e2, $e1, 'results match');
+});
+
+done_testing;
^ permalink raw reply related [relevance 22%]
* [PATCH 3/2] lei: auto-refresh watches in config, cancel missing
2021-07-19 8:59 63% [PATCH 0/2] lei Maildir inotify/kevent support Eric Wong
2021-07-19 8:59 22% ` [PATCH 2/2] lei: start implementing inotify Maildir support Eric Wong
@ 2021-07-21 14:07 47% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-21 14:07 UTC (permalink / raw)
To: meta
This makes behavior less surprising on restarts as we no longer
lose state on restarts, so there's no need to manually run "lei
add-watch" to re-enable watches. This also allows us to
transparently handle changes if somebody edits the lei config
file directly or via git-config(1).
---
lib/PublicInbox/DirIdle.pm | 5 +++-
lib/PublicInbox/LEI.pm | 50 +++++++++++++++++++++++++--------
lib/PublicInbox/LeiNoteEvent.pm | 2 +-
t/lei-watch.t | 39 +++++++++++++++++++++++++
4 files changed, 83 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 7031e5fd..65896f95 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -56,10 +56,13 @@ sub new {
sub add_watches {
my ($self, $dirs, $gone) = @_;
my $fl = $MAIL_IN | ($gone ? $MAIL_GONE : 0);
+ my @ret;
for my $d (@$dirs) {
- $self->{inot}->watch($d, $fl);
+ my $w = $self->{inot}->watch($d, $fl) or next;
+ push @ret, $w;
}
PublicInbox::FakeInotify::poll_once($self) if !$ino_cls;
+ @ret
}
sub rm_watches {
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b92d7512..52c551cf 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -36,7 +36,7 @@ my $GLP_PASS = Getopt::Long::Parser->new;
$GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
-our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => undef }
+our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => [ ino watches ] }
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -820,6 +820,8 @@ sub _lei_cfg ($;$) {
}
}
$self->{cfg} = $PATH2CFG{$f} = $cfg;
+ refresh_watches($self);
+ $cfg;
}
sub _lei_store ($;$) {
@@ -1353,36 +1355,62 @@ sub watch_state_ok ($) {
$state =~ /\Apause|(?:import|index|tag)-(?:ro|rw)\z/;
}
+sub cancel_maildir_watch ($$) {
+ my ($d, $cfg_f) = @_;
+ my $w = delete $MDIR2CFGPATH->{$d}->{$cfg_f};
+ scalar(keys %{$MDIR2CFGPATH->{$d}}) or
+ delete $MDIR2CFGPATH->{$d};
+ for my $x (@{$w // []}) { $x->cancel }
+}
+
sub refresh_watches {
my ($lei) = @_;
my $cfg = _lei_cfg($lei) or return;
- $cfg->{-env} //= { %{$lei->{env}}, PWD => '/' }; # for cfg2lei
+ my $old = $cfg->{-watches};
my $watches = $cfg->{-watches} //= {};
- require PublicInbox::LeiWatch;
+ my %seen;
+ my $cfg_f = $cfg->{'-f'};
for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
my $url = substr($w, length('watch.'), -length('.state'));
- my $lw = $watches->{$w} //= PublicInbox::LeiWatch->new($url);
+ require PublicInbox::LeiWatch;
+ my $lw = $watches->{$url} //= PublicInbox::LeiWatch->new($url);
+ $seen{$url} = undef;
my $state = $cfg->get_1("watch.$url", 'state');
if (!watch_state_ok($state)) {
$lei->err("watch.$url.state=$state not supported");
next;
}
- my $f = $cfg->{'-f'};
if ($url =~ /\Amaildir:(.+)/i) {
my $d = File::Spec->canonpath($1);
if ($state eq 'pause') {
- delete $MDIR2CFGPATH->{$d}->{$f};
- scalar(keys %{$MDIR2CFGPATH->{$d}}) or
- delete $MDIR2CFGPATH->{$d};
- } elsif (!exists($MDIR2CFGPATH->{$d}->{$f})) {
- $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
- $MDIR2CFGPATH->{$d}->{$f} = undef;
+ cancel_maildir_watch($d, $cfg_f);
+ } elsif (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
+ my @w = $dir_idle->add_watches(
+ ["$d/cur", "$d/new"], 1);
+ push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
}
} else { # TODO: imap/nntp/jmap
$lei->child_error(1,
"E: watch $url not supported, yet");
}
}
+ if ($old) { # cull old non-existent entries
+ for my $url (keys %$old) {
+ next if exists $seen{$url};
+ delete $old->{$url};
+ if ($url =~ /\Amaildir:(.+)/i) {
+ my $d = File::Spec->canonpath($1);
+ cancel_maildir_watch($d, $cfg_f);
+ } else { # TODO: imap/nntp/jmap
+ $lei->child_error(1, "E: watch $url TODO");
+ }
+ }
+ }
+ if (scalar keys %$watches) {
+ $cfg->{-env} //= { %{$lei->{env}}, PWD => '/' }; # for cfg2lei
+ } else {
+ delete $cfg->{-watches};
+ }
}
1;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index bf15cd26..d6511cf6 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -14,7 +14,7 @@ sub flush_lei ($) {
my ($lei) = @_;
if (my $lne = delete $lei->{cfg}->{-lei_note_event}) {
$lne->wq_close(1, undef, $lei); # runs _lei_wq_eof;
- } else { # lms_clear_src calls only:
+ } elsif ($lei->{sto}) { # lms_clear_src calls only:
my $wait = $lei->{sto}->ipc_do('done');
}
}
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 3a2f9e64..492f6c1d 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -13,9 +13,27 @@ $have_fast_inotify or
my ($ro_home, $cfg_path) = setup_public_inboxes;
test_lei(sub {
my $md = "$ENV{HOME}/md";
+ my $cfg_f = "$ENV{HOME}/.config/lei/config";
my $md2 = $md.'2';
lei_ok 'ls-watch';
is($lei_out, '', 'nothing in ls-watch, yet');
+
+ my ($ino_fdinfo, $ino_contents);
+ SKIP: {
+ $have_fast_inotify && $^O eq 'linux' or
+ skip 'Linux/inotify-only internals check', 1;
+ lei_ok 'daemon-pid'; chomp(my $pid = $lei_out);
+ skip 'missing /proc/$PID/fd', 1 if !-d "/proc/$pid/fd";
+ my @ino = grep {
+ readlink($_) =~ /\binotify\b/
+ } glob("/proc/$pid/fd/*");
+ is(scalar(@ino), 1, 'only one inotify FD');
+ my $ino_fd = (split('/', $ino[0]))[-1];
+ $ino_fdinfo = "/proc/$pid/fdinfo/$ino_fd";
+ open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
+ $ino_contents = [ <$fh> ];
+ }
+
if (0) { # TODO
my $url = 'imaps://example.com/foo.bar.0';
lei_ok([qw(add-watch --state=pause), $url], undef, {});
@@ -44,6 +62,27 @@ test_lei(sub {
my $e2 = eml_load($f2[0]);
my $e1 = eml_load("$f[0]S");
is_deeply($e2, $e1, 'results match');
+
+ SKIP: {
+ $ino_fdinfo or skip 'Linux/inotify-only watch check', 1;
+ open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
+ my $cmp = [ <$fh> ];
+ ok(scalar(@$cmp) > scalar(@$ino_contents),
+ 'inotify has Maildir watches');
+ }
+
+ is(xsys(qw(git config -f), $cfg_f,
+ '--remove-section', "watch.maildir:$md"),
+ 0, 'unset config state');
+ lei_ok 'ls-watch', \'refresh watches';
+ is($lei_out, '', 'no watches left');
+
+ SKIP: {
+ $ino_fdinfo or skip 'Linux/inotify-only removal removal', 1;
+ open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
+ my $cmp = [ <$fh> ];
+ is_deeply($cmp, $ino_contents, 'inotify Maildir watches gone');
+ };
});
done_testing;
^ permalink raw reply related [relevance 47%]
* [PATCH 0/3] lei rm-watch, some error handling stuff
@ 2021-07-23 10:56 71% Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
I'm considering making "tag-ro" the default watch state
and introducing the idea of implicit watches for all
"lei q" output directories.
Eric Wong (3):
t/lei*: check error messages on failures
lei: avoid SQLite COUNT() for dedupe
lei rm-watch: new command to support removing watches
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 4 +++-
lib/PublicInbox/LeiDedupe.pm | 5 ++---
lib/PublicInbox/LeiInput.pm | 8 ++++++++
lib/PublicInbox/LeiRmWatch.pm | 31 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSavedSearch.pm | 8 ++++----
lib/PublicInbox/LeiToMail.pm | 4 ++--
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/SharedKV.pm | 7 +++++++
t/lei-externals.t | 4 ++++
t/lei-import-http.t | 2 ++
t/lei-import-imap.t | 1 +
t/lei-mirror.t | 7 ++++++-
t/lei-p2q.t | 1 +
t/lei-q-kw.t | 1 +
t/lei-q-remote-import.t | 1 +
t/lei-q-save.t | 4 ++++
t/lei-tag.t | 8 +++++++-
t/lei-watch.t | 18 ++++++++++++++----
t/solver_git.t | 7 +++++++
20 files changed, 107 insertions(+), 17 deletions(-)
create mode 100644 lib/PublicInbox/LeiRmWatch.pm
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
@ 2021-07-23 10:56 58% ` Eric Wong
2021-07-23 10:56 53% ` [PATCH 3/3] lei rm-watch: new command to support removing watches Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
SQLite COUNT() is a slow operation that does a full table scan
with no conditions. There's no need for it, since lei dedupe
only needs to know if it's empty or not to decide between
new/ and cur/ for Maildir outputs.
---
lib/PublicInbox/LeiDedupe.pm | 5 ++---
lib/PublicInbox/LeiSavedSearch.pm | 8 ++++----
lib/PublicInbox/LeiToMail.pm | 4 ++--
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/SharedKV.pm | 7 +++++++
5 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index ed52e417..32f99cd0 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -127,10 +127,9 @@ sub pause_dedupe {
delete($skv->{dbh}) if $skv;
}
-sub dedupe_nr {
+sub has_entries {
my $skv = $_[0]->[0] or return undef;
- my @n = $skv->count;
- $n[0];
+ $skv->has_entries;
}
1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 929380ed..cfbf68c3 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -315,11 +315,11 @@ E: rename($dir_old, $dir_new) error: $!
EOM
}
-# cf. LeiDedupe->dedupe_nr
-sub dedupe_nr {
+# cf. LeiDedupe->has_entries
+sub has_entries {
my $oidx = $_[0]->{oidx} // die 'BUG: no {oidx}';
- my @n = $oidx->{dbh}->selectrow_array('SELECT COUNT(*) FROM over');
- $n[0];
+ my @n = $oidx->{dbh}->selectrow_array('SELECT num FROM over LIMIT 1');
+ scalar(@n) ? 1 : undef;
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9405c0c..d782d4c7 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -198,7 +198,7 @@ sub _mbox_write_cb ($$) {
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe;
my $lse = $lei->{lse}; # may be undef
- my $set_recent = $dedupe->dedupe_nr;
+ my $set_recent = $dedupe->has_entries;
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($buf);
@@ -293,7 +293,7 @@ sub _maildir_write_cb ($$) {
# Favor cur/ and only write to new/ when augmenting. This
# saves MUAs from having to do a mass rename when the initial
# search result set is huge.
- my $dir = $dedupe && $dedupe->dedupe_nr ? 'new/' : 'cur/';
+ my $dir = $dedupe && $dedupe->has_entries ? 'new/' : 'cur/';
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index cac7fb7d..3414e87d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -504,7 +504,7 @@ sub do_query {
my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
if ($l2m->{-wq_nr_workers} > 1 &&
$l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
- # setup two barriers to coordinate dedupe_nr
+ # setup two barriers to coordinate ->has_entries
# between l2m workers
pipe(my ($a_r, $a_w)) or die "pipe: $!";
fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index 8347b195..3487e820 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -154,6 +154,13 @@ SELECT COUNT(k) FROM kv
$sth->fetchrow_array;
}
+# faster than ->count due to how SQLite works
+sub has_entries {
+ my ($self) = @_;
+ my @n = $self->{dbh}->selectrow_array('SELECT k FROM kv LIMIT 1');
+ scalar(@n) ? 1 : undef;
+}
+
sub dbh_release {
my ($self, $lock) = @_;
my $dbh = delete $self->{dbh} or return;
^ permalink raw reply related [relevance 58%]
* [PATCH 3/3] lei rm-watch: new command to support removing watches
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
2021-07-23 10:56 58% ` [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe Eric Wong
@ 2021-07-23 10:56 53% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
Pretty trivial since it just invokes "git-config". It's mainly
intended to make shell completion easier.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 4 +++-
lib/PublicInbox/LeiInput.pm | 8 ++++++++
lib/PublicInbox/LeiRmWatch.pm | 31 +++++++++++++++++++++++++++++++
t/lei-watch.t | 18 ++++++++++++++----
5 files changed, 57 insertions(+), 5 deletions(-)
create mode 100644 lib/PublicInbox/LeiRmWatch.pm
diff --git a/MANIFEST b/MANIFEST
index 1d79b7c9..a3913501 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -233,6 +233,7 @@ lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
+lib/PublicInbox/LeiRmWatch.pm
lib/PublicInbox/LeiSavedSearch.pm
lib/PublicInbox/LeiSearch.pm
lib/PublicInbox/LeiSelfSocket.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 52c551cf..191a0790 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -232,8 +232,10 @@ our %CMD = ( # sorted in order of importance/use:
'remove imported messages from IMAP, Maildirs, and MH',
qw(exact! all jobs:i indexed), @c_opt ],
-'add-watch' => [ 'LOCATION', 'watch for new messages and flag changes',
+'add-watch' => [ 'LOCATION...', 'watch for new messages and flag changes',
qw(poll-interval=s state=s recursive|r), @c_opt ],
+'rm-watch' => [ 'LOCATION...', 'remove specified watch(es)',
+ qw(recursive|r), @c_opt ],
'ls-watch' => [ '[FILTER...]', 'list active watches with numbers and status',
qw(l z|0), @c_opt ],
'pause-watch' => [ '[WATCH_NUMBER_OR_FILTER]', qw(all local remote), @c_opt ],
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index fa330df5..88889f45 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -270,6 +270,10 @@ sub prepare_inputs { # returns undef on error
$sync and $input = 'maildir:'.
$lei->abs_path($input_path);
push @md, $input;
+ } elsif ($self->{missing_ok} && !-e _) {
+ # for "lei rm-watch" on missing Maildir
+ $sync and $input = 'maildir:'.
+ $lei->abs_path($input_path);
} else {
return $lei->fail("Unable to handle $input");
}
@@ -305,6 +309,10 @@ $input is `eml', not --in-format=$in_fmt
push @{$sync->{ok}}, $input;
}
push @md, $input;
+ } elsif ($self->{missing_ok} && !-e $input) {
+ # for lei rm-watch
+ $sync and $input = 'maildir:'.
+ $lei->abs_path($input);
} else {
return $lei->fail("Unable to handle $input")
}
diff --git a/lib/PublicInbox/LeiRmWatch.pm b/lib/PublicInbox/LeiRmWatch.pm
new file mode 100644
index 00000000..c0f336f0
--- /dev/null
+++ b/lib/PublicInbox/LeiRmWatch.pm
@@ -0,0 +1,31 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei rm-watch" command
+package PublicInbox::LeiRmWatch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::LeiInput);
+
+sub lei_rm_watch {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg(1);
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ my $self = bless { missing_ok => 1 }, __PACKAGE__;
+ $self->prepare_inputs($lei, \@argv) or return;
+ for my $w (@{$self->{inputs}}) {
+ $lei->_config('--remove-section', "watch.$w");
+ }
+ delete $lei->{cfg}; # force reload
+ $lei->refresh_watches;
+}
+
+sub _complete_rm_watch {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ my @w = (join("\n", keys %$cfg) =~ m/^watch\.(.+?)\.state$/sgm);
+ map { $match_cb->($_) } @w;
+}
+
+1;
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 492f6c1d..9a3bfd80 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -2,7 +2,7 @@
# Copyright all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
-use File::Path qw(make_path);
+use File::Path qw(make_path remove_tree);
require_mods('lei');
my $have_fast_inotify = eval { require Linux::Inotify2 } ||
eval { require IO::KQueue };
@@ -71,9 +71,19 @@ test_lei(sub {
'inotify has Maildir watches');
}
- is(xsys(qw(git config -f), $cfg_f,
- '--remove-section', "watch.maildir:$md"),
- 0, 'unset config state');
+ lei_ok 'rm-watch', $md;
+ lei_ok 'ls-watch', \'refresh watches';
+ is($lei_out, '', 'no watches left');
+
+ lei_ok 'add-watch', $md2;
+ remove_tree($md2);
+ lei_ok 'rm-watch', "maildir:".$md2, \'with maildir: prefix';
+ lei_ok 'ls-watch', \'refresh watches';
+ is($lei_out, '', 'no watches left');
+
+ lei_ok 'add-watch', $md;
+ remove_tree($md);
+ lei_ok 'rm-watch', $md, \'absolute path w/ missing dir';
lei_ok 'ls-watch', \'refresh watches';
is($lei_out, '', 'no watches left');
^ permalink raw reply related [relevance 53%]
* [PATCH 1/3] t/lei*: check error messages on failures
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
@ 2021-07-23 10:56 36% ` Eric Wong
2021-07-23 10:56 58% ` [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe Eric Wong
2021-07-23 10:56 53% ` [PATCH 3/3] lei rm-watch: new command to support removing watches Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
I just hit an unreproducible failure in t/lei-p2q.t and
lacked $lei_err information to diagnose it. Hopefully
this helps track down odd failures in the future.
---
t/lei-externals.t | 4 ++++
t/lei-import-http.t | 2 ++
t/lei-import-imap.t | 1 +
t/lei-mirror.t | 7 ++++++-
t/lei-p2q.t | 1 +
t/lei-q-kw.t | 1 +
t/lei-q-remote-import.t | 1 +
t/lei-q-save.t | 4 ++++
t/lei-tag.t | 8 +++++++-
t/solver_git.t | 7 +++++++
10 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/t/lei-externals.t b/t/lei-externals.t
index f148fa3c..5e3c67bc 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -211,6 +211,8 @@ test_lei(sub {
like($lei_out, qr/use boolean prefix/, '--stdin on pipe');
}
ok(!lei(qw(q -q --stdin s:use)), "--stdin and argv don't mix");
+ like($lei_err, qr/no query allowed.*--stdin/,
+ '--stdin conflict error message');
for my $fmt (qw(ldjson ndjson jsonl)) {
lei_ok('q', '-f', $fmt, 's:use boolean prefix');
@@ -250,6 +252,8 @@ test_lei(sub {
}
ok(!lei('q', '-o', "$home/mbox", 's:nope'),
'fails if mbox format unspecified');
+ like($lei_err, qr/unable to determine mbox/, 'mbox-related message');
+
ok(!lei(qw(q --no-local s:see)), '--no-local');
is($? >> 8, 1, 'proper exit code');
like($lei_err, qr/no local or remote.+? to search/, 'no inbox');
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
index 2104c778..e9eec1f7 100644
--- a/t/lei-import-http.t
+++ b/t/lei-import-http.t
@@ -17,6 +17,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $url = "http://$host_port/t2";
for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) {
ok(!lei('import', "$url/$p"), "/$p fails properly");
+ like($lei_err, qr/curl.*404/, 'got curl 404');
}
for my $p (qw(/ /T/ /t/ /t.atom)) {
ok(!lei('import', "$url/m\@example$p"), "/$p fails");
@@ -42,5 +43,6 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(!lei(qw(import --mail-sync), "$url/x\@example.com/raw"),
'--mail-sync fails on HTTP');
+ like($lei_err, qr/--mail-sync/, 'error message notes --mail-sync');
});
done_testing;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 12f6fad0..315567b3 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -43,6 +43,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
$url = $u;
$u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s;
ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected');
+ like($lei_err, qr/UIDVALIDITY mismatch/, 'mismatch noted');
lei_ok('inspect', $url);
my $inspect = json_utf8->decode($lei_out);
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index dfd35e1a..80bc6ed5 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -27,6 +27,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(!lei('add-external', $t2, '--mirror', "$http/t2/"),
'--mirror fails if reused') or diag "$lei_err.$lei_out = $?";
+ like($lei_err, qr/\Q$t2\E' already exists/, 'destination in error');
ok(!lei('add-external', "$home/t2\nnewline", '--mirror', "$http/t2/"),
'--mirror fails on newline');
@@ -37,13 +38,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
unlike($lei_out, qr!\Qnewline\E!, 'newline entry not added');
ok(!lei('add-external', "$t2-fail", '-Lmedium'), '--mirror v2');
+ like($lei_err, qr/not a directory/, 'non-directory noted');
ok(!-d "$t2-fail", 'destination not created on failure');
lei_ok('ls-external');
unlike($lei_out, qr!\Q$t2-fail\E!, 'not added to ls-external');
my %phail = (
HTTPS => 'https://public-inbox.org/' . 'phail',
- ONION => 'http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/' . 'phail,'
+ ONION =>
+'http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/' .
+'phail,'
);
for my $t (qw(HTTPS ONION)) {
SKIP: {
@@ -56,6 +60,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
is($? >> 8, 22, 'curl 404');
ok(!-d $dir, 'directory not created');
unlike($lei_err, qr/# mirrored/, 'no success message');
+ like($lei_err, qr/curl.*404/, "curl 404 shown for $k");
} # SKIP
} # for
});
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index 58506f94..495d81de 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -8,6 +8,7 @@ require_mods(qw(json DBD::SQLite Search::Xapian));
test_lei(sub {
ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
'p2q fails on bogus arg');
+ like($lei_err, qr/format-patch.*failed/, 'notes format-patch failure');
lei_ok(qw(p2q -w dfpost t/data/0001.patch));
is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname') or diag $lei_err;
open my $fh, '+<', 't/data/0001.patch' or xbail "open: $!";
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 528751b4..2e6be1f0 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -48,6 +48,7 @@ SKIP: {
my $cat = popen_rd(['cat', $o]);
ok(!lei(qw(q --import-before bogus -o), "mboxrd:$o"),
'--import-before fails on non-seekable output');
+ like($lei_err, qr/not seekable/, 'unseekable noted in error');
is(do { local $/; <$cat> }, '', 'no output on FIFO');
close $cat;
$cat = popen_rd(['cat', $o]);
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 7db684d9..aaf56e27 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -54,6 +54,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(-f $o && -s _, '--lock=none respected') or diag $lei_err;
unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
ok(!lei(@cmd, '--lock=dotlock,timeout=0.000001'), 'dotlock fails');
+ like($lei_err, qr/dotlock timeout/, 'timeout noted');
ok(-f $o && !-s _, 'nothing output on lock failure');
unlink "$o.lock" or BAIL_OUT $!;
lei_ok(@cmd, '--lock=dotlock,timeout=0.000001',
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 6c592088..b1ca4e92 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -69,6 +69,7 @@ test_lei(sub {
ok(-s "$home/mbcl2" > $size, 'size increased after up');
ok(!lei(qw(up -q), $home), 'up fails w/o --save');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
lei_ok qw(ls-search -z); my @z = split(/\0/, $lei_out);
@@ -115,6 +116,8 @@ test_lei(sub {
lei_ok(qw(up --all=local));
ok(!lei(qw(forget-search), "$home/bogus"), 'bogus forget');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
+
lei_ok qw(_complete lei forget-search);
like($lei_out, qr/mbrd-aug/, 'forget-search completion');
lei_ok(qw(forget-search -v), "$home/mbrd-aug");
@@ -124,6 +127,7 @@ test_lei(sub {
unlike($lei_out, qr/mbrd-aug/,
'forget-search completion cleared after forget');
ok(!lei('up', "$home/mbrd-aug"), 'lei up fails after forget');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
# dedupe=mid
my $o = "$home/dd-mid";
diff --git a/t/lei-tag.t b/t/lei-tag.t
index 5cb6d9ce..44e4659f 100644
--- a/t/lei-tag.t
+++ b/t/lei-tag.t
@@ -32,8 +32,14 @@ test_lei(sub {
lei_ok(qw(ls-label)); is($lei_out, "urgent\n", 'label found');
ok(!lei(qw(tag -F eml t/utf8.eml +kw:seeen)), 'bad kw rejected');
like($lei_err, qr/`seeen' is not one of/, 'got helpful error');
+
ok(!lei(qw(tag -F eml t/utf8.eml +k:seen)), 'bad prefix rejected');
+ like($lei_err, qr/Unable to handle.*\Q+k:seen\E/, 'bad prefix noted');
+
ok(!lei(qw(tag -F eml t/utf8.eml)), 'no keywords');
+ like($lei_err, qr/no keywords or labels specified/,
+ 'lack of kw/L noted');
+
my $mb = "$ENV{HOME}/mb";
my $md = "$ENV{HOME}/md";
lei_ok(qw(q m:testmessage@example.com -o), "mboxrd:$mb");
@@ -78,7 +84,7 @@ test_lei(sub {
lei_ok(qw(ls-label));
is($lei_out, "nope\nqp\nurgent\n", 'ls-label shows qp');
- lei_ok qw(tag -F eml t/utf8.eml +L:INBOX +L:x); diag $lei_err;
+ lei_ok qw(tag -F eml t/utf8.eml +L:INBOX +L:x);
lei_ok qw(q m:testmessage@example.com);
$check_kw->([qw(answered seen)], L => [qw(INBOX nope urgent x)]);
lei_ok(qw(ls-label));
diff --git a/t/solver_git.t b/t/solver_git.t
index fe1aff0b..f5cc592c 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -43,6 +43,7 @@ test_lei({tmpdir => "$tmpdir/blob"}, sub {
is($lei_out, $patch2->as_string, 'blob matches');
ok(!lei('blob', '--mail', '69df7d5', '-I', $ibx->{inboxdir}),
"--mail won't run solver");
+ like($lei_err, qr/\b69df7d5\b/, 'OID in error by git(1)');
lei_ok('blob', '69df7d5', '-I', $ibx->{inboxdir});
is(git_sha(1, \$lei_out)->hexdigest, $expect, 'blob contents output');
@@ -51,6 +52,8 @@ test_lei({tmpdir => "$tmpdir/blob"}, sub {
is($lei_out, $prev, '--no-mail works');
ok(!lei(qw(blob -I), $ibx->{inboxdir}, $non_existent),
'non-existent blob fails');
+ my $abbrev = substr($non_existent, 0, 7);
+ like($lei_err, qr/could not find $abbrev/, 'failed abbreviation noted');
SKIP: {
skip '/.git exists', 1 if -e '/.git';
lei_ok(qw(-C / blob 69df7d5 -I), $ibx->{inboxdir},
@@ -59,9 +62,13 @@ test_lei({tmpdir => "$tmpdir/blob"}, sub {
ok(!lei(qw(-C / blob --no-cwd 69df7d5 -I), $ibx->{inboxdir}),
'--no-cwd works');
+ like($lei_err, qr/no --git-dir to try/,
+ 'lack of --git-dir noted');
ok(!lei(qw(-C / blob -I), $ibx->{inboxdir}, $non_existent),
'non-existent blob fails');
+ like($lei_err, qr/no --git-dir to try/,
+ 'lack of --git-dir noted');
}
# fallbacks
^ permalink raw reply related [relevance 36%]
* [PATCH] t/lei-watch.t: improve test reliability
@ 2021-07-25 11:15 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 11:15 UTC (permalink / raw)
To: meta
On single CPU (and overloaded SMP) systems, we can't rely on
inotify in lei-daemon firing before a "lei note-event done"
client hits it. So force in a single tick() to ensure the
scheduler can yield to lei-daemon and see the inotify wakeup
before "lei note-event done" to commit the write.
---
t/lei-watch.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 9a3bfd80..86fa6649 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -52,7 +52,7 @@ test_lei(sub {
my @f = glob("$md/cur/*:2,");
is(scalar(@f), 1, 'got populated maildir with one result');
rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een
- $have_fast_inotify or tick(2);
+ tick($have_fast_inotify ? 0.1 : 2.1); # always needed for 1 CPU systems
lei_ok qw(note-event done); # flushes immediately (instead of 5s)
lei_ok qw(q mid:testmessage@example.com -o), $md2, '-I', "$ro_home/t1";
^ permalink raw reply related [relevance 71%]
* [PATCH] doc: lei-{p2q,rediff}: note implicit --stdin
@ 2021-07-25 12:03 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 12:03 UTC (permalink / raw)
To: meta
lei actually uses implicit --stdin everywhere, but I thing
these patch-related commands are the most common use of them.
---
Documentation/lei-p2q.pod | 3 ++-
Documentation/lei-rediff.pod | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-p2q.pod b/Documentation/lei-p2q.pod
index f404ede0..44798ac3 100644
--- a/Documentation/lei-p2q.pod
+++ b/Documentation/lei-p2q.pod
@@ -48,7 +48,8 @@ Default: C<dfpost7>
=item --stdin
-Read patch from stdin.
+Read message from stdin. This is implicit if no arguments are given
+and stdin is a pipe or regular file.
=item --debug
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index 5fdde230..e968fb20 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -24,7 +24,8 @@ supported.
=item --stdin
-Read message from stdin.
+Read message from stdin. This is implicit if no arguments are given
+and stdin is a pipe or regular file.
=item --git-dir=DIR
^ permalink raw reply related [relevance 71%]
* lei rediff: add --dequote and --requote?
@ 2021-07-25 12:29 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 12:29 UTC (permalink / raw)
To: meta
I find "lei rediff" useful when writing replies to patch emails.
Like any good *nix tool, my $EDITOR allows piping either the
full or partial editor buffer into arbitrary commands.
Besides typing text, the most important feature of my editor is
the ability to run/pipe-to arbitrary commands (e.g. git-am, lei
blob, git-show); and "lei rediff" is a new one.
I barely customize my editor; however, I do have some editor
macros for quickly stripping and adding quote prefixes (at least
the standard "> " mail quoting style, along with '#' for sh/Perl).
But maybe others do not have these macros, so I wonder if
"rediff" should support --dequote and --requote switches to ease
this usage pattern.
Or/and if "lei dequote" and "lei quote" should exist as
(slower) replacements for `sed 's/^[> ]\+//'` and `sed 's/^/> /'`,
respectively.
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei: die on ECONNRESET
@ 2021-07-28 0:37 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-28 0:37 UTC (permalink / raw)
To: meta
ECONNRESET should be rare on a private local socket, and if
we hit it, it's because we're hitting the listen() limit.
---
script/lei | 1 -
1 file changed, 1 deletion(-)
diff --git a/script/lei b/script/lei
index fce8124a..99d94b4e 100755
--- a/script/lei
+++ b/script/lei
@@ -115,7 +115,6 @@ while (1) {
my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
if (scalar(@fds) == 1 && !defined($fds[0])) {
next if $!{EINTR};
- last if $!{ECONNRESET};
die "recvmsg: $!";
}
last if $buf eq '';
^ permalink raw reply related [relevance 71%]
* [RFC] lei: address lifetime problems from Linux::Inotify2
@ 2021-07-28 12:34 58% Eric Wong
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-07-28 12:34 UTC (permalink / raw)
To: meta
Linux::Inotify2 doesn't supply an explicit close() wrapper,
and Linux::Inotify2::Event objects maintain a reference to the
parent Linux::Inotify2 object, so we were leaking inotify FDs
into the "lei note-event" worker processes.
This delays the fork()-ing code paths into the next tick of the
event loop to ensure the inotify FD is closed when $dir_idle
is undef'ed in ->_lei_atfork_child.
Note: I'm not sure if I want to keep this, since
The correct fix would be to have a way to close inotify FDs,
but the current Linux::Inotify2 API doesn't make it easy.
And leaking an inotify FD isn't the worst thing in the world,
since it won't hang other processes. So maybe the cure
is worse than the disease, in this case.
---
lib/PublicInbox/LEI.pm | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d9fd40fd..b81f95c9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -28,8 +28,8 @@ use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
our $quit = \&CORE::exit;
-our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
- $recv_cmd, $send_cmd);
+our ($current_lei, $errors_log, $listener, $oldset,
+ $dir_idle, $lne_nxt, $lne_q, $recv_cmd, $send_cmd);
my $GLP = Getopt::Long::Parser->new;
$GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
my $GLP_PASS = Getopt::Long::Parser->new;
@@ -557,6 +557,8 @@ sub _lei_atfork_child {
close $listener if $listener;
undef $listener;
undef $dir_idle;
+ undef $lne_nxt;
+ undef $lne_q;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
@@ -1153,6 +1155,21 @@ sub cfg2lei ($) {
$lei;
}
+sub lne_task { # requeue, runs without dir_idle stuff on stack
+ undef $lne_nxt;
+ my $q = $lne_q // return;
+ $lne_q = undef;
+
+ while (my ($loc, $nc, $bn, $fn, $cfg) = splice(@$q, 0, 5)) {
+ eval {
+ local %ENV = %{$cfg->{-env}};
+ my $lei = cfg2lei($cfg);
+ $lei->dispatch('note-event', $loc, $nc, $bn, $fn);
+ };
+ warn "E note-event $cfg->{-f}: $@\n" if $@;
+ }
+}
+
sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
my ($ev) = @_; # Linux::Inotify2::Event or duck type
my $fn = $ev->fullname;
@@ -1161,14 +1178,9 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
$nc = '' if $ev->IN_DELETE;
for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
my $cfg = $PATH2CFG{$f} // next;
- eval {
- local %ENV = %{$cfg->{-env}};
- my $lei = cfg2lei($cfg);
- $lei->dispatch('note-event',
- "maildir:$mdir", $nc, $bn, $fn);
- };
- warn "E note-event $f: $@\n" if $@;
+ push @$lne_q, "maildir:$mdir", $nc, $bn, $fn, $cfg;
}
+ $lne_nxt //= PublicInbox::DS::requeue(\&lne_task) if $lne_q;
}
if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
$ev->cancel;
@@ -1261,6 +1273,7 @@ sub lazy_start {
undef $sig;
local $SIG{PIPE} = 'IGNORE';
require PublicInbox::DirIdle;
+ local ($lne_q, $lne_nxt);
local $dir_idle = PublicInbox::DirIdle->new([$sock_dir], sub {
# just rely on wakeup to hit PostLoopCallback set below
dir_idle_handler($_[0]) if $_[0]->fullname ne $path;
^ permalink raw reply related [relevance 58%]
* [RFC v2] lei: close inotify FD in forked child
2021-07-28 12:34 58% [RFC] lei: address lifetime problems from Linux::Inotify2 Eric Wong
@ 2021-07-29 10:01 62% ` Eric Wong
2021-08-04 10:40 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-07-29 10:01 UTC (permalink / raw)
To: meta
It looks like Linux::Inotify2 2.3+ will include an ->fh method
to give us the ability to safely close an FD without hitting
EBADF (and automatically use FD_CLOEXEC).
We'll still need a new wrapper class (LI2Wrap) to handle it for
users of old versions, though.
http://lists.schmorp.de/pipermail/perl/2021q3/thread.html
---
MANIFEST | 1 +
lib/PublicInbox/DirIdle.pm | 11 +++++++++++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LI2Wrap.pm | 20 ++++++++++++++++++++
4 files changed, 33 insertions(+), 1 deletion(-)
create mode 100644 lib/PublicInbox/LI2Wrap.pm
diff --git a/MANIFEST b/MANIFEST
index a3913501..fb9f16bf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -197,6 +197,7 @@ lib/PublicInbox/InputPipe.pm
lib/PublicInbox/Isearch.pm
lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
+lib/PublicInbox/LI2Wrap.pm
lib/PublicInbox/LeiALE.pm
lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 65896f95..d572c274 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -84,4 +84,15 @@ sub event_step {
warn "$self->{inot}->read err: $@\n" if $@;
}
+sub force_close {
+ my ($self) = @_;
+ my $inot = delete $self->{inot} // return;
+ if ($inot->can('fh')) { # Linux::Inotify2 2.3+
+ close($inot->fh) or warn "CLOSE ERROR: $!";
+ } elsif ($inot->isa('Linux::Inotify2')) {
+ require PublicInbox::LI2Wrap;
+ PublicInbox::LI2Wrap::wrapclose($inot);
+ }
+}
+
1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d9fd40fd..e6f763e1 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -556,7 +556,7 @@ sub _lei_atfork_child {
}
close $listener if $listener;
undef $listener;
- undef $dir_idle;
+ $dir_idle->force_close if $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
diff --git a/lib/PublicInbox/LI2Wrap.pm b/lib/PublicInbox/LI2Wrap.pm
new file mode 100644
index 00000000..b0f4f8b8
--- /dev/null
+++ b/lib/PublicInbox/LI2Wrap.pm
@@ -0,0 +1,20 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Wrapper for Linux::Inotify2 < 2.3 which lacked ->fh and auto-close
+# Remove this when supported LTS/enterprise distros are all
+# Linux::Inotify2 >= 2.3
+package PublicInbox::LI2Wrap;
+use v5.10.1;
+our @ISA = qw(Linux::Inotify2);
+
+sub wrapclose {
+ my ($inot) = @_;
+ my $fd = $inot->fileno;
+ open my $fh, '<&=', $fd or die "open <&= $fd $!";
+
+}
+
+sub DESTROY {} # no-op
+
+1
^ permalink raw reply related [relevance 62%]
* Re: [RFC v2] lei: close inotify FD in forked child
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
@ 2021-08-04 10:40 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-04 10:40 UTC (permalink / raw)
To: meta
Pushed with present tense commit message now that L::I2 2.3+ is out:
https://public-inbox.org/meta/7fc6e30aeab9925bece4bb00f88bb91af5646aa2/s/
^ permalink raw reply [relevance 71%]
* [PATCH] lei export-kw: workaround race in updating Maildir locations
@ 2021-08-05 2:33 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-05 2:33 UTC (permalink / raw)
To: meta
Inotify updates may simultaneously remove or update the location
of a message, so ensure we at least have knowledge of the new
location if the old one cannot be updated.
---
lib/PublicInbox/LeiMailSync.pm | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 82740d59..6dfa03be 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -158,7 +158,13 @@ sub mv_src {
my $sth = $self->{dbh}->prepare_cached(<<'');
UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
- $sth->execute($newbn, $fid, $oidbin, $$id);
+ my $nr = $sth->execute($newbn, $fid, $oidbin, $$id);
+ if ($nr == 0) { # may race with a clear_src, ensure new value exists
+ $sth = $self->{dbh}->prepare_cached(<<'');
+INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?)
+
+ $sth->execute($oidbin, $fid, $newbn);
+ }
}
# read-only, iterates every oidbin + UID or name for a given folder
^ permalink raw reply related [relevance 71%]
* [PATCH 0/3] lei pathname canonicalization fixes
@ 2021-08-11 11:26 71% Eric Wong
2021-08-11 11:26 42% ` [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-08-11 11:26 UTC (permalink / raw)
To: meta
Pathnames with "x/../y" components weren't being canonicalized
properly after creation, and some of it was being put into the
lei_mail_sync.sqlite3 DB and configs for saved searches.
This will become more important as we start using inotify more
to track keyword changes on messages.
Eric Wong (3):
treewide: use *nix-specific dirname regexps
lei_saved_search: canonicalized relative save paths
lei: attempt to canonicalize away "/../" pathnames
lib/PublicInbox/IMAPTracker.pm | 4 ++--
lib/PublicInbox/LEI.pm | 16 +++++++++++-----
lib/PublicInbox/LeiALE.pm | 8 ++++----
lib/PublicInbox/LeiBlob.pm | 4 ++--
lib/PublicInbox/LeiInit.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 9 ++++++++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/OverIdx.pm | 4 ++--
lib/PublicInbox/Xapcmd.pm | 5 ++---
script/public-inbox-init | 3 +--
t/init.t | 1 -
t/lei-q-save.t | 9 +++++++++
t/lei_xsearch.t | 8 +++++---
17 files changed, 52 insertions(+), 32 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames
2021-08-11 11:26 71% [PATCH 0/3] lei pathname canonicalization fixes Eric Wong
@ 2021-08-11 11:26 42% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-11 11:26 UTC (permalink / raw)
To: meta
As documented, File::Spec->canonpath does not canonicalize
"/../". While we want to do our best to preserve symlinks in
pathnames, leaving "/../" can mislead our inotify|kqueue usage.
---
lib/PublicInbox/LEI.pm | 14 ++++++++++----
lib/PublicInbox/LeiALE.pm | 8 ++++----
lib/PublicInbox/LeiBlob.pm | 4 ++--
lib/PublicInbox/LeiInit.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 +-
t/lei_xsearch.t | 8 +++++---
10 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index be4754df..54fac7b4 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -94,6 +94,12 @@ sub rel2abs {
# abs_path resolves symlinks in parent iff all parents exist
sub abs_path { Cwd::abs_path($_[1]) // rel2abs(@_) }
+sub canonpath_harder {
+ my $p = $_[-1]; # $_[0] may be self
+ $p = File::Spec->canonpath($p);
+ $p =~ m!(?:/*|\A)\.\.(?:/*|\z)! && -e $p ? Cwd::abs_path($p) : $p;
+}
+
sub share_path ($) { # $HOME/.local/share/lei/$FOO
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_DATA_HOME} //
@@ -808,8 +814,8 @@ sub _lei_cfg ($;$) {
my $cfg = PublicInbox::Config->git_config_dump($f);
$cfg->{-st} = $cur_st;
$cfg->{'-f'} = $f;
- if ($sto && File::Spec->canonpath($sto_dir // store_path($self))
- eq File::Spec->canonpath($cfg->{'leistore.dir'} //
+ if ($sto && canonpath_harder($sto_dir // store_path($self))
+ eq canonpath_harder($cfg->{'leistore.dir'} //
store_path($self))) {
$cfg->{-lei_store} = $sto;
$cfg->{-lei_note_event} = $lne;
@@ -1382,7 +1388,7 @@ sub refresh_watches {
next;
}
if ($url =~ /\Amaildir:(.+)/i) {
- my $d = File::Spec->canonpath($1);
+ my $d = canonpath_harder($1);
if ($state eq 'pause') {
cancel_maildir_watch($d, $cfg_f);
} elsif (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
@@ -1400,7 +1406,7 @@ sub refresh_watches {
next if exists $seen{$url};
delete $old->{$url};
if ($url =~ /\Amaildir:(.+)/i) {
- my $d = File::Spec->canonpath($1);
+ my $d = canonpath_harder($1);
cancel_maildir_watch($d, $cfg_f);
} else { # TODO: imap/nntp/jmap
$lei->child_error(1, "E: watch $url TODO");
diff --git a/lib/PublicInbox/LeiALE.pm b/lib/PublicInbox/LeiALE.pm
index cb570ab4..cc9a2095 100644
--- a/lib/PublicInbox/LeiALE.pm
+++ b/lib/PublicInbox/LeiALE.pm
@@ -33,7 +33,7 @@ sub new {
for my $loc ($lei->externals_each) { # locals only
$lxs->prepare_external($loc) if -d $loc;
}
- $self->refresh_externals($lxs);
+ $self->refresh_externals($lxs, $lei);
$self;
}
@@ -50,7 +50,7 @@ sub overs_all { # for xoids_for (called only in lei workers?)
}
sub refresh_externals {
- my ($self, $lxs) = @_;
+ my ($self, $lxs, $lei) = @_;
$self->git->cleanup;
my $lk = $self->lock_for_scope;
my $cur_lxs = ref($lxs)->new;
@@ -72,7 +72,7 @@ sub refresh_externals {
}
my @ibxish = $cur_lxs->locals;
for my $x ($lxs->locals) {
- my $d = File::Spec->canonpath($x->{inboxdir} // $x->{topdir});
+ my $d = $lei->canonpath_harder($x->{inboxdir} // $x->{topdir});
$seen_ibxish{$d} //= do {
$new .= "$d\n";
push @ibxish, $x;
@@ -95,7 +95,7 @@ sub refresh_externals {
$old = <$fh> // die "readline($f): $!";
}
for my $x (@ibxish) {
- $new .= File::Spec->canonpath($x->git->{git_dir})."/objects\n";
+ $new .= $lei->canonpath_harder($x->git->{git_dir})."/objects\n";
}
$self->{ibxish} = \@ibxish;
return if $old eq $new;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 09217964..3158ca3b 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -112,7 +112,7 @@ sub lei_blob {
if ($opt->{mail} // ($has_hints ? 0 : 1)) {
if (grep(defined, @$opt{qw(include only)})) {
$lxs = $lei->lxs_prepare;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
}
my $rdr = {};
if ($opt->{mail}) {
@@ -155,7 +155,7 @@ sub lei_blob {
return $lei->fail('no --git-dir to try') unless @$git_dirs;
unless ($lxs) {
$lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
}
if ($lxs->remotes) {
require PublicInbox::LeiRemote;
diff --git a/lib/PublicInbox/LeiInit.pm b/lib/PublicInbox/LeiInit.pm
index c6c0c01b..6558ac0a 100644
--- a/lib/PublicInbox/LeiInit.pm
+++ b/lib/PublicInbox/LeiInit.pm
@@ -4,7 +4,6 @@
# for the "lei init" command, not sure if it's even needed...
package PublicInbox::LeiInit;
use v5.10.1;
-use File::Spec;
sub lei_init {
my ($self, $dir) = @_;
@@ -13,7 +12,7 @@ sub lei_init {
$dir //= $self->store_path;
$dir = $self->rel2abs($dir);
my @cur = stat($cur) if defined($cur);
- $cur = File::Spec->canonpath($cur // $dir);
+ $cur = $self->canonpath_harder($cur // $dir);
my @dir = stat($dir);
my $exists = "# leistore.dir=$cur already initialized" if @dir;
if (@cur) {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index cb5eb5b4..4a0c24a9 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -128,7 +128,7 @@ sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
sub lei_lcat {
my ($lei, @argv) = @_;
my $lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
my $sto = $lei->_lei_store(1);
$lei->{lse} = $sto->search;
my $opt = $lei->{opt};
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index e4242d9b..223db222 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -76,7 +76,7 @@ sub new {
$fmt //= $devfd >= 0 ? 'json' : (detect_fmt($lei, $dst) or return);
if (index($dst, '://') < 0) { # not a URL, so assume path
- $dst = File::Spec->canonpath($dst);
+ $dst = $lei->canonpath_harder($dst);
} # else URL
my $self = bless { fmt => $fmt, dst => $dst }, $class;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index eb7b98d4..37b660f9 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -116,7 +116,7 @@ sub lei_q {
my ($self, @argv) = @_;
PublicInbox::Config->json; # preload before forking
my $lxs = lxs_prepare($self) or return;
- $self->ale->refresh_externals($lxs);
+ $self->ale->refresh_externals($lxs, $self);
my $opt = $self->{opt};
my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 7607b44f..0ba5897c 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -215,7 +215,7 @@ sub lei_rediff {
$lei->{curl} //= which('curl') or return
$lei->fail('curl needed for', $lxs->remotes);
}
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
my $self = bless {
-force_eml => 1, # for LeiInput->input_fh
lxs => $lxs,
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 9069232b..3356d11e 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -42,7 +42,7 @@ sub up1 ($$) {
}
$lei->{lss} = $lss; # for LeiOverview->new
my $lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
$lei->_start_query;
}
diff --git a/t/lei_xsearch.t b/t/lei_xsearch.t
index 3eb44233..d9ddb297 100644
--- a/t/lei_xsearch.t
+++ b/t/lei_xsearch.t
@@ -11,6 +11,7 @@ require PublicInbox::ExtSearchIdx;
require_git 2.6;
require_ok 'PublicInbox::LeiXSearch';
require_ok 'PublicInbox::LeiALE';
+require_ok 'PublicInbox::LEI';
my ($home, $for_destroy) = tmpdir();
my @ibx;
for my $V (1..2) {
@@ -88,18 +89,19 @@ is($lxs->over, undef, '->over fails');
my $smsg = $lxs->smsg_for($mitem) or BAIL_OUT 'smsg_for broken';
my $ale = PublicInbox::LeiALE::_new("$home/ale");
- $ale->refresh_externals($lxs);
+ my $lei = bless {}, 'PublicInbox::LEI';
+ $ale->refresh_externals($lxs, $lei);
my $exp = [ $smsg->{blob}, 'blob', -s 't/utf8.eml' ];
is_deeply([ $ale->git->check($smsg->{blob}) ], $exp, 'ale->git->check');
$lxs = PublicInbox::LeiXSearch->new;
$lxs->prepare_external($v2ibx);
- $ale->refresh_externals($lxs);
+ $ale->refresh_externals($lxs, $lei);
is_deeply([ $ale->git->check($smsg->{blob}) ], $exp,
'ale->git->check remembered inactive external');
rename("$home/v1tmp", "$home/v1moved") or BAIL_OUT "rename: $!";
- $ale->refresh_externals($lxs);
+ $ale->refresh_externals($lxs, $lei);
is($ale->git->check($smsg->{blob}), undef,
'missing after directory gone');
}
^ permalink raw reply related [relevance 42%]
* [PATCH 0/2] "lei up" improvements
@ 2021-08-12 23:40 71% Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
Some obvious things for now, some maybe not-so-obvious things on
the way...
Eric Wong (2):
lei up: support multiple output folders w/o --all=local
lei up: note errors if one output destination fails
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 23 +++++++++++++++++++----
2 files changed, 20 insertions(+), 5 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei up: note errors if one output destination fails
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
@ 2021-08-12 23:40 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
We can keep going if one (out of multiple) output destinations
fail, but the error needs to be communicated to the caller as an
exit code.
---
lib/PublicInbox/LeiUp.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 49b558fd..c5a01527 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -57,7 +57,7 @@ sub up1_redispatch {
up1($l, $out);
$l->qerr("# $out done");
};
- $l->err($@) if $@;
+ $l->child_error(1 << 8, $@) if $@;
}
sub lei_up {
^ permalink raw reply related [relevance 71%]
* [PATCH 1/2] lei up: support multiple output folders w/o --all=local
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
@ 2021-08-12 23:40 63% ` Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
Being able to update 1 folder, or all (local) folders is
sometimes too limiting, so just allow updating any subset
of local folders.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 21 ++++++++++++++++++---
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 54fac7b4..7d0f63dc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -178,7 +178,7 @@ our %CMD = ( # sorted in order of importance/use:
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
-'up' => [ 'OUTPUT|--all', 'update saved search',
+'up' => [ 'OUTPUT...|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 3356d11e..49b558fd 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -61,11 +61,13 @@ sub up1_redispatch {
}
sub lei_up {
- my ($lei, $out) = @_;
+ my ($lei, @outs) = @_;
$lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
$opt->{save} = -1;
+ my @local;
if (defined $opt->{all}) {
+ return $lei->fail("--all and @outs incompatible") if @outs;
length($opt->{mua}//'') and return
$lei->fail('--all and --mua= are incompatible');
@@ -74,7 +76,20 @@ sub lei_up {
$opt->{all} eq 'local' or return
$lei->fail('only --all=local works at the moment');
my @all = PublicInbox::LeiSavedSearch::list($lei);
- my @local = grep(!m!\Aimaps?://!i, @all);
+ @local = grep(!m!\Aimaps?://!i, @all);
+ } else {
+ @local = @outs;
+ }
+ if (scalar(@outs) > 1) {
+ length($opt->{mua}//'') and return $lei->fail(<<EOM);
+multiple outputs and --mua= are incompatible
+EOM
+ # TODO:
+ return $lei->fail(<<EOM) if grep(m!\Aimaps?://!i, @outs);
+multiple destinations only supported for local outputs (FIXME)
+EOM
+ }
+ if (scalar(@local) > 1) {
$lei->_lei_store->write_prepare($lei); # share early
# daemon mode, re-dispatch into our event loop w/o
# creating an extra fork-level
@@ -89,7 +104,7 @@ sub lei_up {
$lei->event_step_init;
$op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
} else {
- up1($lei, $out);
+ up1($lei, $local[0]);
}
}
^ permalink raw reply related [relevance 63%]
* [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors
@ 2021-08-14 0:29 71% Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
2/3 is probably a fix for a long-standing problem, 3/3 was
noticed while working on it. If 2/3 doesn't fix it, then maybe
1/3 will help us narrow it down.
Eric Wong (3):
lei: diagnostics for /Document \d+ not found/ errors
lei <q|up>: wait on remote mboxrd imports synchronously
lei: hexdigest mocks account for unwanted headers
lib/PublicInbox/FakeImport.pm | 3 +++
lib/PublicInbox/IPC.pm | 2 +-
lib/PublicInbox/LEI.pm | 5 +++++
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 9 +++++----
lib/PublicInbox/LeiSearch.pm | 17 ++++++++++-------
lib/PublicInbox/LeiStore.pm | 10 +++++++++-
lib/PublicInbox/LeiXSearch.pm | 11 +++++++----
8 files changed, 41 insertions(+), 18 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei: diagnostics for /Document \d+ not found/ errors
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
@ 2021-08-14 0:29 68% ` Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
This may help diagnose "Exception: Document \d+ not found"
errors I'm seeing from "lei up" with HTTPS endpoints.
---
lib/PublicInbox/IPC.pm | 2 +-
lib/PublicInbox/LeiSearch.pm | 17 ++++++++++-------
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 497a6035..d909dc1c 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -236,7 +236,7 @@ sub recv_and_run {
undef $buf;
my $sub = shift @$args;
eval { $self->$sub(@$args) };
- warn "$$ wq_worker: $@" if $@;
+ warn "$$ $0 wq_worker: $@" if $@;
delete @$self{0..($nfd-1)};
$n;
}
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 79b2fd7d..f9e5c8e9 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -55,13 +55,16 @@ sub _xsmsg_vmd { # retry_reopen
$kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged});
my @num = $self->over->blob_exists($smsg->{blob});
for my $num (@num) { # there should only be one...
- $doc = $xdb->get_document(num2docid($self, $num));
- $x = xap_terms('K', $doc);
- %kw = (%kw, %$x);
- if ($want_label) { # JSON/JMAP only
- $x = xap_terms('L', $doc);
- %L = (%L, %$x);
- }
+ eval {
+ $doc = $xdb->get_document(num2docid($self, $num));
+ $x = xap_terms('K', $doc);
+ %kw = (%kw, %$x);
+ if ($want_label) { # JSON/JMAP only
+ $x = xap_terms('L', $doc);
+ %L = (%L, %$x);
+ }
+ };
+ warn "$$ $0 #$num (nshard=$self->{nshard}) $smsg->{blob}: $@";
}
$smsg->{kw} = [ sort keys %kw ] if scalar(keys(%kw));
$smsg->{L} = [ sort keys %L ] if scalar(keys(%L));
^ permalink raw reply related [relevance 68%]
* [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
@ 2021-08-14 0:29 64% ` Eric Wong
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
2021-08-24 20:14 71% ` [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
This ought to avoid /Document \d+ not found/ errors from Xapian
when seeing a message for the first time by not attempting to
read keywords for totally unseen messages.
---
lib/PublicInbox/LeiRemote.pm | 7 ++++---
lib/PublicInbox/LeiStore.pm | 1 +
lib/PublicInbox/LeiXSearch.pm | 10 +++++++---
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 945d9990..e7deecb8 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -26,11 +26,12 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
my ($eml, $self) = @_;
my $lei = $self->{lei};
my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ my $smsg = bless {}, 'PublicInbox::Smsg';
if ($lei->{sto} && !$xoids) { # memoize locally
- $lei->{sto}->ipc_do('add_eml', $eml);
+ my $res = $lei->{sto}->ipc_do('add_eml', $eml);
+ $smsg = $res if ref($res) eq ref($smsg);
}
- my $smsg = bless {}, 'PublicInbox::Smsg';
- $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e26b622d..ce66014f 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -329,6 +329,7 @@ sub add_eml {
}
\@docids;
} else { # totally new message
+ delete $smsg->{-oidx}; # for IPC-friendliness
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
$oidx->add_overview($eml, $smsg);
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 393f25bf..971f3a06 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -266,11 +266,15 @@ sub _smsg_fill ($$) {
sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- $self->{import_sto}->ipc_do('add_eml', $eml);
+ my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+ if (ref($res) eq ref($smsg)) { # totally new message
+ $smsg = $res;
+ $smsg->{kw} = []; # short-circuit xsmsg_vmd
+ }
}
- my $smsg = bless {}, 'PublicInbox::Smsg';
- $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
_smsg_fill($smsg, $eml);
wait_startq($lei);
^ permalink raw reply related [relevance 64%]
* [PATCH 3/3] lei: hexdigest mocks account for unwanted headers
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
@ 2021-08-14 0:29 58% ` Eric Wong
2021-08-24 20:14 71% ` [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
PublicInbox::Import never imports @UNWANTED_HEADERS, so ensure
our mock blob OIDs do the same. This ought to prevent
duplicates if the PSGI mboxrd download starts setting
"X-Status: F" like "lei q -tt .."
---
lib/PublicInbox/FakeImport.pm | 3 +++
lib/PublicInbox/LEI.pm | 5 +++++
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 9 ++++++++-
lib/PublicInbox/LeiXSearch.pm | 3 +--
6 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/FakeImport.pm b/lib/PublicInbox/FakeImport.pm
index dea25cbe..bccc3321 100644
--- a/lib/PublicInbox/FakeImport.pm
+++ b/lib/PublicInbox/FakeImport.pm
@@ -4,12 +4,15 @@
# pretend to do PublicInbox::Import::add for "lei index"
package PublicInbox::FakeImport;
use strict;
+use v5.10.1;
use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub new { bless { bytes_added => 0 }, __PACKAGE__ }
sub add {
my ($self, $eml, $check_cb, $smsg) = @_;
+ PublicInbox::Import::drop_unwanted_headers($eml);
$smsg->populate($eml);
my $raw = $eml->as_string;
$smsg->{blob} = git_sha(1, \$raw)->hexdigest;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7d0f63dc..347dd280 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1420,4 +1420,9 @@ sub refresh_watches {
}
}
+sub git_blob_id {
+ my ($lei, $eml) = @_;
+ ($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
+}
+
1;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 37b660f9..962ad49e 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -73,7 +73,7 @@ sub lxs_prepare {
my @only = @{$opt->{only} // []};
# --local is enabled by default unless --only is used
# we'll allow "--only $LOCATION --local"
- my $sto = $self->_lei_store(1); # FIXME: should not create
+ my $sto = $self->_lei_store(1);
$self->{lse} = $sto->search;
if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
$lxs->prepare_external($self->{lse});
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index e7deecb8..580787c0 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -32,7 +32,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
$smsg = $res if ref($res) eq ref($smsg);
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : git_sha(1, $eml)->hexdigest;
+ : $lei->git_blob_id($eml);
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
push @{$self->{smsg}}, $smsg;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index ce66014f..3f33d114 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -20,7 +20,7 @@ use PublicInbox::Eml;
use PublicInbox::Import;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::ContentHash qw(content_hash git_sha);
use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
@@ -508,4 +508,11 @@ sub write_prepare {
$lei->{sto} = $self;
}
+# TODO: support SHA-256
+sub git_blob_id { # called via LEI->git_blob_id
+ my ($self, $eml) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+ git_sha(1, $eml)->hexdigest;
+}
+
1;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 971f3a06..5e34d864 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -274,8 +274,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
$smsg->{kw} = []; # short-circuit xsmsg_vmd
}
}
- $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : git_sha(1, $eml)->hexdigest;
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
_smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
^ permalink raw reply related [relevance 58%]
* [PATCH 3/3] lei forget-mail-sync: rely on lei/store process
2021-08-17 8:52 71% [PATCH 0/3] lei: some mail sync stuff Eric Wong
2021-08-17 8:52 54% ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
@ 2021-08-17 8:52 88% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
As implied in commit 6ff03ba2be9247f1
("lei export-kw: do not write directly to mail_sync.sqlite3"),
modifying mail_sync.sqlite3 directly can lead to conflicts
and making everything go through lei/store is easier.
---
lib/PublicInbox/LeiForgetMailSync.pm | 8 ++++----
lib/PublicInbox/LeiStore.pm | 6 ++++++
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index c74ba25d..940ca1b6 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -15,13 +15,13 @@ use PublicInbox::LeiExportKw;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
my $lms = $lei->lms or return;
+ my $sto = $lei->_lei_store or return; # may disappear due to race
+ $sto->write_prepare;
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
- delete $lms->{dbh};
- $lms->lms_begin;
- $lms->forget_folder($_) for @folders;
- $lms->lms_commit;
+ $sto->ipc_do('lms_forget_folders', @folders);
+ my $wait = $sto->ipc_do('done');
}
*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 3f33d114..e8334187 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -216,6 +216,12 @@ sub lms_mv_src {
_lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn);
}
+sub lms_forget_folders {
+ my ($self, @folders) = @_;
+ my $lms = _lms_rw($self);
+ for my $f (@folders) { $lms->forget_folder($f) }
+}
+
sub set_sync_info {
my ($self, $oidhex, $folder, $id) = @_;
_lms_rw($self)->set_src($oidhex, $folder, $id);
^ permalink raw reply related [relevance 88%]
* [PATCH 0/3] lei: some mail sync stuff
@ 2021-08-17 8:52 71% Eric Wong
2021-08-17 8:52 54% ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
2021-08-17 8:52 88% ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
Still trying to wrap my head around some other stuff related to
mail_sync.sqlite3, but 3/3 is a necessary fix and 2/3 is an obvious
cleanup. The ipc_do('done') stuff could probably be made
faster, but probably after the 1.7 release...
Eric Wong (3):
lei: add ->lms shortcut for LeiMailSync
ipc: remove WQ_MAX_WORKERS
lei forget-mail-sync: rely on lei/store process
lib/PublicInbox/IPC.pm | 5 -----
lib/PublicInbox/LEI.pm | 9 +++++++++
lib/PublicInbox/LeiBlob.pm | 3 +--
lib/PublicInbox/LeiForgetMailSync.pm | 11 +++++------
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 10 ++++------
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
lib/PublicInbox/LeiStore.pm | 6 ++++++
8 files changed, 27 insertions(+), 23 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync
2021-08-17 8:52 71% [PATCH 0/3] lei: some mail sync stuff Eric Wong
@ 2021-08-17 8:52 54% ` Eric Wong
2021-08-17 8:52 88% ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
We access this read-only in many places (and will in more),
so provide a shortcut to simplify callers.
---
lib/PublicInbox/LEI.pm | 9 +++++++++
lib/PublicInbox/LeiBlob.pm | 3 +--
lib/PublicInbox/LeiForgetMailSync.pm | 3 +--
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 10 ++++------
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
6 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 347dd280..e5232f1b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1425,4 +1425,13 @@ sub git_blob_id {
($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
}
+sub lms { # read-only LeiMailSync
+ my ($lei) = @_;
+ my $lse = $lei->{lse} // do {
+ my $sto = $lei->{sto} // _lei_store($lei);
+ $sto ? $sto->search : undef
+ };
+ $lse ? $lse->lms : undef;
+}
+
1;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 3158ca3b..21003894 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -133,8 +133,7 @@ sub lei_blob {
}
my $ce = $?;
return if $ce == 0;
- my $sto = $lei->_lei_store;
- my $lms = $sto ? $sto->search->lms : undef;
+ my $lms = $lei->lms;
if (my $bref = $lms ? $lms->local_blob($blob, 1) : undef) {
defined($lei->{-attach_idx}) and
return extract_attach($lei, $blob, $bref);
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 46dde1a7..c74ba25d 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -14,8 +14,7 @@ use PublicInbox::LeiExportKw;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index bf7a4836..2d2ff1a0 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -41,8 +41,7 @@ sub inspect_imap_uid ($$) {
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
- my $lse = $lei->{lse} or return $ent;
- my $lms = $lse->lms or return $ent;
+ my $lms = $lei->lms or return $ent;
my $folders = [ $folder ];
my $err = $lms->arg2folder($lei, $folders);
if ($err) {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 4a0c24a9..9d95e899 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -13,7 +13,7 @@ use PublicInbox::MID qw($MID_EXTRACT);
sub lcat_folder ($$$) {
my ($lei, $lms, $folder) = @_;
- $lms //= $lei->{lse}->lms // return;
+ $lms //= $lei->lms or return;
my $folders = [ $folder];
my $err = $lms->arg2folder($lei, $folders);
$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
@@ -29,7 +29,7 @@ sub lcat_folder ($$$) {
sub lcat_imap_uri ($$) {
my ($lei, $uri) = @_;
- my $lms = $lei->{lse}->lms or return;
+ my $lms = $lei->lms or return;
# cf. LeiXsearch->lcat_dump
if (defined $uri->uid) {
my $oidhex = $lms->imap_oid($lei, $uri);
@@ -129,8 +129,7 @@ sub lei_lcat {
my ($lei, @argv) = @_;
my $lxs = $lei->lxs_prepare or return;
$lei->ale->refresh_externals($lxs, $lei);
- my $sto = $lei->_lei_store(1);
- $lei->{lse} = $sto->search;
+ $lei->_lei_store(1);
my $opt = $lei->{opt};
my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
@@ -153,8 +152,7 @@ no args allowed on command-line with --stdin
sub _complete_lcat {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } $lms->folders;
}
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index cadc61ed..2d8913ac 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -103,8 +103,7 @@ sub _complete_ls_mail_source {
my $match_cb = $lei->complete_url_prepare(\@argv);
my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
my %f = map { $_ => 1 } @m;
- my $sto = $lei->_lei_store;
- if (my $lms = $sto ? $sto->search->lms : undef) {
+ if (my $lms = $lei->lms) {
@m = map { $match_cb->($_) } grep(
m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
@f{@m} = @m;
^ permalink raw reply related [relevance 54%]
* "lei q --save" - should it be the default?
@ 2021-08-18 11:56 71% Eric Wong
2021-08-19 1:36 31% ` [PATCH] lei q: make --save the default Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-08-18 11:56 UTC (permalink / raw)
To: meta
I've often found "lei up" useful after-the-fact, but it requires
"lei q --save" to be used initially. Perhaps "--save" should be
the default when writing to mbox/Maildir/IMAP...
Thoughts?
^ permalink raw reply [relevance 71%]
* [PATCH] lei q: make --save the default
2021-08-18 11:56 71% "lei q --save" - should it be the default? Eric Wong
@ 2021-08-19 1:36 31% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-19 1:36 UTC (permalink / raw)
To: meta
Since "lei up" is more often useful than not and incurs neglible
overhead; enable --save by default and allow --no-save to work.
This also fixes a long-standing when overwriting --output
destinations with saved searches: dedupe data from previous
searches are reset and no longer influences the new (changed)
search, so results no longer go missing if two sequential
invocations of "lei q --save" point to the same --output.
---
Documentation/lei-q.pod | 4 ++--
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiSavedSearch.pm | 10 ++++++++++
lib/PublicInbox/LeiToMail.pm | 31 +++++++++++++++++++++++++++----
lib/PublicInbox/LeiUp.pm | 1 -
t/lei-q-kw.t | 3 ++-
t/lei-q-save.t | 26 ++++++++++++++------------
t/lei.t | 2 +-
t/lei_to_mail.t | 2 +-
9 files changed, 59 insertions(+), 24 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index fbe61920..69a6cdf2 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -26,9 +26,9 @@ TODO: mention curl options?
Read search terms from stdin.
-=item --save
+=item --no-save
-Save a search for L<lei-up(1)>.
+Do not save the search for L<lei-up(1)>.
=item --output=MFOLDER
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e5232f1b..79dc9bf9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -173,7 +173,7 @@ our %CMD = ( # sorted in order of importance/use:
'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms',
'stdin|', # /|\z/ must be first for lone dash
@lxs_opt,
- qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+ qw(save! output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
@@ -337,7 +337,7 @@ my %OPTDESC = (
'torsocks=s' => ['VAL|auto|no|yes',
'whether or not to wrap git and curl commands with torsocks'],
'no-torsocks' => 'alias for --torsocks=no',
-'save' => "save a search for `lei up'",
+'save!' => "do not save a search for `lei up'",
'import-remote!' => 'do not memoize remote messages into local store',
'type=s' => [ 'any|mid|git', 'disambiguate type' ],
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 2a0e9321..fd51fe38 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -243,6 +243,16 @@ sub pause_dedupe {
$oidx->commit_lazy;
}
+sub reset_dedupe {
+ my ($self) = @_;
+ prepare_dedupe($self);
+ my $lk = $self->lock_for_scope_fast;
+ for my $t (qw(xref3 over id2num)) {
+ $self->{oidx}->{dbh}->do("DELETE FROM $t");
+ }
+ pause_dedupe($self);
+}
+
sub mm { undef }
sub altid_map { {} }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d782d4c7..be6e178f 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -390,10 +390,16 @@ sub new {
-e $dst && !-d _ and die
"$dst exists and is not a directory\n";
$lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
+ $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
} elsif (substr($fmt, 0, 4) eq 'mbox') {
require PublicInbox::MboxReader;
$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
$self->{base_type} = 'mbox';
+ if ($lei->{cmd} eq 'q' &&
+ (($lei->path_to_fd($dst) // -1) < 0) &&
+ (-f $dst || !-e _)) {
+ $lei->{opt}->{save} //= \1;
+ }
} elsif ($fmt =~ /\Aimaps?\z/) {
require PublicInbox::NetWriter;
require PublicInbox::URIimap;
@@ -408,6 +414,7 @@ sub new {
$dst = $lei->{ovv}->{dst} = $$uri; # canonicalized
$lei->{net} = $net;
$self->{base_type} = 'imap';
+ $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
} elsif ($fmt eq 'text') {
require PublicInbox::LeiViewText;
$lei->{lvt} = PublicInbox::LeiViewText->new($lei);
@@ -454,9 +461,17 @@ sub _pre_augment_maildir {
open $self->{poke_dh}, '<', "${dst}cur" or die "open ${dst}cur: $!";
}
+sub clobber_dst_prepare ($;$) {
+ my ($lei, $f) = @_;
+ my $wait = (defined($f) && $lei->{sto}) ?
+ $lei->{sto}->ipc_do('lms_forget_folders', $f) : undef;
+ my $dedupe = $lei->{dedupe} or return;
+ $dedupe->reset_dedupe if $dedupe->can('reset_dedupe');
+}
+
sub _do_augment_maildir {
my ($self, $lei) = @_;
- return if ($lei->{opt}->{save} // 0) < 0;
+ return if $lei->{cmd} eq 'up';
my $dst = $lei->{ovv}->{dst};
my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
my $mdr = PublicInbox::MdirReader->new;
@@ -468,9 +483,11 @@ sub _do_augment_maildir {
$dedupe->pause_dedupe;
}
} elsif ($lse) {
+ clobber_dst_prepare($lei, "maildir:$dst");
$mdr->{shard_info} = $self->{shard_info};
$mdr->maildir_each_eml($dst, \&_md_update, $lei, $lse, 1);
} else {# clobber existing Maildir
+ clobber_dst_prepare($lei, "maildir:$dst");
$mdr->maildir_each_file($dst, \&_unlink);
}
}
@@ -487,7 +504,7 @@ sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
sub _do_augment_imap {
my ($self, $lei) = @_;
- return if ($lei->{opt}->{save} // 0) < 0;
+ return if $lei->{cmd} eq 'up';
my $net = $lei->{net};
my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
if ($lei->{opt}->{augment}) {
@@ -499,11 +516,13 @@ sub _do_augment_imap {
}
} elsif ($lse) {
my $delete_mic;
+ clobber_dst_prepare($lei, "$self->{uri}");
$net->imap_each($self->{uri}, \&_imap_augment_or_delete,
$lei, $lse, \$delete_mic);
$delete_mic->expunge if $delete_mic;
} elsif (!$self->{-wq_worker_nr}) { # undef or 0
# clobber existing IMAP folder
+ clobber_dst_prepare($lei, "$self->{uri}");
$net->imap_delete_all($self->{uri});
}
}
@@ -563,6 +582,8 @@ sub _pre_augment_mbox {
if $imp_before && !ref($imp_before);
die "--augment specified but $dst is not seekable\n" if
$lei->{opt}->{augment};
+ die "cannot --save with unseekable $dst\n" if
+ $lei->{dedupe} && $lei->{dedupe}->can('reset_dedupe');
}
if ($self->{zsfx} = PublicInbox::MboxReader::zsfx($dst)) {
pipe(my ($r, $w)) or die "pipe: $!";
@@ -581,10 +602,10 @@ sub _do_augment_mbox {
my ($self, $lei) = @_;
return unless $self->{seekable};
my $opt = $lei->{opt};
- return if ($opt->{save} // 0) < 0;
+ return if $lei->{cmd} eq 'up';
my $out = $lei->{1};
my ($fmt, $dst) = @{$lei->{ovv}}{qw(fmt dst)};
- return unless -s $out;
+ return clobber_dst_prepare($lei) unless -s $out;
unless ($opt->{augment} || $opt->{'import-before'}) {
truncate($out, 0) or die "truncate($dst): $!";
return;
@@ -599,6 +620,8 @@ sub _do_augment_mbox {
if ($opt->{augment}) {
$dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
+ } else {
+ clobber_dst_prepare($lei);
}
if ($opt->{'import-before'}) { # the default
my $lse = $lei->{lse};
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index c5a01527..ba11761a 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -64,7 +64,6 @@ sub lei_up {
my ($lei, @outs) = @_;
$lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
- $opt->{save} = -1;
my @local;
if (defined $opt->{all}) {
return $lei->fail("--all and @outs incompatible") if @outs;
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 2e6be1f0..4edee72a 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -28,7 +28,8 @@ ok(!glob("$o/cur/*"), 'last result cleared after augment-import');
lei_ok(qw(q -o), "maildir:$o", qw(m:qp@example.com));
@fn = glob("$o/cur/*:2,S");
-is(scalar(@fn), 1, "`seen' flag set on Maildir file");
+is(scalar(@fn), 1, "`seen' flag set on Maildir file") or
+ diag "$o contents: ", explain([glob("$o/*/*")]);
# ensure --no-import-before works
my $n = $fn[0];
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index eada2dd4..7aa9b84e 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -25,7 +25,7 @@ test_lei(sub {
my $home = $ENV{HOME};
my $in = $doc1->as_string;
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
- lei_ok qw(q -q --save z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
+ lei_ok qw(q -q z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
my %before = map { $_ => 1 } glob("$home/md/cur/*");
my $f = (keys %before)[0] or xbail({before => \%before});
is_deeply(eml_load($f), $doc1, 'doc1 matches');
@@ -52,8 +52,8 @@ test_lei(sub {
is_deeply(eml_load($f), $doc2, 'doc2 matches');
# check stdin
- lei_ok [qw(q --save - -o), "mboxcl2:mbcl2" ],
- undef, { -C => $home, %$lei_opt, 0 => \'d:last.week..'};
+ lei_ok [qw(q - -o), "mboxcl2:mbcl2" ], undef,
+ { -C => $home, %$lei_opt, 0 => \'d:last.week..'};
@s = glob("$home/.local/share/lei/saved-searches/mbcl2-*");
$cfg = PublicInbox::Config->new("$s[0]/lei.saved-search");
is_deeply $cfg->{'lei.q'}, 'd:last.week..',
@@ -68,7 +68,11 @@ test_lei(sub {
lei_ok([qw(up mbcl2)], undef, { -C => $home, %$lei_opt });
ok(-s "$home/mbcl2" > $size, 'size increased after up');
- ok(!lei(qw(up -q), $home), 'up fails w/o --save');
+ ok(!lei(qw(up -q), $home), 'up fails on unknown dir');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
+
+ lei_ok(qw(q --no-save d:last.week.. -q -o), "$home/no-save");
+ ok(!lei(qw(up -q), "$home/no-save"), 'up fails on --no-save');
like($lei_err, qr/--save was not used/, 'error noted --save');
lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
@@ -94,7 +98,7 @@ test_lei(sub {
open my $mb, '>', "$home/mbrd" or xbail "open $!";
print $mb $pre_existing;
close $mb or xbail "close: $!";
- lei_ok(qw(q --save -o mboxrd:mbrd m:qp@example.com -C), $home);
+ lei_ok(qw(q -o mboxrd:mbrd m:qp@example.com -C), $home);
open $mb, '<', "$home/mbrd" or xbail "open $!";
is_deeply([grep(/pre-existing/, <$mb>)], [],
'pre-existing messsage gone w/o augment');
@@ -107,7 +111,7 @@ test_lei(sub {
open $mb, '>', "$home/mbrd-aug" or xbail "open $!";
print $mb $pre_existing;
close $mb or xbail "close: $!";
- lei_ok(qw(q -a --save -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
+ lei_ok(qw(q -a -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
open $mb, '<', "$home/mbrd-aug" or xbail "open $!";
$mb = do { local $/; <$mb> };
like($mb, qr/pre-existing/, 'pre-existing message preserved w/ -a');
@@ -133,7 +137,7 @@ test_lei(sub {
my $o = "$home/dd-mid";
$in = $doc2->as_string . "\n-------\nappended list sig\n";
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
- lei_ok(qw(q --dedupe=mid --save m:testmessage@example.com -o), $o);
+ lei_ok(qw(q --dedupe=mid m:testmessage@example.com -o), $o);
my @m = glob("$o/cur/*");
is(scalar(@m), 1, '--dedupe=mid w/ --save');
$in = $doc2->as_string . "\n-------\nanother list sig\n";
@@ -143,8 +147,7 @@ test_lei(sub {
for my $dd (qw(content)) {
$o = "$home/dd-$dd";
- lei_ok(qw(q --save m:testmessage@example.com -o), $o,
- "--dedupe=$dd");
+ lei_ok(qw(q m:testmessage@example.com -o), $o, "--dedupe=$dd");
@m = glob("$o/cur/*");
is(scalar(@m), 3, 'all 3 matches with dedupe='.$dd);
}
@@ -153,7 +156,7 @@ test_lei(sub {
$o = "$home/dd-oid";
my $ibx = create_inbox 'ibx', indexlevel => 'medium',
tmpdir => "$home/v1", sub {};
- lei_ok(qw(q --save --dedupe=oid m:qp@example.com -o), $o,
+ lei_ok(qw(q --dedupe=oid m:qp@example.com -o), $o,
'-I', $ibx->{inboxdir});
@m = glob("$o/cur/*");
is(scalar(@m), 1, 'got first result');
@@ -203,8 +206,7 @@ test_lei(sub {
lei_ok([qw(edit-search), $v2s], { VISUAL => 'cat', EDITOR => 'cat' });
like($lei_out, qr/^\[lei/sm, 'edit-search can cat');
- lei_ok('-C', "$home/v2s",
- qw(q -q --save -o ../s m:testmessage@example.com));
+ lei_ok('-C', "$home/v2s", qw(q -q -o ../s m:testmessage@example.com));
lei_ok qw(ls-search);
unlike $lei_out, qr{/\.\./s$}sm, 'relative path not in ls-search';
like $lei_out, qr{^\Q$home\E/s$}sm,
diff --git a/t/lei.t b/t/lei.t
index 8211c01d..dfbcb1f3 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -114,7 +114,7 @@ my $test_completion = sub {
%out = map { $_ => 1 } split(/\s+/s, $lei_out);
for my $sw (qw(-f --format -o --output --mfolder --augment -a
--mua --no-local --local --verbose -v
- --save --no-remote --remote --torsocks
+ --save --no-save --no-remote --remote --torsocks
--reverse -r )) {
ok($out{$sw}, "$sw offered as `lei q' completion");
}
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 35904706..e8958c64 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -75,7 +75,7 @@ for my $mbox (@MBOX) {
my ($tmpdir, $for_destroy) = tmpdir();
local $ENV{TMPDIR} = $tmpdir;
open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!;
-my $lei = bless { 2 => $err }, 'PublicInbox::LEI';
+my $lei = bless { 2 => $err, cmd => 'test' }, 'PublicInbox::LEI';
my $commit = sub {
$_[0] = undef; # wcb
delete $lei->{1};
^ permalink raw reply related [relevance 31%]
* [PATCH] lei: implicitly watch all Maildirs it knows about
@ 2021-08-19 9:49 54% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-19 9:49 UTC (permalink / raw)
To: meta
This allows MUA-made flag changes to Maildirs to be instantly
read and acknowledged for future search results.
In the future, it may be used to speed up --augment and
--import-before (the default) with with "lei q".
---
lib/PublicInbox/LEI.pm | 42 +++++++++++++++++++++++++++------
lib/PublicInbox/LeiMailSync.pm | 10 ++++++++
lib/PublicInbox/LeiNoteEvent.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 5 ++++
4 files changed, 51 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 79dc9bf9..4d5c61fe 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1370,6 +1370,14 @@ sub cancel_maildir_watch ($$) {
for my $x (@{$w // []}) { $x->cancel }
}
+sub add_maildir_watch ($$) {
+ my ($d, $cfg_f) = @_;
+ if (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
+ my @w = $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
+ push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
+ }
+}
+
sub refresh_watches {
my ($lei) = @_;
my $cfg = _lei_cfg($lei) or return;
@@ -1380,7 +1388,7 @@ sub refresh_watches {
for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
my $url = substr($w, length('watch.'), -length('.state'));
require PublicInbox::LeiWatch;
- my $lw = $watches->{$url} //= PublicInbox::LeiWatch->new($url);
+ $watches->{$url} //= PublicInbox::LeiWatch->new($url);
$seen{$url} = undef;
my $state = $cfg->get_1("watch.$url", 'state');
if (!watch_state_ok($state)) {
@@ -1391,16 +1399,36 @@ sub refresh_watches {
my $d = canonpath_harder($1);
if ($state eq 'pause') {
cancel_maildir_watch($d, $cfg_f);
- } elsif (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
- my @w = $dir_idle->add_watches(
- ["$d/cur", "$d/new"], 1);
- push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
+ } else {
+ add_maildir_watch($d, $cfg_f);
}
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1,
- "E: watch $url not supported, yet");
+ $lei->child_error(1, "E: watch $url not supported, yet")
+ }
+ }
+
+ # add all known Maildir folders as implicit watches
+ my $sto = $lei->_lei_store;
+ my $renames = 0;
+ if (my $lms = $sto ? $sto->search->lms : undef) {
+ for my $d ($lms->folders('maildir:')) {
+ substr($d, 0, length('maildir:')) = '';
+ my $cd = canonpath_harder($d);
+ my $f = "maildir:$cd";
+
+ # fixup old bugs while we're iterating:
+ if ($d ne $cd) {
+ $sto->ipc_do('lms_rename_folder',
+ "maildir:$d", $f);
+ ++$renames;
+ }
+ next if $watches->{$f}; # may be set to pause
+ $watches->{$f} = PublicInbox::LeiWatch->new($f);
+ $seen{$f} = undef;
+ add_maildir_watch($cd, $cfg_f);
}
}
+ my $wait = $renames ? $sto->ipc_do('done') : undef;
if ($old) { # cull old non-existent entries
for my $url (keys %$old) {
next if exists $seen{$url};
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 6dfa03be..80e1bb9d 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -412,6 +412,16 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
+# only used for changing canonicalization errors
+sub rename_folder {
+ my ($self, $old, $new) = @_;
+ my $fid = delete($self->{fmap}->{$old}) //
+ fid_for($self, $old) // return;
+ $self->{dbh}->do(<<EOM, undef, $new, $fid);
+UPDATE folders SET loc = ? WHERE fid = ?
+EOM
+}
+
sub imap_oidbin ($$$) {
my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY
my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index d6511cf6..1cd15296 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -74,7 +74,7 @@ sub lei_note_event {
my $err = $lms->arg2folder($lei, [ $folder ]);
return if $err->{fail};
undef $lms;
- my $state = $cfg->get_1("watch.$folder", 'state') // 'pause';
+ my $state = $cfg->get_1("watch.$folder", 'state') // 'tag-rw';
return if $state eq 'pause';
$lei->ale; # prepare
$sto->write_prepare($lei);
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e8334187..bbd853e5 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -222,6 +222,11 @@ sub lms_forget_folders {
for my $f (@folders) { $lms->forget_folder($f) }
}
+sub lms_rename_folder {
+ my ($self, $old, $new) = @_;
+ _lms_rw($self)->rename_folder($old, $new);
+}
+
sub set_sync_info {
my ($self, $oidhex, $folder, $id) = @_;
_lms_rw($self)->set_src($oidhex, $folder, $id);
^ permalink raw reply related [relevance 54%]
* [PATCH] lei: add missing LeiWatch lazy-load
@ 2021-08-24 13:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-24 13:04 UTC (permalink / raw)
To: meta
I'm not sure if this class will actually be needed, but
we need to load it while we're using it.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 4d5c61fe..ea3ec0fe 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1423,6 +1423,7 @@ sub refresh_watches {
++$renames;
}
next if $watches->{$f}; # may be set to pause
+ require PublicInbox::LeiWatch;
$watches->{$f} = PublicInbox::LeiWatch->new($f);
$seen{$f} = undef;
add_maildir_watch($cd, $cfg_f);
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: non-blocking lei/store->done in lei-daemon
@ 2021-08-24 13:06 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-24 13:06 UTC (permalink / raw)
To: meta
This allows client sockets to wait for "done" commits to
lei/store while the daemon reacts asynchronously. The goal
of this change is to keep the script/lei client alive until
lei/store commits changes to the filesystem, but without
blocking the lei-daemon event loop. It depends on Perl
refcounting to close the socket.
This change also highlighted our over-use of "done" requests to
lei/store processes, which is now corrected so we only issue it
on collective socket EOF rather than upon reaping every single
worker.
This also fixes "lei forget-mail-sync" when it is the initial
command.
This took several iterations and much debugging to arrive at the
current implementation:
1. The initial iteration of this change utilized socket passing
from lei-daemon to lei/store, which necessitated switching
from faster pipes to slower Unix sockets.
2. The second iteration switched to registering notification sockets
independently of "done" requests, but that could lead to early
wakeups when "done" was requested by other workers. This
appeared to work most of the time, but suffered races under
high load which were difficult to track down.
Finally, this iteration passes the stringified socket GLOB ref
to lei/store which is echoed back to lei-daemon upon completion
of that particular "done" request.
---
lib/PublicInbox/LEI.pm | 19 ++++++++++++++-
lib/PublicInbox/LeiForgetMailSync.pm | 4 ++--
lib/PublicInbox/LeiImportKw.pm | 3 ++-
lib/PublicInbox/LeiNoteEvent.pm | 4 ++--
lib/PublicInbox/LeiPmdir.pm | 5 ++--
lib/PublicInbox/LeiStore.pm | 35 +++++++++++++++++++---------
lib/PublicInbox/LeiXSearch.pm | 4 ++--
lib/PublicInbox/PktOp.pm | 9 ++++---
8 files changed, 59 insertions(+), 24 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ea3ec0fe..5694e92c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -37,6 +37,7 @@ $GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => [ ino watches ] }
+our %LIVE_SOCK; # "GLOB(0x....)" => $lei->{sock}
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -565,6 +566,7 @@ sub _lei_atfork_child {
$dir_idle->force_close if $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
+ %LIVE_SOCK = ();
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
undef $errors_log;
$quit = \&CORE::exit;
@@ -1429,7 +1431,7 @@ sub refresh_watches {
add_maildir_watch($cd, $cfg_f);
}
}
- my $wait = $renames ? $sto->ipc_do('done') : undef;
+ $lei->sto_done_request if $renames;
if ($old) { # cull old non-existent entries
for my $url (keys %$old) {
next if exists $seen{$url};
@@ -1463,4 +1465,19 @@ sub lms { # read-only LeiMailSync
$lse ? $lse->lms : undef;
}
+sub sto_done_request { # only call this from lei-daemon process (not workers)
+ my ($lei, $sock) = @_;
+ if ($sock //= $lei->{sock}) {
+ $LIVE_SOCK{"$sock"} = $sock;
+ $lei->{sto}->ipc_do('done', "$sock"); # issue, async wait
+ } else { # forcibly wait
+ my $wait = $lei->{sto}->ipc_do('done');
+ }
+}
+
+sub sto_done_complete { # called in lei-daemon when LeiStore->done is complete
+ my ($sock_str) = @_;
+ delete $LIVE_SOCK{$sock_str}; # frees {sock} for waiting lei clients
+}
+
1;
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 940ca1b6..2b4e58a9 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -16,12 +16,12 @@ sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
my $lms = $lei->lms or return;
my $sto = $lei->_lei_store or return; # may disappear due to race
- $sto->write_prepare;
+ $sto->write_prepare($lei);
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
$sto->ipc_do('lms_forget_folders', @folders);
- my $wait = $sto->ipc_do('done');
+ $lei->sto_done_request;
}
*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 2878cbdf..402125cf 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -13,6 +13,7 @@ sub new {
my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
$op_c->{ops} = $ops; # for PktOp->event_step
+ $self->{lei_sock} = $lei->{sock};
$lei->{ikw} = $self;
}
@@ -42,13 +43,13 @@ sub ck_update_kw { # via wq_io_do
sub ikw_done_wait {
my ($arg, $pid) = @_;
my ($self, $lei) = @$arg;
- my $wait = $lei->{sto}->ipc_do('done');
$lei->can('wq_done_wait')->($arg, $pid);
}
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $ikw = delete $lei->{ikw} or return $lei->fail;
+ $lei->sto_done_request($ikw->{lei_sock});
$ikw->wq_wait_old(\&ikw_done_wait, $lei);
}
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 1cd15296..6a40ba39 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -15,7 +15,7 @@ sub flush_lei ($) {
if (my $lne = delete $lei->{cfg}->{-lei_note_event}) {
$lne->wq_close(1, undef, $lei); # runs _lei_wq_eof;
} elsif ($lei->{sto}) { # lms_clear_src calls only:
- my $wait = $lei->{sto}->ipc_do('done');
+ $lei->sto_done_request;
}
}
@@ -117,7 +117,7 @@ sub lne_done_wait {
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $lne = delete $lei->{lne} or return $lei->fail;
- my $wait = $lei->{sto}->ipc_do('done');
+ $lei->sto_done_request;
$lne->wq_wait_old(\&lne_done_wait, $lei);
}
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index 760f276c..59cf886e 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -25,6 +25,7 @@ sub new {
my ($op_c, $ops) = $lei->workers_start($self, $nproc,
undef, { ipt => $ipt }); # LeiInput subclass
$op_c->{ops} = $ops; # for PktOp->event_step
+ $self->{lei_sock} = $lei->{sock}; # keep client for pmd_done_wait
$lei->{pmd} = $self;
}
@@ -32,7 +33,7 @@ sub ipc_atfork_child {
my ($self) = @_;
my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}';
$ipt->{lei} = $self->{lei};
- $ipt->ipc_atfork_child;
+ $ipt->ipc_atfork_child; # calls _lei_atfork_child;
}
sub each_mdir_fn { # maildir_each_file callback
@@ -48,13 +49,13 @@ sub mdir_iter { # via wq_io_do
sub pmd_done_wait {
my ($arg, $pid) = @_;
my ($self, $lei) = @$arg;
- my $wait = $lei->{sto}->ipc_do('done');
$lei->can('wq_done_wait')->($arg, $pid);
}
sub _lei_wq_eof { # EOF callback for main lei daemon
my ($lei) = @_;
my $pmd = delete $lei->{pmd} or return $lei->fail;
+ $lei->sto_done_request($pmd->{lei_sock});
$pmd->wq_wait_old(\&pmd_done_wait, $lei);
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index bbd853e5..28e36e89 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -471,7 +471,7 @@ sub xchg_stderr {
}
sub done {
- my ($self) = @_;
+ my ($self, $sock_ref) = @_;
my $err = '';
if (my $im = delete($self->{im})) {
eval { $im->done };
@@ -486,6 +486,10 @@ sub done {
$self->{priv_eidx}->done; # V2Writable::done
xchg_stderr($self);
die $err if $err;
+
+ # notify clients ->done has been issued
+ defined($sock_ref) and
+ $self->{s2d_op_p}->pkt_do('sto_done_complete', $sock_ref);
}
sub ipc_atfork_child {
@@ -493,28 +497,37 @@ sub ipc_atfork_child {
my $lei = $self->{lei};
$lei->_lei_atfork_child(1) if $lei;
xchg_stderr($self);
- if (my $err = delete($self->{err_pipe})) {
- close $err->[0];
- $self->{-err_wr} = $err->[1];
+ if (my $to_close = delete($self->{to_close})) {
+ close($_) for @$to_close;
}
$self->SUPER::ipc_atfork_child;
}
sub write_prepare {
my ($self, $lei) = @_;
+ $lei // die 'BUG: $lei not passed';
unless ($self->{-ipc_req}) {
- my $d = $lei->store_path;
- $self->ipc_lock_init("$d/ipc.lock");
- substr($d, -length('/lei/store'), 10, '');
+ # s2d => store-to-daemon messages
+ require PublicInbox::PktOp;
+ my ($s2d_op_c, $s2d_op_p) = PublicInbox::PktOp->pair;
+ my $dir = $lei->store_path;
+ $self->ipc_lock_init("$dir/ipc.lock");
+ substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
- my $err_pipe = [ $r, $w ];
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
- $self->ipc_worker_spawn("lei/store $d", $lei->oldset,
- { lei => $lei, err_pipe => $err_pipe });
+ $self->ipc_worker_spawn("lei/store $dir", $lei->oldset, {
+ lei => $lei,
+ -err_wr => $w,
+ to_close => [ $r, $s2d_op_c->{sock} ],
+ s2d_op_p => $s2d_op_p,
+ });
require PublicInbox::LeiStoreErr;
- PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
+ PublicInbox::LeiStoreErr->new($r, $lei);
+ $s2d_op_c->{ops} = {
+ sto_done_complete => [ $lei->can('sto_done_complete') ]
+ };
}
$lei->{sto} = $self;
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 5e34d864..1f83e582 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -374,8 +374,8 @@ sub query_done { # EOF callback for main daemon
if ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) {
warn "BUG: {sto} missing with --mail-sync";
}
- my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
- $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
+ $lei->sto_done_request if $lei->{sto};
+ my $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
diff --git a/lib/PublicInbox/PktOp.pm b/lib/PublicInbox/PktOp.pm
index 92e150a4..10942dd1 100644
--- a/lib/PublicInbox/PktOp.pm
+++ b/lib/PublicInbox/PktOp.pm
@@ -56,9 +56,12 @@ sub event_step {
($cmd, @pargs) = split(/ /, $msg);
}
my $op = $self->{ops}->{$cmd //= $msg};
- die "BUG: unknown message: `$cmd'" unless $op;
- my ($sub, @args) = @$op;
- $sub->(@args, @pargs);
+ if ($op) {
+ my ($sub, @args) = @$op;
+ $sub->(@args, @pargs);
+ } elsif ($msg ne '') {
+ die "BUG: unknown message: `$cmd'";
+ }
return $self->close if $msg eq ''; # close on EOF
}
}
^ permalink raw reply related [relevance 70%]
* [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
` (2 preceding siblings ...)
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
@ 2021-08-24 20:14 71% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-24 20:14 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> 2/3 is probably a fix for a long-standing problem, 3/3 was
> noticed while working on it. If 2/3 doesn't fix it, then maybe
> 1/3 will help us narrow it down.
s/is probably/was almost certainly/ \o/
^ permalink raw reply [relevance 71%]
* [PATCH 0/2] minor lei usability tweaks
@ 2021-08-25 8:40 71% Eric Wong
2021-08-25 8:40 59% ` [PATCH 1/2] lei up: improve --all=local stderr output Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-08-25 8:40 UTC (permalink / raw)
To: meta
Some things I noticed replacing much of my existing mail
stack with lei...
Eric Wong (2):
lei up: improve --all=local stderr output
lei_mail_sync: remove warning message from caller
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiFinmsg.pm | 21 +++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 ++
lib/PublicInbox/LeiXSearch.pm | 9 ++++++---
6 files changed, 37 insertions(+), 4 deletions(-)
create mode 100644 lib/PublicInbox/LeiFinmsg.pm
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei up: improve --all=local stderr output
2021-08-25 8:40 71% [PATCH 0/2] minor lei usability tweaks Eric Wong
@ 2021-08-25 8:40 59% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-25 8:40 UTC (permalink / raw)
To: meta
The "# $NR written to $DEST ($total matches)" messages are
arguably the most useful output of "lei up --all=local",
but they get intermixed with progress messages from various
workers. Queue up these finalization messages and only spit
them out on ->DESTROY.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiFinmsg.pm | 21 +++++++++++++++++++++
lib/PublicInbox/LeiUp.pm | 2 ++
lib/PublicInbox/LeiXSearch.pm | 9 ++++++---
5 files changed, 36 insertions(+), 3 deletions(-)
create mode 100644 lib/PublicInbox/LeiFinmsg.pm
diff --git a/MANIFEST b/MANIFEST
index fb9f16bf..cf7268ed 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiDedupe.pm
lib/PublicInbox/LeiEditSearch.pm
lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
+lib/PublicInbox/LeiFinmsg.pm
lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5694e92c..28fe0c83 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -486,6 +486,12 @@ sub err ($;@) {
sub qerr ($;@) { $_[0]->{opt}->{quiet} or err(shift, @_) }
+sub qfin { # show message on finalization (LeiFinmsg)
+ my ($lei, $msg) = @_;
+ return if $lei->{opt}->{quiet};
+ $lei->{fmsg} ? push(@{$lei->{fmsg}}, "$msg\n") : qerr($lei, $msg);
+}
+
sub fail_handler ($;$$) {
my ($lei, $code, $io) = @_;
close($io) if $io; # needed to avoid warnings on SIGPIPE
diff --git a/lib/PublicInbox/LeiFinmsg.pm b/lib/PublicInbox/LeiFinmsg.pm
new file mode 100644
index 00000000..0ef5f070
--- /dev/null
+++ b/lib/PublicInbox/LeiFinmsg.pm
@@ -0,0 +1,21 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Finalization messages, used to queue up a bunch of messages which
+# only get written out on ->DESTROY
+package PublicInbox::LeiFinmsg;
+use strict;
+use v5.10.1;
+
+sub new {
+ my ($cls, $io) = @_;
+ bless [ $io ], $cls;
+}
+
+sub DESTROY {
+ my ($self) = @_;
+ my $io = shift @$self;
+ print $io @$self;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index ba11761a..85efd9f5 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -48,6 +48,8 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
+ require PublicInbox::LeiFinmsg;
+ $lei->{fmsg} //= PublicInbox::LeiFinmsg->new($lei->{2});
my $l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} };
delete $l->{sock};
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 1f83e582..b9f0d692 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -398,9 +398,12 @@ Error closing $lei->{ovv}->{dst}: $!
if ($lei->{-progress}) {
my $tot = $lei->{-mset_total} // 0;
my $nr = $lei->{-nr_write} // 0;
- $lei->qerr($l2m ?
- "# $nr written to $lei->{ovv}->{dst} ($tot matches)" :
- "# $tot matches");
+ if ($l2m) {
+ $lei->qfin("# $nr written to " .
+ "$lei->{ovv}->{dst} ($tot matches)");
+ } else {
+ $lei->qerr("# $tot matches");
+ }
}
$lei->start_mua if $start_mua;
$lei->dclose;
^ permalink raw reply related [relevance 59%]
* [PATCH 00/10] lei: several bug fixes and refinements
@ 2021-08-31 11:21 71% Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
` (5 more replies)
0 siblings, 6 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Another pile of things found while working on better
synchronization.
Eric Wong (10):
lei_mail_sync: forget_folder: simplify code
lei prune-mail-sync: handle --all (no args)
lei_mail_sync: simplify group2folders
lei_mail_sync: make rename_folder more robust
t/lei-watch: avoid race between glob + readlink
lei note-event: always flush changes on daemon exit
lei: refresh watches before MUA spawn for Maildir
lei_mail_sync: set_src uses binary OIDs
lei: fix error reporting from lei/store -> lei-daemon
lei/store: correctly delete entries from over
lib/PublicInbox/LEI.pm | 4 ++++
lib/PublicInbox/LeiMailSync.pm | 36 ++++++++++++++++++++++------------
lib/PublicInbox/LeiStore.pm | 8 +++++---
lib/PublicInbox/LeiStoreErr.pm | 14 +++++++++++--
t/lei-watch.t | 2 +-
t/lei_mail_sync.t | 15 +++++++-------
6 files changed, 53 insertions(+), 26 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 02/10] lei prune-mail-sync: handle --all (no args)
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 05/10] t/lei-watch: avoid race between glob + readlink Eric Wong
` (4 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
This still needs tests, but I noticed "--all" w/o "local" or
"remote" was not working correctly since split() returned
an empty array.
---
lib/PublicInbox/LeiMailSync.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 57b56b3c..bf8fb7de 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -330,6 +330,7 @@ sub group2folders {
EOM
my %x = map { $_ => $_ } split(/,/, $all);
my @ok = grep(defined, delete(@x{qw(local remote), ''}));
+ push(@ok, '') if $all eq '';
my @no = keys %x;
if (@no) {
@no = (join(',', @no));
^ permalink raw reply related [relevance 71%]
* [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
` (3 preceding siblings ...)
2021-08-31 11:21 71% ` [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir Eric Wong
@ 2021-08-31 11:21 94% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 10/10] lei/store: correctly delete entries from over Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
We must set autoflush to ensure timely notification of clients;
and lei-daemon must not block when waiting on reads in case of
spurious wakeups.
Finally, if no clients are connected to lei-daemon, write to
syslog to ensure the error is visible.
---
lib/PublicInbox/LeiStore.pm | 2 ++
lib/PublicInbox/LeiStoreErr.pm | 14 ++++++++++++--
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 5a48c064..0652137e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -28,6 +28,7 @@ use PublicInbox::Spawn qw(spawn);
use List::Util qw(max);
use File::Temp ();
use POSIX ();
+use IO::Handle (); # ->autoflush
sub new {
my (undef, $dir, $opt) = @_;
@@ -514,6 +515,7 @@ sub write_prepare {
$self->ipc_lock_init("$dir/ipc.lock");
substr($dir, -length('/lei/store'), 10, '');
pipe(my ($r, $w)) or die "pipe: $!";
+ $w->autoflush(1);
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index 68ce96d6..5f9ba24d 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -8,22 +8,32 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::DS);
use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use Sys::Syslog qw(openlog syslog closelog);
+use IO::Handle (); # ->blocking
sub new {
my ($cls, $rd, $lei) = @_;
my $self = bless { sock => $rd, store_path => $lei->store_path }, $cls;
+ $rd->blocking(0);
$self->SUPER::new($rd, EPOLLIN | EPOLLONESHOT);
}
sub event_step {
my ($self) = @_;
- $self->do_read(\(my $rbuf), 4096) or return;
+ my $rbuf = $self->{rbuf} // \(my $x = '');
+ $self->do_read($rbuf, 8192, length($$rbuf)) or return;
my $cb;
+ my $printed;
for my $lei (values %PublicInbox::DS::DescriptorMap) {
$cb = $lei->can('store_path') // next;
next if $cb->($lei) ne $self->{store_path};
my $err = $lei->{2} // next;
- print $err $rbuf;
+ print $err $$rbuf and $printed = 1;
+ }
+ if (!$printed) {
+ openlog('lei-store', 'pid,nowait,nofatal,ndelay', 'user');
+ for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
+ closelog(); # don't share across fork
}
}
^ permalink raw reply related [relevance 94%]
* [PATCH 05/10] t/lei-watch: avoid race between glob + readlink
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 06/10] lei note-event: always flush changes on daemon exit Eric Wong
` (3 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Open file handles in lei-daemon may be unstable so we need to
account for readlink() returning undef.
---
t/lei-watch.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 86fa6649..a881fbb9 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -25,7 +25,7 @@ test_lei(sub {
lei_ok 'daemon-pid'; chomp(my $pid = $lei_out);
skip 'missing /proc/$PID/fd', 1 if !-d "/proc/$pid/fd";
my @ino = grep {
- readlink($_) =~ /\binotify\b/
+ (readlink($_) // '') =~ /\binotify\b/
} glob("/proc/$pid/fd/*");
is(scalar(@ino), 1, 'only one inotify FD');
my $ino_fd = (split('/', $ino[0]))[-1];
^ permalink raw reply related [relevance 71%]
* [PATCH 06/10] lei note-event: always flush changes on daemon exit
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
2021-08-31 11:21 71% ` [PATCH 05/10] t/lei-watch: avoid race between glob + readlink Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 71% ` [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir Eric Wong
` (2 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Because the timer may not fire in time before daemon shutdown.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 28fe0c83..520fb519 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1254,6 +1254,7 @@ sub lazy_start {
my (undef, $eof_p) = PublicInbox::PktOp->pair;
sub {
$exit_code //= shift;
+ eval 'PublicInbox::LeiNoteEvent::flush_task()';
my $lis = $pil or exit($exit_code);
# closing eof_p triggers \&noop wakeup
$listener = $eof_p = $pil = $path = undef;
^ permalink raw reply related [relevance 71%]
* [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
` (2 preceding siblings ...)
2021-08-31 11:21 71% ` [PATCH 06/10] lei note-event: always flush changes on daemon exit Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
2021-08-31 11:21 94% ` [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon Eric Wong
2021-08-31 11:21 71% ` [PATCH 10/10] lei/store: correctly delete entries from over Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
If we possibly just wrote or created a Maildir, ensure it's
monitored by the lei watch mechanism.
---
lib/PublicInbox/LEI.pm | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 520fb519..41e811ca 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -951,6 +951,9 @@ sub exec_buf ($$) {
sub start_mua {
my ($self) = @_;
+ if ($self->{ovv}->{fmt} =~ /\A(?:maildir)\z/) { # TODO: IMAP
+ refresh_watches($self);
+ }
my $mua = $self->{opt}->{mua} // return;
my $mfolder = $self->{ovv}->{dst};
my (@cmd, $replaced);
^ permalink raw reply related [relevance 71%]
* [PATCH 10/10] lei/store: correctly delete entries from over
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
` (4 preceding siblings ...)
2021-08-31 11:21 94% ` [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon Eric Wong
@ 2021-08-31 11:21 71% ` Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 11:21 UTC (permalink / raw)
To: meta
Some of these errors were inadvertantly lost due to delayed
error reporting in the past.
---
lib/PublicInbox/LeiStore.pm | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 0652137e..ab39043e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -243,8 +243,8 @@ sub remove_docids ($;@) {
my $eidx = eidx_init($self);
for my $docid (@docids) {
$eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
- $self->{oidx}->delete_by_num($docid);
- $self->{oidx}->{dbh}->do(<<EOF, undef, $docid);
+ $eidx->{oidx}->delete_by_num($docid);
+ $eidx->{oidx}->{dbh}->do(<<EOF, undef, $docid);
DELETE FROM xref3 WHERE docid = ?
EOF
}
^ permalink raw reply related [relevance 71%]
* [PATCH] lei up: only show finmsg in top-level lei-daemon
@ 2021-08-31 19:38 90% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-31 19:38 UTC (permalink / raw)
To: meta
->DESTROY can get triggered in child processes, which
unnecessarily duplicates messages queued up for display
when lei spawns extra workers.
---
lib/PublicInbox/LeiFinmsg.pm | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiFinmsg.pm b/lib/PublicInbox/LeiFinmsg.pm
index 0ef5f070..395e7d3c 100644
--- a/lib/PublicInbox/LeiFinmsg.pm
+++ b/lib/PublicInbox/LeiFinmsg.pm
@@ -9,13 +9,13 @@ use v5.10.1;
sub new {
my ($cls, $io) = @_;
- bless [ $io ], $cls;
+ bless [ $io, $$ ], $cls;
}
sub DESTROY {
my ($self) = @_;
my $io = shift @$self;
- print $io @$self;
+ shift(@$self) == $$ and print $io @$self;
}
1;
^ permalink raw reply related [relevance 90%]
* [PATCH 0/3] lei: auto keyword propagation to Maildirs
@ 2021-09-02 10:17 71% Eric Wong
2021-09-02 10:17 51% ` [PATCH 3/3] lei: propagate keyword changes from lei/store Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-02 10:17 UTC (permalink / raw)
To: meta
At least the tests pass, and getting t/lei-export-kw.t to pass
after 3/3 was no small feat, but I believe everything is more
correct now (especially after the 10-patch series posted
yesterday-ish).
Patches 1 and 2 were developed while fixing 3/3, since the stuff
in t/lei-auto-watch.t happened to work right away while
preserving the behavior of t/lei-export-kw.t (unchanged) proved
extremely challenging in my current mental state.
Eric Wong (3):
lei_mail_sync: do not use transactions
lei_input: set and prepare watches early
lei: propagate keyword changes from lei/store
MANIFEST | 1 +
lib/PublicInbox/LeiExportKw.pm | 24 +++----
lib/PublicInbox/LeiInput.pm | 20 ++++--
lib/PublicInbox/LeiMailSync.pm | 14 +---
lib/PublicInbox/LeiNoteEvent.pm | 14 ++--
lib/PublicInbox/LeiStore.pm | 122 +++++++++++++++++++++++++++++---
t/lei-auto-watch.t | 45 ++++++++++++
t/lei_mail_sync.t | 18 ++---
8 files changed, 196 insertions(+), 62 deletions(-)
create mode 100644 t/lei-auto-watch.t
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei: propagate keyword changes from lei/store
2021-09-02 10:17 71% [PATCH 0/3] lei: auto keyword propagation to Maildirs Eric Wong
@ 2021-09-02 10:17 51% ` Eric Wong
2021-09-02 10:25 67% ` [SQUASH 4/3] t/lei-auto-watch: workaround for FreeBSD kevent Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-02 10:17 UTC (permalink / raw)
To: meta
This works with existing inotify/EVFILT_VNODE functionality to
propagate changes made from one Maildir to another Maildir.
I chose the lei/store worker process to handle this since
propagating changes back into lei-daemon on a massive scale
could lead to dead-locking while both processes are attempting
to write to each other. Eliminating IPC overhead is a nice
side effect, but could hurt performance if Maildirs are slow.
The code for "lei export-kw" is significantly revamped to match
the new code used in the "lei/store" daemon. It should be more
correct w.r.t. corner-cases and stale entries, but perhaps
better tests need to be written.
---
MANIFEST | 1 +
lib/PublicInbox/LeiExportKw.pm | 24 ++++----
lib/PublicInbox/LeiNoteEvent.pm | 14 ++---
lib/PublicInbox/LeiStore.pm | 105 ++++++++++++++++++++++++++++++--
t/lei-auto-watch.t | 45 ++++++++++++++
5 files changed, 165 insertions(+), 24 deletions(-)
create mode 100644 t/lei-auto-watch.t
diff --git a/MANIFEST b/MANIFEST
index cf7268ed..be6ec927 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -423,6 +423,7 @@ t/init.t
t/ipc.t
t/iso-2202-jp.eml
t/kqnotify.t
+t/lei-auto-watch.t
t/lei-convert.t
t/lei-daemon.t
t/lei-export-kw.t
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 42a5ff22..78c6c6f9 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -25,12 +25,11 @@ sub export_kw_md { # LeiMailSync->each_src callback
}
$bn .= ':2,'.
PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown);
+ return if $bn eq $$id;
my $dst = "$mdir/cur/$bn";
- my @fail;
my $lei = $self->{lei};
for my $d (@try) {
my $src = "$mdir/$d/$$id";
- next if $src eq $dst;
# we use link(2) + unlink(2) since rename(2) may
# inadvertently clobber if the "uniquefilename" part wasn't
@@ -44,20 +43,19 @@ sub export_kw_md { # LeiMailSync->each_src callback
$lei->{sto}->ipc_do('lms_mv_src', "maildir:$mdir",
$oidbin, $id, $bn);
return; # success anyways if link(2) worked
- }
- if ($! == ENOENT && !-e $src) { # some other process moved it
- $lei->{sto}->ipc_do('lms_clear_src',
- "maildir:$mdir", $id);
- next;
- }
- push @fail, $src if $! != EEXIST;
+ } elsif ($! == EEXIST) { # lost race with lei/store?
+ return;
+ } elsif ($! != ENOENT) {
+ $lei->child_error(1, "E: link($src -> $dst): $!");
+ } # else loop @try
}
- return unless @fail;
- # both tries failed
my $e = $!;
- my $orig = '['.join('|', @fail).']';
+ # both tries failed
my $oidhex = unpack('H*', $oidbin);
- $lei->child_error(1, "link($orig, $dst) ($oidhex): $e");
+ my $src = "$mdir/{".join(',', @try)."}/$$id";
+ $lei->child_error(1, "link($src -> $dst) ($oidhex): $e");
+ for (@try) { return if -e "$mdir/$_/$$id" }
+ $lei->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
}
sub export_kw_imap { # LeiMailSync->each_src callback
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 6a40ba39..41415346 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -36,32 +36,31 @@ sub note_event_arm_done ($) {
}
sub eml_event ($$$$) {
- my ($self, $eml, $kw, $state) = @_;
+ my ($self, $eml, $vmd, $state) = @_;
my $sto = $self->{lei}->{sto};
my $lse = $self->{lse} //= $sto->search;
- my $vmd = { kw => $kw };
if ($state =~ /\Aimport-(?:rw|ro)\z/) {
$sto->ipc_do('set_eml', $eml, $vmd);
} elsif ($state =~ /\Aindex-(?:rw|ro)\z/) {
my $xoids = $self->{lei}->ale->xoids_for($eml);
$sto->ipc_do('index_eml_only', $eml, $vmd, $xoids);
} elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
- my $c = $lse->kw_changed($eml, $kw, my $docids = []);
+ my $c = $lse->kw_changed($eml, $vmd->{kw}, my $docids = []);
if (scalar @$docids) { # already in lei/store
$sto->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
} elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
# it's in an external, only set kw, here
$sto->ipc_do('set_xvmd', $xoids, $eml, $vmd);
- } # else { totally unknown
+ } # else { totally unknown: ignore
} else {
warn "unknown state: $state (in $self->{lei}->{cfg}->{'-f'})\n";
}
}
sub maildir_event { # via wq_io_do
- my ($self, $fn, $kw, $state) = @_;
+ my ($self, $fn, $vmd, $state) = @_;
my $eml = PublicInbox::InboxWritable::eml_from_path($fn) // return;
- eml_event($self, $eml, $kw, $state);
+ eml_event($self, $eml, $vmd, $state);
}
sub lei_note_event {
@@ -98,7 +97,8 @@ sub lei_note_event {
// return;
return if index($fl, 'T') >= 0;
my $kw = PublicInbox::MdirReader::flags2kw($fl);
- $self->wq_io_do('maildir_event', [], $fn, $kw, $state);
+ my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] };
+ $self->wq_io_do('maildir_event', [], $fn, $vmd, $state);
} # else: TODO: imap
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 0fa2d3c0..a91b30f7 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -25,10 +25,14 @@ use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
use PublicInbox::Spawn qw(spawn);
+use PublicInbox::MdirReader;
+use PublicInbox::LeiToMail;
use List::Util qw(max);
use File::Temp ();
use POSIX ();
use IO::Handle (); # ->autoflush
+use Sys::Syslog qw(syslog openlog);
+use Errno qw(EEXIST ENOENT);
sub new {
my (undef, $dir, $opt) = @_;
@@ -165,12 +169,92 @@ sub _docids_for ($$) {
sort { $a <=> $b } values %docids;
}
+# n.b. similar to LeiExportKw->export_kw_md, but this is for a single eml
+sub export1_kw_md ($$$$$) {
+ my ($self, $mdir, $bn, $oidbin, $vmdish) = @_; # vmd/vmd_mod
+ my $orig = $bn;
+ my (@try, $unkn, $kw);
+ if ($bn =~ s/:2,([a-zA-Z]*)\z//) {
+ ($kw, $unkn) = PublicInbox::MdirReader::flags2kw($1);
+ if (my $set = $vmdish->{kw}) {
+ $kw = $set;
+ } elsif (my $add = $vmdish->{'+kw'}) {
+ @$kw{@$add} = ();
+ } elsif (my $del = $vmdish->{-kw}) {
+ delete @$kw{@$del};
+ } # else no changes...
+ @try = qw(cur new);
+ } else { # no keywords, yet, could be in new/
+ @try = qw(new cur);
+ $unkn = [];
+ if (my $set = $vmdish->{kw}) {
+ $kw = $set;
+ } elsif (my $add = $vmdish->{'+kw'}) {
+ @$kw{@$add} = (); # auto-vivify
+ } else { # ignore $vmdish->{-kw}
+ $kw = [];
+ }
+ }
+ $kw = [ keys %$kw ] if ref($kw) eq 'HASH';
+ $bn .= ':2,'. PublicInbox::LeiToMail::kw2suffix($kw, @$unkn);
+ return if $orig eq $bn; # no change
+
+ # we use link(2) + unlink(2) since rename(2) may
+ # inadvertently clobber if the "uniquefilename" part wasn't
+ # actually unique.
+ my $dst = "$mdir/cur/$bn";
+ for my $d (@try) {
+ my $src = "$mdir/$d/$orig";
+ if (link($src, $dst)) {
+ if (!unlink($src) and $! != ENOENT) {
+ syslog('warning', "unlink($src): $!");
+ }
+ # TODO: verify oidbin?
+ lms_mv_src($self, "maildir:$mdir",
+ $oidbin, \$orig, $bn);
+ return;
+ } elsif ($! == EEXIST) { # lost race with "lei export-kw"?
+ return;
+ } elsif ($! == ENOENT) {
+ syslog('warning', "link($src -> $dst): $!")
+ } # else loop @try
+ }
+ my $e = $!;
+ my $src = "$mdir/{".join(',', @try)."}/$orig";
+ my $oidhex = unpack('H*', $oidbin);
+ syslog('warning', "link($src -> $dst) ($oidhex): $e");
+ for (@try) { return if -e "$mdir/$_/$orig" };
+ lms_clear_src($self, "maildir:$mdir", \$orig);
+}
+
+sub sto_export_kw ($$$) {
+ my ($self, $docid, $vmdish) = @_; # vmdish (vmd or vmd_mod)
+ my ($eidx, $tl) = eidx_init($self);
+ my $lms = _lms_rw($self) // return;
+ my $xr3 = $eidx->{oidx}->get_xref3($docid, 1);
+ for my $row (@$xr3) {
+ my (undef, undef, $oidbin) = @$row;
+ my $locs = $lms->locations_for($oidbin) // next;
+ while (my ($loc, $ids) = each %$locs) {
+ if ($loc =~ s!\Amaildir:!!i) {
+ for my $id (@$ids) {
+ export1_kw_md($self, $loc, $id,
+ $oidbin, $vmdish);
+ }
+ }
+ # TODO: IMAP
+ }
+ }
+}
+
+# vmd = { kw => [ qw(seen ...) ], L => [ qw(inbox ...) ] }
sub set_eml_vmd {
my ($self, $eml, $vmd, $docids) = @_;
my ($eidx, $tl) = eidx_init($self);
$docids //= [ _docids_for($self, $eml) ];
for my $docid (@$docids) {
$eidx->idx_shard($docid)->ipc_do('set_vmd', $docid, $vmd);
+ sto_export_kw($self, $docid, $vmd);
}
$docids;
}
@@ -284,6 +368,12 @@ EOF
$docid;
}
+sub _add_vmd ($$$$) {
+ my ($self, $idx, $docid, $vmd) = @_;
+ $idx->ipc_do('add_vmd', $docid, $vmd);
+ sto_export_kw($self, $docid, $vmd);
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
@@ -310,7 +400,7 @@ sub add_eml {
@$vivify_xvmd = sort { $a <=> $b } keys(%docids);
}
}
- if (@$vivify_xvmd) {
+ if (@$vivify_xvmd) { # docids list
$xoids //= {};
$xoids->{$smsg->{blob}} = 1;
for my $docid (@$vivify_xvmd) {
@@ -327,7 +417,7 @@ sub add_eml {
for my $oid (keys %$xoids) {
$oidx->add_xref3($docid, -1, $oid, '.');
}
- $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd;
+ _add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
$vivify_xvmd;
} elsif (my @docids = _docids_for($self, $eml)) {
@@ -337,7 +427,7 @@ sub add_eml {
$oidx->add_xref3($docid, -1, $smsg->{blob}, '.');
# add_eidx_info for List-Id
$idx->ipc_do('add_eidx_info', $docid, '.', $eml);
- $idx->ipc_do('add_vmd', $docid, $vmd) if $vmd;
+ _add_vmd($self, $idx, $docid, $vmd) if $vmd;
}
\@docids;
} else { # totally new message
@@ -347,7 +437,7 @@ sub add_eml {
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
my $idx = $eidx->idx_shard($smsg->{num});
$idx->index_eml($eml, $smsg);
- $idx->ipc_do('add_vmd', $smsg->{num}, $vmd) if $vmd;
+ _add_vmd($self, $idx, $smsg->{num}, $vmd) if $vmd;
$smsg;
}
}
@@ -365,6 +455,7 @@ sub index_eml_only {
set_eml($self, $eml, $vmd, $xoids);
}
+# store {kw} / {L} info for a message which is only in an external
sub _external_only ($$$) {
my ($self, $xoids, $eml) = @_;
my $eidx = $self->{priv_eidx};
@@ -398,6 +489,7 @@ sub update_xvmd {
next if $seen{$docid}++;
my $idx = $eidx->idx_shard($docid);
$idx->ipc_do('update_vmd', $docid, $vmd_mod);
+ sto_export_kw($self, $docid, $vmd_mod);
}
return unless scalar(keys(%$xoids));
@@ -410,12 +502,14 @@ sub update_xvmd {
}
my $idx = $eidx->idx_shard($docid);
$idx->ipc_do('update_vmd', $docid, $vmd_mod);
+ sto_export_kw($self, $docid, $vmd_mod);
}
return;
}
# totally unseen
my ($smsg, $idx) = _external_only($self, $xoids, $eml);
$idx->ipc_do('update_vmd', $smsg->{num}, $vmd_mod);
+ sto_export_kw($self, $smsg->{num}, $vmd_mod);
}
# set or update keywords for external message, called via ipc_do
@@ -433,6 +527,7 @@ sub set_xvmd {
next if $seen{$docid}++;
my $idx = $eidx->idx_shard($docid);
$idx->ipc_do('set_vmd', $docid, $vmd);
+ sto_export_kw($self, $docid, $vmd);
}
return unless scalar(keys(%$xoids));
@@ -443,6 +538,7 @@ sub set_xvmd {
# totally unseen:
my ($smsg, $idx) = _external_only($self, $xoids, $eml);
$idx->ipc_do('add_vmd', $smsg->{num}, $vmd);
+ sto_export_kw($self, $smsg->{num}, $vmd);
}
sub checkpoint {
@@ -497,6 +593,7 @@ sub ipc_atfork_child {
if (my $to_close = delete($self->{to_close})) {
close($_) for @$to_close;
}
+ openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
$self->SUPER::ipc_atfork_child;
}
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
new file mode 100644
index 00000000..321c0ab3
--- /dev/null
+++ b/t/lei-auto-watch.t
@@ -0,0 +1,45 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Basename qw(basename);
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+ eval { require IO::KQueue };
+
+$have_fast_inotify or
+ diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
+
+test_lei(sub {
+ my $x = "$ENV{HOME}/x";
+ my $y = "$ENV{HOME}/y";
+ lei_ok qw(add-external), "$ro_home/t1";
+ lei_ok qw(q mid:testmessage@example.com -o), $x;
+ lei_ok qw(q mid:testmessage@example.com -o), $y;
+ my @x = glob("$x/cur/*");
+ my @y = glob("$y/cur/*");
+ scalar(@x) == 1 or xbail 'expected 1 file', \@x;
+ scalar(@y) == 1 or xbail 'expected 1 file', \@y;
+
+ my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+ lei_ok qw(inspect), "blob:$oid";
+ my $ins = json_utf8->decode($lei_out);
+ my $exp = { "maildir:$x" => [ map { basename($_) } @x ],
+ "maildir:$y" => [ map { basename($_) } @y ] };
+ is_deeply($ins->{'mail-sync'}, $exp, 'inspect as expected');
+ lei_ok qw(add-watch), $x;
+ my $dst = $x[0] . 'S';
+ rename($x[0], $dst) or xbail "rename($x[0], $dst): $!";
+ tick($have_fast_inotify ? undef : 2.1); # wait for inotify
+ my @y2 = glob("$y/*/*");
+ is_deeply(\@y2, [ "$y[0]S" ], "`seen' kw propagated to `y' dir");
+ lei_ok qw(note-event done);
+ tick; # XXX why is this needed?
+ lei_ok qw(inspect), "blob:$oid";
+ $ins = json_utf8->decode($lei_out);
+ $exp = { "maildir:$x" => [ map { basename($_) } glob("$x/*/*") ],
+ "maildir:$y" => [ map { basename($_) } glob("$y/*/*") ] };
+ is_deeply($ins->{'mail-sync'}, $exp, 'mail_sync matches FS');
+});
+
+done_testing;
^ permalink raw reply related [relevance 51%]
* [SQUASH 4/3] t/lei-auto-watch: workaround for FreeBSD kevent
2021-09-02 10:17 51% ` [PATCH 3/3] lei: propagate keyword changes from lei/store Eric Wong
@ 2021-09-02 10:25 67% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-02 10:25 UTC (permalink / raw)
To: meta
My FreeBSD VM seems to need longer for this test than inotify
under Linux, likely because the kevent support code is more
complicated in userspace and needs extra file handles.
And drop unnecessary tick delay after "note-event done" since
that seems unneeded with transactions eliminated for
mail_sync.sqlite3
---
t/lei-auto-watch.t | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index 321c0ab3..146402a6 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -4,8 +4,9 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Basename qw(basename);
my ($ro_home, $cfg_path) = setup_public_inboxes;
-my $have_fast_inotify = eval { require Linux::Inotify2 } ||
- eval { require IO::KQueue };
+my $tick = 2.1;
+my $have_fast_inotify = eval { require Linux::Inotify2; $tick = 0.1 } ||
+ eval { require IO::KQueue; $tick = 0.5 };
$have_fast_inotify or
diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
@@ -30,11 +31,10 @@ test_lei(sub {
lei_ok qw(add-watch), $x;
my $dst = $x[0] . 'S';
rename($x[0], $dst) or xbail "rename($x[0], $dst): $!";
- tick($have_fast_inotify ? undef : 2.1); # wait for inotify
+ tick($tick); # wait for inotify or kevent
my @y2 = glob("$y/*/*");
is_deeply(\@y2, [ "$y[0]S" ], "`seen' kw propagated to `y' dir");
lei_ok qw(note-event done);
- tick; # XXX why is this needed?
lei_ok qw(inspect), "blob:$oid";
$ins = json_utf8->decode($lei_out);
$exp = { "maildir:$x" => [ map { basename($_) } glob("$x/*/*") ],
^ permalink raw reply related [relevance 67%]
* Showcasing lei at Linux Plumbers
@ 2021-09-02 21:12 61% Konstantin Ryabitsev
2021-09-02 21:58 65% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-02 21:12 UTC (permalink / raw)
To: meta
Eric:
I am getting ready for my presentation to the Linux Plumbers (happening in a
few weeks, eek), which is based around lore, lei (I see what you did there)
and search-based subscriptions. I want to make it hands-on with practical
examples, which is what developers would appreciate more than just dry
manpages.
I am in the process of wrapping my head around lei tooling, but I may have
some questions in the process, so I wanted to start this thread as a record of
my poking at it. :)
What I currently have:
- an imap mailbox
- lei configured and installed locally (in a debian container)
The goal is to illustrate how to use this to start "receiving" mail for a
subsystem without subscribing to any of the lists. The example I have in mind
is the LANDLOCK subsystem, and the reason I picked it is because it already
has a well-defined set of search criteria we can use:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/MAINTAINERS#n10462
LANDLOCK SECURITY MODULE
...
F: Documentation/security/landlock.rst
F: Documentation/userspace-api/landlock.rst
F: include/uapi/linux/landlock.h
F: samples/landlock/
F: security/landlock/
F: tools/testing/selftests/landlock/
K: landlock
K: LANDLOCK
This means we want to configure lei to grab any mail from lore.kernel.org/all/
that matches this query:
dfn:Documentation/security/landlock.rst OR
dfn:Documentation/userspace-api/landlock.rst OR
dfn:include/uapi/linux/landlock.h OR
dfn:samples/landlock/ OR
dfn:security/landlock/ OR
dfn:tools/testing/selftests/landlock/ OR
dfhh:landlock
https://lore.kernel.org/all/?q=dfn%3ADocumentation%2Fsecurity%2Flandlock.rst+OR+dfn%3ADocumentation%2Fuserspace-api%2Flandlock.rst+OR+dfn%3Ainclude%2Fuapi%2Flinux%2Flandlock.h+OR+dfn%3Asamples%2Flandlock%2F+OR+dfn%3Asecurity%2Flandlock%2F+OR+dfn%3Atools%2Ftesting%2Fselftests%2Flandlock%2F+OR+dfhh%3Alandlock
I'll want to retrieve any threads and follow-ups and upload them to my imap
landlock folder -- and then run in the background and just continuously update
things as more mail comes in, so I don't have to remember to run anything
manually.
What succession of lei commands would accomplish this?
Thanks for your continued help.
-K
^ permalink raw reply [relevance 61%]
* Re: Showcasing lei at Linux Plumbers
2021-09-02 21:12 61% Showcasing lei at Linux Plumbers Konstantin Ryabitsev
@ 2021-09-02 21:58 65% ` Eric Wong
2021-09-03 15:15 71% ` Konstantin Ryabitsev
2021-09-07 21:33 71% ` Konstantin Ryabitsev
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-02 21:58 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Eric:
>
> I am getting ready for my presentation to the Linux Plumbers (happening in a
> few weeks, eek), which is based around lore, lei (I see what you did there)
> and search-based subscriptions. I want to make it hands-on with practical
> examples, which is what developers would appreciate more than just dry
> manpages.
>
> I am in the process of wrapping my head around lei tooling, but I may have
> some questions in the process, so I wanted to start this thread as a record of
> my poking at it. :)
Yeah, I'm still trying to figure out how some things are
supposed to work myself...
> What I currently have:
>
> - an imap mailbox
> - lei configured and installed locally (in a debian container)
Fwiw, most of the functionality works much better with Maildir
because of potential password prompts needed for IMAP and
interactivity required.
> The goal is to illustrate how to use this to start "receiving" mail for a
> subsystem without subscribing to any of the lists. The example I have in mind
> is the LANDLOCK subsystem, and the reason I picked it is because it already
> has a well-defined set of search criteria we can use:
> https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/MAINTAINERS#n10462
>
> LANDLOCK SECURITY MODULE
> ...
> F: Documentation/security/landlock.rst
> F: Documentation/userspace-api/landlock.rst
> F: include/uapi/linux/landlock.h
> F: samples/landlock/
> F: security/landlock/
> F: tools/testing/selftests/landlock/
> K: landlock
> K: LANDLOCK
>
> This means we want to configure lei to grab any mail from lore.kernel.org/all/
> that matches this query:
>
> dfn:Documentation/security/landlock.rst OR
> dfn:Documentation/userspace-api/landlock.rst OR
> dfn:include/uapi/linux/landlock.h OR
> dfn:samples/landlock/ OR
> dfn:security/landlock/ OR
> dfn:tools/testing/selftests/landlock/ OR
> dfhh:landlock
>
> https://lore.kernel.org/all/?q=dfn%3ADocumentation%2Fsecurity%2Flandlock.rst+OR+dfn%3ADocumentation%2Fuserspace-api%2Flandlock.rst+OR+dfn%3Ainclude%2Fuapi%2Flinux%2Flandlock.h+OR+dfn%3Asamples%2Flandlock%2F+OR+dfn%3Asecurity%2Flandlock%2F+OR+dfn%3Atools%2Ftesting%2Fselftests%2Flandlock%2F+OR+dfhh%3Alandlock
For HTTP(S)-based queries, I would add rt: (received-time)
around the whole thing and maybe use "lei edit-search" to tweak
for subsequent runs. Not sure if the rt: handling should be
automatic for HTTP(S) (local Xapian searches track max docid, instead)
> I'll want to retrieve any threads and follow-ups and upload them to my imap
> landlock folder -- and then run in the background and just continuously update
> things as more mail comes in, so I don't have to remember to run anything
> manually.
>
> What succession of lei commands would accomplish this?
OK, there's two main commands, "lei q" and "lei up".
Both of which may prompt for passwords depending on how
git-credential is set up:
# the destination, could be Maildir
MFOLDER=imaps://user@example.com/INBOX.landlock
# initial search:
lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
(
dfn:Documentation/security/landlock.rst OR
dfn:Documentation/userspace-api/landlock.rst OR
dfn:include/uapi/linux/landlock.h OR
dfn:samples/landlock/ OR
dfn:security/landlock/ OR
dfn:tools/testing/selftests/landlock/ OR
dfhh:landlock
) AND rt:2.months.ago..
EOF
# update whenever, may prompt for IMAP password, but could be
# cron-ed or similar if passwords are cached
lei up $MFOLDER
# Optional: tweaking the search parameters can be done via
lei edit-search $MFOLDER
For Maildirs, "lei up --all=local" works as it should.
"lei up --all" and "lei up --all=remote" don't work, yet,
because prompting for multiple IMAP folders (with potentially
different accounts) can get a bit complicated. But
"lei up $ONE_IMAP_FOLDER" already works.
> Thanks for your continued help.
No problem, thanks for your patience since everything seems
overwhelming :<
^ permalink raw reply [relevance 65%]
* [PATCH 1/2] t/lei-auto-watch: improve test reliability
@ 2021-09-02 22:36 68% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-02 22:36 UTC (permalink / raw)
To: meta
On slower systems, even a 100ms delay may not be enough;
so loop and retry in hopes of an early exit for faster
systems.
---
t/lei-auto-watch.t | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index 146402a6..3b0c1b10 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -4,9 +4,8 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Basename qw(basename);
my ($ro_home, $cfg_path) = setup_public_inboxes;
-my $tick = 2.1;
-my $have_fast_inotify = eval { require Linux::Inotify2; $tick = 0.1 } ||
- eval { require IO::KQueue; $tick = 0.5 };
+my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+ eval { require IO::KQueue };
$have_fast_inotify or
diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
@@ -31,9 +30,13 @@ test_lei(sub {
lei_ok qw(add-watch), $x;
my $dst = $x[0] . 'S';
rename($x[0], $dst) or xbail "rename($x[0], $dst): $!";
- tick($tick); # wait for inotify or kevent
+ my $ys = "$y[0]S";
+ for (0..50) {
+ last if -f $ys;
+ tick; # wait for inotify or kevent
+ }
my @y2 = glob("$y/*/*");
- is_deeply(\@y2, [ "$y[0]S" ], "`seen' kw propagated to `y' dir");
+ is_deeply(\@y2, [ $ys ], "`seen' kw propagated to `y' dir");
lei_ok qw(note-event done);
lei_ok qw(inspect), "blob:$oid";
$ins = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 68%]
* [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (2 preceding siblings ...)
2021-09-03 8:54 52% ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
@ 2021-09-03 8:54 83% ` Eric Wong
2021-09-03 8:54 57% ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
` (2 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
We can golf out some code and refcounts this way.
---
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiPruneMailSync.pm | 26 +++++++++++++-------------
2 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2d2ff1a0..2ade17af 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -206,8 +206,7 @@ sub lei_inspect {
sub _complete_inspect {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } $lms->folders;
}
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
index 79f3325d..98239a13 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -40,7 +40,7 @@ sub prune_imap { # lms->each_src callback
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
- my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
my $mdir = $1;
$lms->each_src($input, \&prune_mdir, $self, $mdir);
@@ -59,24 +59,24 @@ sub lei_prune_mail_sync {
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
EOM
- my $lse = $sto->search;
- my $lms = $lse->lms or return $lei->fail(<<EOM);
-lei mail_sync uninitialized, see lei-import(1)
-EOM
- if (defined(my $all = $lei->{opt}->{all})) {
- $lms->group2folders($lei, $all, \@folders) or return;
+ if (my $lms = $lei->lms) {
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
+ }
} else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ return $lei->fail(<<EOM);
+lei mail_sync.sqlite3 uninitialized, see lei-import(1)
+EOM
}
- delete $lms->{dbh};
$sto->write_prepare($lei);
- my $self = bless { lse => $lse }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
- undef $lms; # for fork
my $ops = {};
$sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
^ permalink raw reply related [relevance 83%]
* [PATCH 2/8] lei/store: quiet down link(2) warnings
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
@ 2021-09-03 8:54 71% ` Eric Wong
2021-09-03 8:54 52% ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
` (4 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
ENOENT can be too common due to timing and concurrent access
from MUAs and "lei export-kw", and other mail synchronization
tools (e.g. mbsync and offlineimap).
---
lib/PublicInbox/LeiStore.pm | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index a91b30f7..f81a8dae 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -215,14 +215,10 @@ sub export1_kw_md ($$$$$) {
return;
} elsif ($! == EEXIST) { # lost race with "lei export-kw"?
return;
- } elsif ($! == ENOENT) {
- syslog('warning', "link($src -> $dst): $!")
- } # else loop @try
+ } elsif ($! != ENOENT) {
+ syslog('warning', "link($src -> $dst): $!");
+ }
}
- my $e = $!;
- my $src = "$mdir/{".join(',', @try)."}/$orig";
- my $oidhex = unpack('H*', $oidbin);
- syslog('warning', "link($src -> $dst) ($oidhex): $e");
for (@try) { return if -e "$mdir/$_/$orig" };
lms_clear_src($self, "maildir:$mdir", \$orig);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives
@ 2021-09-03 8:54 66% Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
` (6 more replies)
0 siblings, 7 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
Back to using syslog more for errors, since it's still less
instrusive. IMAP R/W is fixed (ugh, lack of automatic tests
since we only have a read-only IMAP server for testing).
Some error/warning fixes, and false-positives on L: and kw:
searches on local (but not remote) externals are avoided.
"lei inspect --stdin" was somewhat useful for figuring out why
L: was false-positiving an external result on me.
Eric Wong (8):
lei: dump errors to syslog, and not to CLI
lei/store: quiet down link(2) warnings
lei: ->child_error less error-prone
lei: use lei->lms in place of lse->lms in a few places
lei up --all: avoid double-close on shared STDOUT
lei inspect: support reading eml from --stdin
lei_xsearch: avoid false-positives on externals w/ L: and kw:
lei: fix read/write IMAP access
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 27 ++++++++++--------
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 4 +--
lib/PublicInbox/LeiInspect.pm | 43 ++++++++++++++++++++++++-----
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiPruneMailSync.pm | 26 ++++++++---------
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 10 ++-----
lib/PublicInbox/LeiStoreErr.pm | 4 +--
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 6 +++-
lib/PublicInbox/LeiXSearch.pm | 1 +
lib/PublicInbox/NetReader.pm | 5 +++-
lib/PublicInbox/NetWriter.pm | 2 ++
lib/PublicInbox/Watch.pm | 2 ++
t/lei-daemon.t | 5 ----
t/lei-up.t | 39 ++++++++++++++++++++++++++
19 files changed, 130 insertions(+), 55 deletions(-)
create mode 100644 t/lei-up.t
^ permalink raw reply [relevance 66%]
* [PATCH 8/8] lei: fix read/write IMAP access
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (5 preceding siblings ...)
2021-09-03 8:54 61% ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
@ 2021-09-03 8:54 63% ` Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
xt/net_writer-imap.t was completely broken in recent months and
I completely forgot this test. net->add_url still only accepts
bare scalars (and not scalar refs), so we must set that up
properly. Furthermore, our changes to do FLAGS-only
synchronization in lei of old messages was causing us to not
handle FLAGS properly for the test.
---
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/NetReader.pm | 5 ++++-
lib/PublicInbox/NetWriter.pm | 2 ++
lib/PublicInbox/Watch.pm | 2 ++
4 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index be6e178f..6e102a1d 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -406,7 +406,7 @@ sub new {
my $net = PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
- $net->add_url($uri);
+ $net->add_url($$uri);
my $err = $net->errors($lei);
return $lei->fail($err) if $err;
$uri->mailbox or return $lei->fail("No mailbox: $dst");
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 23445e7a..c050c60f 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -493,6 +493,9 @@ sub perm_fl_ok ($) {
undef;
}
+# may be overridden in NetWriter or Watch
+sub folder_select { $_[0]->{each_old} ? 'select' : 'examine' }
+
sub _imap_fetch_all ($$$) {
my ($self, $mic, $orig_uri) = @_;
my $sec = uri_section($orig_uri);
@@ -501,7 +504,7 @@ sub _imap_fetch_all ($$$) {
# we need to check for mailbox writability to see if we care about
# FLAGS from already-imported messages.
- my $cmd = $self->{each_old} ? 'select' : 'examine';
+ my $cmd = $self->folder_select;
$mic->$cmd($mbx) or return "E: \U$cmd\E $mbx ($sec) failed: $!";
my ($r_uidval, $r_uidnext, $perm_fl);
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 82288e6b..629a752a 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,6 +26,8 @@ sub imap_append {
die "APPEND $folder: $@";
}
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
sub imap_delete_all {
my ($self, $uri) = @_;
my $mic = $self->mic_for_folder($uri) or return;
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 86dae91f..482d35c4 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -682,4 +682,6 @@ EOF
undef;
}
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
1;
^ permalink raw reply related [relevance 63%]
* [PATCH 6/8] lei inspect: support reading eml from --stdin
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (4 preceding siblings ...)
2021-09-03 8:54 57% ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
@ 2021-09-03 8:54 61% ` Eric Wong
2021-09-03 8:54 63% ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
This can be useful inside mutt since I was diagnosing why
a label ("L:$FOO") search was giving me a false-positive
search result...
---
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiInspect.pm | 40 ++++++++++++++++++++++++++++++-----
2 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8b6c1d36..098a45ba 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -279,8 +279,8 @@ our %CMD = ( # sorted in order of importance/use:
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
qw(c=s@ C=s@), pass_through('git config') ],
-'inspect' => [ 'ITEMS...', 'inspect lei/store and/or local external',
- qw(pretty ascii dir=s), @c_opt ],
+'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
+ qw(stdin| pretty ascii dir=s), @c_opt ],
'init' => [ '[DIRNAME]', sub {
"initialize storage, default: ".store_path($_[0]);
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2ade17af..25bd47e7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -9,6 +9,7 @@ package PublicInbox::LeiInspect;
use strict;
use v5.10.1;
use PublicInbox::Config;
+use PublicInbox::MID qw(mids);
sub inspect_blob ($$) {
my ($lei, $oidhex) = @_;
@@ -184,6 +185,32 @@ sub inspect1 ($$$) {
1;
}
+sub _inspect_argv ($$) {
+ my ($lei, $argv) = @_;
+ my $multi = scalar(@$argv) > 1;
+ $lei->out('[') if $multi;
+ while (defined(my $x = shift @$argv)) {
+ inspect1($lei, $x, scalar(@$argv)) or return;
+ }
+ $lei->out(']') if $multi;
+}
+
+sub ins_add { # InputPipe->consume callback
+ my ($lei) = @_; # $_[1] = $rbuf
+ if (defined $_[1]) {
+ $_[1] eq '' and return eval {
+ my $str = delete $lei->{istr};
+ $str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+ my $eml = PublicInbox::Eml->new(\$str);
+ _inspect_argv($lei, [ 'blob:'.$lei->git_blob_id($eml),
+ map { "mid:$_" } @{mids($eml)} ]);
+ };
+ $lei->{istr} .= $_[1];
+ } else {
+ $lei->fail("error reading stdin: $!");
+ }
+}
+
sub lei_inspect {
my ($lei, @argv) = @_;
$lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
@@ -196,12 +223,15 @@ sub lei_inspect {
}
$lei->start_pager if -t $lei->{1};
$lei->{1}->autoflush(0);
- my $multi = scalar(@argv) > 1;
- $lei->out('[') if $multi;
- while (defined(my $x = shift @argv)) {
- inspect1($lei, $x, scalar(@argv)) or return;
+ if ($lei->{opt}->{stdin}) {
+ return $lei->fail(<<'') if @argv;
+no args allowed on command-line with --stdin
+
+ require PublicInbox::InputPipe;
+ PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
+ return;
}
- $lei->out(']') if $multi;
+ _inspect_argv($lei, \@argv);
}
sub _complete_inspect {
^ permalink raw reply related [relevance 61%]
* [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (3 preceding siblings ...)
2021-09-03 8:54 83% ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
@ 2021-09-03 8:54 57% ` Eric Wong
2021-09-03 8:54 61% ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
2021-09-03 8:54 63% ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
This is merely to avoid perl setting errors internally which
were not user visible. The double-close wasn't a problem in
practice since we open a new file hanlde for the mbox or
mbox.gz anyways, so the new t/lei-up.t test case shows no
regressions nor fixes.
---
MANIFEST | 1 +
lib/PublicInbox/LeiUp.pm | 4 ++++
t/lei-up.t | 39 +++++++++++++++++++++++++++++++++++++++
3 files changed, 44 insertions(+)
create mode 100644 t/lei-up.t
diff --git a/MANIFEST b/MANIFEST
index be6ec927..fad29622 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -443,6 +443,7 @@ t/lei-q-save.t
t/lei-q-thread.t
t/lei-sigpipe.t
t/lei-tag.t
+t/lei-up.t
t/lei-watch.t
t/lei.t
t/lei_dedupe.t
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index e1da64aa..a39d6047 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -54,6 +54,10 @@ sub up1_redispatch {
$l->{opt} = { %{$l->{opt}} };
delete $l->{sock};
$l->{''} = $op_p; # daemon only
+
+ # make close($l->{1}) happy in lei->dclose
+ open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ $l->{1} = $fh;
eval {
$l->qerr("# updating $out");
up1($l, $out);
diff --git a/t/lei-up.t b/t/lei-up.t
new file mode 100644
index 00000000..c6f31c74
--- /dev/null
+++ b/t/lei-up.t
@@ -0,0 +1,39 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
+test_lei(sub {
+ my $s = eml_load('t/plack-qp.eml')->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a.mbox.gz";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b.mbox.gz";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b";
+ lei_ok qw(ls-search);
+ $s = eml_load('t/utf8.eml')->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+ lei_ok qw(up --all=local);
+ open my $fh, "$ENV{HOME}/a.mbox.gz" or xbail "open: $!";
+ my $gz = do { local $/; <$fh> };
+ my $uc;
+ gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+ open $fh, "$ENV{HOME}/a" or xbail "open: $!";
+
+ my $exp = do { local $/; <$fh> };
+ is($uc, $exp, 'compressed and uncompressed match (a.gz)');
+ like($exp, qr/testmessage\@example.com/, '2nd message added');
+ open $fh, "$ENV{HOME}/b.mbox.gz" or xbail "open: $!";
+
+ $gz = do { local $/; <$fh> };
+ undef $uc;
+ gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+ is($uc, $exp, 'compressed and uncompressed match (b.gz)');
+
+ open $fh, "$ENV{HOME}/b" or xbail "open: $!";
+ $uc = do { local $/; <$fh> };
+ is($uc, $exp, 'uncompressed both match');
+});
+
+done_testing;
^ permalink raw reply related [relevance 57%]
* [PATCH 1/8] lei: dump errors to syslog, and not to CLI
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
@ 2021-09-03 8:54 54% ` Eric Wong
2021-09-03 8:54 71% ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
` (5 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
Dumping errors from the previous run can often get lost, so just
spew to syslog since it's a standard place to put errors that
don't make it to a client. Note: we don't rely on $SIG{__WARN__}
since some of the Net:: stuff will write directly to STDERR
(as will external processes).
---
lib/PublicInbox/LEI.pm | 16 +++++++++-------
lib/PublicInbox/LeiStoreErr.pm | 4 ++--
t/lei-daemon.t | 5 -----
3 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 41e811ca..9e9aa165 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -27,6 +27,7 @@ use PublicInbox::Eml;
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
+use Sys::Syslog qw(openlog syslog closelog);
our $quit = \&CORE::exit;
our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
$recv_cmd, $send_cmd);
@@ -464,7 +465,6 @@ sub x_it ($$) {
my ($self, $code) = @_;
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
- dump_and_clear_log();
stop_pager($self);
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('x_it', $code);
@@ -765,7 +765,6 @@ sub dispatch {
my ($self, $cmd, @argv) = @_;
local $current_lei = $self; # for __WARN__
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
- dump_and_clear_log("from previous run\n");
return _help($self, 'no command given') unless defined($cmd);
# do not support Getopt bundling for this
while ($cmd eq '-C' || $cmd eq '-c') {
@@ -1147,10 +1146,12 @@ sub oldset { $oldset }
sub dump_and_clear_log {
if (defined($errors_log) && -s STDIN && seek(STDIN, 0, SEEK_SET)) {
- my @pfx = @_;
- unshift(@pfx, "$errors_log ") if @pfx;
- warn @pfx, do { local $/; <STDIN> };
- truncate(STDIN, 0) or warn "ftruncate ($errors_log): $!";
+ openlog('lei-daemon', 'pid,nowait,nofatal,ndelay', 'user');
+ chomp(my @lines = <STDIN>);
+ truncate(STDIN, 0) or
+ syslog('warning', "ftruncate (%s): %m", $errors_log);
+ for my $l (@lines) { syslog('warning', '%s', $l) }
+ closelog(); # don't share across fork
}
}
@@ -1243,7 +1244,7 @@ sub lazy_start {
(-p STDOUT) or die "E: stdout must be a pipe\n";
open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
STDIN->autoflush(1);
- dump_and_clear_log("from previous daemon process:\n");
+ dump_and_clear_log();
POSIX::setsid() > 0 or die "setsid: $!";
my $pid = fork // die "fork: $!";
return if $pid;
@@ -1345,6 +1346,7 @@ sub DESTROY {
}
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
+ dump_and_clear_log();
# preserve $? for ->fail or ->x_it code
}
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index 5f9ba24d..cc085fdc 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -2,7 +2,7 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# forwards stderr from lei/store process to any lei clients using
-# the same store
+# the same store, falls back to syslog if no matching clients exist.
package PublicInbox::LeiStoreErr;
use strict;
use v5.10.1;
@@ -31,7 +31,7 @@ sub event_step {
print $err $$rbuf and $printed = 1;
}
if (!$printed) {
- openlog('lei-store', 'pid,nowait,nofatal,ndelay', 'user');
+ openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
closelog(); # don't share across fork
}
diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index a7c4b799..b0c94a87 100644
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -21,14 +21,9 @@ test_lei({ daemon_only => 1 }, sub {
ok(kill(0, $pid), 'pid is valid');
ok(-S $sock, 'sock created');
is(-s $err_log, 0, 'nothing in errors.log');
- open my $efh, '>>', $err_log or BAIL_OUT $!;
- print $efh "phail\n" or BAIL_OUT $!;
- close $efh or BAIL_OUT $!;
-
lei_ok('daemon-pid');
chomp(my $pid_again = $lei_out);
is($pid, $pid_again, 'daemon-pid idempotent');
- like($lei_err, qr/phail/, 'got mock "phail" error previous run');
SKIP: {
skip 'only testing open files on Linux', 1 if $^O ne 'linux';
^ permalink raw reply related [relevance 54%]
* [PATCH 3/8] lei: ->child_error less error-prone
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
2021-09-03 8:54 71% ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
@ 2021-09-03 8:54 52% ` Eric Wong
2021-09-03 8:54 83% ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
` (3 subsequent siblings)
6 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
I was calling "child_error(1, ...)" in a few places where I meant
to be calling "child_error(1 << 8, ...)" and inadvertantly
triggering SIGHUP in script/lei. Since giving a zero exit code
to child_error makes no sense, just allow falsy values to
default to 1 << 8.
---
lib/PublicInbox/LEI.pm | 7 ++++---
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 4 ++--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 +-
7 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9e9aa165..8b6c1d36 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -532,6 +532,7 @@ sub puts ($;@) { out(shift, map { "$_\n" } @_) }
sub child_error { # passes non-fatal curl exit codes to user
my ($self, $child_error, $msg) = @_; # child_error is $?
+ $child_error ||= 1 << 8;
$self->err($msg) if $msg;
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('child_error', $child_error);
@@ -1341,7 +1342,7 @@ sub DESTROY {
if (my $counters = delete $self->{counters}) {
for my $k (sort keys %$counters) {
my $nr = $counters->{$k};
- $self->child_error(1 << 8, "$nr $k messages");
+ $self->child_error(0, "$nr $k messages");
}
}
$self->{1}->autoflush(1) if $self->{1};
@@ -1417,7 +1418,7 @@ sub refresh_watches {
add_maildir_watch($d, $cfg_f);
}
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1, "E: watch $url not supported, yet")
+ $lei->child_error(0, "E: watch $url not supported, yet")
}
}
@@ -1452,7 +1453,7 @@ sub refresh_watches {
my $d = canonpath_harder($1);
cancel_maildir_watch($d, $cfg_f);
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1, "E: watch $url TODO");
+ $lei->child_error(0, "E: watch $url TODO");
}
}
}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 21003894..b94f67a0 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -32,7 +32,7 @@ sub solver_user_cb { # called by solver when done
my $lei = $self->{lei};
my $log_buf = delete $lei->{'log_buf'};
$$log_buf =~ s/^/# /sgm;
- ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+ ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
$lei->qerr($$log_buf);
my ($git, $oid, $type, $size, $di) = @$res;
my $gd = $git->{git_dir};
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 5b545998..ef3e4d0b 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -21,7 +21,7 @@ sub input_eml_cb { # used by input_maildir_cb and input_net_cb
sub input_fh { # overrides PublicInbox::LeiInput::input_fh
my ($self, $ifmt, $fh, $input, @args) = @_;
- $self->{lei}->child_error(1<<8, <<EOM);
+ $self->{lei}->child_error(0, <<EOM);
$input ($ifmt) not yet supported, try `lei import'
EOM
}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 1b28f36f..cb71e97c 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -63,7 +63,7 @@ sub input_fh {
my ($self, $ifmt, $fh, $name, @args) = @_;
if ($ifmt eq 'eml') {
my $buf = do { local $/; <$fh> } //
- return $self->{lei}->child_error(1 << 8, <<"");
+ return $self->{lei}->child_error(0, <<"");
error reading $name: $!
# mutt pipes single RFC822 messages with a "From " line,
@@ -104,7 +104,7 @@ sub handle_http_input ($$@) {
my $err = $@;
waitpid($pid, 0);
$? || $err and
- $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+ $lei->child_error($?, "@$cmd failed".$err ? " $err" : '');
}
sub input_path_url {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 9d95e899..1e54c3bf 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -18,7 +18,7 @@ sub lcat_folder ($$$) {
my $err = $lms->arg2folder($lei, $folders);
$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
if ($err && $err->{fail}) {
- $lei->child_error(1 << 8, "# unknown folder: $folder");
+ $lei->child_error(0, "# unknown folder: $folder");
} else {
for my $f (@$folders) {
my $fid = $lms->fid_for($f);
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 0ba5897c..60286b06 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -23,7 +23,7 @@ sub rediff_user_cb { # called by solver when done
my $lei = $self->{lei};
my $log_buf = delete $lei->{log_buf};
$$log_buf =~ s/^/# /sgm;
- ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+ ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
$lei->qerr($$log_buf);
my ($git, $oid, $type, $size, $di) = @$res;
my $oid_want = delete $self->{cur_oid_want};
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 85efd9f5..e1da64aa 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -59,7 +59,7 @@ sub up1_redispatch {
up1($l, $out);
$l->qerr("# $out done");
};
- $l->child_error(1 << 8, $@) if $@;
+ $l->child_error(0, $@) if $@;
}
sub lei_up {
^ permalink raw reply related [relevance 52%]
* Re: Showcasing lei at Linux Plumbers
2021-09-02 21:58 65% ` Eric Wong
@ 2021-09-03 15:15 71% ` Konstantin Ryabitsev
2021-09-07 21:33 71% ` Konstantin Ryabitsev
1 sibling, 0 replies; 200+ results
From: Konstantin Ryabitsev @ 2021-09-03 15:15 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 02, 2021 at 09:58:50PM +0000, Eric Wong wrote:
> Fwiw, most of the functionality works much better with Maildir
> because of potential password prompts needed for IMAP and
> interactivity required.
Okay, I'll try this out with maildir for now -- it's easy to hook mbsync into
the process if desired.
> OK, there's two main commands, "lei q" and "lei up".
> Both of which may prompt for passwords depending on how
> git-credential is set up:
>
> # the destination, could be Maildir
> MFOLDER=imaps://user@example.com/INBOX.landlock
>
> # initial search:
> lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
> (
> dfn:Documentation/security/landlock.rst OR
> dfn:Documentation/userspace-api/landlock.rst OR
> dfn:include/uapi/linux/landlock.h OR
> dfn:samples/landlock/ OR
> dfn:security/landlock/ OR
> dfn:tools/testing/selftests/landlock/ OR
> dfhh:landlock
> ) AND rt:2.months.ago..
> EOF
>
> # update whenever, may prompt for IMAP password, but could be
> # cron-ed or similar if passwords are cached
> lei up $MFOLDER
>
> # Optional: tweaking the search parameters can be done via
> lei edit-search $MFOLDER
Yep, that seems to work fine. Question -- I noticed that lei just issues a
regular query, retrieves results with curl and then parses the output. Is
there a danger of potentially running into issues with parsing the regular
HTML output if it changes in the future?
-K
^ permalink raw reply [relevance 71%]
* [PATCH 0/4] lei up --all support for IMAP
@ 2021-09-07 11:32 71% Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
It's not efficient since it wastes an IMAP connection for
auth-only (unlike every other lei command which reuses the
connection once authed). However, there may be ways we can
work around it in the future.
Eric Wong (4):
xt/net_writer_imap: test "lei up" on single IMAP output
lei: dump and clear log at exit
lei up: support --all for IMAP folders
doc: lei-*.pod: update to Tor v3 .onion address
Documentation/lei-convert.pod | 2 +-
Documentation/lei-edit-search.pod | 2 +-
Documentation/lei-forget-search.pod | 2 +-
Documentation/lei-lcat.pod | 2 +-
Documentation/lei-ls-mail-sync.pod | 2 +-
Documentation/lei-ls-search.pod | 2 +-
Documentation/lei-rediff.pod | 2 +-
Documentation/lei-up.pod | 17 +++--
lib/PublicInbox/LEI.pm | 20 +++---
lib/PublicInbox/LeiAuth.pm | 15 ++--
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 104 +++++++++++++++++-----------
xt/net_writer-imap.t | 4 ++
13 files changed, 114 insertions(+), 63 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
@ 2021-09-07 11:32 71% ` Eric Wong
2021-09-07 11:32 71% ` [PATCH 2/4] lei: dump and clear log at exit Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
That's the minimum, at least...
---
xt/net_writer-imap.t | 3 +++
1 file changed, 3 insertions(+)
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index ec8f80d1..aeed3144 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -239,6 +239,9 @@ EOM
lei_ok qw(q m:forwarded@test.example.com);
is_deeply(json_utf8->decode($lei_out)->[0]->{kw}, ['forwarded'],
'forwarded kw imported from IMAP');
+
+ lei_ok qw(q m:testmessage --no-external -o), $folder_url;
+ lei_ok qw(up), $folder_url;
});
undef $cleanup; # remove temporary folder
^ permalink raw reply related [relevance 71%]
* [PATCH 2/4] lei: dump and clear log at exit
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
@ 2021-09-07 11:32 71% ` Eric Wong
2021-09-07 11:32 38% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
2021-09-07 11:32 51% ` [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
This may be helpful for diagnosing errors in case we missed any.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 098a45ba..bd44cfae 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1330,6 +1330,7 @@ sub lazy_start {
open STDOUT, '>&STDIN' or die "redirect stdout failed: $!";
# $daemon pipe to `lei' closed, main loop begins:
PublicInbox::DS->EventLoop;
+ dump_and_clear_log();
exit($exit_code // 0);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 3/4] lei up: support --all for IMAP folders
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
2021-09-07 11:32 71% ` [PATCH 2/4] lei: dump and clear log at exit Eric Wong
@ 2021-09-07 11:32 38% ` Eric Wong
2021-09-07 11:32 51% ` [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
Since "lei up" is expected to be a heavily-used command,
better support for IMAP seems like a reasonable idea.
This is inefficient since we waste an IMAP(S) TCP connection
since it dies when an auth-only LeiUp worker process dies, but
it's better than not working at all, right now.
---
Documentation/lei-up.pod | 15 ++++-
lib/PublicInbox/LEI.pm | 19 ++++---
lib/PublicInbox/LeiAuth.pm | 15 +++--
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 104 ++++++++++++++++++++++-------------
xt/net_writer-imap.t | 1 +
6 files changed, 102 insertions(+), 55 deletions(-)
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index cea0f619..ca4cf4fe 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -6,15 +6,24 @@ lei-up - update a saved search
lei up [OPTIONS] OUTPUT
-lei up [OPTIONS] --all=TYPE
+lei up [OPTIONS] --all[=<local|remote>]
=head1 DESCRIPTION
-Update the saved search at C<OUTPUT> or all saved searches of C<TYPE>
-(currently C<local> is the only supported value).
+Update the saved search at C<OUTPUT> or all saved searches.
=head1 OPTIONS
+=over
+
+=item --all[=<local|remote>]
+
+C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
+C<--all=local> only updates local mailboxes, C<--all=remote> only
+updates remote mailboxes (currently C<imap://> and C<imaps://>).
+
+=back
+
The following options, described in L<lei-q(1)>, are supported.
=over
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bd44cfae..a258722e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -605,16 +605,19 @@ sub incr {
$self->{counters}->{$field} += $nr;
}
+sub pkt_ops {
+ my ($lei, $ops) = @_;
+ $ops->{'!'} = [ \&fail_handler, $lei ];
+ $ops->{'|'} = [ \&sigpipe_handler, $lei ];
+ $ops->{x_it} = [ \&x_it, $lei ];
+ $ops->{child_error} = [ \&child_error, $lei ];
+ $ops->{incr} = [ \&incr, $lei ];
+ $ops;
+}
+
sub workers_start {
my ($lei, $wq, $jobs, $ops, $flds) = @_;
- $ops = {
- '!' => [ \&fail_handler, $lei ],
- '|' => [ \&sigpipe_handler, $lei ],
- 'x_it' => [ \&x_it, $lei ],
- 'child_error' => [ \&child_error, $lei ],
- 'incr' => [ \&incr, $lei ],
- ($ops ? %$ops : ()),
- };
+ $ops = pkt_ops($lei, { ($ops ? %$ops : ()) });
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm
index 465a2758..9b09cecf 100644
--- a/lib/PublicInbox/LeiAuth.pm
+++ b/lib/PublicInbox/LeiAuth.pm
@@ -30,10 +30,16 @@ sub do_auth_atfork { # used by IPC WQ workers
return if $wq->{-wq_worker_nr} != 0; # only first worker calls this
my $lei = $wq->{lei};
my $net = $lei->{net};
+ if ($net->{-auth_done}) { # from previous worker... (ugly)
+ $lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
+ $lei->fail("pkt_do net_merge_continue: $!");
+ return;
+ }
eval { # fill auth info (may prompt user or read netrc)
my $mics = $net->imap_common_init($lei);
my $nn = $net->nntp_common_init($lei);
# broadcast successful auth info to lei-daemon:
+ $net->{-auth_done} = 1;
$lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
die "pkt_do net_merge_continue: $!";
$net->{mics_cached} = $mics if $mics;
@@ -51,14 +57,15 @@ sub net_merge_all { # called in wq worker via wq_broadcast
# called by top-level lei-daemon when first worker is done with auth
# passes updated net auth info to current workers
sub net_merge_continue {
- my ($wq, $net_new) = @_;
+ my ($wq, $lei, $net_new) = @_;
+ $wq->{-net_new} = $net_new; # for "lei up"
$wq->wq_broadcast('PublicInbox::LeiAuth::net_merge_all', $net_new);
- $wq->net_merge_all_done; # defined per-WQ
+ $wq->net_merge_all_done($lei); # defined per-WQ
}
sub op_merge { # prepares PktOp->pair ops
- my ($self, $ops, $wq) = @_;
- $ops->{net_merge_continue} = [ \&net_merge_continue, $wq ];
+ my ($self, $ops, $wq, $lei) = @_;
+ $ops->{net_merge_continue} = [ \&net_merge_continue, $wq, $lei ];
}
sub new { bless \(my $x), __PACKAGE__ }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 2c7a92de..01f08384 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -396,7 +396,8 @@ sub new {
} elsif ($fmt =~ /\Aimaps?\z/) {
require PublicInbox::NetWriter;
require PublicInbox::URIimap;
- my $net = PublicInbox::NetWriter->new;
+ # {net} may exist from "lei up" for auth
+ my $net = $lei->{net} // PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
$net->add_url($$uri);
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index a39d6047..30358e9d 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -5,8 +5,14 @@
package PublicInbox::LeiUp;
use strict;
use v5.10.1;
+# n.b. we use LeiInput to setup IMAP auth
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiSavedSearch;
-use parent qw(PublicInbox::IPC);
+use PublicInbox::DS;
+use PublicInbox::PktOp;
+use PublicInbox::LeiFinmsg;
+use PublicInbox::LEI;
+my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
sub up1 ($$) {
my ($lei, $out) = @_;
@@ -48,75 +54,95 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
- require PublicInbox::LeiFinmsg;
- $lei->{fmsg} //= PublicInbox::LeiFinmsg->new($lei->{2});
my $l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} };
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
+ delete $l->{sock}; # do not close
+ $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ local $PublicInbox::LEI::current_lei = $l;
$l->{1} = $fh;
eval {
$l->qerr("# updating $out");
up1($l, $out);
- $l->qerr("# $out done");
};
- $l->child_error(0, $@) if $@;
+ $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
+}
+
+sub redispatch_all ($$) {
+ my ($self, $lei) = @_;
+ # re-dispatch into our event loop w/o creating an extra fork-level
+ $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei->{2});
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
+ PublicInbox::DS::requeue(sub {
+ up1_redispatch($lei, $o, $op_p);
+ });
+ }
+ $lei->event_step_init;
+ $lei->pkt_ops($op_c->{ops} = { '' => [$lei->can('dclose'), $lei] });
}
sub lei_up {
my ($lei, @outs) = @_;
- $lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
- my @local;
- if (defined $opt->{all}) {
+ my $self = bless { -mail_sync => 1 }, __PACKAGE__;
+ $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
+ if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
length($opt->{mua}//'') and return
$lei->fail('--all and --mua= are incompatible');
-
- # supporting IMAP outputs is more involved due to
- # git-credential prompts. TODO: add this in 1.8
- $opt->{all} eq 'local' or return
- $lei->fail('only --all=local works at the moment');
- my @all = PublicInbox::LeiSavedSearch::list($lei);
- @local = grep(!m!\Aimaps?://!i, @all);
+ @outs = PublicInbox::LeiSavedSearch::list($lei);
+ if ($all eq 'local') {
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq 'remote') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq '') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } else {
+ $lei->fail("only --all=$all not understood");
+ }
} else {
- @local = @outs;
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
- if (scalar(@outs) > 1) {
- length($opt->{mua}//'') and return $lei->fail(<<EOM);
+ ((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
+ length($opt->{mua} // '')) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
- # TODO:
- return $lei->fail(<<EOM) if grep(m!\Aimaps?://!i, @outs);
-multiple destinations only supported for local outputs (FIXME)
-EOM
+ if ($self->{remote}) { # setup lei->{auth}
+ $self->prepare_inputs($lei, $self->{remote}) or return;
}
- if (scalar(@local) > 1) {
- $lei->_lei_store->write_prepare($lei); # share early
- # daemon mode, re-dispatch into our event loop w/o
- # creating an extra fork-level
- require PublicInbox::DS;
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@local) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
- }
- $lei->event_step_init;
- $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
+ if ($lei->{auth}) { # start auth worker
+ require PublicInbox::NetWriter;
+ bless $lei->{net}, 'PublicInbox::NetWriter';
+ $lei->{auth}->op_merge(my $ops = {}, $self, $lei);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
+ $lei->{wq1} = $self;
+ $lei->wait_wq_events($op_c, $ops);
+ # net_merge_all_done will fire when auth is done
} else {
- up1($lei, $local[0]);
+ redispatch_all($self, $lei); # see below
}
}
+# called in top-level lei-daemon when LeiAuth is done
+sub net_merge_all_done {
+ my ($self, $lei) = @_;
+ $lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
+ $self->wq_close(1);
+ redispatch_all($self, $lei);
+}
+
sub _complete_up {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
}
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+
1;
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index aeed3144..0a4cea68 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -242,6 +242,7 @@ EOM
lei_ok qw(q m:testmessage --no-external -o), $folder_url;
lei_ok qw(up), $folder_url;
+ lei_ok qw(up --all=remote);
});
undef $cleanup; # remove temporary folder
^ permalink raw reply related [relevance 38%]
* [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
` (2 preceding siblings ...)
2021-09-07 11:32 38% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
@ 2021-09-07 11:32 51% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
We missed a few when new documentation came in, and there's no
going back to v2 onions.
Followup-to: 0b15dfc58ceaecdc ("treewide: update to v3 Tor onions")
---
Documentation/lei-convert.pod | 2 +-
Documentation/lei-edit-search.pod | 2 +-
Documentation/lei-forget-search.pod | 2 +-
Documentation/lei-lcat.pod | 2 +-
Documentation/lei-ls-mail-sync.pod | 2 +-
Documentation/lei-ls-search.pod | 2 +-
Documentation/lei-rediff.pod | 2 +-
Documentation/lei-up.pod | 2 +-
8 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/Documentation/lei-convert.pod b/Documentation/lei-convert.pod
index e8a71393..7f372327 100644
--- a/Documentation/lei-convert.pod
+++ b/Documentation/lei-convert.pod
@@ -59,7 +59,7 @@ L<lei-q(1)>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-edit-search.pod b/Documentation/lei-edit-search.pod
index 7908b5a2..21cb11aa 100644
--- a/Documentation/lei-edit-search.pod
+++ b/Documentation/lei-edit-search.pod
@@ -15,7 +15,7 @@ Invoke C<git config --edit> to edit the saved search at C<OUTPUT>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
index 49bc1d68..f3f043f9 100644
--- a/Documentation/lei-forget-search.pod
+++ b/Documentation/lei-forget-search.pod
@@ -15,7 +15,7 @@ Forget a saved search at C<OUTPUT>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
index 5a2bdb5a..656df489 100644
--- a/Documentation/lei-lcat.pod
+++ b/Documentation/lei-lcat.pod
@@ -66,7 +66,7 @@ The following options, described in L<lei-q(1)>, are supported.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-ls-mail-sync.pod b/Documentation/lei-ls-mail-sync.pod
index 37aa910f..86aede40 100644
--- a/Documentation/lei-ls-mail-sync.pod
+++ b/Documentation/lei-ls-mail-sync.pod
@@ -42,7 +42,7 @@ Use C<\0> (NUL) instead of newline (CR) to delimit lines.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-ls-search.pod b/Documentation/lei-ls-search.pod
index 138dbbff..a56611bf 100644
--- a/Documentation/lei-ls-search.pod
+++ b/Documentation/lei-ls-search.pod
@@ -51,7 +51,7 @@ is incompatible with C<--format>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index e968fb20..c7db6c1e 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -67,7 +67,7 @@ The options below, described in L<lei-q(1)>, are also supported.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index ca4cf4fe..e5d97f43 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -43,7 +43,7 @@ This option is incompatible with C<--all>.
Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
The mail archives are hosted at L<https://public-inbox.org/meta/>
-and L<http://hjrcffqmbrq6wope.onion/meta/>
+and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
=head1 COPYRIGHT
^ permalink raw reply related [relevance 51%]
* Re: Showcasing lei at Linux Plumbers
2021-09-02 21:58 65% ` Eric Wong
2021-09-03 15:15 71% ` Konstantin Ryabitsev
@ 2021-09-07 21:33 71% ` Konstantin Ryabitsev
2021-09-07 22:14 71% ` Eric Wong
1 sibling, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-07 21:33 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 02, 2021 at 09:58:50PM +0000, Eric Wong wrote:
> OK, there's two main commands, "lei q" and "lei up".
> Both of which may prompt for passwords depending on how
> git-credential is set up:
>
> # the destination, could be Maildir
> MFOLDER=imaps://user@example.com/INBOX.landlock
>
> # initial search:
> lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
> (
> dfn:Documentation/security/landlock.rst OR
> dfn:Documentation/userspace-api/landlock.rst OR
> dfn:include/uapi/linux/landlock.h OR
> dfn:samples/landlock/ OR
> dfn:security/landlock/ OR
> dfn:tools/testing/selftests/landlock/ OR
> dfhh:landlock
> ) AND rt:2.months.ago..
> EOF
>
> # update whenever, may prompt for IMAP password, but could be
> # cron-ed or similar if passwords are cached
> lei up $MFOLDER
>
> # Optional: tweaking the search parameters can be done via
> lei edit-search $MFOLDER
If I had a local mirror with extindex and I wanted to do the same thing, would
I just modify the -I flag to point at the extindex location? One of the
options I want to investigate is making IMAP/POP3 accessible individual
mailboxes fed by lei, such that a new subsystem maintainer could have a
ready-made mailbox available to them without needing to subscribe/unsubscribe
to a bunch of mailing lists. (This would be different from read-only imap
mailboxes offered by public-inbox-imapd, since we'll be tracking individual
message state. The POP3 bit would allow them to plug it into something like
Gmail which allows sucking down remote POPs.)
-K
^ permalink raw reply [relevance 71%]
* Re: Showcasing lei at Linux Plumbers
2021-09-07 21:33 71% ` Konstantin Ryabitsev
@ 2021-09-07 22:14 71% ` Eric Wong
2021-09-08 13:36 65% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-07 22:14 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Thu, Sep 02, 2021 at 09:58:50PM +0000, Eric Wong wrote:
> > # the destination, could be Maildir
> > MFOLDER=imaps://user@example.com/INBOX.landlock
> >
> > # initial search:
> > lei q -o $MFOLDER -t -I https://lore.kernel.org/all/ --stdin <<EOF
>
> If I had a local mirror with extindex and I wanted to do the same thing, would
> I just modify the -I flag to point at the extindex location?
Yes. For local stuff that's permanently mounted, I tend to do
"lei add-external $PATHNAME" so it's included by default.
> One of the
> options I want to investigate is making IMAP/POP3 accessible individual
> mailboxes fed by lei, such that a new subsystem maintainer could have a
> ready-made mailbox available to them without needing to subscribe/unsubscribe
> to a bunch of mailing lists. (This would be different from read-only imap
> mailboxes offered by public-inbox-imapd, since we'll be tracking individual
> message state. The POP3 bit would allow them to plug it into something like
> Gmail which allows sucking down remote POPs.)
I think using the "-o v2:..." option for now would be the way to
go for making a v2 inbox available via -imapd (and it'll get
JMAP/POP3 support in the future).
We don't have POP3 support in client nor server form, yet. Not
sure how account/state management would work, nor how to
prioritize it vs JMAP support. I'm thinking POP3 takes priority
since there's more clients for it...
Existing POP3 servers would work, too; since lei can output
to Maildir/mbox* which can work with them.
On a side note, I'm not aware of IMAP sync tools accounting for
read-only IMAP servers well, since they attempt bidirectional
sync. "lei import" seems alright in that regard, treating IMAP
the same way it will (eventually) treat POP3.
^ permalink raw reply [relevance 71%]
* [PATCH] lei q|up: fix write counter for v2
@ 2021-09-07 22:41 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-07 22:41 UTC (permalink / raw)
To: meta
It's a bit confusing to see "0 written to ..." when we actually
wrote something.
---
lib/PublicInbox/LeiToMail.pm | 1 +
t/lei-q-save.t | 2 ++
2 files changed, 3 insertions(+)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 01f08384..dbf58df9 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -361,6 +361,7 @@ sub _v2_write_cb ($$) {
$eml //= PublicInbox::Eml->new($bref);
return if $dedupe && $dedupe->is_dup($eml, $smsg);
$lei->{v2w}->ipc_do('add', $eml); # V2Writable->add
+ ++$lei->{-nr_write};
}
}
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 7aa9b84e..743a7b70 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -179,6 +179,8 @@ test_lei(sub {
my (@before, @after);
require PublicInbox::MboxReader;
lei_ok(qw(q z:0.. -o), "v2:$v2");
+ like($lei_err, qr/^# ([1-9][0-9]*) written to \Q$v2\E/sm,
+ 'non-zero write output to stderr');
lei_ok(qw(q z:0.. -o), "mboxrd:$home/before", '--only', $v2, '-j1,1');
open my $fh, '<', "$home/before";
PublicInbox::MboxReader->mboxrd($fh, sub { push @before, $_[0] });
^ permalink raw reply related [relevance 71%]
* Re: Showcasing lei at Linux Plumbers
2021-09-07 22:14 71% ` Eric Wong
@ 2021-09-08 13:36 65% ` Konstantin Ryabitsev
2021-09-08 14:49 66% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-08 13:36 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Tue, Sep 07, 2021 at 10:14:04PM +0000, Eric Wong wrote:
> > One of the
> > options I want to investigate is making IMAP/POP3 accessible individual
> > mailboxes fed by lei, such that a new subsystem maintainer could have a
> > ready-made mailbox available to them without needing to subscribe/unsubscribe
> > to a bunch of mailing lists. (This would be different from read-only imap
> > mailboxes offered by public-inbox-imapd, since we'll be tracking individual
> > message state. The POP3 bit would allow them to plug it into something like
> > Gmail which allows sucking down remote POPs.)
>
> I think using the "-o v2:..." option for now would be the way to
> go for making a v2 inbox available via -imapd (and it'll get
> JMAP/POP3 support in the future).
I'm worried that read-only imap folders are going to cause problems for dumber
imap clients, including mbsync. My goal is to make it easy for folks to use
existing tools to which they are already accustomed, since my experience is
that if the learning curve is too steep or requires too much fiddling to
configure, the uptake is going to be extremely limited.
On the other hand, a service that offers full search-based imap/pop3 folders
is going to be an easy sell:
- it works with any imap client as a simple extra account
- it can be mirrored locally and synced two-ways via mbsync
- it can be incorporated into existing services like gmail, so people can
monitor things on the go
- I can do clever things like suspend "lei up" runs if there was no access to
the folder for over N weeks
- we can use FS dedupe features since all messages are going to be
identical after writing them out to maildirs
The slightly harder part is making it easy for people to configure their
search parameters, but I'm hoping to expose this via a git repo.
I'm not implementing this right away, but I'm going to float this idea at
plumbers to see what the reception is going to be. I believe this will be of
interest to many devs, since this would allow them to no longer depend on
their corporate mail servers and their mail-mangling ways.
-K
^ permalink raw reply [relevance 65%]
* Re: Showcasing lei at Linux Plumbers
2021-09-08 13:36 65% ` Konstantin Ryabitsev
@ 2021-09-08 14:49 66% ` Eric Wong
2021-09-08 17:17 66% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-08 14:49 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Tue, Sep 07, 2021 at 10:14:04PM +0000, Eric Wong wrote:
> > > One of the
> > > options I want to investigate is making IMAP/POP3 accessible individual
> > > mailboxes fed by lei, such that a new subsystem maintainer could have a
> > > ready-made mailbox available to them without needing to subscribe/unsubscribe
> > > to a bunch of mailing lists. (This would be different from read-only imap
> > > mailboxes offered by public-inbox-imapd, since we'll be tracking individual
> > > message state. The POP3 bit would allow them to plug it into something like
> > > Gmail which allows sucking down remote POPs.)
> >
> > I think using the "-o v2:..." option for now would be the way to
> > go for making a v2 inbox available via -imapd (and it'll get
> > JMAP/POP3 support in the future).
>
> I'm worried that read-only imap folders are going to cause problems for dumber
> imap clients, including mbsync. My goal is to make it easy for folks to use
> existing tools to which they are already accustomed, since my experience is
> that if the learning curve is too steep or requires too much fiddling to
> configure, the uptake is going to be extremely limited.
Agreed with read-only IMAP being a problem for existing clients.
lei is gradual in that approach in you can pick and choose which
parts to use. It's actually close to being able to offer
<mbsync||offlineimap> functionality, but it's a bit clumsy
usage-wise atm:
lei import imaps://example.com/folder
# lei <q|lcat> results dumped to Maildir
# inotify reads Maildir keyword updates done by MUA
lei export-kw imaps://example.com/folder
I'm working on making the "export-kw" part transparent like it
mostly is with Maildirs.
The one thing lei doesn't do right now is deleting messages
from IMAP folders (unless it's overwriting search results).
That will probably be a separate command:
lei prune-mfolder [--expire=...]
I hope to stop using <mbsync||offlineimap> myself, soon...
> On the other hand, a service that offers full search-based imap/pop3 folders
> is going to be an easy sell:
>
> - it works with any imap client as a simple extra account
> - it can be mirrored locally and synced two-ways via mbsync
POP3 would be significantly easier to support server-side with
multiple users since it won't need to store per-user keywords.
Since lei is a daemon and can support multiple users, it could
have an R/W JMAP||IMAP front-end, though...
> - it can be incorporated into existing services like gmail, so people can
> monitor things on the go
POP3 seems excellent for integrating into large mail providers.
I mainly haven't gotten around to implementing it, nor figuring
out how to deal with account management...
> - I can do clever things like suspend "lei up" runs if there was no access to
> the folder for over N weeks
> - we can use FS dedupe features since all messages are going to be
> identical after writing them out to maildirs
I've been thinking of making lei storage accessible as Maildirs
via FUSE, as well.
> The slightly harder part is making it easy for people to configure their
> search parameters, but I'm hoping to expose this via a git repo.
*shrug* I've been trying to keep the learning curve as low as
possible by using most of the same prefixes as mairix (lei only
adds L: and kw: for labels and keywords).
^ permalink raw reply [relevance 66%]
* Re: Showcasing lei at Linux Plumbers
2021-09-08 14:49 66% ` Eric Wong
@ 2021-09-08 17:17 66% ` Konstantin Ryabitsev
2021-09-08 17:32 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-08 17:17 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Wed, Sep 08, 2021 at 02:49:48PM +0000, Eric Wong wrote:
> > On the other hand, a service that offers full search-based imap/pop3 folders
> > is going to be an easy sell:
> >
> > - it works with any imap client as a simple extra account
> > - it can be mirrored locally and synced two-ways via mbsync
>
> POP3 would be significantly easier to support server-side with
> multiple users since it won't need to store per-user keywords.
Okay, then perhaps I should sit on my hands for a bit. I'll showcase lei with
remote searches as a feature preview, but buffer it with the following
statements:
- We're working on making it easy to add search-based inboxes that would allow
developers to closely match subsystem MAINTAINERS entries. In fact, we can
probably automate the creation of such feeds by watching the MAINTAINERS
file and automatically converting F:/X: lines into queries (not so easily
done for K: and N: lines unless they aren't using actual regex expressions).
- Developers will be able to easily access these feeds via multiple ways, e.g:
- read-only imap folders
- pseudo mailing list subscriptions
- nntp groups
- pop3 mailboxes (coming in the future)
The goal is to solve the following several problems:
- remove content-mangling corporate mail gateways out of the picture
- make it unnecessary for patch submitters to know where they should send the
patches ("just send them to patches@linux.dev").
- reduce the need for new mailing lists as new subsystems are introduced
("just send email to discuss@linux.dev with somekeyword: in the subject")
I think that sounds pretty reasonable and I can get most of it done by EOY.
> > - I can do clever things like suspend "lei up" runs if there was no access to
> > the folder for over N weeks
> > - we can use FS dedupe features since all messages are going to be
> > identical after writing them out to maildirs
>
> I've been thinking of making lei storage accessible as Maildirs
> via FUSE, as well.
That's a pretty cool idea, actually -- would that be readonly or with full
deletes/renames support?
-K
^ permalink raw reply [relevance 66%]
* Re: Showcasing lei at Linux Plumbers
2021-09-08 17:17 66% ` Konstantin Ryabitsev
@ 2021-09-08 17:32 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-08 17:32 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Wed, Sep 08, 2021 at 02:49:48PM +0000, Eric Wong wrote:
> > I've been thinking of making lei storage accessible as Maildirs
> > via FUSE, as well.
>
> That's a pretty cool idea, actually -- would that be readonly or with full
> deletes/renames support?
Renames for sure. Likely deletes, at least on a per-label basis.
Haven't thought much about deletes/purge...
^ permalink raw reply [relevance 71%]
* [PATCH] lei-rm: add man page, support LeiInput args
@ 2021-09-08 18:48 54% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-08 18:48 UTC (permalink / raw)
To: meta
-F/--in-format and --lock=TYPE(S) are easily supported by
all classes using LeiInput.
---
Documentation/lei-rm.pod | 72 ++++++++++++++++++++++++++++++++++++++++
MANIFEST | 1 +
Makefile.PL | 2 +-
lib/PublicInbox/LEI.pm | 2 +-
4 files changed, 75 insertions(+), 2 deletions(-)
create mode 100644 Documentation/lei-rm.pod
diff --git a/Documentation/lei-rm.pod b/Documentation/lei-rm.pod
new file mode 100644
index 00000000..f2a0c0f0
--- /dev/null
+++ b/Documentation/lei-rm.pod
@@ -0,0 +1,72 @@
+=head1 NAME
+
+lei-rm - unindex a message in lei/store
+
+=head1 SYNOPSIS
+
+lei rm [OPTIONS] (-|--stdin)
+
+lei rm [OPTIONS] LOCATION
+
+=head1 DESCRIPTION
+
+Removes message(s) and associated private metadata from lei/store
+indices. It does not affect messages stored in externals, so it's
+still possible to get "removed" messages from externals in L<lei-q>
+search results.
+
+This does not remove the message from underlying git storage nor
+does it remove messages from Maildir/mbox/IMAP/etc. sources.
+
+=head1 OPTIONS
+
+=over
+
+=item -
+
+=item --stdin
+
+Read input from standard input. This is the default if standard
+input is a pipe or regular file and there are no arguments on
+the command-line.
+
+=item -F MAIL_FORMAT
+
+=item --in-format=MAIL_FORMAT
+
+Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
+C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+
+Default: C<eml> when reading from stdin
+
+=item --lock=METHOD
+
+L<mbox(5)> locking method(s) to use: C<dotlock>, C<fcntl>, C<flock> or
+C<none>.
+
+Default: fcntl,dotlock
+
+=item -q
+
+=item --quiet
+
+Suppress feedback messages.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-store-format(5)>
diff --git a/MANIFEST b/MANIFEST
index fad29622..531f8c46 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -42,6 +42,7 @@ Documentation/lei-overview.pod
Documentation/lei-p2q.pod
Documentation/lei-q.pod
Documentation/lei-rediff.pod
+Documentation/lei-rm.pod
Documentation/lei-store-format.pod
Documentation/lei-tag.pod
Documentation/lei-up.pod
diff --git a/Makefile.PL b/Makefile.PL
index 2af8c2f1..82b50543 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -48,7 +48,7 @@ $v->{-m1} = [ map {
lei-add-external lei-blob lei-config lei-convert lei-edit-search
lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
lei-import lei-init lei-lcat lei-ls-external lei-ls-label
- lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-tag
+ lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-rm lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a258722e..3dce0236 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -226,7 +226,7 @@ our %CMD = ( # sorted in order of importance/use:
'rm' => [ '--stdin|LOCATION...',
'remove a message from the index and prevent reindexing',
'stdin|', # /|\z/ must be first for lone dash
- @c_opt ],
+ qw(in-format|F=s lock=s@), @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
^ permalink raw reply related [relevance 54%]
* [PATCH] lei prune-mail-sync: ignore missing locations
@ 2021-09-08 19:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-08 19:04 UTC (permalink / raw)
To: meta
"lei prune-mail-sync --all" shouldn't abort if a location
isn't available, and maybe it should prune harder...
---
lib/PublicInbox/LeiPruneMailSync.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
index 1a277122..3678bd04 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -73,7 +73,7 @@ lei mail_sync.sqlite3 uninitialized, see lei-import(1)
EOM
}
$sto->write_prepare($lei);
- my $self = bless {}, __PACKAGE__;
+ my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
^ permalink raw reply related [relevance 71%]
* [PATCH] lei up: print messages before disconnecting
@ 2021-09-09 5:34 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 5:34 UTC (permalink / raw)
To: meta
Closing the socket for script/lei needs to be done AFTER the
final message(s) are printed.
---
lib/PublicInbox/LeiFinmsg.pm | 9 +++++----
lib/PublicInbox/LeiUp.pm | 2 +-
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiFinmsg.pm b/lib/PublicInbox/LeiFinmsg.pm
index 395e7d3c..7ed58c24 100644
--- a/lib/PublicInbox/LeiFinmsg.pm
+++ b/lib/PublicInbox/LeiFinmsg.pm
@@ -8,14 +8,15 @@ use strict;
use v5.10.1;
sub new {
- my ($cls, $io) = @_;
- bless [ $io, $$ ], $cls;
+ my ($cls, $lei) = @_;
+ bless [ @$lei{qw(2 sock)}, $$ ], $cls;
}
sub DESTROY {
my ($self) = @_;
- my $io = shift @$self;
- shift(@$self) == $$ and print $io @$self;
+ my ($stderr, $sock, $pid) = splice(@$self, 0, 3);
+ print $stderr @$self if $pid == $$;
+ # script/lei disconnects when $sock SvREFCNT drops to zero
}
1;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 30358e9d..be476427 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -73,7 +73,7 @@ sub up1_redispatch {
sub redispatch_all ($$) {
my ($self, $lei) = @_;
# re-dispatch into our event loop w/o creating an extra fork-level
- $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei->{2});
+ $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei);
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
PublicInbox::DS::requeue(sub {
^ permalink raw reply related [relevance 70%]
* lei refresh-mail-sync [vs. prune-mail-sync]
@ 2021-09-09 5:47 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 5:47 UTC (permalink / raw)
To: meta
I'm thinking "lei prune-mail-sync" should be replaced with
"lei refresh-mail-sync". Some things move around in
Maildir and get pruned from mail_sync.sqlite3 instead
of updated if lei-daemon wasn't running (or the inotify
queue overflows)
^ permalink raw reply [relevance 71%]
* Tracking one-off threads with lei
@ 2021-09-09 15:19 71% Konstantin Ryabitsev
2021-09-09 20:06 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-09 15:19 UTC (permalink / raw)
To: meta
Eric:
Is there a way to "tag" a single thread that isn't matching a saved search and
have it be followed for any new updates? E.g. someone pings a developer on IRC
and says "you may be interested in following this discussion" -- what's the
best course of action for them to pull that into their MFOLDER and get all the
new updates?
-K
^ permalink raw reply [relevance 71%]
* Re: Tracking one-off threads with lei
2021-09-09 15:19 71% Tracking one-off threads with lei Konstantin Ryabitsev
@ 2021-09-09 20:06 71% ` Eric Wong
2021-09-09 20:56 71% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-09 20:06 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Eric:
>
> Is there a way to "tag" a single thread that isn't matching a saved search and
> have it be followed for any new updates? E.g. someone pings a developer on IRC
> and says "you may be interested in following this discussion" -- what's the
> best course of action for them to pull that into their MFOLDER and get all the
> new updates?
-t includes every message in the thread strictly (same with mairix):
lei q -t -o $MFOLDER mid:$MSGID
For a fuzzy subject match (like /t/ and /T/ in WWW), add ` OR s:"..." '
lei q -t -o $MFOLDER mid:$MSGID OR s:"subject phrase"
And the usual: lei up $MFOLDER
^ permalink raw reply [relevance 71%]
* Re: Tracking one-off threads with lei
2021-09-09 20:06 71% ` Eric Wong
@ 2021-09-09 20:56 71% ` Konstantin Ryabitsev
2021-09-09 21:06 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-09 20:56 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 09, 2021 at 08:06:28PM +0000, Eric Wong wrote:
> Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> > Eric:
> >
> > Is there a way to "tag" a single thread that isn't matching a saved search and
> > have it be followed for any new updates? E.g. someone pings a developer on IRC
> > and says "you may be interested in following this discussion" -- what's the
> > best course of action for them to pull that into their MFOLDER and get all the
> > new updates?
>
> -t includes every message in the thread strictly (same with mairix):
>
> lei q -t -o $MFOLDER mid:$MSGID
Ah, sorry, I wasn't clear -- this would be into an existing $MFOLDER with a
regular q already defined. E.g. I have ~/Maildir/foofunc with:
q = dfhh:foofunc
However, I am now suddenly interested in a thread with msgid foo@bar. I can
modify the q= parameter to be "dfhh:foofunc OR mid:foo@bar", or I can define a
new $MFOLDER just for mid:foo@bar, but it doesn't look like I can just one-off
cherry-pick a thread into an existing ~/Maildir/foofunc, right?
Is there a way to feed multiple saved searches into the same local maildir?
(Not saying there should be a way, just trying to get a clear picture in my
head.)
Thanks,
-K
^ permalink raw reply [relevance 71%]
* Re: Tracking one-off threads with lei
2021-09-09 20:56 71% ` Konstantin Ryabitsev
@ 2021-09-09 21:06 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 21:06 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Thu, Sep 09, 2021 at 08:06:28PM +0000, Eric Wong wrote:
> > Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> > > Eric:
> > >
> > > Is there a way to "tag" a single thread that isn't matching a saved search and
> > > have it be followed for any new updates? E.g. someone pings a developer on IRC
> > > and says "you may be interested in following this discussion" -- what's the
> > > best course of action for them to pull that into their MFOLDER and get all the
> > > new updates?
> >
> > -t includes every message in the thread strictly (same with mairix):
> >
> > lei q -t -o $MFOLDER mid:$MSGID
>
> Ah, sorry, I wasn't clear -- this would be into an existing $MFOLDER with a
> regular q already defined. E.g. I have ~/Maildir/foofunc with:
>
> q = dfhh:foofunc
>
> However, I am now suddenly interested in a thread with msgid foo@bar. I can
> modify the q= parameter to be "dfhh:foofunc OR mid:foo@bar", or I can define a
> new $MFOLDER just for mid:foo@bar, but it doesn't look like I can just one-off
> cherry-pick a thread into an existing ~/Maildir/foofunc, right?
You can modify existing searches with:
lei edit-search $MFOLDER
And possibly reset the external.$FOO.maxuid for local externals
to so old messages aren't excluded (dedupe should still work).
> Is there a way to feed multiple saved searches into the same local maildir?
> (Not saying there should be a way, just trying to get a clear picture in my
> head.)
Not sure, maybe --augment works, but I haven't thought about how
it interacts with saved searches at all
Of course Xapian OR lets you combine as many queries as you want
(but things like -t and the external list used is global across
all subqueries).
^ permalink raw reply [relevance 71%]
* Using lei with podman + toolbox
@ 2021-09-09 21:39 64% Konstantin Ryabitsev
2021-09-09 23:36 68% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-09 21:39 UTC (permalink / raw)
To: meta
Hi, all:
These are my quickie instructions for how to use lei in a toolbox environment
if you are running a distribution like Fedora and don't want to install a lot
of perl dependencies into your main OS.
1. Grab the dockerfile:
https://gist.github.com/mricon/046ba7c8b03bd92176dbe83e04f2466c
Right now, it's as below, though it may change in the future:
--- start: public-inbox.dockerfile ---
# Podman/Toolbox container for public-inbox
FROM docker.io/library/debian
LABEL com.github.containers.toolbox="true" \
com.github.debarshiray.toolbox="true"
RUN apt-get update && \
apt-get -y install sudo libcap2-bin locales vim \
git liburi-perl libemail-mime-perl libplack-perl libtimedate-perl \
libdbd-sqlite3-perl libsearch-xapian-perl libnet-server-perl \
libinline-c-perl libemail-address-xs-perl libparse-recdescent-perl \
xapian-tools libencode-perl libdbi-perl liblinux-inotify2-perl \
libio-compress-perl curl libmail-imapclient-perl libsocket-msghdr-perl \
sqlite3 libgit2-dev make eatmydata man-db pkg-config
# Change this to your locale, if you're not en_CA
RUN echo "en_CA.UTF-8 UTF-8" >> /etc/locale.gen && \
locale-gen && \
sed -i -e 's/ ALL$/ NOPASSWD:ALL/' /etc/sudoers && \
touch /etc/localtime && \
echo VARIANT_ID=container >> /etc/os-release
RUN git clone https://public-inbox.org /usr/local/public-inbox && \
cd /usr/local/public-inbox && \
perl Makefile.PL && \
make && \
make install && \
make clean
CMD /bin/bash
--- end: public-inbox.dockerfile ---
2. podman build -t public-inbox -f public-inbox.dockerfile
3. toolbox create --image localhost/public-inbox:latest lei
4. toolbox enter lei
That should let you use "lei" commands right after entering the container. To
update public-inbox, just run "git pull" in /usr/local/public-inbox and build
it again.
-K
^ permalink raw reply [relevance 64%]
* Re: Using lei with podman + toolbox
2021-09-09 21:39 64% Using lei with podman + toolbox Konstantin Ryabitsev
@ 2021-09-09 23:36 68% ` Eric Wong
2021-09-09 23:51 71% ` native C++ Xapian wrapper [was: Using lei with podman + toolbox] Eric Wong
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-09 23:36 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Hi, all:
>
> These are my quickie instructions for how to use lei in a toolbox environment
> if you are running a distribution like Fedora and don't want to install a lot
> of perl dependencies into your main OS.
Off the top of my head, I think Search::Xapian // Xapian.pm was
the main thing that was missing from CentOS 7. Does Fedora have
that?
(disclaimer: I don't care for Docker, seems like a giant waste
of space and bandwidth compared to just using the distro)
> 1. Grab the dockerfile:
> https://gist.github.com/mricon/046ba7c8b03bd92176dbe83e04f2466c
>
> Right now, it's as below, though it may change in the future:
> --- start: public-inbox.dockerfile ---
> # Podman/Toolbox container for public-inbox
> FROM docker.io/library/debian
>
> LABEL com.github.containers.toolbox="true" \
> com.github.debarshiray.toolbox="true"
>
> RUN apt-get update && \
> apt-get -y install sudo libcap2-bin locales vim \
> git liburi-perl libemail-mime-perl libplack-perl libtimedate-perl \
Email::MIME isn't used at all outside of tests (but it's widely packaged).
No idea why libcap2-bin and vim are explicit dependencies (any
editor will do). Don't need Plack for lei, either.
> libdbd-sqlite3-perl libsearch-xapian-perl libnet-server-perl \
> libinline-c-perl libemail-address-xs-perl libparse-recdescent-perl \
No need for Net::Server nor Parse::RecDescent for lei. I don't
use Net::Server at all outside of tests, since I use systemd.
Email::Address::XS and TimeDate can be useful for messed up
messages, but low importance (I think they're widely packaged).
E:A:X and P:RD are required for -imapd but nothing else.
> xapian-tools libencode-perl libdbi-perl liblinux-inotify2-perl \
> libio-compress-perl curl libmail-imapclient-perl libsocket-msghdr-perl \
Socket::Msghdr makes lei a teeny bit faster, but I don't think
it's worth using another distro or running a compiler to get
since Inline::C is already available in all distros. Everything
else should be in Fedora...
> sqlite3 libgit2-dev make eatmydata man-db pkg-config
eatmydata shouldn't be useful outside of development, and
libgit2+pkg-config isn't used by lei, yet
(it is for -httpd/-imapd/-nntpd)
^ permalink raw reply [relevance 68%]
* native C++ Xapian wrapper [was: Using lei with podman + toolbox]
2021-09-09 23:36 68% ` Eric Wong
@ 2021-09-09 23:51 71% ` Eric Wong
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-09 23:51 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Eric Wong <e@80x24.org> wrote:
> Off the top of my head, I think Search::Xapian // Xapian.pm was
> the main thing that was missing from CentOS 7. Does Fedora have
> that?
Btw, I've been considering a Just-Ahead-Of-Time C++ wrapper for
Xapian. SWIG-or-not, bindings always seem behind and limited in
functionality compared to the native Xapian API. I still need
to learn the "++" parts of C++...
^ permalink raw reply [relevance 71%]
* [PATCH] lei add-external --mirror: deduce paths for PSGI mount prefixes
@ 2021-09-10 5:51 51% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 5:51 UTC (permalink / raw)
To: meta
The current manifest.js.gz generation in WWW doesn't account for
PSGI mount prefixes (and grokmirror 1.x appears to work fine).
In other words, <https://yhbt.net/lore/lkml/manifest.js.gz>
currently has keys like "/lkml/git/0.git" and not
"/lore/lkml/git/0.git" where "/lore" is the PSGI mount prefix.
This works fine with the prefix accounted for in my grokmirror
(1.x) repos.conf like this:
site = https://yhbt.net/lore/
manifest = https://yhbt.net/lore/manifest.js.gz
Adding the PSGI mount prefix in manifest.js.gz is probably not
desirable since it would force the prefix into the locally
cloned path by grokmirror, and all the cloned directories
would have the remote PSGI mount prefix prepended to the
toplevel.
So, "lei add-external --mirror" needs to account for PSGI
mount prefixes by deducing the prefix based on available keys
in the manifest.js.gz hash table.
---
MANIFEST | 1 +
lib/PublicInbox/LeiMirror.pm | 28 +++++++++++++++++++++-------
t/lei-mirror.psgi | 9 +++++++++
t/lei-mirror.t | 6 +++++-
4 files changed, 36 insertions(+), 8 deletions(-)
create mode 100644 t/lei-mirror.psgi
diff --git a/MANIFEST b/MANIFEST
index 531f8c46..a22672e7 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -436,6 +436,7 @@ t/lei-import-nntp.t
t/lei-import.t
t/lei-index.t
t/lei-lcat.t
+t/lei-mirror.psgi
t/lei-mirror.t
t/lei-p2q.t
t/lei-q-kw.t
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 39671f90..fca11ccf 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -200,6 +200,19 @@ failed to extract epoch number from $src
index_cloned_inbox($self, 2);
}
+# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+sub deduce_epochs ($$) {
+ my ($m, $path) = @_;
+ my ($v1_bare, @v2_epochs);
+ my $path_pfx = '';
+ do {
+ $v1_bare = $m->{$path};
+ @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+ } while (!defined($v1_bare) && !@v2_epochs &&
+ $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
+ ($path_pfx, $v1_bare, @v2_epochs);
+}
+
sub try_manifest {
my ($self) = @_;
my $uri = URI->new($self->{src});
@@ -229,8 +242,7 @@ sub try_manifest {
die "$uri: error decoding `$js': $@" if $@;
ref($m) eq 'HASH' or die "$uri unknown type: ".ref($m);
- my $v1_bare = $m->{$path};
- my @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+ my ($path_pfx, $v1_bare, @v2_epochs) = deduce_epochs($m, $path);
if (@v2_epochs) {
# It may be possible to have v1 + v2 in parallel someday:
$lei->err(<<EOM) if defined $v1_bare;
@@ -238,14 +250,16 @@ sub try_manifest {
# @v2_epochs
# ignoring $v1_bare (use --inbox-version=1 to force v1 instead)
EOM
- @v2_epochs = map { $uri->path($_); $uri->clone } @v2_epochs;
+ @v2_epochs = map {
+ $uri->path($path_pfx.$_);
+ $uri->clone
+ } @v2_epochs;
clone_v2($self, \@v2_epochs);
- } elsif ($v1_bare) {
+ } elsif (defined $v1_bare) {
clone_v1($self);
- } elsif (my @maybe = grep(m!\Q$path\E!, keys %$m)) {
- die "E: confused by <$uri>, possible matches:\n@maybe";
} else {
- die "E: confused by <$uri>";
+ die "E: confused by <$uri>, possible matches:\n\t",
+ join(', ', sort keys %$m), "\n";
}
}
diff --git a/t/lei-mirror.psgi b/t/lei-mirror.psgi
new file mode 100644
index 00000000..6b4bbfec
--- /dev/null
+++ b/t/lei-mirror.psgi
@@ -0,0 +1,9 @@
+use Plack::Builder;
+use PublicInbox::WWW;
+my $www = PublicInbox::WWW->new;
+$www->preload;
+builder {
+ enable 'Head';
+ mount '/pfx' => builder { sub { $www->call(@_) } };
+ mount '/' => builder { sub { $www->call(@_) } };
+};
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 80bc6ed5..65b6068c 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -7,7 +7,8 @@ my $sock = tcp_server();
my ($tmpdir, $for_destroy) = tmpdir();
my $http = 'http://'.tcp_host_port($sock);
my ($ro_home, $cfg_path) = setup_public_inboxes;
-my $cmd = [ qw(-httpd -W0), "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
+my $cmd = [ qw(-httpd -W0 ./t/lei-mirror.psgi),
+ "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
my $td = start_script($cmd, { PI_CONFIG => $cfg_path }, { 3 => $sock });
test_lei({ tmpdir => $tmpdir }, sub {
my $home = $ENV{HOME};
@@ -43,6 +44,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('ls-external');
unlike($lei_out, qr!\Q$t2-fail\E!, 'not added to ls-external');
+ lei_ok('add-external', "$t1-pfx", '--mirror', "$http/pfx/t1/",
+ \'--mirror v1 w/ PSGI prefix');
+
my %phail = (
HTTPS => 'https://public-inbox.org/' . 'phail',
ONION =>
^ permalink raw reply related [relevance 51%]
* [PATCH 0/4] lei: some net-related things
@ 2021-09-10 9:08 71% Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
After some consideration, ~/.netrc will no longer be read by
default to match the behavior of existing IMAP/NNTP clients.
And lei-index is pretty limited, but still useful for Maildir
users, so it's documented (mainly for its limitations).
Eric Wong (4):
lei_query: fix comment about %lei2curl commands
lei: split out @net_opt for curl/torsocks use
lei: do not read ~/.netrc by default
doc: lei-index manpage
Documentation/lei-index.pod | 69 ++++++++++++++++++++++++++++++++
MANIFEST | 1 +
Makefile.PL | 2 +-
lib/PublicInbox/GitCredential.pm | 8 +++-
lib/PublicInbox/LEI.pm | 34 ++++++++--------
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/NetReader.pm | 4 +-
7 files changed, 96 insertions(+), 24 deletions(-)
create mode 100644 Documentation/lei-index.pod
^ permalink raw reply [relevance 71%]
* [PATCH 3/4] lei: do not read ~/.netrc by default
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
@ 2021-09-10 9:08 65% ` Eric Wong
2021-09-10 9:08 57% ` [PATCH 4/4] doc: lei-index manpage Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
Since ~/.netrc isn't widely used by most (if any) NNTP and IMAP
clients, we won't read it by default for lei. AFAIK, ~/.netrc
is mainly by FTP clients (e.g. ftp(1) and lftp(1)). wget uses
it by default for HTTP(S) (and FTP), but curl does not.
To avoid breaking stable release use cases, public-inbox-watch
continues to read ~/.netrc by default.
The --netrc switch is supported by all existing lei commands
which may use curl.
---
lib/PublicInbox/GitCredential.pm | 8 ++++++--
lib/PublicInbox/NetReader.pm | 4 ++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/GitCredential.pm b/lib/PublicInbox/GitCredential.pm
index c83fed43..b18bba1e 100644
--- a/lib/PublicInbox/GitCredential.pm
+++ b/lib/PublicInbox/GitCredential.pm
@@ -31,8 +31,12 @@ sub run ($$;$) {
close $out_r or die "`git credential $op' failed: \$!=$! \$?=$?\n";
}
-sub check_netrc ($) {
- my ($self) = @_;
+sub check_netrc {
+ my ($self, $lei) = @_;
+
+ # n.b. lei doesn't load ~/.netrc by default, public-inbox-watch does,
+ # which may've been a mistake, but we have to live with it.
+ return if ($lei && !$lei->{opt}->{netrc});
# part of the standard library, but distributions may split it out
eval { require Net::Netrc };
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index a0e52fc5..f0f56431 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -96,7 +96,7 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
$cred = undef;
}
if ($cred) {
- my $p = $cred->{password} // $cred->check_netrc;
+ my $p = $cred->{password} // $cred->check_netrc($lei);
$cred->fill($lei) unless defined($p); # may prompt user here
$mic->User($mic_arg->{User} = $cred->{username});
$mic->Password($mic_arg->{Password} = $cred->{password});
@@ -191,7 +191,7 @@ sub nn_for ($$$$) { # nn = Net::NNTP
}, 'PublicInbox::GitCredential';
($u, $p) = split(/:/, $ui, 2);
($cred->{username}, $cred->{password}) = ($u, $p);
- $p //= $cred->check_netrc;
+ $p //= $cred->check_netrc($lei);
}
my $common = $nn_common->{$sec} // {};
my $nn_arg = {
^ permalink raw reply related [relevance 65%]
* [PATCH 4/4] doc: lei-index manpage
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
2021-09-10 9:08 65% ` [PATCH 3/4] lei: do not read ~/.netrc by default Eric Wong
@ 2021-09-10 9:08 57% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
It's a pretty incomplete command, so it's important to document
its incompleteness.
---
Documentation/lei-index.pod | 69 +++++++++++++++++++++++++++++++++++++
MANIFEST | 1 +
Makefile.PL | 2 +-
3 files changed, 71 insertions(+), 1 deletion(-)
create mode 100644 Documentation/lei-index.pod
diff --git a/Documentation/lei-index.pod b/Documentation/lei-index.pod
new file mode 100644
index 00000000..bd125bcc
--- /dev/null
+++ b/Documentation/lei-index.pod
@@ -0,0 +1,69 @@
+=head1 NAME
+
+lei-index - index messages without importing them into lei/store
+
+=head1 SYNOPSIS
+
+lei index [OPTIONS] FOLDER
+
+lei index [OPTIONS] --stdin
+
+=head1 DESCRIPTION
+
+Similar to L<lei-import(1)>, but does not store a copy of
+messages into C<lei/store>.
+
+This command only makes sense for messages stored in Maildir
+folders. Other folder types may be supported in the future
+(they can all be indexed, but the message isn't automatically
+retrieved by L<lei-q(1)> or L<lei-lcat(1)>).
+
+Combined with L<lei-q(1)>, C<lei index> allows Maildir users to
+have similar functionality to L<mairix(1)> by not duplicating
+messages into C<lei/store>.
+
+=head1 OPTIONS
+
+=over
+
+=item -
+
+=item --stdin
+
+Read input from standard input. This is the default if standard
+input is a pipe or regular file and there are no arguments on
+the command-line.
+
+=item -F MAIL_FORMAT
+
+=item --in-format=MAIL_FORMAT
+
+Message input format: C<eml>, C<maildir>, C<imap>, C<imaps>, C<nntp>,
+C<nntps>, C<mboxrd>, C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+
+Default: C<eml> when reading from stdin
+
+=item -q
+
+=item --quiet
+
+Suppress feedback messages.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-store-format(5)>, L<lei-import(1)>
diff --git a/MANIFEST b/MANIFEST
index 531f8c46..c0e3e855 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -31,6 +31,7 @@ Documentation/lei-edit-search.pod
Documentation/lei-forget-external.pod
Documentation/lei-forget-search.pod
Documentation/lei-import.pod
+Documentation/lei-index.pod
Documentation/lei-init.pod
Documentation/lei-lcat.pod
Documentation/lei-ls-external.pod
diff --git a/Makefile.PL b/Makefile.PL
index 82b50543..bfabb171 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -47,7 +47,7 @@ $v->{-m1} = [ map {
qw(
lei-add-external lei-blob lei-config lei-convert lei-edit-search
lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
- lei-import lei-init lei-lcat lei-ls-external lei-ls-label
+ lei-import lei-index lei-init lei-lcat lei-ls-external lei-ls-label
lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-rm lei-tag
lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
^ permalink raw reply related [relevance 57%]
* [PATCH 2/4] lei: split out @net_opt for curl/torsocks use
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
@ 2021-09-10 9:08 47% ` Eric Wong
2021-09-10 9:08 65% ` [PATCH 3/4] lei: do not read ~/.netrc by default Eric Wong
2021-09-10 9:08 57% ` [PATCH 4/4] doc: lei-index manpage Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:08 UTC (permalink / raw)
To: meta
IMAP and NNTP connections share some curl(1) options for TLS,
IPv4/IPv6, or netrc, etc...
---
lib/PublicInbox/LEI.pm | 34 ++++++++++++++++------------------
1 file changed, 16 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3dce0236..bbb6ab7e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -147,9 +147,9 @@ sub index_opt {
}
my @c_opt = qw(c=s@ C=s@ quiet|q);
-my @lxs_opt = (qw(remote! local! external! include|I=s@ exclude=s@ only=s@
- import-remote! no-torsocks torsocks=s),
- PublicInbox::LeiQuery::curl_opt());
+my @net_opt = (qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt());
+my @lxs_opt = qw(remote! local! external! include|I=s@ exclude=s@ only=s@
+ import-remote!);
# we don't support -C as an alias for --find-copies since it's already
# used for chdir
@@ -174,7 +174,7 @@ our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
our %CMD = ( # sorted in order of importance/use:
'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms',
'stdin|', # /|\z/ must be first for lone dash
- @lxs_opt,
+ @lxs_opt, @net_opt,
qw(save! output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
@@ -186,26 +186,26 @@ our %CMD = ( # sorted in order of importance/use:
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
# some of these options are ridiculous for lcat
- @lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+ @lxs_opt, @net_opt,
+ qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'blob' => [ 'OID', 'show a git blob, reconstructing from mail if necessary',
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
- @lxs_opt, @c_opt ],
+ @lxs_opt, @net_opt, @c_opt ],
'rediff' => [ '--stdin|LOCATION...',
'regenerate a diff with different options',
'stdin|', # /|\z/ must be first for lone dash
qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
- @diff_opt, @lxs_opt, @c_opt ],
+ @diff_opt, @lxs_opt, @net_opt, @c_opt ],
'add-external' => [ 'LOCATION',
'add/set priority of a publicinbox|extindex for extra matches',
- qw(boost=i mirror=s no-torsocks torsocks=s inbox-version=i
- verbose|v+), @c_opt, index_opt(),
- PublicInbox::LeiQuery::curl_opt() ],
+ qw(boost=i mirror=s inbox-version=i verbose|v+),
+ @c_opt, index_opt(), @net_opt ],
'ls-external' => [ '[FILTER]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
@@ -226,15 +226,14 @@ our %CMD = ( # sorted in order of importance/use:
'rm' => [ '--stdin|LOCATION...',
'remove a message from the index and prevent reindexing',
'stdin|', # /|\z/ must be first for lone dash
- qw(in-format|F=s lock=s@), @c_opt ],
+ qw(in-format|F=s lock=s@), @net_opt, @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
'tag' => [ 'KEYWORDS...',
'set/unset keywords and/or labels on message(s)',
qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
- qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt,
- pass_through('-kw:foo for delete') ],
+ @net_opt, @c_opt, pass_through('-kw:foo for delete') ],
'purge-mailsource' => [ 'LOCATION|--all',
'remove imported messages from IMAP, Maildirs, and MH',
@@ -253,25 +252,24 @@ our %CMD = ( # sorted in order of importance/use:
'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
- verbose|v+ incremental!),
- PublicInbox::LeiQuery::curl_opt(), # mainly for --proxy=
+ verbose|v+ incremental!), @net_opt, # mainly for --proxy=
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
- qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+ @net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
'prune-mail-sync' => [ 'LOCATION...|--all',
'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
- qw(all:s mode=s), @c_opt ],
+ qw(all:s mode=s), @net_opt, @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
- qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+ @net_opt, @c_opt ],
'p2q' => [ 'FILE|COMMIT_OID|--stdin',
"use a patch to generate a query for `lei q --stdin'",
qw(stdin| want|w=s@ uri debug), @c_opt ],
^ permalink raw reply related [relevance 47%]
* [PATCH] lei add-external --mirror: quiet unlink error on ENOENT
@ 2021-09-10 9:15 68% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 9:15 UTC (permalink / raw)
To: meta
If the mirror.done file doesn't exist for unlink, it's because
we already got another error, so don't confuse users by noting
an unlink error since the ENOENT is expected in the face of
other errors.
---
lib/PublicInbox/LeiMirror.pm | 2 +-
t/lei-mirror.t | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index fca11ccf..638add42 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -16,7 +16,7 @@ sub do_finish_mirror { # dwaitpid callback
if ($?) {
$lei->child_error($?);
} elsif (!unlink($f)) {
- $lei->err("unlink($f): $!");
+ $lei->err("unlink($f): $!") unless $!{ENOENT};
} else {
$lei->add_external_finish($mrr->{dst});
$lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 65b6068c..a61a7565 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -47,6 +47,14 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('add-external', "$t1-pfx", '--mirror', "$http/pfx/t1/",
\'--mirror v1 w/ PSGI prefix');
+ my $d = "$home/404";
+ ok(!lei(qw(add-external --mirror), "$http/404", $d), 'mirror 404');
+ unlike($lei_err, qr!unlink.*?404/mirror\.done!,
+ 'no unlink failure message');
+ ok(!-d $d, "`404' dir not created");
+ lei_ok('ls-external');
+ unlike($lei_out, qr!\Q$d\E!s, 'not added to ls-external');
+
my %phail = (
HTTPS => 'https://public-inbox.org/' . 'phail',
ONION =>
^ permalink raw reply related [relevance 68%]
* [PATCH] lei up: only delay non-zero "# $NR written to ..."
@ 2021-09-10 11:46 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-09-10 11:46 UTC (permalink / raw)
To: meta
"# 0 written to $FOLDER" messages aren't important to the
user, so we can show them in real time and allow them to
be lost in the terminal scroll. When >0 messages are
written to a folder, we'll show them last so a user
will know which folders to open with their MUA.
---
lib/PublicInbox/LeiXSearch.pm | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index b6d7bf2b..709a3b3a 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -400,8 +400,9 @@ Error closing $lei->{ovv}->{dst}: $!
my $tot = $lei->{-mset_total} // 0;
my $nr = $lei->{-nr_write} // 0;
if ($l2m) {
- $lei->qfin("# $nr written to " .
- "$lei->{ovv}->{dst} ($tot matches)");
+ my $m = "# $nr written to " .
+ "$lei->{ovv}->{dst} ($tot matches)";
+ $nr ? $lei->qfin($m) : $lei->qerr($m);
} else {
$lei->qerr("# $tot matches");
}
^ permalink raw reply related [relevance 71%]
* Re: Using lei with podman + toolbox
2021-09-09 23:36 68% ` Eric Wong
2021-09-09 23:51 71% ` native C++ Xapian wrapper [was: Using lei with podman + toolbox] Eric Wong
@ 2021-09-10 12:42 70% ` Konstantin Ryabitsev
2021-09-10 13:56 71% ` Eric Wong
1 sibling, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-10 12:42 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Sep 09, 2021 at 11:36:14PM +0000, Eric Wong wrote:
> > These are my quickie instructions for how to use lei in a toolbox environment
> > if you are running a distribution like Fedora and don't want to install a lot
> > of perl dependencies into your main OS.
>
> Off the top of my head, I think Search::Xapian // Xapian.pm was
> the main thing that was missing from CentOS 7. Does Fedora have
> that?
Seems to have it as perl-Search-Xapian, which provides Xapian.pm.
> (disclaimer: I don't care for Docker, seems like a giant waste
> of space and bandwidth compared to just using the distro)
Well, this is for toolbox which uses podman, not docker. Toolbox is actually
the preferred mechanism in Fedora for setting up quickie work environments,
especially on something like Fedora Silverblue with its immutable root
partition.
I don't intend these instructions as the preferred mechanism for getting lei
up and running, just to be clear. Eventually, it will be packaged for most
distros -- but for now it's a convenient way to get the latest version on the
platform most likely to be most tested (Debian).
> > RUN apt-get update && \
> > apt-get -y install sudo libcap2-bin locales vim \
> > git liburi-perl libemail-mime-perl libplack-perl libtimedate-perl \
>
> Email::MIME isn't used at all outside of tests (but it's widely packaged).
> No idea why libcap2-bin and vim are explicit dependencies (any
> editor will do). Don't need Plack for lei, either.
Yeah, some of these were mostly for "make test" runs and others are adding
basic packages to the container image to make it slightly more usable (I don't
like nano or whatever is the default editor in the container image).
> No need for Net::Server nor Parse::RecDescent for lei. I don't
> use Net::Server at all outside of tests, since I use systemd.
>
> Email::Address::XS and TimeDate can be useful for messed up
> messages, but low importance (I think they're widely packaged).
> E:A:X and P:RD are required for -imapd but nothing else.
>
> Socket::Msghdr makes lei a teeny bit faster, but I don't think
> it's worth using another distro or running a compiler to get
> since Inline::C is already available in all distros. Everything
> else should be in Fedora...
>
> > sqlite3 libgit2-dev make eatmydata man-db pkg-config
>
> eatmydata shouldn't be useful outside of development, and
> libgit2+pkg-config isn't used by lei, yet
> (it is for -httpd/-imapd/-nntpd)
All noted -- I may be the one who packages things for Fedora at some point, so
this is useful info.
-K
^ permalink raw reply [relevance 70%]
* Re: Using lei with podman + toolbox
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
@ 2021-09-10 13:56 71% ` Eric Wong
2021-09-10 14:48 71% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-09-10 13:56 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Thu, Sep 09, 2021 at 11:36:14PM +0000, Eric Wong wrote:
> > > These are my quickie instructions for how to use lei in a toolbox environment
> > > if you are running a distribution like Fedora and don't want to install a lot
> > > of perl dependencies into your main OS.
> >
> > Off the top of my head, I think Search::Xapian // Xapian.pm was
> > the main thing that was missing from CentOS 7. Does Fedora have
> > that?
>
> Seems to have it as perl-Search-Xapian, which provides Xapian.pm.
Oh I meant the newer SWIG Xapian.pm. Search::Xapian is
Search/Xapian.pm which uses XS and isn't getting new features.
The XS version is far better-tested, but the SWIG version gives
access to some newer APIs. AFAIK neither lets us do custom
query parsing like notmuch does in C++ (our approxidate
rt:/dt:/d: handling is a huge hack)
> > (disclaimer: I don't care for Docker, seems like a giant waste
> > of space and bandwidth compared to just using the distro)
>
> Well, this is for toolbox which uses podman, not docker. Toolbox is actually
> the preferred mechanism in Fedora for setting up quickie work environments,
> especially on something like Fedora Silverblue with its immutable root
> partition.
Ah, still seems like a waste of space and bandwidth :>
(It just took me several hours to download upgrades from
buster => bullseye on a small dev VM)
> I don't intend these instructions as the preferred mechanism for getting lei
> up and running, just to be clear. Eventually, it will be packaged for most
> distros -- but for now it's a convenient way to get the latest version on the
> platform most likely to be most tested (Debian).
I actually test everything on FreeBSD to force myself into doing
things portably. Probably 95% of my "git push" is done on
FreeBSD (IOW, whenever I have connectivity to that VM).
> All noted -- I may be the one who packages things for Fedora at some point, so
> this is useful info.
Cool. I'm wondering how to better arrange INSTALL to suit
different usages (lei-only vs daemons vs mda/watch-only).
^ permalink raw reply [relevance 71%]
* RFC: normalize whitespace in lei queries
@ 2021-09-10 14:11 65% Konstantin Ryabitsev
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-09-10 14:11 UTC (permalink / raw)
To: meta
Just a quickie note. I was trying to make my query more readable, so I
switched my test example to the following:
[lei]
q = (dfn:Documentation/security/landlock.rst \
OR dfn:Documentation/userspace-api/landlock.rst \
OR dfn:include/uapi/linux/landlock.h \
OR dfn:samples/landlock/ \
OR dfn:security/landlock/ \
OR dfn:tools/testing/selftests/landlock/ \
OR dfhh:landlock OR dfctx:landlock) \
AND rt:2.months.ago..
However, I noticed that curl queries ended up gaining a lot of ++++:
# /usr/bin/curl -Sf -s -d '' https://lore.kernel.org/all/?q=(dfn%3ADocumentation%2Fsecurity%2Flandlock.rst++++++++++OR+dfn%3ADocumentation%2Fuserspace-api%2Flandlock.rst++++++++++OR+dfn%3Ainclude%2Fuapi%2Flinux%2Flandlock.h++++++++++OR+dfn%3Asamples%2Flandlock%2F++++++++++OR+dfn%3Asecurity%2Flandlock%2F++++++++++OR+dfn%3Atools%2Ftesting%2Fselftests%2Flandlock%2F++++++++++OR+dfhh%3Alandlock+OR+dfctx%3Alandlock)+++++++++AND+rt%3A1625943784..&x=m&t=1
I think it's reasonable to normalize \s+ into a single space for all queries,
right?
Another observation is that when I missed a \ on one of the lines, I managed
to make lei unusable. :)
$ lei up --all
failed to close (git config -z -l --includes --file=/home/user/.cache/lei/saved-tmp.1581950.1631282889.config) pipe: 32768 at /usr/local/share/perl/5.32.1/PublicInbox/Config.pm line 172.
$ lei edit-search ~/work/temp/lei/landlock
failed to close (git config -z -l --includes --file=/home/user/.local/share/lei/saved-searches/landlock-1804cfad691a409f55598a8528566d5f1539b2632e1db7e206cb147396582631/lei.saved-search) pipe: 32768 at /usr/local/share/perl/5.32.1/PublicInbox/Config.pm line 172.
I fixed it by manually editing lei.saved-search and adding the missing
backslash. Not sure what the proper solution for this is -- perhaps attempting
to parse the edited config file and refusing to save it if it doesn't work?
-K
^ permalink raw reply [relevance 65%]
* Re: Using lei with podman + toolbox
2021-09-10 13:56 71% ` Eric Wong
@ 2021-09-10 14:48 71% ` Konstantin Ryabitsev
0 siblings, 0 replies; 200+ results
From: Konstantin Ryabitsev @ 2021-09-10 14:48 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Fri, Sep 10, 2021 at 01:56:53PM +0000, Eric Wong wrote:
> > Seems to have it as perl-Search-Xapian, which provides Xapian.pm.
>
> Oh I meant the newer SWIG Xapian.pm. Search::Xapian is
> Search/Xapian.pm which uses XS and isn't getting new features.
Ah, no, doesn't look like xapian-bindings-perl is built.
I may open an RFE with the packager for this once I get around to that.
> > All noted -- I may be the one who packages things for Fedora at some point, so
> > this is useful info.
>
> Cool. I'm wondering how to better arrange INSTALL to suit
> different usages (lei-only vs daemons vs mda/watch-only).
I mean, it's not like it pulls in huge dependency trees. Perl cpan packages
are a few KB in size, so pulling in some extras isn't going to inconvenience a
lot of people or anything. :)
-K
^ permalink raw reply [relevance 71%]
* [PATCH 0/3] lei saved-search fixes
2021-09-10 14:11 65% RFC: normalize whitespace in lei queries Konstantin Ryabitsev
@ 2021-09-11 0:19 71% ` Eric Wong
2021-09-11 0:19 63% ` [PATCH 2/3] lei: pass client stderr to git-config in more places Eric Wong
2021-09-11 0:19 71% ` [PATCH 3/3] lei: normalize whitespace in remote queries Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta
Konstantin Ryabitsev wrote:
> I think it's reasonable to normalize \s+ into a single space for all queries,
> right?
Yes, 3/3 does it for HTTP(S). Don't think it's needed for local
queries. I'm leaving non-ASCII whitespace alone for now since I
don't think it'd be a real problem.
> Another observation is that when I missed a \ on one of the lines, I managed
> to make lei unusable. :)
That's a far more important problem fixed in 1/3 :>
Eric Wong (3):
lei: fix handling of broken lei.saved-search config files
lei: pass client stderr to git-config in more places
lei: normalize whitespace in remote queries
lib/PublicInbox/Config.pm | 10 ++--
lib/PublicInbox/LEI.pm | 23 +++++++--
lib/PublicInbox/LeiEditSearch.pm | 79 +++++++++++++++++++++++++++++--
lib/PublicInbox/LeiMirror.pm | 4 +-
lib/PublicInbox/LeiSavedSearch.pm | 77 ++++++------------------------
lib/PublicInbox/LeiXSearch.pm | 4 +-
script/lei | 2 +
t/lei-q-save.t | 17 +++++++
8 files changed, 136 insertions(+), 80 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei: normalize whitespace in remote queries
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
2021-09-11 0:19 63% ` [PATCH 2/3] lei: pass client stderr to git-config in more places Eric Wong
@ 2021-09-11 0:19 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta; +Cc: Konstantin Ryabitsev
Having redundant "+" in URLs is ugly and can hurt cacheability
of queries. Even with "quoted phrase searches", Xapian seems
unaffected by redundant spaces, so just normalize the ASCII
white spaces to ' ' (%20) when fed via STDIN or saved-search
config file.
Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/20210910141157.6u5adehpx7wftkor@meerkat.local/
---
lib/PublicInbox/LeiXSearch.pm | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 709a3b3a..9f7f3885 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -297,7 +297,9 @@ sub query_remote_mboxrd {
local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
my $lei = $self->{lei};
my $opt = $lei->{opt};
- my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
+ my $qstr = $lei->{mset_opt}->{qstr};
+ $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
+ my @qform = (q => $qstr, x => 'm');
push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
my ($reap_tail, $reap_curl);
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] lei: pass client stderr to git-config in more places
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
@ 2021-09-11 0:19 63% ` Eric Wong
2021-09-11 0:19 71% ` [PATCH 3/3] lei: normalize whitespace in remote queries Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-09-11 0:19 UTC (permalink / raw)
To: meta
This should improve the users' chances of seeing errors in
various git config files we use.
---
lib/PublicInbox/Config.pm | 4 ++--
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 74a1a6f5..ee5322fe 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -19,7 +19,7 @@ sub _array ($) { ref($_[0]) eq 'ARRAY' ? $_[0] : [ $_[0] ] }
# returns key-value pairs of config directives in a hash
# if keys may be multi-value, the value is an array ref containing all values
sub new {
- my ($class, $file) = @_;
+ my ($class, $file, $errfh) = @_;
$file //= default_file();
my $self;
if (ref($file) eq 'SCALAR') { # used by some tests
@@ -27,7 +27,7 @@ sub new {
$self = config_fh_parse($fh, "\n", '=');
bless $self, $class;
} else {
- $self = git_config_dump($class, $file);
+ $self = git_config_dump($class, $file, $errfh);
$self->{'-f'} = $file;
}
# caches
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0b4c99dc..aff2bf19 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -820,7 +820,7 @@ sub _lei_cfg ($;$) {
$cur_st = pack('dd', $st[10], $st[7]);
qerr($self, "# $f created") if $self->{cmd} ne 'config';
}
- my $cfg = PublicInbox::Config->git_config_dump($f);
+ my $cfg = PublicInbox::Config->git_config_dump($f, $self->{2});
$cfg->{-st} = $cur_st;
$cfg->{'-f'} = $f;
if ($sto && canonpath_harder($sto_dir // store_path($self))
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 638add42..8689b825 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -110,7 +110,7 @@ sub _try_config {
}
return $lei->err("# @$cmd failed (non-fatal)") if $cerr;
rename($f, $ce) or return $lei->err("link($f, $ce): $! (non-fatal)");
- my $cfg = PublicInbox::Config->git_config_dump($f);
+ my $cfg = PublicInbox::Config->git_config_dump($f, $lei->{2});
my $ibx = $self->{ibx} = {};
for my $sec (grep(/\Apublicinbox\./, @{$cfg->{-section_order}})) {
for (qw(address newsgroup nntpmirror)) {
@@ -136,7 +136,7 @@ sub index_cloned_inbox {
}
# force synchronous dwaitpid for v2:
local $PublicInbox::DS::in_loop = 0;
- my $cfg = PublicInbox::Config->new;
+ my $cfg = PublicInbox::Config->new(undef, $lei->{2});
my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
local %ENV = (%ENV, %$env) if $env;
PublicInbox::Admin::progress_prepare($opt, $lei->{2});
^ permalink raw reply related [relevance 63%]
Results 601-800 of ~1329 next (newer) | prev (older) | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-05-05 17:49 [PATCH 0/3] lei rediff fixes Eric Wong
2021-05-05 17:49 55% ` [PATCH 2/3] lei rediff: capture and regenerate file modes Eric Wong
2021-05-05 17:49 65% ` [PATCH 3/3] lei rediff: do not automatically store patches/mails Eric Wong
2021-05-07 0:13 71% lei timestamp resolution for mail synchronization Eric Wong
2021-05-16 2:41 71% [PATCH] lei rediff: mark --color-moved value as optional Kyle Meyer
2021-05-16 2:42 71% [PATCH] lei rediff: handle stdin like other commands Kyle Meyer
2021-05-16 15:23 71% ` Eric Wong
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
2021-05-17 3:35 82% ` [PATCH 3/9] doc lei blob: point to lei-q for shared options Kyle Meyer
2021-05-17 3:35 90% ` [PATCH 4/9] doc lei: resort lei-tag entries Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 5/9] doc lei q: fix a typo Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 6/9] doc lei q: add missing value for --lock Kyle Meyer
2021-05-17 3:35 54% ` [PATCH 7/9] doc lei: add manpage for convert Kyle Meyer
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
2021-05-17 8:09 71% ` Eric Wong
2021-05-17 23:24 71% ` Kyle Meyer
2021-05-17 3:35 66% ` [PATCH 9/9] doc lei: update manpages with new options Kyle Meyer
2021-05-17 3:37 71% [PATCH] lei lcat: fix handling of multiple MSGID_OR_URL arguments Kyle Meyer
2021-05-19 8:54 57% [PATCH] lei: relax rules for "new" in Maildir Eric Wong
2021-05-21 4:38 70% [PATCH] lei rediff: fix construction of git-diff options Kyle Meyer
2021-05-21 9:52 71% ` Eric Wong
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
2021-05-21 10:28 53% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
2021-05-21 10:28 71% ` [PATCH 5/8] lei index: support command-line options Eric Wong
2021-05-21 10:28 32% ` [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs Eric Wong
2021-05-21 10:28 46% ` [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Eric Wong
2021-05-23 1:38 71% [PATCH 0/3] lei export-kw: IMAP support Eric Wong
2021-05-23 1:38 49% ` [PATCH 2/3] lei export-kw: support exporting keywords to IMAP Eric Wong
2021-05-23 1:38 47% ` [PATCH 3/3] lei export-kw: relax IMAP URL matching Eric Wong
2021-05-23 8:01 35% [PATCH] lei <q|up>: set \Recent on non-empty mbox and Maildir Eric Wong
2021-05-23 21:36 71% [PATCH 0/2] lei IMAP usability stuff Eric Wong
2021-05-23 21:36 46% ` [PATCH 1/2] lei inspect: use LeiMailSync->match_imap_url Eric Wong
2021-05-24 22:48 71% [PATCH] lei export-kw: fix case-insensitivity typo for IMAP URLs Eric Wong
2021-05-25 11:01 71% [PATCH 0/3] lei forget-mail-sync: drop sync information Eric Wong
2021-05-25 11:01 90% ` [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs Eric Wong
2021-05-25 11:01 55% ` [PATCH 3/3] lei forget-mail-sync: new command to drop sync information Eric Wong
2021-05-25 22:19 71% [PATCH 0/2] ipc: fix "lei q" w/ HTTP(S) externals Eric Wong
2021-05-26 18:08 28% [PATCH] lei: require Socket::MsgHdr or Inline::C, drop oneshot Eric Wong
2021-05-26 23:50 50% [PATCH] lei rm: new command to remove messages from index Eric Wong
2021-05-27 10:49 44% [PATCH] lei: handle a single IMAP message in most places Eric Wong
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
2021-05-28 9:37 71% ` Eric Wong
2021-05-28 0:07 71% ` [PATCH 6/6] lei: add TODO item for FUSE mount Eric Wong
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
2021-05-28 9:45 62% ` [PATCH 2/3] lei: restore working directory in more places Eric Wong
2021-05-28 9:45 71% ` [PATCH 3/3] script/lei: drop leftover message about fallback Eric Wong
2021-05-28 19:47 49% [PATCH] lei: retry_reopen on read-only Xapian access Eric Wong
2021-05-28 22:39 39% [PATCH] lei q|up: support v2:/path/to/inboxdir destination Eric Wong
2021-05-29 20:20 71% [PATCH 0/3] lei: v2 and lcat/import/IMAP things Eric Wong
2021-05-29 20:20 36% ` [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support Eric Wong
2021-05-29 20:20 61% ` [PATCH 3/3] lei q: --sort and --save|v2 are incompatible Eric Wong
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
2021-05-30 6:33 68% ` [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY Eric Wong
2021-05-30 6:33 63% ` [PATCH 3/4] lei lcat: support maildir: paths, too Eric Wong
2021-05-30 6:33 61% ` [PATCH 4/4] lei: support implicit stdin by default Eric Wong
2021-05-30 11:52 71% ` [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 11:45 42% [PATCH] lei import: import IMAP flag changes from old messages Eric Wong
2021-05-31 10:20 86% [PATCH] lei import: reduce writes to lei/store on IMAP sync Eric Wong
2021-06-01 19:20 71% [PATCH] lei: remove "forget" (old name for "rm") Eric Wong
2021-06-02 10:03 58% [PATCH] lei export-kw: do not write directly to mail_sync.sqlite3 Eric Wong
2021-06-03 0:17 40% [PATCH] lei import: speed up kw updates for old IMAP messages Eric Wong
2021-06-03 1:05 38% ` [PATCH v2] " Eric Wong
2021-06-03 5:00 67% [RFC] lei import: support --new-only Eric Wong
2021-06-09 22:39 62% ` [PATCH v2] lei import: support --new-only for IMAP Eric Wong
2021-06-05 21:04 71% [PATCH] INSTALL: note about lei metadata storage Eric Wong
2021-06-05 21:06 71% [PATCH] lei: don't drop WQ workers on normal exit Eric Wong
2021-06-07 19:06 71% [PATCH] lei/store: checkpoint commits mail_sync.sqlite3 Eric Wong
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
2021-06-08 9:50 48% ` [PATCH 2/3] lei: generalize auxiliary WQ handling Eric Wong
2021-06-08 9:50 32% ` [PATCH 3/3] lei import: speed up repeated Maildir imports Eric Wong
2021-06-08 23:56 70% [PATCH] lei pmdir: fix nproc for <= 4 CPUs Eric Wong
2021-06-09 0:11 70% [PATCH] lei edit-search: fix and add a (weak) test Eric Wong
2021-06-09 7:47 71% [PATCH 0/5] lei Maildir stuff Eric Wong
2021-06-09 7:47 58% ` [PATCH 3/5] lei tag: parallelize Maildir access Eric Wong
2021-06-09 7:47 45% ` [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data Eric Wong
2021-06-09 10:03 69% [PATCH] lei/store: do eidx_init before creating R/W lms dbh Eric Wong
2021-06-09 23:27 58% [PATCH] lei tag: less confusing warning about unimported messages Eric Wong
2021-06-11 9:42 41% [PATCH] lei ls-mail-source: list IMAP folders and NNTP groups Eric Wong
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
2021-06-12 0:10 48% ` [PATCH 2/5] lei ls-mail-source: write through to URL folder cache Eric Wong
2021-06-12 0:10 71% ` [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message Eric Wong
2021-06-12 0:10 71% ` [PATCH 4/5] lei import: use url_folder_cache for completion Eric Wong
2021-06-13 18:12 71% [PATCH 0/2] lei: support keywords off a single Maildir file Eric Wong
2021-06-13 18:12 59% ` [PATCH 2/2] lei index+import: reject keywords from R/O IMAP Eric Wong
2021-06-17 22:00 71% [PATCH 0/3] lei: internal bug fixups Eric Wong
2021-06-17 22:00 60% ` [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes Eric Wong
2021-06-17 22:00 55% ` [PATCH 3/3] lei/store: cull redundant docids based on blob OID Eric Wong
2021-06-18 19:20 71% [PATCH] lei/store: do not put NULL into over.num column Eric Wong
2021-06-20 4:33 71% [PATCH] lei sucks: don't warn or error out on missing dependencies Eric Wong
2021-06-20 8:39 71% [PATCH] lei import: help + completion for --new-only Eric Wong
2021-06-22 10:04 71% [PATCH] lei: use open() perlop for -C (chdir) Eric Wong
2021-07-01 11:31 71% [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
2021-07-01 11:31 71% ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
2021-07-01 11:31 62% ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
2021-07-02 20:42 71% [PATCH] lei inspect: help+completion for --dir option Eric Wong
2021-07-02 21:02 71% [PATCH] lei import: increase flags search batch size, display progress Eric Wong
2021-07-04 19:35 71% [PATCH] lei: drop workers on EOF from clients Eric Wong
2021-07-08 8:42 71% deduplication doesn't always work with "lei up" Eric Wong
2021-07-19 8:59 63% [PATCH 0/2] lei Maildir inotify/kevent support Eric Wong
2021-07-19 8:59 22% ` [PATCH 2/2] lei: start implementing inotify Maildir support Eric Wong
2021-07-21 14:07 47% ` [PATCH 3/2] lei: auto-refresh watches in config, cancel missing Eric Wong
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
2021-07-23 10:56 58% ` [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe Eric Wong
2021-07-23 10:56 53% ` [PATCH 3/3] lei rm-watch: new command to support removing watches Eric Wong
2021-07-25 11:15 71% [PATCH] t/lei-watch.t: improve test reliability Eric Wong
2021-07-25 12:03 71% [PATCH] doc: lei-{p2q,rediff}: note implicit --stdin Eric Wong
2021-07-25 12:29 71% lei rediff: add --dequote and --requote? Eric Wong
2021-07-28 0:37 [PATCH 0/2] fix "make check-run" reliability Eric Wong
2021-07-28 0:37 71% ` [PATCH 1/2] lei: die on ECONNRESET Eric Wong
2021-07-28 12:34 58% [RFC] lei: address lifetime problems from Linux::Inotify2 Eric Wong
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
2021-08-04 10:40 71% ` Eric Wong
2021-08-05 2:33 71% [PATCH] lei export-kw: workaround race in updating Maildir locations Eric Wong
2021-08-11 11:26 71% [PATCH 0/3] lei pathname canonicalization fixes Eric Wong
2021-08-11 11:26 42% ` [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames Eric Wong
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
2021-08-24 20:14 71% ` [PATCH 0/3] lei: hopefully^W kill /Document \d+ not found/ errors Eric Wong
2021-08-17 8:52 71% [PATCH 0/3] lei: some mail sync stuff Eric Wong
2021-08-17 8:52 54% ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
2021-08-17 8:52 88% ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
2021-08-18 11:56 71% "lei q --save" - should it be the default? Eric Wong
2021-08-19 1:36 31% ` [PATCH] lei q: make --save the default Eric Wong
2021-08-19 9:49 54% [PATCH] lei: implicitly watch all Maildirs it knows about Eric Wong
2021-08-24 13:04 71% [PATCH] lei: add missing LeiWatch lazy-load Eric Wong
2021-08-24 13:06 70% [PATCH] lei: non-blocking lei/store->done in lei-daemon Eric Wong
2021-08-25 8:40 71% [PATCH 0/2] minor lei usability tweaks Eric Wong
2021-08-25 8:40 59% ` [PATCH 1/2] lei up: improve --all=local stderr output Eric Wong
2021-08-31 11:21 71% [PATCH 00/10] lei: several bug fixes and refinements Eric Wong
2021-08-31 11:21 71% ` [PATCH 02/10] lei prune-mail-sync: handle --all (no args) Eric Wong
2021-08-31 11:21 71% ` [PATCH 05/10] t/lei-watch: avoid race between glob + readlink Eric Wong
2021-08-31 11:21 71% ` [PATCH 06/10] lei note-event: always flush changes on daemon exit Eric Wong
2021-08-31 11:21 71% ` [PATCH 07/10] lei: refresh watches before MUA spawn for Maildir Eric Wong
2021-08-31 11:21 94% ` [PATCH 09/10] lei: fix error reporting from lei/store -> lei-daemon Eric Wong
2021-08-31 11:21 71% ` [PATCH 10/10] lei/store: correctly delete entries from over Eric Wong
2021-08-31 19:38 90% [PATCH] lei up: only show finmsg in top-level lei-daemon Eric Wong
2021-09-02 10:17 71% [PATCH 0/3] lei: auto keyword propagation to Maildirs Eric Wong
2021-09-02 10:17 51% ` [PATCH 3/3] lei: propagate keyword changes from lei/store Eric Wong
2021-09-02 10:25 67% ` [SQUASH 4/3] t/lei-auto-watch: workaround for FreeBSD kevent Eric Wong
2021-09-02 21:12 61% Showcasing lei at Linux Plumbers Konstantin Ryabitsev
2021-09-02 21:58 65% ` Eric Wong
2021-09-03 15:15 71% ` Konstantin Ryabitsev
2021-09-07 21:33 71% ` Konstantin Ryabitsev
2021-09-07 22:14 71% ` Eric Wong
2021-09-08 13:36 65% ` Konstantin Ryabitsev
2021-09-08 14:49 66% ` Eric Wong
2021-09-08 17:17 66% ` Konstantin Ryabitsev
2021-09-08 17:32 71% ` Eric Wong
2021-09-02 22:36 [PATCH 0/2] test reliability fixes Eric Wong
2021-09-02 22:36 68% ` [PATCH 1/2] t/lei-auto-watch: improve test reliability Eric Wong
2021-09-03 8:54 66% [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 54% ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
2021-09-03 8:54 71% ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
2021-09-03 8:54 52% ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
2021-09-03 8:54 83% ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
2021-09-03 8:54 57% ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
2021-09-03 8:54 61% ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
2021-09-03 8:54 63% ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
2021-09-07 11:32 71% [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 71% ` [PATCH 1/4] xt/net_writer_imap: test "lei up" on single IMAP output Eric Wong
2021-09-07 11:32 71% ` [PATCH 2/4] lei: dump and clear log at exit Eric Wong
2021-09-07 11:32 38% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
2021-09-07 11:32 51% ` [PATCH 4/4] doc: lei-*.pod: update to Tor v3 .onion address Eric Wong
2021-09-07 22:41 71% [PATCH] lei q|up: fix write counter for v2 Eric Wong
2021-09-08 18:48 54% [PATCH] lei-rm: add man page, support LeiInput args Eric Wong
2021-09-08 19:04 71% [PATCH] lei prune-mail-sync: ignore missing locations Eric Wong
2021-09-09 5:34 70% [PATCH] lei up: print messages before disconnecting Eric Wong
2021-09-09 5:47 71% lei refresh-mail-sync [vs. prune-mail-sync] Eric Wong
2021-09-09 15:19 71% Tracking one-off threads with lei Konstantin Ryabitsev
2021-09-09 20:06 71% ` Eric Wong
2021-09-09 20:56 71% ` Konstantin Ryabitsev
2021-09-09 21:06 71% ` Eric Wong
2021-09-09 21:39 64% Using lei with podman + toolbox Konstantin Ryabitsev
2021-09-09 23:36 68% ` Eric Wong
2021-09-09 23:51 71% ` native C++ Xapian wrapper [was: Using lei with podman + toolbox] Eric Wong
2021-09-10 12:42 70% ` Using lei with podman + toolbox Konstantin Ryabitsev
2021-09-10 13:56 71% ` Eric Wong
2021-09-10 14:48 71% ` Konstantin Ryabitsev
2021-09-10 5:51 51% [PATCH] lei add-external --mirror: deduce paths for PSGI mount prefixes Eric Wong
2021-09-10 9:08 71% [PATCH 0/4] lei: some net-related things Eric Wong
2021-09-10 9:08 47% ` [PATCH 2/4] lei: split out @net_opt for curl/torsocks use Eric Wong
2021-09-10 9:08 65% ` [PATCH 3/4] lei: do not read ~/.netrc by default Eric Wong
2021-09-10 9:08 57% ` [PATCH 4/4] doc: lei-index manpage Eric Wong
2021-09-10 9:15 68% [PATCH] lei add-external --mirror: quiet unlink error on ENOENT Eric Wong
2021-09-10 11:46 71% [PATCH] lei up: only delay non-zero "# $NR written to ..." Eric Wong
2021-09-10 14:11 65% RFC: normalize whitespace in lei queries Konstantin Ryabitsev
2021-09-11 0:19 71% ` [PATCH 0/3] lei saved-search fixes Eric Wong
2021-09-11 0:19 63% ` [PATCH 2/3] lei: pass client stderr to git-config in more places Eric Wong
2021-09-11 0:19 71% ` [PATCH 3/3] lei: normalize whitespace in remote queries Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).