* [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
@ 2021-08-14 0:29 64% ` Eric Wong
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
This ought to avoid /Document \d+ not found/ errors from Xapian
when seeing a message for the first time by not attempting to
read keywords for totally unseen messages.
---
lib/PublicInbox/LeiRemote.pm | 7 ++++---
lib/PublicInbox/LeiStore.pm | 1 +
lib/PublicInbox/LeiXSearch.pm | 10 +++++++---
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 945d9990..e7deecb8 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -26,11 +26,12 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
my ($eml, $self) = @_;
my $lei = $self->{lei};
my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ my $smsg = bless {}, 'PublicInbox::Smsg';
if ($lei->{sto} && !$xoids) { # memoize locally
- $lei->{sto}->ipc_do('add_eml', $eml);
+ my $res = $lei->{sto}->ipc_do('add_eml', $eml);
+ $smsg = $res if ref($res) eq ref($smsg);
}
- my $smsg = bless {}, 'PublicInbox::Smsg';
- $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e26b622d..ce66014f 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -329,6 +329,7 @@ sub add_eml {
}
\@docids;
} else { # totally new message
+ delete $smsg->{-oidx}; # for IPC-friendliness
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
$oidx->add_overview($eml, $smsg);
$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 393f25bf..971f3a06 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -266,11 +266,15 @@ sub _smsg_fill ($$) {
sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
my $xoids = $lei->{ale}->xoids_for($eml, 1);
+ my $smsg = bless {}, 'PublicInbox::Smsg';
if ($self->{import_sto} && !$xoids) {
- $self->{import_sto}->ipc_do('add_eml', $eml);
+ my $res = $self->{import_sto}->ipc_do('add_eml', $eml);
+ if (ref($res) eq ref($smsg)) { # totally new message
+ $smsg = $res;
+ $smsg->{kw} = []; # short-circuit xsmsg_vmd
+ }
}
- my $smsg = bless {}, 'PublicInbox::Smsg';
- $smsg->{blob} = $xoids ? (keys(%$xoids))[0]
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
_smsg_fill($smsg, $eml);
wait_startq($lei);
^ permalink raw reply related [relevance 64%]
* [PATCH 3/3] lei: hexdigest mocks account for unwanted headers
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
@ 2021-08-14 0:29 58% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
PublicInbox::Import never imports @UNWANTED_HEADERS, so ensure
our mock blob OIDs do the same. This ought to prevent
duplicates if the PSGI mboxrd download starts setting
"X-Status: F" like "lei q -tt .."
---
lib/PublicInbox/FakeImport.pm | 3 +++
lib/PublicInbox/LEI.pm | 5 +++++
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 9 ++++++++-
lib/PublicInbox/LeiXSearch.pm | 3 +--
6 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/FakeImport.pm b/lib/PublicInbox/FakeImport.pm
index dea25cbe..bccc3321 100644
--- a/lib/PublicInbox/FakeImport.pm
+++ b/lib/PublicInbox/FakeImport.pm
@@ -4,12 +4,15 @@
# pretend to do PublicInbox::Import::add for "lei index"
package PublicInbox::FakeImport;
use strict;
+use v5.10.1;
use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
sub new { bless { bytes_added => 0 }, __PACKAGE__ }
sub add {
my ($self, $eml, $check_cb, $smsg) = @_;
+ PublicInbox::Import::drop_unwanted_headers($eml);
$smsg->populate($eml);
my $raw = $eml->as_string;
$smsg->{blob} = git_sha(1, \$raw)->hexdigest;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7d0f63dc..347dd280 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1420,4 +1420,9 @@ sub refresh_watches {
}
}
+sub git_blob_id {
+ my ($lei, $eml) = @_;
+ ($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
+}
+
1;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 37b660f9..962ad49e 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -73,7 +73,7 @@ sub lxs_prepare {
my @only = @{$opt->{only} // []};
# --local is enabled by default unless --only is used
# we'll allow "--only $LOCATION --local"
- my $sto = $self->_lei_store(1); # FIXME: should not create
+ my $sto = $self->_lei_store(1);
$self->{lse} = $sto->search;
if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
$lxs->prepare_external($self->{lse});
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index e7deecb8..580787c0 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -32,7 +32,7 @@ sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
$smsg = $res if ref($res) eq ref($smsg);
}
$smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : git_sha(1, $eml)->hexdigest;
+ : $lei->git_blob_id($eml);
$smsg->populate($eml);
$smsg->{mid} //= '(none)';
push @{$self->{smsg}}, $smsg;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index ce66014f..3f33d114 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -20,7 +20,7 @@ use PublicInbox::Eml;
use PublicInbox::Import;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::V2Writable;
-use PublicInbox::ContentHash qw(content_hash);
+use PublicInbox::ContentHash qw(content_hash git_sha);
use PublicInbox::MID qw(mids);
use PublicInbox::LeiSearch;
use PublicInbox::MDA;
@@ -508,4 +508,11 @@ sub write_prepare {
$lei->{sto} = $self;
}
+# TODO: support SHA-256
+sub git_blob_id { # called via LEI->git_blob_id
+ my ($self, $eml) = @_;
+ $eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
+ git_sha(1, $eml)->hexdigest;
+}
+
1;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 971f3a06..5e34d864 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -274,8 +274,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
$smsg->{kw} = []; # short-circuit xsmsg_vmd
}
}
- $smsg->{blob} //= $xoids ? (keys(%$xoids))[0]
- : git_sha(1, $eml)->hexdigest;
+ $smsg->{blob} //= $xoids ? (keys(%$xoids))[0] : $lei->git_blob_id($eml);
_smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
^ permalink raw reply related [relevance 58%]
* [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors
@ 2021-08-14 0:29 71% Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
2/3 is probably a fix for a long-standing problem, 3/3 was
noticed while working on it. If 2/3 doesn't fix it, then maybe
1/3 will help us narrow it down.
Eric Wong (3):
lei: diagnostics for /Document \d+ not found/ errors
lei <q|up>: wait on remote mboxrd imports synchronously
lei: hexdigest mocks account for unwanted headers
lib/PublicInbox/FakeImport.pm | 3 +++
lib/PublicInbox/IPC.pm | 2 +-
lib/PublicInbox/LEI.pm | 5 +++++
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 9 +++++----
lib/PublicInbox/LeiSearch.pm | 17 ++++++++++-------
lib/PublicInbox/LeiStore.pm | 10 +++++++++-
lib/PublicInbox/LeiXSearch.pm | 11 +++++++----
8 files changed, 41 insertions(+), 18 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei: diagnostics for /Document \d+ not found/ errors
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
@ 2021-08-14 0:29 68% ` Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-14 0:29 UTC (permalink / raw)
To: meta
This may help diagnose "Exception: Document \d+ not found"
errors I'm seeing from "lei up" with HTTPS endpoints.
---
lib/PublicInbox/IPC.pm | 2 +-
lib/PublicInbox/LeiSearch.pm | 17 ++++++++++-------
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 497a6035..d909dc1c 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -236,7 +236,7 @@ sub recv_and_run {
undef $buf;
my $sub = shift @$args;
eval { $self->$sub(@$args) };
- warn "$$ wq_worker: $@" if $@;
+ warn "$$ $0 wq_worker: $@" if $@;
delete @$self{0..($nfd-1)};
$n;
}
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 79b2fd7d..f9e5c8e9 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -55,13 +55,16 @@ sub _xsmsg_vmd { # retry_reopen
$kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged});
my @num = $self->over->blob_exists($smsg->{blob});
for my $num (@num) { # there should only be one...
- $doc = $xdb->get_document(num2docid($self, $num));
- $x = xap_terms('K', $doc);
- %kw = (%kw, %$x);
- if ($want_label) { # JSON/JMAP only
- $x = xap_terms('L', $doc);
- %L = (%L, %$x);
- }
+ eval {
+ $doc = $xdb->get_document(num2docid($self, $num));
+ $x = xap_terms('K', $doc);
+ %kw = (%kw, %$x);
+ if ($want_label) { # JSON/JMAP only
+ $x = xap_terms('L', $doc);
+ %L = (%L, %$x);
+ }
+ };
+ warn "$$ $0 #$num (nshard=$self->{nshard}) $smsg->{blob}: $@";
}
$smsg->{kw} = [ sort keys %kw ] if scalar(keys(%kw));
$smsg->{L} = [ sort keys %L ] if scalar(keys(%L));
^ permalink raw reply related [relevance 68%]
* [PATCH 0/2] "lei up" improvements
@ 2021-08-12 23:40 71% Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
Some obvious things for now, some maybe not-so-obvious things on
the way...
Eric Wong (2):
lei up: support multiple output folders w/o --all=local
lei up: note errors if one output destination fails
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 23 +++++++++++++++++++----
2 files changed, 20 insertions(+), 5 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei up: note errors if one output destination fails
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
@ 2021-08-12 23:40 71% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
We can keep going if one (out of multiple) output destinations
fail, but the error needs to be communicated to the caller as an
exit code.
---
lib/PublicInbox/LeiUp.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 49b558fd..c5a01527 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -57,7 +57,7 @@ sub up1_redispatch {
up1($l, $out);
$l->qerr("# $out done");
};
- $l->err($@) if $@;
+ $l->child_error(1 << 8, $@) if $@;
}
sub lei_up {
^ permalink raw reply related [relevance 71%]
* [PATCH 1/2] lei up: support multiple output folders w/o --all=local
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
@ 2021-08-12 23:40 63% ` Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-08-12 23:40 UTC (permalink / raw)
To: meta
Being able to update 1 folder, or all (local) folders is
sometimes too limiting, so just allow updating any subset
of local folders.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 21 ++++++++++++++++++---
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 54fac7b4..7d0f63dc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -178,7 +178,7 @@ our %CMD = ( # sorted in order of importance/use:
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
-'up' => [ 'OUTPUT|--all', 'update saved search',
+'up' => [ 'OUTPUT...|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 3356d11e..49b558fd 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -61,11 +61,13 @@ sub up1_redispatch {
}
sub lei_up {
- my ($lei, $out) = @_;
+ my ($lei, @outs) = @_;
$lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
$opt->{save} = -1;
+ my @local;
if (defined $opt->{all}) {
+ return $lei->fail("--all and @outs incompatible") if @outs;
length($opt->{mua}//'') and return
$lei->fail('--all and --mua= are incompatible');
@@ -74,7 +76,20 @@ sub lei_up {
$opt->{all} eq 'local' or return
$lei->fail('only --all=local works at the moment');
my @all = PublicInbox::LeiSavedSearch::list($lei);
- my @local = grep(!m!\Aimaps?://!i, @all);
+ @local = grep(!m!\Aimaps?://!i, @all);
+ } else {
+ @local = @outs;
+ }
+ if (scalar(@outs) > 1) {
+ length($opt->{mua}//'') and return $lei->fail(<<EOM);
+multiple outputs and --mua= are incompatible
+EOM
+ # TODO:
+ return $lei->fail(<<EOM) if grep(m!\Aimaps?://!i, @outs);
+multiple destinations only supported for local outputs (FIXME)
+EOM
+ }
+ if (scalar(@local) > 1) {
$lei->_lei_store->write_prepare($lei); # share early
# daemon mode, re-dispatch into our event loop w/o
# creating an extra fork-level
@@ -89,7 +104,7 @@ sub lei_up {
$lei->event_step_init;
$op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
} else {
- up1($lei, $out);
+ up1($lei, $local[0]);
}
}
^ permalink raw reply related [relevance 63%]
* [PATCH 0/3] lei pathname canonicalization fixes
@ 2021-08-11 11:26 71% Eric Wong
2021-08-11 11:26 42% ` [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-08-11 11:26 UTC (permalink / raw)
To: meta
Pathnames with "x/../y" components weren't being canonicalized
properly after creation, and some of it was being put into the
lei_mail_sync.sqlite3 DB and configs for saved searches.
This will become more important as we start using inotify more
to track keyword changes on messages.
Eric Wong (3):
treewide: use *nix-specific dirname regexps
lei_saved_search: canonicalized relative save paths
lei: attempt to canonicalize away "/../" pathnames
lib/PublicInbox/IMAPTracker.pm | 4 ++--
lib/PublicInbox/LEI.pm | 16 +++++++++++-----
lib/PublicInbox/LeiALE.pm | 8 ++++----
lib/PublicInbox/LeiBlob.pm | 4 ++--
lib/PublicInbox/LeiInit.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 9 ++++++++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/OverIdx.pm | 4 ++--
lib/PublicInbox/Xapcmd.pm | 5 ++---
script/public-inbox-init | 3 +--
t/init.t | 1 -
t/lei-q-save.t | 9 +++++++++
t/lei_xsearch.t | 8 +++++---
17 files changed, 52 insertions(+), 32 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames
2021-08-11 11:26 71% [PATCH 0/3] lei pathname canonicalization fixes Eric Wong
@ 2021-08-11 11:26 42% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-11 11:26 UTC (permalink / raw)
To: meta
As documented, File::Spec->canonpath does not canonicalize
"/../". While we want to do our best to preserve symlinks in
pathnames, leaving "/../" can mislead our inotify|kqueue usage.
---
lib/PublicInbox/LEI.pm | 14 ++++++++++----
lib/PublicInbox/LeiALE.pm | 8 ++++----
lib/PublicInbox/LeiBlob.pm | 4 ++--
lib/PublicInbox/LeiInit.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 +-
t/lei_xsearch.t | 8 +++++---
10 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index be4754df..54fac7b4 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -94,6 +94,12 @@ sub rel2abs {
# abs_path resolves symlinks in parent iff all parents exist
sub abs_path { Cwd::abs_path($_[1]) // rel2abs(@_) }
+sub canonpath_harder {
+ my $p = $_[-1]; # $_[0] may be self
+ $p = File::Spec->canonpath($p);
+ $p =~ m!(?:/*|\A)\.\.(?:/*|\z)! && -e $p ? Cwd::abs_path($p) : $p;
+}
+
sub share_path ($) { # $HOME/.local/share/lei/$FOO
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_DATA_HOME} //
@@ -808,8 +814,8 @@ sub _lei_cfg ($;$) {
my $cfg = PublicInbox::Config->git_config_dump($f);
$cfg->{-st} = $cur_st;
$cfg->{'-f'} = $f;
- if ($sto && File::Spec->canonpath($sto_dir // store_path($self))
- eq File::Spec->canonpath($cfg->{'leistore.dir'} //
+ if ($sto && canonpath_harder($sto_dir // store_path($self))
+ eq canonpath_harder($cfg->{'leistore.dir'} //
store_path($self))) {
$cfg->{-lei_store} = $sto;
$cfg->{-lei_note_event} = $lne;
@@ -1382,7 +1388,7 @@ sub refresh_watches {
next;
}
if ($url =~ /\Amaildir:(.+)/i) {
- my $d = File::Spec->canonpath($1);
+ my $d = canonpath_harder($1);
if ($state eq 'pause') {
cancel_maildir_watch($d, $cfg_f);
} elsif (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
@@ -1400,7 +1406,7 @@ sub refresh_watches {
next if exists $seen{$url};
delete $old->{$url};
if ($url =~ /\Amaildir:(.+)/i) {
- my $d = File::Spec->canonpath($1);
+ my $d = canonpath_harder($1);
cancel_maildir_watch($d, $cfg_f);
} else { # TODO: imap/nntp/jmap
$lei->child_error(1, "E: watch $url TODO");
diff --git a/lib/PublicInbox/LeiALE.pm b/lib/PublicInbox/LeiALE.pm
index cb570ab4..cc9a2095 100644
--- a/lib/PublicInbox/LeiALE.pm
+++ b/lib/PublicInbox/LeiALE.pm
@@ -33,7 +33,7 @@ sub new {
for my $loc ($lei->externals_each) { # locals only
$lxs->prepare_external($loc) if -d $loc;
}
- $self->refresh_externals($lxs);
+ $self->refresh_externals($lxs, $lei);
$self;
}
@@ -50,7 +50,7 @@ sub overs_all { # for xoids_for (called only in lei workers?)
}
sub refresh_externals {
- my ($self, $lxs) = @_;
+ my ($self, $lxs, $lei) = @_;
$self->git->cleanup;
my $lk = $self->lock_for_scope;
my $cur_lxs = ref($lxs)->new;
@@ -72,7 +72,7 @@ sub refresh_externals {
}
my @ibxish = $cur_lxs->locals;
for my $x ($lxs->locals) {
- my $d = File::Spec->canonpath($x->{inboxdir} // $x->{topdir});
+ my $d = $lei->canonpath_harder($x->{inboxdir} // $x->{topdir});
$seen_ibxish{$d} //= do {
$new .= "$d\n";
push @ibxish, $x;
@@ -95,7 +95,7 @@ sub refresh_externals {
$old = <$fh> // die "readline($f): $!";
}
for my $x (@ibxish) {
- $new .= File::Spec->canonpath($x->git->{git_dir})."/objects\n";
+ $new .= $lei->canonpath_harder($x->git->{git_dir})."/objects\n";
}
$self->{ibxish} = \@ibxish;
return if $old eq $new;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 09217964..3158ca3b 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -112,7 +112,7 @@ sub lei_blob {
if ($opt->{mail} // ($has_hints ? 0 : 1)) {
if (grep(defined, @$opt{qw(include only)})) {
$lxs = $lei->lxs_prepare;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
}
my $rdr = {};
if ($opt->{mail}) {
@@ -155,7 +155,7 @@ sub lei_blob {
return $lei->fail('no --git-dir to try') unless @$git_dirs;
unless ($lxs) {
$lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
}
if ($lxs->remotes) {
require PublicInbox::LeiRemote;
diff --git a/lib/PublicInbox/LeiInit.pm b/lib/PublicInbox/LeiInit.pm
index c6c0c01b..6558ac0a 100644
--- a/lib/PublicInbox/LeiInit.pm
+++ b/lib/PublicInbox/LeiInit.pm
@@ -4,7 +4,6 @@
# for the "lei init" command, not sure if it's even needed...
package PublicInbox::LeiInit;
use v5.10.1;
-use File::Spec;
sub lei_init {
my ($self, $dir) = @_;
@@ -13,7 +12,7 @@ sub lei_init {
$dir //= $self->store_path;
$dir = $self->rel2abs($dir);
my @cur = stat($cur) if defined($cur);
- $cur = File::Spec->canonpath($cur // $dir);
+ $cur = $self->canonpath_harder($cur // $dir);
my @dir = stat($dir);
my $exists = "# leistore.dir=$cur already initialized" if @dir;
if (@cur) {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index cb5eb5b4..4a0c24a9 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -128,7 +128,7 @@ sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
sub lei_lcat {
my ($lei, @argv) = @_;
my $lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
my $sto = $lei->_lei_store(1);
$lei->{lse} = $sto->search;
my $opt = $lei->{opt};
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index e4242d9b..223db222 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -76,7 +76,7 @@ sub new {
$fmt //= $devfd >= 0 ? 'json' : (detect_fmt($lei, $dst) or return);
if (index($dst, '://') < 0) { # not a URL, so assume path
- $dst = File::Spec->canonpath($dst);
+ $dst = $lei->canonpath_harder($dst);
} # else URL
my $self = bless { fmt => $fmt, dst => $dst }, $class;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index eb7b98d4..37b660f9 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -116,7 +116,7 @@ sub lei_q {
my ($self, @argv) = @_;
PublicInbox::Config->json; # preload before forking
my $lxs = lxs_prepare($self) or return;
- $self->ale->refresh_externals($lxs);
+ $self->ale->refresh_externals($lxs, $self);
my $opt = $self->{opt};
my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 7607b44f..0ba5897c 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -215,7 +215,7 @@ sub lei_rediff {
$lei->{curl} //= which('curl') or return
$lei->fail('curl needed for', $lxs->remotes);
}
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
my $self = bless {
-force_eml => 1, # for LeiInput->input_fh
lxs => $lxs,
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 9069232b..3356d11e 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -42,7 +42,7 @@ sub up1 ($$) {
}
$lei->{lss} = $lss; # for LeiOverview->new
my $lxs = $lei->lxs_prepare or return;
- $lei->ale->refresh_externals($lxs);
+ $lei->ale->refresh_externals($lxs, $lei);
$lei->_start_query;
}
diff --git a/t/lei_xsearch.t b/t/lei_xsearch.t
index 3eb44233..d9ddb297 100644
--- a/t/lei_xsearch.t
+++ b/t/lei_xsearch.t
@@ -11,6 +11,7 @@ require PublicInbox::ExtSearchIdx;
require_git 2.6;
require_ok 'PublicInbox::LeiXSearch';
require_ok 'PublicInbox::LeiALE';
+require_ok 'PublicInbox::LEI';
my ($home, $for_destroy) = tmpdir();
my @ibx;
for my $V (1..2) {
@@ -88,18 +89,19 @@ is($lxs->over, undef, '->over fails');
my $smsg = $lxs->smsg_for($mitem) or BAIL_OUT 'smsg_for broken';
my $ale = PublicInbox::LeiALE::_new("$home/ale");
- $ale->refresh_externals($lxs);
+ my $lei = bless {}, 'PublicInbox::LEI';
+ $ale->refresh_externals($lxs, $lei);
my $exp = [ $smsg->{blob}, 'blob', -s 't/utf8.eml' ];
is_deeply([ $ale->git->check($smsg->{blob}) ], $exp, 'ale->git->check');
$lxs = PublicInbox::LeiXSearch->new;
$lxs->prepare_external($v2ibx);
- $ale->refresh_externals($lxs);
+ $ale->refresh_externals($lxs, $lei);
is_deeply([ $ale->git->check($smsg->{blob}) ], $exp,
'ale->git->check remembered inactive external');
rename("$home/v1tmp", "$home/v1moved") or BAIL_OUT "rename: $!";
- $ale->refresh_externals($lxs);
+ $ale->refresh_externals($lxs, $lei);
is($ale->git->check($smsg->{blob}), undef,
'missing after directory gone');
}
^ permalink raw reply related [relevance 42%]
* [PATCH] lei export-kw: workaround race in updating Maildir locations
@ 2021-08-05 2:33 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-05 2:33 UTC (permalink / raw)
To: meta
Inotify updates may simultaneously remove or update the location
of a message, so ensure we at least have knowledge of the new
location if the old one cannot be updated.
---
lib/PublicInbox/LeiMailSync.pm | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 82740d59..6dfa03be 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -158,7 +158,13 @@ sub mv_src {
my $sth = $self->{dbh}->prepare_cached(<<'');
UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
- $sth->execute($newbn, $fid, $oidbin, $$id);
+ my $nr = $sth->execute($newbn, $fid, $oidbin, $$id);
+ if ($nr == 0) { # may race with a clear_src, ensure new value exists
+ $sth = $self->{dbh}->prepare_cached(<<'');
+INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?)
+
+ $sth->execute($oidbin, $fid, $newbn);
+ }
}
# read-only, iterates every oidbin + UID or name for a given folder
^ permalink raw reply related [relevance 71%]
* Re: [RFC v2] lei: close inotify FD in forked child
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
@ 2021-08-04 10:40 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-08-04 10:40 UTC (permalink / raw)
To: meta
Pushed with present tense commit message now that L::I2 2.3+ is out:
https://public-inbox.org/meta/7fc6e30aeab9925bece4bb00f88bb91af5646aa2/s/
^ permalink raw reply [relevance 71%]
* [RFC v2] lei: close inotify FD in forked child
2021-07-28 12:34 58% [RFC] lei: address lifetime problems from Linux::Inotify2 Eric Wong
@ 2021-07-29 10:01 62% ` Eric Wong
2021-08-04 10:40 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-07-29 10:01 UTC (permalink / raw)
To: meta
It looks like Linux::Inotify2 2.3+ will include an ->fh method
to give us the ability to safely close an FD without hitting
EBADF (and automatically use FD_CLOEXEC).
We'll still need a new wrapper class (LI2Wrap) to handle it for
users of old versions, though.
http://lists.schmorp.de/pipermail/perl/2021q3/thread.html
---
MANIFEST | 1 +
lib/PublicInbox/DirIdle.pm | 11 +++++++++++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LI2Wrap.pm | 20 ++++++++++++++++++++
4 files changed, 33 insertions(+), 1 deletion(-)
create mode 100644 lib/PublicInbox/LI2Wrap.pm
diff --git a/MANIFEST b/MANIFEST
index a3913501..fb9f16bf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -197,6 +197,7 @@ lib/PublicInbox/InputPipe.pm
lib/PublicInbox/Isearch.pm
lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
+lib/PublicInbox/LI2Wrap.pm
lib/PublicInbox/LeiALE.pm
lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 65896f95..d572c274 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -84,4 +84,15 @@ sub event_step {
warn "$self->{inot}->read err: $@\n" if $@;
}
+sub force_close {
+ my ($self) = @_;
+ my $inot = delete $self->{inot} // return;
+ if ($inot->can('fh')) { # Linux::Inotify2 2.3+
+ close($inot->fh) or warn "CLOSE ERROR: $!";
+ } elsif ($inot->isa('Linux::Inotify2')) {
+ require PublicInbox::LI2Wrap;
+ PublicInbox::LI2Wrap::wrapclose($inot);
+ }
+}
+
1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d9fd40fd..e6f763e1 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -556,7 +556,7 @@ sub _lei_atfork_child {
}
close $listener if $listener;
undef $listener;
- undef $dir_idle;
+ $dir_idle->force_close if $dir_idle;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
diff --git a/lib/PublicInbox/LI2Wrap.pm b/lib/PublicInbox/LI2Wrap.pm
new file mode 100644
index 00000000..b0f4f8b8
--- /dev/null
+++ b/lib/PublicInbox/LI2Wrap.pm
@@ -0,0 +1,20 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Wrapper for Linux::Inotify2 < 2.3 which lacked ->fh and auto-close
+# Remove this when supported LTS/enterprise distros are all
+# Linux::Inotify2 >= 2.3
+package PublicInbox::LI2Wrap;
+use v5.10.1;
+our @ISA = qw(Linux::Inotify2);
+
+sub wrapclose {
+ my ($inot) = @_;
+ my $fd = $inot->fileno;
+ open my $fh, '<&=', $fd or die "open <&= $fd $!";
+
+}
+
+sub DESTROY {} # no-op
+
+1
^ permalink raw reply related [relevance 62%]
* [RFC] lei: address lifetime problems from Linux::Inotify2
@ 2021-07-28 12:34 58% Eric Wong
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-07-28 12:34 UTC (permalink / raw)
To: meta
Linux::Inotify2 doesn't supply an explicit close() wrapper,
and Linux::Inotify2::Event objects maintain a reference to the
parent Linux::Inotify2 object, so we were leaking inotify FDs
into the "lei note-event" worker processes.
This delays the fork()-ing code paths into the next tick of the
event loop to ensure the inotify FD is closed when $dir_idle
is undef'ed in ->_lei_atfork_child.
Note: I'm not sure if I want to keep this, since
The correct fix would be to have a way to close inotify FDs,
but the current Linux::Inotify2 API doesn't make it easy.
And leaking an inotify FD isn't the worst thing in the world,
since it won't hang other processes. So maybe the cure
is worse than the disease, in this case.
---
lib/PublicInbox/LEI.pm | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d9fd40fd..b81f95c9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -28,8 +28,8 @@ use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
our $quit = \&CORE::exit;
-our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
- $recv_cmd, $send_cmd);
+our ($current_lei, $errors_log, $listener, $oldset,
+ $dir_idle, $lne_nxt, $lne_q, $recv_cmd, $send_cmd);
my $GLP = Getopt::Long::Parser->new;
$GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
my $GLP_PASS = Getopt::Long::Parser->new;
@@ -557,6 +557,8 @@ sub _lei_atfork_child {
close $listener if $listener;
undef $listener;
undef $dir_idle;
+ undef $lne_nxt;
+ undef $lne_q;
%PATH2CFG = ();
$MDIR2CFGPATH = {};
eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
@@ -1153,6 +1155,21 @@ sub cfg2lei ($) {
$lei;
}
+sub lne_task { # requeue, runs without dir_idle stuff on stack
+ undef $lne_nxt;
+ my $q = $lne_q // return;
+ $lne_q = undef;
+
+ while (my ($loc, $nc, $bn, $fn, $cfg) = splice(@$q, 0, 5)) {
+ eval {
+ local %ENV = %{$cfg->{-env}};
+ my $lei = cfg2lei($cfg);
+ $lei->dispatch('note-event', $loc, $nc, $bn, $fn);
+ };
+ warn "E note-event $cfg->{-f}: $@\n" if $@;
+ }
+}
+
sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
my ($ev) = @_; # Linux::Inotify2::Event or duck type
my $fn = $ev->fullname;
@@ -1161,14 +1178,9 @@ sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
$nc = '' if $ev->IN_DELETE;
for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
my $cfg = $PATH2CFG{$f} // next;
- eval {
- local %ENV = %{$cfg->{-env}};
- my $lei = cfg2lei($cfg);
- $lei->dispatch('note-event',
- "maildir:$mdir", $nc, $bn, $fn);
- };
- warn "E note-event $f: $@\n" if $@;
+ push @$lne_q, "maildir:$mdir", $nc, $bn, $fn, $cfg;
}
+ $lne_nxt //= PublicInbox::DS::requeue(\&lne_task) if $lne_q;
}
if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
$ev->cancel;
@@ -1261,6 +1273,7 @@ sub lazy_start {
undef $sig;
local $SIG{PIPE} = 'IGNORE';
require PublicInbox::DirIdle;
+ local ($lne_q, $lne_nxt);
local $dir_idle = PublicInbox::DirIdle->new([$sock_dir], sub {
# just rely on wakeup to hit PostLoopCallback set below
dir_idle_handler($_[0]) if $_[0]->fullname ne $path;
^ permalink raw reply related [relevance 58%]
* [PATCH 1/2] lei: die on ECONNRESET
@ 2021-07-28 0:37 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-28 0:37 UTC (permalink / raw)
To: meta
ECONNRESET should be rare on a private local socket, and if
we hit it, it's because we're hitting the listen() limit.
---
script/lei | 1 -
1 file changed, 1 deletion(-)
diff --git a/script/lei b/script/lei
index fce8124a..99d94b4e 100755
--- a/script/lei
+++ b/script/lei
@@ -115,7 +115,6 @@ while (1) {
my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
if (scalar(@fds) == 1 && !defined($fds[0])) {
next if $!{EINTR};
- last if $!{ECONNRESET};
die "recvmsg: $!";
}
last if $buf eq '';
^ permalink raw reply related [relevance 71%]
* lei rediff: add --dequote and --requote?
@ 2021-07-25 12:29 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 12:29 UTC (permalink / raw)
To: meta
I find "lei rediff" useful when writing replies to patch emails.
Like any good *nix tool, my $EDITOR allows piping either the
full or partial editor buffer into arbitrary commands.
Besides typing text, the most important feature of my editor is
the ability to run/pipe-to arbitrary commands (e.g. git-am, lei
blob, git-show); and "lei rediff" is a new one.
I barely customize my editor; however, I do have some editor
macros for quickly stripping and adding quote prefixes (at least
the standard "> " mail quoting style, along with '#' for sh/Perl).
But maybe others do not have these macros, so I wonder if
"rediff" should support --dequote and --requote switches to ease
this usage pattern.
Or/and if "lei dequote" and "lei quote" should exist as
(slower) replacements for `sed 's/^[> ]\+//'` and `sed 's/^/> /'`,
respectively.
^ permalink raw reply [relevance 71%]
* [PATCH] doc: lei-{p2q,rediff}: note implicit --stdin
@ 2021-07-25 12:03 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 12:03 UTC (permalink / raw)
To: meta
lei actually uses implicit --stdin everywhere, but I thing
these patch-related commands are the most common use of them.
---
Documentation/lei-p2q.pod | 3 ++-
Documentation/lei-rediff.pod | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-p2q.pod b/Documentation/lei-p2q.pod
index f404ede0..44798ac3 100644
--- a/Documentation/lei-p2q.pod
+++ b/Documentation/lei-p2q.pod
@@ -48,7 +48,8 @@ Default: C<dfpost7>
=item --stdin
-Read patch from stdin.
+Read message from stdin. This is implicit if no arguments are given
+and stdin is a pipe or regular file.
=item --debug
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index 5fdde230..e968fb20 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -24,7 +24,8 @@ supported.
=item --stdin
-Read message from stdin.
+Read message from stdin. This is implicit if no arguments are given
+and stdin is a pipe or regular file.
=item --git-dir=DIR
^ permalink raw reply related [relevance 71%]
* [PATCH] t/lei-watch.t: improve test reliability
@ 2021-07-25 11:15 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-25 11:15 UTC (permalink / raw)
To: meta
On single CPU (and overloaded SMP) systems, we can't rely on
inotify in lei-daemon firing before a "lei note-event done"
client hits it. So force in a single tick() to ensure the
scheduler can yield to lei-daemon and see the inotify wakeup
before "lei note-event done" to commit the write.
---
t/lei-watch.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 9a3bfd80..86fa6649 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -52,7 +52,7 @@ test_lei(sub {
my @f = glob("$md/cur/*:2,");
is(scalar(@f), 1, 'got populated maildir with one result');
rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een
- $have_fast_inotify or tick(2);
+ tick($have_fast_inotify ? 0.1 : 2.1); # always needed for 1 CPU systems
lei_ok qw(note-event done); # flushes immediately (instead of 5s)
lei_ok qw(q mid:testmessage@example.com -o), $md2, '-I', "$ro_home/t1";
^ permalink raw reply related [relevance 71%]
* [PATCH 0/3] lei rm-watch, some error handling stuff
@ 2021-07-23 10:56 71% Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
I'm considering making "tag-ro" the default watch state
and introducing the idea of implicit watches for all
"lei q" output directories.
Eric Wong (3):
t/lei*: check error messages on failures
lei: avoid SQLite COUNT() for dedupe
lei rm-watch: new command to support removing watches
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 4 +++-
lib/PublicInbox/LeiDedupe.pm | 5 ++---
lib/PublicInbox/LeiInput.pm | 8 ++++++++
lib/PublicInbox/LeiRmWatch.pm | 31 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSavedSearch.pm | 8 ++++----
lib/PublicInbox/LeiToMail.pm | 4 ++--
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/SharedKV.pm | 7 +++++++
t/lei-externals.t | 4 ++++
t/lei-import-http.t | 2 ++
t/lei-import-imap.t | 1 +
t/lei-mirror.t | 7 ++++++-
t/lei-p2q.t | 1 +
t/lei-q-kw.t | 1 +
t/lei-q-remote-import.t | 1 +
t/lei-q-save.t | 4 ++++
t/lei-tag.t | 8 +++++++-
t/lei-watch.t | 18 ++++++++++++++----
t/solver_git.t | 7 +++++++
20 files changed, 107 insertions(+), 17 deletions(-)
create mode 100644 lib/PublicInbox/LeiRmWatch.pm
^ permalink raw reply [relevance 71%]
* [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
@ 2021-07-23 10:56 58% ` Eric Wong
2021-07-23 10:56 53% ` [PATCH 3/3] lei rm-watch: new command to support removing watches Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
SQLite COUNT() is a slow operation that does a full table scan
with no conditions. There's no need for it, since lei dedupe
only needs to know if it's empty or not to decide between
new/ and cur/ for Maildir outputs.
---
lib/PublicInbox/LeiDedupe.pm | 5 ++---
lib/PublicInbox/LeiSavedSearch.pm | 8 ++++----
lib/PublicInbox/LeiToMail.pm | 4 ++--
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/SharedKV.pm | 7 +++++++
5 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index ed52e417..32f99cd0 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -127,10 +127,9 @@ sub pause_dedupe {
delete($skv->{dbh}) if $skv;
}
-sub dedupe_nr {
+sub has_entries {
my $skv = $_[0]->[0] or return undef;
- my @n = $skv->count;
- $n[0];
+ $skv->has_entries;
}
1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 929380ed..cfbf68c3 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -315,11 +315,11 @@ E: rename($dir_old, $dir_new) error: $!
EOM
}
-# cf. LeiDedupe->dedupe_nr
-sub dedupe_nr {
+# cf. LeiDedupe->has_entries
+sub has_entries {
my $oidx = $_[0]->{oidx} // die 'BUG: no {oidx}';
- my @n = $oidx->{dbh}->selectrow_array('SELECT COUNT(*) FROM over');
- $n[0];
+ my @n = $oidx->{dbh}->selectrow_array('SELECT num FROM over LIMIT 1');
+ scalar(@n) ? 1 : undef;
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9405c0c..d782d4c7 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -198,7 +198,7 @@ sub _mbox_write_cb ($$) {
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe;
my $lse = $lei->{lse}; # may be undef
- my $set_recent = $dedupe->dedupe_nr;
+ my $set_recent = $dedupe->has_entries;
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($buf);
@@ -293,7 +293,7 @@ sub _maildir_write_cb ($$) {
# Favor cur/ and only write to new/ when augmenting. This
# saves MUAs from having to do a mass rename when the initial
# search result set is huge.
- my $dir = $dedupe && $dedupe->dedupe_nr ? 'new/' : 'cur/';
+ my $dir = $dedupe && $dedupe->has_entries ? 'new/' : 'cur/';
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index cac7fb7d..3414e87d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -504,7 +504,7 @@ sub do_query {
my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
if ($l2m->{-wq_nr_workers} > 1 &&
$l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
- # setup two barriers to coordinate dedupe_nr
+ # setup two barriers to coordinate ->has_entries
# between l2m workers
pipe(my ($a_r, $a_w)) or die "pipe: $!";
fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index 8347b195..3487e820 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -154,6 +154,13 @@ SELECT COUNT(k) FROM kv
$sth->fetchrow_array;
}
+# faster than ->count due to how SQLite works
+sub has_entries {
+ my ($self) = @_;
+ my @n = $self->{dbh}->selectrow_array('SELECT k FROM kv LIMIT 1');
+ scalar(@n) ? 1 : undef;
+}
+
sub dbh_release {
my ($self, $lock) = @_;
my $dbh = delete $self->{dbh} or return;
^ permalink raw reply related [relevance 58%]
* [PATCH 3/3] lei rm-watch: new command to support removing watches
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
2021-07-23 10:56 58% ` [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe Eric Wong
@ 2021-07-23 10:56 53% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
Pretty trivial since it just invokes "git-config". It's mainly
intended to make shell completion easier.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 4 +++-
lib/PublicInbox/LeiInput.pm | 8 ++++++++
lib/PublicInbox/LeiRmWatch.pm | 31 +++++++++++++++++++++++++++++++
t/lei-watch.t | 18 ++++++++++++++----
5 files changed, 57 insertions(+), 5 deletions(-)
create mode 100644 lib/PublicInbox/LeiRmWatch.pm
diff --git a/MANIFEST b/MANIFEST
index 1d79b7c9..a3913501 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -233,6 +233,7 @@ lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
+lib/PublicInbox/LeiRmWatch.pm
lib/PublicInbox/LeiSavedSearch.pm
lib/PublicInbox/LeiSearch.pm
lib/PublicInbox/LeiSelfSocket.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 52c551cf..191a0790 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -232,8 +232,10 @@ our %CMD = ( # sorted in order of importance/use:
'remove imported messages from IMAP, Maildirs, and MH',
qw(exact! all jobs:i indexed), @c_opt ],
-'add-watch' => [ 'LOCATION', 'watch for new messages and flag changes',
+'add-watch' => [ 'LOCATION...', 'watch for new messages and flag changes',
qw(poll-interval=s state=s recursive|r), @c_opt ],
+'rm-watch' => [ 'LOCATION...', 'remove specified watch(es)',
+ qw(recursive|r), @c_opt ],
'ls-watch' => [ '[FILTER...]', 'list active watches with numbers and status',
qw(l z|0), @c_opt ],
'pause-watch' => [ '[WATCH_NUMBER_OR_FILTER]', qw(all local remote), @c_opt ],
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index fa330df5..88889f45 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -270,6 +270,10 @@ sub prepare_inputs { # returns undef on error
$sync and $input = 'maildir:'.
$lei->abs_path($input_path);
push @md, $input;
+ } elsif ($self->{missing_ok} && !-e _) {
+ # for "lei rm-watch" on missing Maildir
+ $sync and $input = 'maildir:'.
+ $lei->abs_path($input_path);
} else {
return $lei->fail("Unable to handle $input");
}
@@ -305,6 +309,10 @@ $input is `eml', not --in-format=$in_fmt
push @{$sync->{ok}}, $input;
}
push @md, $input;
+ } elsif ($self->{missing_ok} && !-e $input) {
+ # for lei rm-watch
+ $sync and $input = 'maildir:'.
+ $lei->abs_path($input);
} else {
return $lei->fail("Unable to handle $input")
}
diff --git a/lib/PublicInbox/LeiRmWatch.pm b/lib/PublicInbox/LeiRmWatch.pm
new file mode 100644
index 00000000..c0f336f0
--- /dev/null
+++ b/lib/PublicInbox/LeiRmWatch.pm
@@ -0,0 +1,31 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei rm-watch" command
+package PublicInbox::LeiRmWatch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::LeiInput);
+
+sub lei_rm_watch {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg(1);
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ my $self = bless { missing_ok => 1 }, __PACKAGE__;
+ $self->prepare_inputs($lei, \@argv) or return;
+ for my $w (@{$self->{inputs}}) {
+ $lei->_config('--remove-section', "watch.$w");
+ }
+ delete $lei->{cfg}; # force reload
+ $lei->refresh_watches;
+}
+
+sub _complete_rm_watch {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ my @w = (join("\n", keys %$cfg) =~ m/^watch\.(.+?)\.state$/sgm);
+ map { $match_cb->($_) } @w;
+}
+
+1;
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 492f6c1d..9a3bfd80 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -2,7 +2,7 @@
# Copyright all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
-use File::Path qw(make_path);
+use File::Path qw(make_path remove_tree);
require_mods('lei');
my $have_fast_inotify = eval { require Linux::Inotify2 } ||
eval { require IO::KQueue };
@@ -71,9 +71,19 @@ test_lei(sub {
'inotify has Maildir watches');
}
- is(xsys(qw(git config -f), $cfg_f,
- '--remove-section', "watch.maildir:$md"),
- 0, 'unset config state');
+ lei_ok 'rm-watch', $md;
+ lei_ok 'ls-watch', \'refresh watches';
+ is($lei_out, '', 'no watches left');
+
+ lei_ok 'add-watch', $md2;
+ remove_tree($md2);
+ lei_ok 'rm-watch', "maildir:".$md2, \'with maildir: prefix';
+ lei_ok 'ls-watch', \'refresh watches';
+ is($lei_out, '', 'no watches left');
+
+ lei_ok 'add-watch', $md;
+ remove_tree($md);
+ lei_ok 'rm-watch', $md, \'absolute path w/ missing dir';
lei_ok 'ls-watch', \'refresh watches';
is($lei_out, '', 'no watches left');
^ permalink raw reply related [relevance 53%]
* [PATCH 1/3] t/lei*: check error messages on failures
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
@ 2021-07-23 10:56 36% ` Eric Wong
2021-07-23 10:56 58% ` [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe Eric Wong
2021-07-23 10:56 53% ` [PATCH 3/3] lei rm-watch: new command to support removing watches Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-23 10:56 UTC (permalink / raw)
To: meta
I just hit an unreproducible failure in t/lei-p2q.t and
lacked $lei_err information to diagnose it. Hopefully
this helps track down odd failures in the future.
---
t/lei-externals.t | 4 ++++
t/lei-import-http.t | 2 ++
t/lei-import-imap.t | 1 +
t/lei-mirror.t | 7 ++++++-
t/lei-p2q.t | 1 +
t/lei-q-kw.t | 1 +
t/lei-q-remote-import.t | 1 +
t/lei-q-save.t | 4 ++++
t/lei-tag.t | 8 +++++++-
t/solver_git.t | 7 +++++++
10 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/t/lei-externals.t b/t/lei-externals.t
index f148fa3c..5e3c67bc 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -211,6 +211,8 @@ test_lei(sub {
like($lei_out, qr/use boolean prefix/, '--stdin on pipe');
}
ok(!lei(qw(q -q --stdin s:use)), "--stdin and argv don't mix");
+ like($lei_err, qr/no query allowed.*--stdin/,
+ '--stdin conflict error message');
for my $fmt (qw(ldjson ndjson jsonl)) {
lei_ok('q', '-f', $fmt, 's:use boolean prefix');
@@ -250,6 +252,8 @@ test_lei(sub {
}
ok(!lei('q', '-o', "$home/mbox", 's:nope'),
'fails if mbox format unspecified');
+ like($lei_err, qr/unable to determine mbox/, 'mbox-related message');
+
ok(!lei(qw(q --no-local s:see)), '--no-local');
is($? >> 8, 1, 'proper exit code');
like($lei_err, qr/no local or remote.+? to search/, 'no inbox');
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
index 2104c778..e9eec1f7 100644
--- a/t/lei-import-http.t
+++ b/t/lei-import-http.t
@@ -17,6 +17,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $url = "http://$host_port/t2";
for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) {
ok(!lei('import', "$url/$p"), "/$p fails properly");
+ like($lei_err, qr/curl.*404/, 'got curl 404');
}
for my $p (qw(/ /T/ /t/ /t.atom)) {
ok(!lei('import', "$url/m\@example$p"), "/$p fails");
@@ -42,5 +43,6 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(!lei(qw(import --mail-sync), "$url/x\@example.com/raw"),
'--mail-sync fails on HTTP');
+ like($lei_err, qr/--mail-sync/, 'error message notes --mail-sync');
});
done_testing;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 12f6fad0..315567b3 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -43,6 +43,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
$url = $u;
$u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s;
ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected');
+ like($lei_err, qr/UIDVALIDITY mismatch/, 'mismatch noted');
lei_ok('inspect', $url);
my $inspect = json_utf8->decode($lei_out);
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index dfd35e1a..80bc6ed5 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -27,6 +27,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(!lei('add-external', $t2, '--mirror', "$http/t2/"),
'--mirror fails if reused') or diag "$lei_err.$lei_out = $?";
+ like($lei_err, qr/\Q$t2\E' already exists/, 'destination in error');
ok(!lei('add-external', "$home/t2\nnewline", '--mirror', "$http/t2/"),
'--mirror fails on newline');
@@ -37,13 +38,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
unlike($lei_out, qr!\Qnewline\E!, 'newline entry not added');
ok(!lei('add-external', "$t2-fail", '-Lmedium'), '--mirror v2');
+ like($lei_err, qr/not a directory/, 'non-directory noted');
ok(!-d "$t2-fail", 'destination not created on failure');
lei_ok('ls-external');
unlike($lei_out, qr!\Q$t2-fail\E!, 'not added to ls-external');
my %phail = (
HTTPS => 'https://public-inbox.org/' . 'phail',
- ONION => 'http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/' . 'phail,'
+ ONION =>
+'http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/' .
+'phail,'
);
for my $t (qw(HTTPS ONION)) {
SKIP: {
@@ -56,6 +60,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
is($? >> 8, 22, 'curl 404');
ok(!-d $dir, 'directory not created');
unlike($lei_err, qr/# mirrored/, 'no success message');
+ like($lei_err, qr/curl.*404/, "curl 404 shown for $k");
} # SKIP
} # for
});
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index 58506f94..495d81de 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -8,6 +8,7 @@ require_mods(qw(json DBD::SQLite Search::Xapian));
test_lei(sub {
ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
'p2q fails on bogus arg');
+ like($lei_err, qr/format-patch.*failed/, 'notes format-patch failure');
lei_ok(qw(p2q -w dfpost t/data/0001.patch));
is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname') or diag $lei_err;
open my $fh, '+<', 't/data/0001.patch' or xbail "open: $!";
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 528751b4..2e6be1f0 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -48,6 +48,7 @@ SKIP: {
my $cat = popen_rd(['cat', $o]);
ok(!lei(qw(q --import-before bogus -o), "mboxrd:$o"),
'--import-before fails on non-seekable output');
+ like($lei_err, qr/not seekable/, 'unseekable noted in error');
is(do { local $/; <$cat> }, '', 'no output on FIFO');
close $cat;
$cat = popen_rd(['cat', $o]);
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 7db684d9..aaf56e27 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -54,6 +54,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(-f $o && -s _, '--lock=none respected') or diag $lei_err;
unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
ok(!lei(@cmd, '--lock=dotlock,timeout=0.000001'), 'dotlock fails');
+ like($lei_err, qr/dotlock timeout/, 'timeout noted');
ok(-f $o && !-s _, 'nothing output on lock failure');
unlink "$o.lock" or BAIL_OUT $!;
lei_ok(@cmd, '--lock=dotlock,timeout=0.000001',
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 6c592088..b1ca4e92 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -69,6 +69,7 @@ test_lei(sub {
ok(-s "$home/mbcl2" > $size, 'size increased after up');
ok(!lei(qw(up -q), $home), 'up fails w/o --save');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
lei_ok qw(ls-search -z); my @z = split(/\0/, $lei_out);
@@ -115,6 +116,8 @@ test_lei(sub {
lei_ok(qw(up --all=local));
ok(!lei(qw(forget-search), "$home/bogus"), 'bogus forget');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
+
lei_ok qw(_complete lei forget-search);
like($lei_out, qr/mbrd-aug/, 'forget-search completion');
lei_ok(qw(forget-search -v), "$home/mbrd-aug");
@@ -124,6 +127,7 @@ test_lei(sub {
unlike($lei_out, qr/mbrd-aug/,
'forget-search completion cleared after forget');
ok(!lei('up', "$home/mbrd-aug"), 'lei up fails after forget');
+ like($lei_err, qr/--save was not used/, 'error noted --save');
# dedupe=mid
my $o = "$home/dd-mid";
diff --git a/t/lei-tag.t b/t/lei-tag.t
index 5cb6d9ce..44e4659f 100644
--- a/t/lei-tag.t
+++ b/t/lei-tag.t
@@ -32,8 +32,14 @@ test_lei(sub {
lei_ok(qw(ls-label)); is($lei_out, "urgent\n", 'label found');
ok(!lei(qw(tag -F eml t/utf8.eml +kw:seeen)), 'bad kw rejected');
like($lei_err, qr/`seeen' is not one of/, 'got helpful error');
+
ok(!lei(qw(tag -F eml t/utf8.eml +k:seen)), 'bad prefix rejected');
+ like($lei_err, qr/Unable to handle.*\Q+k:seen\E/, 'bad prefix noted');
+
ok(!lei(qw(tag -F eml t/utf8.eml)), 'no keywords');
+ like($lei_err, qr/no keywords or labels specified/,
+ 'lack of kw/L noted');
+
my $mb = "$ENV{HOME}/mb";
my $md = "$ENV{HOME}/md";
lei_ok(qw(q m:testmessage@example.com -o), "mboxrd:$mb");
@@ -78,7 +84,7 @@ test_lei(sub {
lei_ok(qw(ls-label));
is($lei_out, "nope\nqp\nurgent\n", 'ls-label shows qp');
- lei_ok qw(tag -F eml t/utf8.eml +L:INBOX +L:x); diag $lei_err;
+ lei_ok qw(tag -F eml t/utf8.eml +L:INBOX +L:x);
lei_ok qw(q m:testmessage@example.com);
$check_kw->([qw(answered seen)], L => [qw(INBOX nope urgent x)]);
lei_ok(qw(ls-label));
diff --git a/t/solver_git.t b/t/solver_git.t
index fe1aff0b..f5cc592c 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -43,6 +43,7 @@ test_lei({tmpdir => "$tmpdir/blob"}, sub {
is($lei_out, $patch2->as_string, 'blob matches');
ok(!lei('blob', '--mail', '69df7d5', '-I', $ibx->{inboxdir}),
"--mail won't run solver");
+ like($lei_err, qr/\b69df7d5\b/, 'OID in error by git(1)');
lei_ok('blob', '69df7d5', '-I', $ibx->{inboxdir});
is(git_sha(1, \$lei_out)->hexdigest, $expect, 'blob contents output');
@@ -51,6 +52,8 @@ test_lei({tmpdir => "$tmpdir/blob"}, sub {
is($lei_out, $prev, '--no-mail works');
ok(!lei(qw(blob -I), $ibx->{inboxdir}, $non_existent),
'non-existent blob fails');
+ my $abbrev = substr($non_existent, 0, 7);
+ like($lei_err, qr/could not find $abbrev/, 'failed abbreviation noted');
SKIP: {
skip '/.git exists', 1 if -e '/.git';
lei_ok(qw(-C / blob 69df7d5 -I), $ibx->{inboxdir},
@@ -59,9 +62,13 @@ test_lei({tmpdir => "$tmpdir/blob"}, sub {
ok(!lei(qw(-C / blob --no-cwd 69df7d5 -I), $ibx->{inboxdir}),
'--no-cwd works');
+ like($lei_err, qr/no --git-dir to try/,
+ 'lack of --git-dir noted');
ok(!lei(qw(-C / blob -I), $ibx->{inboxdir}, $non_existent),
'non-existent blob fails');
+ like($lei_err, qr/no --git-dir to try/,
+ 'lack of --git-dir noted');
}
# fallbacks
^ permalink raw reply related [relevance 36%]
* [PATCH 3/2] lei: auto-refresh watches in config, cancel missing
2021-07-19 8:59 63% [PATCH 0/2] lei Maildir inotify/kevent support Eric Wong
2021-07-19 8:59 22% ` [PATCH 2/2] lei: start implementing inotify Maildir support Eric Wong
@ 2021-07-21 14:07 47% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-21 14:07 UTC (permalink / raw)
To: meta
This makes behavior less surprising on restarts as we no longer
lose state on restarts, so there's no need to manually run "lei
add-watch" to re-enable watches. This also allows us to
transparently handle changes if somebody edits the lei config
file directly or via git-config(1).
---
lib/PublicInbox/DirIdle.pm | 5 +++-
lib/PublicInbox/LEI.pm | 50 +++++++++++++++++++++++++--------
lib/PublicInbox/LeiNoteEvent.pm | 2 +-
t/lei-watch.t | 39 +++++++++++++++++++++++++
4 files changed, 83 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 7031e5fd..65896f95 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -56,10 +56,13 @@ sub new {
sub add_watches {
my ($self, $dirs, $gone) = @_;
my $fl = $MAIL_IN | ($gone ? $MAIL_GONE : 0);
+ my @ret;
for my $d (@$dirs) {
- $self->{inot}->watch($d, $fl);
+ my $w = $self->{inot}->watch($d, $fl) or next;
+ push @ret, $w;
}
PublicInbox::FakeInotify::poll_once($self) if !$ino_cls;
+ @ret
}
sub rm_watches {
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b92d7512..52c551cf 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -36,7 +36,7 @@ my $GLP_PASS = Getopt::Long::Parser->new;
$GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
-our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => undef }
+our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => [ ino watches ] }
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -820,6 +820,8 @@ sub _lei_cfg ($;$) {
}
}
$self->{cfg} = $PATH2CFG{$f} = $cfg;
+ refresh_watches($self);
+ $cfg;
}
sub _lei_store ($;$) {
@@ -1353,36 +1355,62 @@ sub watch_state_ok ($) {
$state =~ /\Apause|(?:import|index|tag)-(?:ro|rw)\z/;
}
+sub cancel_maildir_watch ($$) {
+ my ($d, $cfg_f) = @_;
+ my $w = delete $MDIR2CFGPATH->{$d}->{$cfg_f};
+ scalar(keys %{$MDIR2CFGPATH->{$d}}) or
+ delete $MDIR2CFGPATH->{$d};
+ for my $x (@{$w // []}) { $x->cancel }
+}
+
sub refresh_watches {
my ($lei) = @_;
my $cfg = _lei_cfg($lei) or return;
- $cfg->{-env} //= { %{$lei->{env}}, PWD => '/' }; # for cfg2lei
+ my $old = $cfg->{-watches};
my $watches = $cfg->{-watches} //= {};
- require PublicInbox::LeiWatch;
+ my %seen;
+ my $cfg_f = $cfg->{'-f'};
for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
my $url = substr($w, length('watch.'), -length('.state'));
- my $lw = $watches->{$w} //= PublicInbox::LeiWatch->new($url);
+ require PublicInbox::LeiWatch;
+ my $lw = $watches->{$url} //= PublicInbox::LeiWatch->new($url);
+ $seen{$url} = undef;
my $state = $cfg->get_1("watch.$url", 'state');
if (!watch_state_ok($state)) {
$lei->err("watch.$url.state=$state not supported");
next;
}
- my $f = $cfg->{'-f'};
if ($url =~ /\Amaildir:(.+)/i) {
my $d = File::Spec->canonpath($1);
if ($state eq 'pause') {
- delete $MDIR2CFGPATH->{$d}->{$f};
- scalar(keys %{$MDIR2CFGPATH->{$d}}) or
- delete $MDIR2CFGPATH->{$d};
- } elsif (!exists($MDIR2CFGPATH->{$d}->{$f})) {
- $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
- $MDIR2CFGPATH->{$d}->{$f} = undef;
+ cancel_maildir_watch($d, $cfg_f);
+ } elsif (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
+ my @w = $dir_idle->add_watches(
+ ["$d/cur", "$d/new"], 1);
+ push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
}
} else { # TODO: imap/nntp/jmap
$lei->child_error(1,
"E: watch $url not supported, yet");
}
}
+ if ($old) { # cull old non-existent entries
+ for my $url (keys %$old) {
+ next if exists $seen{$url};
+ delete $old->{$url};
+ if ($url =~ /\Amaildir:(.+)/i) {
+ my $d = File::Spec->canonpath($1);
+ cancel_maildir_watch($d, $cfg_f);
+ } else { # TODO: imap/nntp/jmap
+ $lei->child_error(1, "E: watch $url TODO");
+ }
+ }
+ }
+ if (scalar keys %$watches) {
+ $cfg->{-env} //= { %{$lei->{env}}, PWD => '/' }; # for cfg2lei
+ } else {
+ delete $cfg->{-watches};
+ }
}
1;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index bf15cd26..d6511cf6 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -14,7 +14,7 @@ sub flush_lei ($) {
my ($lei) = @_;
if (my $lne = delete $lei->{cfg}->{-lei_note_event}) {
$lne->wq_close(1, undef, $lei); # runs _lei_wq_eof;
- } else { # lms_clear_src calls only:
+ } elsif ($lei->{sto}) { # lms_clear_src calls only:
my $wait = $lei->{sto}->ipc_do('done');
}
}
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 3a2f9e64..492f6c1d 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -13,9 +13,27 @@ $have_fast_inotify or
my ($ro_home, $cfg_path) = setup_public_inboxes;
test_lei(sub {
my $md = "$ENV{HOME}/md";
+ my $cfg_f = "$ENV{HOME}/.config/lei/config";
my $md2 = $md.'2';
lei_ok 'ls-watch';
is($lei_out, '', 'nothing in ls-watch, yet');
+
+ my ($ino_fdinfo, $ino_contents);
+ SKIP: {
+ $have_fast_inotify && $^O eq 'linux' or
+ skip 'Linux/inotify-only internals check', 1;
+ lei_ok 'daemon-pid'; chomp(my $pid = $lei_out);
+ skip 'missing /proc/$PID/fd', 1 if !-d "/proc/$pid/fd";
+ my @ino = grep {
+ readlink($_) =~ /\binotify\b/
+ } glob("/proc/$pid/fd/*");
+ is(scalar(@ino), 1, 'only one inotify FD');
+ my $ino_fd = (split('/', $ino[0]))[-1];
+ $ino_fdinfo = "/proc/$pid/fdinfo/$ino_fd";
+ open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
+ $ino_contents = [ <$fh> ];
+ }
+
if (0) { # TODO
my $url = 'imaps://example.com/foo.bar.0';
lei_ok([qw(add-watch --state=pause), $url], undef, {});
@@ -44,6 +62,27 @@ test_lei(sub {
my $e2 = eml_load($f2[0]);
my $e1 = eml_load("$f[0]S");
is_deeply($e2, $e1, 'results match');
+
+ SKIP: {
+ $ino_fdinfo or skip 'Linux/inotify-only watch check', 1;
+ open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
+ my $cmp = [ <$fh> ];
+ ok(scalar(@$cmp) > scalar(@$ino_contents),
+ 'inotify has Maildir watches');
+ }
+
+ is(xsys(qw(git config -f), $cfg_f,
+ '--remove-section', "watch.maildir:$md"),
+ 0, 'unset config state');
+ lei_ok 'ls-watch', \'refresh watches';
+ is($lei_out, '', 'no watches left');
+
+ SKIP: {
+ $ino_fdinfo or skip 'Linux/inotify-only removal removal', 1;
+ open my $fh, '<', $ino_fdinfo or xbail "open $ino_fdinfo: $!";
+ my $cmp = [ <$fh> ];
+ is_deeply($cmp, $ino_contents, 'inotify Maildir watches gone');
+ };
});
done_testing;
^ permalink raw reply related [relevance 47%]
* [PATCH 0/2] lei Maildir inotify/kevent support
@ 2021-07-19 8:59 63% Eric Wong
2021-07-19 8:59 22% ` [PATCH 2/2] lei: start implementing inotify Maildir support Eric Wong
2021-07-21 14:07 47% ` [PATCH 3/2] lei: auto-refresh watches in config, cancel missing Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-07-19 8:59 UTC (permalink / raw)
To: meta
One of the most important features of lei that drove the
decision towards its daemonic architecture.
My brain still has some wires loose or crossed :<
So I've had a very tough time mapping out the internal design
for this that could deal with rename() storms and reduce the
need to handle IN_Q_OVERFLOW conditions. Eventually,
introducing a new concept in the form of LeiNoteEvent helped my
crippled brain make the necessary connections and cobble
together something that seems halfway working.
LeiNoteEvent represents a significant conceptual shift in lei
internals. Having long-lived workers is closer to my original
vision of how lei internals would've been. That old model had
some major weaknesses, however:
1) error handling/reporting back to the CLI required FD passing
(and large-scale FD passing got confusing pretty quick)
2) excessive FD passing overflowed default kernel buffers
and made parallelism difficult for unprivileged users
Since there's no CLI to report errors to with internal
notifications, we don't have FDs to pass. Of course, that
means we can lose errors right now, but it's probably not
a big deal for (mostly) innocuous keyword changes.
Eric Wong (2):
config: s/_one_val/get_1/ for public use
lei: start implementing inotify Maildir support
MANIFEST | 6 ++
lib/PublicInbox/Config.pm | 6 +-
lib/PublicInbox/DirIdle.pm | 17 +++++
lib/PublicInbox/LEI.pm | 116 ++++++++++++++++++++++++++---
lib/PublicInbox/LeiAddWatch.pm | 41 ++++++++++
lib/PublicInbox/LeiInput.pm | 1 +
lib/PublicInbox/LeiLsWatch.pm | 15 ++++
lib/PublicInbox/LeiNoteEvent.pm | 124 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSearch.pm | 2 +-
lib/PublicInbox/LeiSelfSocket.pm | 45 +++++++++++
lib/PublicInbox/LeiWatch.pm | 13 ++++
t/lei-watch.t | 49 ++++++++++++
12 files changed, 419 insertions(+), 16 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddWatch.pm
create mode 100644 lib/PublicInbox/LeiLsWatch.pm
create mode 100644 lib/PublicInbox/LeiNoteEvent.pm
create mode 100644 lib/PublicInbox/LeiSelfSocket.pm
create mode 100644 lib/PublicInbox/LeiWatch.pm
create mode 100644 t/lei-watch.t
^ permalink raw reply [relevance 63%]
* [PATCH 2/2] lei: start implementing inotify Maildir support
2021-07-19 8:59 63% [PATCH 0/2] lei Maildir inotify/kevent support Eric Wong
@ 2021-07-19 8:59 22% ` Eric Wong
2021-07-21 14:07 47% ` [PATCH 3/2] lei: auto-refresh watches in config, cancel missing Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-19 8:59 UTC (permalink / raw)
To: meta
This allows lei to automatically note keyword (message flag)
changes made to a Maildir and propagate it into lei/store:
lei add-watch --state=tag-ro /path/to/Maildir
This doesn't persist across restarts, yet. In the future,
it will be applied automatically to "lei q" output Maildirs
by default (with an option to disable it).
State values of tag-rw, index-<ro|rw>, import-<ro|rw> will all
be supported for Maildir.
This represents a fairly major internal change that's fairly
intrusive, but the whole daemon-oriented design was to
facilitate being able to automatically monitor (and propagate)
Maildir/IMAP flag changes.
---
MANIFEST | 6 ++
lib/PublicInbox/DirIdle.pm | 17 +++++
lib/PublicInbox/LEI.pm | 116 ++++++++++++++++++++++++++---
lib/PublicInbox/LeiAddWatch.pm | 41 ++++++++++
lib/PublicInbox/LeiInput.pm | 1 +
lib/PublicInbox/LeiLsWatch.pm | 15 ++++
lib/PublicInbox/LeiNoteEvent.pm | 124 +++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSearch.pm | 2 +-
lib/PublicInbox/LeiSelfSocket.pm | 45 +++++++++++
lib/PublicInbox/LeiWatch.pm | 13 ++++
t/lei-watch.t | 49 ++++++++++++
11 files changed, 416 insertions(+), 13 deletions(-)
create mode 100644 lib/PublicInbox/LeiAddWatch.pm
create mode 100644 lib/PublicInbox/LeiLsWatch.pm
create mode 100644 lib/PublicInbox/LeiNoteEvent.pm
create mode 100644 lib/PublicInbox/LeiSelfSocket.pm
create mode 100644 lib/PublicInbox/LeiWatch.pm
create mode 100644 t/lei-watch.t
diff --git a/MANIFEST b/MANIFEST
index 146a32ab..1d79b7c9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -198,6 +198,7 @@ lib/PublicInbox/Isearch.pm
lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
lib/PublicInbox/LeiALE.pm
+lib/PublicInbox/LeiAddWatch.pm
lib/PublicInbox/LeiAuth.pm
lib/PublicInbox/LeiBlob.pm
lib/PublicInbox/LeiConvert.pm
@@ -220,8 +221,10 @@ lib/PublicInbox/LeiLsLabel.pm
lib/PublicInbox/LeiLsMailSource.pm
lib/PublicInbox/LeiLsMailSync.pm
lib/PublicInbox/LeiLsSearch.pm
+lib/PublicInbox/LeiLsWatch.pm
lib/PublicInbox/LeiMailSync.pm
lib/PublicInbox/LeiMirror.pm
+lib/PublicInbox/LeiNoteEvent.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiPmdir.pm
@@ -232,6 +235,7 @@ lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiSavedSearch.pm
lib/PublicInbox/LeiSearch.pm
+lib/PublicInbox/LeiSelfSocket.pm
lib/PublicInbox/LeiStore.pm
lib/PublicInbox/LeiStoreErr.pm
lib/PublicInbox/LeiSucks.pm
@@ -239,6 +243,7 @@ lib/PublicInbox/LeiTag.pm
lib/PublicInbox/LeiToMail.pm
lib/PublicInbox/LeiUp.pm
lib/PublicInbox/LeiViewText.pm
+lib/PublicInbox/LeiWatch.pm
lib/PublicInbox/LeiXSearch.pm
lib/PublicInbox/Linkify.pm
lib/PublicInbox/Listener.pm
@@ -434,6 +439,7 @@ t/lei-q-save.t
t/lei-q-thread.t
t/lei-sigpipe.t
t/lei-tag.t
+t/lei-watch.t
t/lei.t
t/lei_dedupe.t
t/lei_external.t
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 5142d005..7031e5fd 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -53,6 +53,23 @@ sub new {
$self;
}
+sub add_watches {
+ my ($self, $dirs, $gone) = @_;
+ my $fl = $MAIL_IN | ($gone ? $MAIL_GONE : 0);
+ for my $d (@$dirs) {
+ $self->{inot}->watch($d, $fl);
+ }
+ PublicInbox::FakeInotify::poll_once($self) if !$ino_cls;
+}
+
+sub rm_watches {
+ my ($self, $dir) = @_;
+ my $inot = $self->{inot};
+ if (my $cb = $inot->can('rm_watches')) { # TODO for fake watchers
+ $cb->($inot, $dir);
+ }
+}
+
sub event_step {
my ($self) = @_;
my $cb = $self->{cb};
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a9f5edae..b92d7512 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -28,14 +28,15 @@ use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
our $quit = \&CORE::exit;
-our ($current_lei, $errors_log, $listener, $oldset, $dir_idle);
-my ($recv_cmd, $send_cmd);
+our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
+ $recv_cmd, $send_cmd);
my $GLP = Getopt::Long::Parser->new;
$GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
my $GLP_PASS = Getopt::Long::Parser->new;
$GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
our %PATH2CFG; # persistent for socket daemon
+our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => undef }
# TBD: this is a documentation mechanism to show a subcommand
# (may) pass options through to another command:
@@ -232,10 +233,9 @@ our %CMD = ( # sorted in order of importance/use:
qw(exact! all jobs:i indexed), @c_opt ],
'add-watch' => [ 'LOCATION', 'watch for new messages and flag changes',
- qw(import! kw! interval=s recursive|r
- exclude=s include=s), @c_opt ],
+ qw(poll-interval=s state=s recursive|r), @c_opt ],
'ls-watch' => [ '[FILTER...]', 'list active watches with numbers and status',
- qw(format|f=s z), @c_opt ],
+ qw(l z|0), @c_opt ],
'pause-watch' => [ '[WATCH_NUMBER_OR_FILTER]', qw(all local remote), @c_opt ],
'resume-watch' => [ '[WATCH_NUMBER_OR_FILTER]', qw(all local remote), @c_opt ],
'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
@@ -391,6 +391,7 @@ my %OPTDESC = (
'format|f=s ls-search' => ['OUT|json|jsonl|concatjson',
'listing output format' ],
'l ls-search' => 'long listing format',
+'l ls-watch' => 'long listing format',
'l ls-mail-source' => 'long listing format',
'url ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
'format|f=s ls-external' => $ls_format,
@@ -435,7 +436,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m ikw pmd wq1); # internal workers
+my @WQ_KEYS = qw(lxs l2m ikw pmd wq1 lne); # internal workers
sub _drop_wq {
my ($self) = @_;
@@ -538,7 +539,7 @@ sub _lei_atfork_child {
chdir '/' or die "chdir(/): $!";
close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
if (my $cfg = $self->{cfg}) {
- delete $cfg->{-lei_store};
+ delete @$cfg{qw(-lei_store -watches -lei_note_event)};
}
} else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
@@ -555,6 +556,8 @@ sub _lei_atfork_child {
undef $listener;
undef $dir_idle;
%PATH2CFG = ();
+ $MDIR2CFGPATH = {};
+ eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
undef $errors_log;
$quit = \&CORE::exit;
$self->{-eml_noisy} or # only "lei import" sets this atm
@@ -781,10 +784,12 @@ sub _lei_cfg ($;$) {
my $f = _config_path($self);
my @st = stat($f);
my $cur_st = @st ? pack('dd', $st[10], $st[7]) : ''; # 10:ctime, 7:size
- my ($sto, $sto_dir);
+ my ($sto, $sto_dir, $watches, $lne);
if (my $cfg = $PATH2CFG{$f}) { # reuse existing object in common case
return ($self->{cfg} = $cfg) if $cur_st eq $cfg->{-st};
- ($sto, $sto_dir) = @$cfg{qw(-lei_store leistore.dir)};
+ ($sto, $sto_dir, $watches, $lne) =
+ @$cfg{qw(-lei_store leistore.dir -watches
+ -lei_note_event)};
}
if (!@st) {
unless ($creat) {
@@ -805,6 +810,8 @@ sub _lei_cfg ($;$) {
eq File::Spec->canonpath($cfg->{'leistore.dir'} //
store_path($self))) {
$cfg->{-lei_store} = $sto;
+ $cfg->{-lei_note_event} = $lne;
+ $cfg->{-watches} = $watches if $watches;
}
if (scalar(keys %PATH2CFG) > 5) {
# FIXME: use inotify/EVFILT_VNODE to detect unlinked configs
@@ -817,7 +824,7 @@ sub _lei_cfg ($;$) {
sub _lei_store ($;$) {
my ($self, $creat) = @_;
- my $cfg = _lei_cfg($self, $creat);
+ my $cfg = _lei_cfg($self, $creat) // return;
$cfg->{-lei_store} //= do {
require PublicInbox::LeiStore;
my $dir = $cfg->{'leistore.dir'} // store_path($self);
@@ -1126,6 +1133,53 @@ sub dump_and_clear_log {
}
}
+sub cfg2lei ($) {
+ my ($cfg) = @_;
+ my $lei = bless { env => { %{$cfg->{-env}} } }, __PACKAGE__;
+ open($lei->{0}, '<&', \*STDIN) or die "dup 0: $!";
+ open($lei->{1}, '>>&', \*STDOUT) or die "dup 1: $!";
+ open($lei->{2}, '>>&', \*STDERR) or die "dup 2: $!";
+ open($lei->{3}, '/') or die "open /: $!";
+ chdir($lei->{3}) or die "chdir /': $!";
+ my ($x, $y);
+ socketpair($x, $y, AF_UNIX, SOCK_SEQPACKET, 0) or die "socketpair: $!";
+ $lei->{sock} = $x;
+ require PublicInbox::LeiSelfSocket;
+ PublicInbox::LeiSelfSocket->new($y); # adds to event loop
+ $lei;
+}
+
+sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
+ my ($ev) = @_; # Linux::Inotify2::Event or duck type
+ my $fn = $ev->fullname;
+ if ($fn =~ m!\A(.+)/(new|cur)/([^/]+)\z!) { # Maildir file
+ my ($mdir, $nc, $bn) = ($1, $2, $3);
+ $nc = '' if $ev->IN_DELETE;
+ for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
+ my $cfg = $PATH2CFG{$f} // next;
+ eval {
+ local %ENV = %{$cfg->{-env}};
+ my $lei = cfg2lei($cfg);
+ $lei->dispatch('note-event',
+ "maildir:$mdir", $nc, $bn, $fn);
+ };
+ warn "E note-event $f: $@\n" if $@;
+ }
+ }
+ if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
+ $ev->cancel;
+ }
+ if ($fn =~ m!\A(.+)/(?:new|cur)\z! && !-e $fn) {
+ delete $MDIR2CFGPATH->{$1};
+ }
+ if (!-e $fn) { # config file or Maildir gone
+ for my $cfgpaths (values %$MDIR2CFGPATH) {
+ delete $cfgpaths->{$fn};
+ }
+ delete $PATH2CFG{$fn};
+ }
+}
+
# lei(1) calls this when it can't connect
sub lazy_start {
my ($path, $errno, $narg) = @_;
@@ -1175,6 +1229,7 @@ sub lazy_start {
return if $pid;
$0 = "lei-daemon $path";
local %PATH2CFG;
+ local $MDIR2CFGPATH;
$listener->blocking(0);
my $exit_code;
my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
@@ -1204,8 +1259,8 @@ sub lazy_start {
local $SIG{PIPE} = 'IGNORE';
require PublicInbox::DirIdle;
local $dir_idle = PublicInbox::DirIdle->new([$sock_dir], sub {
- # just rely on wakeup ot hit PostLoopCallback set below
- _dir_idle_handler(@_) if $_[0]->fullname ne $path;
+ # just rely on wakeup to hit PostLoopCallback set below
+ dir_idle_handler($_[0]) if $_[0]->fullname ne $path;
}, 1);
if ($sigfd) {
undef $sigfd; # unref, already in DS::DescriptorMap
@@ -1293,4 +1348,41 @@ sub wq_eof { # EOF callback for main daemon
$wq1->wq_wait_old(\&wq_done_wait, $lei);
}
+sub watch_state_ok ($) {
+ my ($state) = $_[-1]; # $_[0] may be $self
+ $state =~ /\Apause|(?:import|index|tag)-(?:ro|rw)\z/;
+}
+
+sub refresh_watches {
+ my ($lei) = @_;
+ my $cfg = _lei_cfg($lei) or return;
+ $cfg->{-env} //= { %{$lei->{env}}, PWD => '/' }; # for cfg2lei
+ my $watches = $cfg->{-watches} //= {};
+ require PublicInbox::LeiWatch;
+ for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
+ my $url = substr($w, length('watch.'), -length('.state'));
+ my $lw = $watches->{$w} //= PublicInbox::LeiWatch->new($url);
+ my $state = $cfg->get_1("watch.$url", 'state');
+ if (!watch_state_ok($state)) {
+ $lei->err("watch.$url.state=$state not supported");
+ next;
+ }
+ my $f = $cfg->{'-f'};
+ if ($url =~ /\Amaildir:(.+)/i) {
+ my $d = File::Spec->canonpath($1);
+ if ($state eq 'pause') {
+ delete $MDIR2CFGPATH->{$d}->{$f};
+ scalar(keys %{$MDIR2CFGPATH->{$d}}) or
+ delete $MDIR2CFGPATH->{$d};
+ } elsif (!exists($MDIR2CFGPATH->{$d}->{$f})) {
+ $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
+ $MDIR2CFGPATH->{$d}->{$f} = undef;
+ }
+ } else { # TODO: imap/nntp/jmap
+ $lei->child_error(1,
+ "E: watch $url not supported, yet");
+ }
+ }
+}
+
1;
diff --git a/lib/PublicInbox/LeiAddWatch.pm b/lib/PublicInbox/LeiAddWatch.pm
new file mode 100644
index 00000000..671d54f9
--- /dev/null
+++ b/lib/PublicInbox/LeiAddWatch.pm
@@ -0,0 +1,41 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei add-watch" command
+package PublicInbox::LeiAddWatch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::LeiInput);
+
+sub lei_add_watch {
+ my ($lei, @argv) = @_;
+ my $cfg = $lei->_lei_cfg(1);
+ my $self = bless {}, __PACKAGE__;
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ my $state = $lei->{opt}->{'state'} // 'import-rw';
+ $lei->watch_state_ok($state) or
+ return $lei->fail("invalid state: $state");
+ my $vmd_mod = $self->vmd_mod_extract(\@argv);
+ return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
+ $self->prepare_inputs($lei, \@argv) or return;
+ my @vmd;
+ while (my ($type, $vals) = each %$vmd_mod) {
+ push @vmd, "$type:$_" for @$vals;
+ }
+ my $vmd0 = shift @vmd;
+ for my $w (@{$self->{inputs}}) {
+ # clobber existing, allow multiple
+ if (defined($vmd0)) {
+ $lei->_config("watch.$w.vmd", '--replace-all', $vmd0);
+ for my $v (@vmd) {
+ $lei->_config("watch.$w.vmd", $v);
+ }
+ }
+ next if defined $cfg->{"watch.$w.state"};
+ $lei->_config("watch.$w.state", $state);
+ }
+ delete $lei->{cfg}; # force reload
+ $lei->refresh_watches;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index de2a8ff1..fa330df5 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -372,6 +372,7 @@ sub input_only_net_merge_all_done {
# like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare
# for update_xvmd -> update_vmd
+# returns something like { "+L" => [ @Labels ], ... }
sub vmd_mod_extract {
my $argv = $_[-1];
my $vmd_mod = {};
diff --git a/lib/PublicInbox/LeiLsWatch.pm b/lib/PublicInbox/LeiLsWatch.pm
new file mode 100644
index 00000000..f96dc4ec
--- /dev/null
+++ b/lib/PublicInbox/LeiLsWatch.pm
@@ -0,0 +1,15 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+package PublicInbox::LeiLsWatch;
+use strict;
+use v5.10.1;
+
+sub lei_ls_watch {
+ my ($lei) = @_;
+ my $cfg = $lei->_lei_cfg or return;
+ my @w = (join("\n", keys %$cfg) =~ m/^watch\.(.+?)\.state$/sgm);
+ $lei->puts(join("\n", @w)) if @w;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
new file mode 100644
index 00000000..bf15cd26
--- /dev/null
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -0,0 +1,124 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# internal command for dealing with inotify, kqueue vnodes, etc
+package PublicInbox::LeiNoteEvent;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+my $flush_timer;
+our $to_flush; # { cfgpath => $lei }
+
+sub flush_lei ($) {
+ my ($lei) = @_;
+ if (my $lne = delete $lei->{cfg}->{-lei_note_event}) {
+ $lne->wq_close(1, undef, $lei); # runs _lei_wq_eof;
+ } else { # lms_clear_src calls only:
+ my $wait = $lei->{sto}->ipc_do('done');
+ }
+}
+
+# we batch up writes and flush every 5s (matching Linux default
+# writeback behavior) since MUAs can trigger a storm of inotify events
+sub flush_task { # PublicInbox::DS timer callback
+ undef $flush_timer;
+ my $todo = $to_flush // return;
+ $to_flush = undef;
+ for my $lei (values %$todo) { flush_lei($lei) }
+}
+
+# sets a timer to flush
+sub note_event_arm_done ($) {
+ my ($lei) = @_;
+ $flush_timer //= PublicInbox::DS::add_timer(5, \&flush_task);
+ $to_flush->{$lei->{cfg}->{'-f'}} //= $lei;
+}
+
+sub eml_event ($$$$) {
+ my ($self, $eml, $kw, $state) = @_;
+ my $sto = $self->{lei}->{sto};
+ my $lse = $self->{lse} //= $sto->search;
+ my $vmd = { kw => $kw };
+ if ($state =~ /\Aimport-(?:rw|ro)\z/) {
+ $sto->ipc_do('set_eml', $eml, $vmd);
+ } elsif ($state =~ /\Aindex-(?:rw|ro)\z/) {
+ my $xoids = $self->{lei}->ale->xoids_for($eml);
+ $sto->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+ } elsif ($state =~ /\Atag-(?:rw|ro)\z/) {
+ my $c = $lse->kw_changed($eml, $kw, my $docids = []);
+ if (scalar @$docids) { # already in lei/store
+ $sto->ipc_do('set_eml_vmd', undef, $vmd, $docids) if $c;
+ } elsif (my $xoids = $self->{lei}->ale->xoids_for($eml)) {
+ # it's in an external, only set kw, here
+ $sto->ipc_do('set_xvmd', $xoids, $eml, $vmd);
+ } # else { totally unknown
+ } else {
+ warn "unknown state: $state (in $self->{lei}->{cfg}->{'-f'})\n";
+ }
+}
+
+sub maildir_event { # via wq_io_do
+ my ($self, $fn, $kw, $state) = @_;
+ my $eml = PublicInbox::InboxWritable::eml_from_path($fn) // return;
+ eml_event($self, $eml, $kw, $state);
+}
+
+sub lei_note_event {
+ my ($lei, $folder, $new_cur, $bn, $fn, @rest) = @_;
+ die "BUG: unexpected: @rest" if @rest;
+ my $cfg = $lei->_lei_cfg or return; # gone (race)
+ my $sto = $lei->_lei_store or return; # gone
+ return flush_lei($lei) if $folder eq 'done'; # special case
+ my $lms = $sto->search->lms or return;
+ my $err = $lms->arg2folder($lei, [ $folder ]);
+ return if $err->{fail};
+ undef $lms;
+ my $state = $cfg->get_1("watch.$folder", 'state') // 'pause';
+ return if $state eq 'pause';
+ $lei->ale; # prepare
+ $sto->write_prepare($lei);
+ if ($new_cur eq '') {
+ $sto->ipc_do('lms_clear_src', $folder, \$bn);
+ return note_event_arm_done($lei);
+ }
+ require PublicInbox::MdirReader;
+ my $self = $cfg->{-lei_note_event} //= do {
+ my $wq = bless {}, __PACKAGE__;
+ # MUAs such as mutt can trigger massive rename() storms so
+ # use all CPU power available:
+ my $jobs = $wq->detect_nproc // 1;
+ my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
+ $lei->wait_wq_events($op_c, $ops);
+ note_event_arm_done($lei);
+ $lei->{lne} = $wq;
+ };
+ if ($folder =~ /\Amaildir:/i) {
+ my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn)
+ // return;
+ return if index($fl, 'T') >= 0;
+ my $kw = PublicInbox::MdirReader::flags2kw($fl);
+ $self->wq_io_do('maildir_event', [], $fn, $kw, $state);
+ } # else: TODO: imap
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ $self->{lei}->_lei_atfork_child(1); # persistent, for a while
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub lne_done_wait {
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
+ $lei->can('wq_done_wait')->($arg, $pid);
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $lne = delete $lei->{lne} or return $lei->fail;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lne->wq_wait_old(\&lne_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 06ea6299..37bfc65e 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -5,7 +5,7 @@
package PublicInbox::LeiSearch;
use strict;
use v5.10.1;
-use parent qw(PublicInbox::ExtSearch);
+use parent qw(PublicInbox::ExtSearch); # PublicInbox::Search->reopen
use PublicInbox::Search qw(xap_terms);
use PublicInbox::ContentHash qw(content_digest content_hash);
use PublicInbox::MID qw(mids mids_for_index);
diff --git a/lib/PublicInbox/LeiSelfSocket.pm b/lib/PublicInbox/LeiSelfSocket.pm
new file mode 100644
index 00000000..3d847649
--- /dev/null
+++ b/lib/PublicInbox/LeiSelfSocket.pm
@@ -0,0 +1,45 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# dummy placeholder socket for internal lei commands.
+# This receives what script/lei receives, but isn't connected
+# to an interactive terminal so I'm not sure what to do with it...
+package PublicInbox::LeiSelfSocket;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::DS);
+use Data::Dumper;
+$Data::Dumper::Useqq = 1; # should've been the Perl default :P
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use PublicInbox::Spawn;
+my $recv_cmd;
+
+sub new {
+ my ($cls, $r) = @_;
+ my $self = bless { sock => $r }, $cls;
+ $r->blocking(0);
+ no warnings 'once';
+ $recv_cmd = $PublicInbox::LEI::recv_cmd;
+ $self->SUPER::new($r, EPOLLIN|EPOLLET);
+}
+
+sub event_step {
+ my ($self) = @_;
+ while (1) {
+ my (@fds) = $recv_cmd->($self->{sock}, my $buf, 4096 * 33);
+ if (scalar(@fds) == 1 && !defined($fds[0])) {
+ return if $!{EAGAIN};
+ next if $!{EINTR};
+ die "recvmsg: $!";
+ }
+ # open so perl can auto-close them:
+ for my $fd (@fds) {
+ open(my $newfh, '+<&=', $fd) or die "open +<&=$fd: $!";
+ }
+ return $self->close if $buf eq '';
+ warn Dumper({ 'unexpected self msg' => $buf, fds => \@fds });
+ # TODO: figure out what to do with these messages...
+ }
+}
+
+1;
diff --git a/lib/PublicInbox/LeiWatch.pm b/lib/PublicInbox/LeiWatch.pm
new file mode 100644
index 00000000..35267b58
--- /dev/null
+++ b/lib/PublicInbox/LeiWatch.pm
@@ -0,0 +1,13 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# represents a Maildir or IMAP "watch" item
+package PublicInbox::LeiWatch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+# "url" may be something like "maildir:/path/to/dir"
+sub new { bless { url => $_[1] }, $_[0] }
+
+1;
diff --git a/t/lei-watch.t b/t/lei-watch.t
new file mode 100644
index 00000000..3a2f9e64
--- /dev/null
+++ b/t/lei-watch.t
@@ -0,0 +1,49 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Path qw(make_path);
+require_mods('lei');
+my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+ eval { require IO::KQueue };
+
+$have_fast_inotify or
+ diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
+
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+test_lei(sub {
+ my $md = "$ENV{HOME}/md";
+ my $md2 = $md.'2';
+ lei_ok 'ls-watch';
+ is($lei_out, '', 'nothing in ls-watch, yet');
+ if (0) { # TODO
+ my $url = 'imaps://example.com/foo.bar.0';
+ lei_ok([qw(add-watch --state=pause), $url], undef, {});
+ lei_ok 'ls-watch';
+ is($lei_out, "$url\n", 'ls-watch shows added watch');
+ ok(!lei(qw(add-watch --state=pause), 'bogus'.$url),
+ 'bogus URL rejected');
+ }
+
+ # first, make sure tag-ro works
+ make_path("$md/new", "$md/cur", "$md/tmp");
+ lei_ok qw(add-watch --state=tag-ro), $md;
+ lei_ok 'ls-watch';
+ like($lei_out, qr/^\Qmaildir:$md\E$/sm, 'maildir shown');
+ lei_ok qw(q mid:testmessage@example.com -o), $md, '-I', "$ro_home/t1";
+ my @f = glob("$md/cur/*:2,");
+ is(scalar(@f), 1, 'got populated maildir with one result');
+ rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een
+ $have_fast_inotify or tick(2);
+ lei_ok qw(note-event done); # flushes immediately (instead of 5s)
+
+ lei_ok qw(q mid:testmessage@example.com -o), $md2, '-I', "$ro_home/t1";
+ my @f2 = glob("$md2/*/*");
+ is(scalar(@f2), 1, 'got one result');
+ like($f2[0], qr/S\z/, 'seen set from rename');
+ my $e2 = eml_load($f2[0]);
+ my $e1 = eml_load("$f[0]S");
+ is_deeply($e2, $e1, 'results match');
+});
+
+done_testing;
^ permalink raw reply related [relevance 22%]
* deduplication doesn't always work with "lei up"
@ 2021-07-08 8:42 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-08 8:42 UTC (permalink / raw)
To: meta
It seems to only happen with HTTP(S) externals, and
not in a FreeBSD VM, only bare metal Debian.
I can't seem to reproduce it consistently, at all.
Been staring at the LeiSavedSearch and query_remote_mboxrd
off and on for a few days while testing dedupe and can't
seem to notice any problems.
Maybe the maxuid guard for locals is hiding the bug...
^ permalink raw reply [relevance 71%]
* [PATCH] lei: drop workers on EOF from clients
@ 2021-07-04 19:35 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-04 19:35 UTC (permalink / raw)
To: meta
Sometimes a user will be bored waiting for a command to finish,
so ensure we drop disconnect workers in this case.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 42e02efb..a9f5edae 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1095,6 +1095,7 @@ sub event_step {
}
die "unrecognized client signal: $buf";
}
+ _drop_wq($self); # EOF, client disconnected
dclose($self);
};
if (my $err = $@) {
^ permalink raw reply related [relevance 71%]
* [PATCH] lei import: increase flags search batch size, display progress
@ 2021-07-02 21:02 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-02 21:02 UTC (permalink / raw)
To: meta
IMAP flag-only synchronization doesn't fetch entire messages,
so we can safely bump the batch size iff a user specified one
for full messages to 10000 times that.
Since I sometimes wonder why nothing happens for several seconds
after starting "lei import $URL", we'll also show some progress
during the flag synchronization phase.
---
lib/PublicInbox/NetReader.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 0c2288d8..23445e7a 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -462,8 +462,9 @@ sub each_old_flags ($$$$) {
my ($self, $mic, $uri, $l_uid) = @_;
$l_uid ||= 1;
my $sec = uri_section($uri);
- my $bs = $self->{imap_opt}->{$sec}->{batch_size} // 10000;
+ my $bs = ($self->{imap_opt}->{$sec}->{batch_size} // 1) * 10000;
my ($eml_cb, @args) = @{$self->{eml_each}};
+ $self->{quiet} or warn "# $uri syncing flags 1:$l_uid\n";
for (my $n = 1; $n <= $l_uid; $n += $bs) {
my $end = $n + $bs;
$end = $l_uid if $end > $l_uid;
^ permalink raw reply related [relevance 71%]
* [PATCH] lei inspect: help+completion for --dir option
@ 2021-07-02 20:42 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-07-02 20:42 UTC (permalink / raw)
To: meta
It's the most generic name I could find for it since it can
mean so many things...
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index cffe4dce..42e02efb 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -323,6 +323,7 @@ my %OPTDESC = (
'path-a|a=s' => 'pre-image pathname associated with OID',
'path-b|b=s' => 'post-image pathname associated with OID',
'git-dir=s@' => 'additional git repository to scan',
+'dir=s inspect' => 'specify a inboxdir, extindex topdir or Xapian shard',
'proxy=s' => [ 'PROTO://HOST[:PORT]', # shared with curl(1)
"proxy for (e.g. `socks5h://0:9050')" ],
'torsocks=s' => ['VAL|auto|no|yes',
^ permalink raw reply related [relevance 71%]
* [PATCH 0/2] lei inspect: "mid:" prefix and pager
@ 2021-07-01 11:31 71% Eric Wong
2021-07-01 11:31 71% ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
2021-07-01 11:31 62% ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
To: meta
Some things that came up while chasing -extindex deduplication
weirdness (and my head hurts because of that...)
Eric Wong (2):
lei inspect: support automatic pager in output
lei inspect: support "mid:" (and "m:") prefix
lib/PublicInbox/LeiInspect.pm | 60 +++++++++++++++++++++++++++--------
1 file changed, 46 insertions(+), 14 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei inspect: support automatic pager in output
2021-07-01 11:31 71% [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
@ 2021-07-01 11:31 71% ` Eric Wong
2021-07-01 11:31 62% ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
To: meta
All commands which output non-trivial amounts of data to
the terminal should support this.
---
lib/PublicInbox/LeiInspect.pm | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 30714764..9a7900c7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -155,9 +155,6 @@ sub inspect1 ($$$) {
sub lei_inspect {
my ($lei, @argv) = @_;
- $lei->{1}->autoflush(0);
- my $multi = scalar(@argv) > 1;
- $lei->out('[') if $multi;
$lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
$lei->{lse} = ($lei->{opt}->{external} // 1) ? do {
my $sto = $lei->_lei_store;
@@ -166,6 +163,10 @@ sub lei_inspect {
if ($lei->{opt}->{pretty} || -t $lei->{1}) {
$lei->{json}->pretty(1)->indent(2);
}
+ $lei->start_pager if -t $lei->{1};
+ $lei->{1}->autoflush(0);
+ my $multi = scalar(@argv) > 1;
+ $lei->out('[') if $multi;
while (defined(my $x = shift @argv)) {
inspect1($lei, $x, scalar(@argv)) or return;
}
^ permalink raw reply related [relevance 71%]
* [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix
2021-07-01 11:31 71% [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
2021-07-01 11:31 71% ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
@ 2021-07-01 11:31 62% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
To: meta
Using this to track down deduplication failures in -extindex...
---
lib/PublicInbox/LeiInspect.pm | 53 +++++++++++++++++++++++++++--------
1 file changed, 42 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 9a7900c7..574da7a7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -98,22 +98,29 @@ sub inspect_docid ($$;$) {
$ent;
}
+sub dir2ibx ($$) {
+ my ($lei, $dir) = @_;
+ if (-f "$dir/ei.lock") {
+ require PublicInbox::ExtSearch;
+ PublicInbox::ExtSearch->new($dir);
+ } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
+ require PublicInbox::Inbox; # v2, v1
+ bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+ } else {
+ $lei->fail("no (indexed) inbox or extindex at $dir");
+ }
+}
+
sub inspect_num ($$) {
my ($lei, $num) = @_;
my ($docid, $ibx);
my $ent = { num => $num };
if (defined(my $dir = $lei->{opt}->{dir})) {
- my $num2docid = $lei->{lse}->can('num2docid');
- if (-f "$dir/ei.lock") {
- require PublicInbox::ExtSearch;
- $ibx = PublicInbox::ExtSearch->new($dir);
- } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
- require PublicInbox::Inbox; # v2, v1
- $ibx = bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+ $ibx = dir2ibx($lei, $dir) or return;
+ if ($ent->{xdb} = $ibx->xdb) {
+ my $num2docid = $lei->{lse}->can('num2docid');
+ $docid = $num2docid->($ibx, $num);
}
- $ent->{xdb} = $ibx->xdb //
- return $lei->fail("no Xapian DB for $dir");
- $docid = $num2docid->($ibx, $num);
} else {
$ibx = $lei->{lse};
$lei->{lse}->xdb; # set {nshard} for num2docid
@@ -123,7 +130,29 @@ sub inspect_num ($$) {
my $smsg = $ibx->over->get_art($num);
$ent->{smsg} = { %$smsg } if $smsg;
}
- inspect_docid($lei, $docid, $ent);
+ defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent;
+}
+
+sub inspect_mid ($$) {
+ my ($lei, $mid) = @_;
+ my ($ibx, $over);
+ my $ent = { mid => $mid };
+ if (defined(my $dir = $lei->{opt}->{dir})) {
+ my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid');
+ $ibx = dir2ibx($lei, $dir) or return;
+ # $ent->{xdb} = $ibx->xdb //
+ # return $lei->fail("no Xapian DB for $dir");
+ } else {
+ $ibx = $lei->{lse};
+ $lei->{lse}->xdb; # set {nshard} for num2docid
+ }
+ if ($ibx && $ibx->over) {
+ my ($id, $prev);
+ while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
+ push @{$ent->{smsg}}, { %$smsg }
+ }
+ }
+ $ent;
}
sub inspect1 ($$$) {
@@ -145,6 +174,8 @@ sub inspect1 ($$$) {
$ent = inspect_docid($lei, $1 + 0);
} elsif ($item =~ m!\Anum:([0-9]+)\z!) {
$ent = inspect_num($lei, $1 + 0);
+ } elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
+ $ent = inspect_mid($lei, $1);
} else { # TODO: more things
return $lei->fail("$item not understood");
}
^ permalink raw reply related [relevance 62%]
* [PATCH] lei: use open() perlop for -C (chdir)
@ 2021-06-22 10:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-22 10:04 UTC (permalink / raw)
To: meta
This is for consistency with the open() at initial accept, in
case we hit a code path which expects Perl directory handles
rather than "file handles". Both work with the chdir() perlop
(fchdir(2), in our case).
---
lib/PublicInbox/LEI.pm | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 546fa773..cffe4dce 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -762,11 +762,7 @@ sub dispatch {
next if $d eq ''; # same as git(1)
chdir $d or return fail($self, "cd $d: $!");
}
- if (delete $self->{3}) { # update cwd for rel2abs
- opendir my $dh, '.' or
- return fail($self, "opendir . $!");
- $self->{3} = $dh;
- }
+ open $self->{3}, '.' or return fail($self, "open . $!");
}
$cb->($self, @argv);
} elsif (grep(/\A-/, $cmd, @argv)) { # --help or -h only
^ permalink raw reply related [relevance 71%]
* [PATCH] lei import: help + completion for --new-only
@ 2021-06-20 8:39 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-20 8:39 UTC (permalink / raw)
To: meta
I've found it's very helpful for large IMAP folders.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 50a7fdad..546fa773 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -353,6 +353,7 @@ my %OPTDESC = (
"or\xa0`-'\x{a0}for\x{a0}stdout)" ],
'mua=s' => [ 'CMD',
"MUA to run on --output Maildir or mbox (e.g.\xa0`mutt\xa0-f\xa0%f')" ],
+'new-only import' => 'only import new messages from IMAP source',
'inbox-version=i' => [ 'NUM|1|2',
'force a public-inbox version with --mirror'],
^ permalink raw reply related [relevance 71%]
* [PATCH] lei sucks: don't warn or error out on missing dependencies
@ 2021-06-20 4:33 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-20 4:33 UTC (permalink / raw)
To: meta
%INC can hold undef. This can be hit on a Linux machine missing
Linux::Inotify2. Loading PublicInbox::KQNotify is attempted and
PublicInbox/KQNotify.pm always exists, causing the `undef' entry
in %INC when it fails to load IO::KQueue.
$ perl -MData::Dumper -I lib \
-E 'eval { require PublicInbox::KQNotify }; say Dumper(\%INC)'
---
lib/PublicInbox/LeiSucks.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index a71158f3..3e945d0b 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -55,7 +55,7 @@ sub lei_sucks {
my $dig = Digest::SHA->new(1);
push @out, "public-inbox blob OIDs of loaded features:\n";
for my $m (grep(m{^PublicInbox/}, sort keys %INC)) {
- my $f = $INC{$m};
+ my $f = $INC{$m} // next; # lazy require failed (missing dep)
$dig->add('blob '.(-s $f)."\0");
$dig->addfile($f);
push @out, ' '.$dig->hexdigest.' '.$m."\n";
^ permalink raw reply related [relevance 71%]
* [PATCH] lei/store: do not put NULL into over.num column
@ 2021-06-18 19:20 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-18 19:20 UTC (permalink / raw)
To: meta
Simplify oid2docid and filter out undefined docids in ->add_eml,
instead. This avoids SQLite "datatype mismatch" errors in
OverIdx->add_over
Fixes: d1052f03ea85d4af ("lei/store: cull redundant docids based on blob OID")
---
lib/PublicInbox/LeiStore.pm | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 4ba1e647..e26b622d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -269,7 +269,7 @@ W: $oid indexed as multiple docids: $docid @cull, culling to fixup old bugs
EOF
remove_docids($self, @cull);
}
- wantarray ? ($docid) : $docid;
+ $docid;
}
sub add_eml {
@@ -292,8 +292,8 @@ sub add_eml {
if (scalar keys %$xoids) {
my %docids = map { $_ => 1 } @$vivify_xvmd;
for my $oid (keys %$xoids) {
- my @id = oid2docid($self, $oid);
- @docids{@id} = @id;
+ my $docid = oid2docid($self, $oid);
+ $docids{$docid} = $docid if defined($docid);
}
@$vivify_xvmd = sort { $a <=> $b } keys(%docids);
}
^ permalink raw reply related [relevance 71%]
* [PATCH 3/3] lei/store: cull redundant docids based on blob OID
2021-06-17 22:00 71% [PATCH 0/3] lei: internal bug fixups Eric Wong
2021-06-17 22:00 60% ` [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes Eric Wong
@ 2021-06-17 22:00 55% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-17 22:00 UTC (permalink / raw)
To: meta
I'm not sure how this happened (only once for me in March), but
it should not happen... In any case, we'll operate on the
lowest numbered docid and cull redundant index entries when
lei/store is open for read-write.
This also fixes the normal lei/store removal path to clean up
the xref3 table (since it's not done automatically for
public-facing -eidx due to the multi-list nature of it).
---
lib/PublicInbox/LeiStore.pm | 54 +++++++++++++++++++++++-------------
lib/PublicInbox/SearchIdx.pm | 2 +-
2 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index f978288a..4ba1e647 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -226,6 +226,18 @@ sub _remove_if_local { # git->cat_async arg
$self->{im}->remove($bref) if $bref;
}
+sub remove_docids ($;@) {
+ my ($self, @docids) = @_;
+ my $eidx = eidx_init($self);
+ for my $docid (@docids) {
+ $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
+ $self->{oidx}->delete_by_num($docid);
+ $self->{oidx}->{dbh}->do(<<EOF, undef, $docid);
+DELETE FROM xref3 WHERE docid = ?
+EOF
+ }
+}
+
# remove the entire message from the index, does not touch mail_sync.sqlite3
sub remove_eml {
my ($self, $eml) = @_;
@@ -241,13 +253,25 @@ sub remove_eml {
my $oidhex = unpack('H*', $oidbin);
$git->cat_async($oidhex, \&_remove_if_local, $self);
}
- $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
- $oidx->delete_by_num($docid);
}
$git->cat_async_wait;
+ remove_docids($self, @docids);
\@docids;
}
+sub oid2docid ($$) {
+ my ($self, $oid) = @_;
+ my $eidx = eidx_init($self);
+ my ($docid, @cull) = $eidx->{oidx}->blob_exists($oid);
+ if (@cull) { # fixup old bugs...
+ warn <<EOF;
+W: $oid indexed as multiple docids: $docid @cull, culling to fixup old bugs
+EOF
+ remove_docids($self, @cull);
+ }
+ wantarray ? ($docid) : $docid;
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
@@ -268,7 +292,7 @@ sub add_eml {
if (scalar keys %$xoids) {
my %docids = map { $_ => 1 } @$vivify_xvmd;
for my $oid (keys %$xoids) {
- my @id = $oidx->blob_exists($oid);
+ my @id = oid2docid($self, $oid);
@docids{@id} = @id;
}
@$vivify_xvmd = sort { $a <=> $b } keys(%docids);
@@ -356,15 +380,11 @@ sub update_xvmd {
my $oidx = $eidx->{oidx};
my %seen;
for my $oid (keys %$xoids) {
- my @docids = $oidx->blob_exists($oid) or next;
- scalar(@docids) > 1 and
- warn "W: $oid indexed as multiple docids: @docids\n";
- for my $docid (@docids) {
- next if $seen{$docid}++;
- my $idx = $eidx->idx_shard($docid);
- $idx->ipc_do('update_vmd', $docid, $vmd_mod);
- }
+ my $docid = oid2docid($self, $oid) // next;
delete $xoids->{$oid};
+ next if $seen{$docid}++;
+ my $idx = $eidx->idx_shard($docid);
+ $idx->ipc_do('update_vmd', $docid, $vmd_mod);
}
return unless scalar(keys(%$xoids));
@@ -395,15 +415,11 @@ sub set_xvmd {
# see if we can just update existing docs
for my $oid (keys %$xoids) {
- my @docids = $oidx->blob_exists($oid) or next;
- scalar(@docids) > 1 and
- warn "W: $oid indexed as multiple docids: @docids\n";
- for my $docid (@docids) {
- next if $seen{$docid}++;
- my $idx = $eidx->idx_shard($docid);
- $idx->ipc_do('set_vmd', $docid, $vmd);
- }
+ my $docid = oid2docid($self, $oid) // next;
delete $xoids->{$oid}; # all done with this oid
+ next if $seen{$docid}++;
+ my $idx = $eidx->idx_shard($docid);
+ $idx->ipc_do('set_vmd', $docid, $vmd);
}
return unless scalar(keys(%$xoids));
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f066cc92..f553eda6 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -572,7 +572,7 @@ sub apply_vmd_mod ($$) {
my $updated = 0;
my @x = @VMD_MAP;
while (my ($field, $pfx) = splice(@x, 0, 2)) {
- # field: "label" or "kw"
+ # field: "L" or "kw"
for my $val (@{$vmd_mod->{"-$field"} // []}) {
eval {
$doc->remove_term($pfx . $val);
^ permalink raw reply related [relevance 55%]
* [PATCH 0/3] lei: internal bug fixups
@ 2021-06-17 22:00 71% Eric Wong
2021-06-17 22:00 60% ` [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes Eric Wong
2021-06-17 22:00 55% ` [PATCH 3/3] lei/store: cull redundant docids based on blob OID Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-06-17 22:00 UTC (permalink / raw)
To: meta
Still chasing some oddness in day-to-day usage; but I think
3/3 is safe (1/3 helped me inspect things)
Eric Wong (3):
lei inspect: learn "num:" and "docid:" prefixes
lei_input: prefix bare Maildir paths w/ "maildir:"
lei/store: cull redundant docids based on blob OID
lib/PublicInbox/LeiInput.pm | 3 +-
lib/PublicInbox/LeiInspect.pm | 73 +++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiStore.pm | 54 +++++++++++++++++---------
lib/PublicInbox/SearchIdx.pm | 2 +-
4 files changed, 111 insertions(+), 21 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes
2021-06-17 22:00 71% [PATCH 0/3] lei: internal bug fixups Eric Wong
@ 2021-06-17 22:00 60% ` Eric Wong
2021-06-17 22:00 55% ` [PATCH 3/3] lei/store: cull redundant docids based on blob OID Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-17 22:00 UTC (permalink / raw)
To: meta
"num:" is useful for inspecting Inbox-ish directories, while
"docid:" can be used for any Xapian DB (not just stuff managed
by our code).
---
lib/PublicInbox/LeiInspect.pm | 73 +++++++++++++++++++++++++++++++++++
1 file changed, 73 insertions(+)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index eb2634b4..30714764 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -57,6 +57,75 @@ sub inspect_sync_folder ($$) {
$ent
}
+sub inspect_docid ($$;$) {
+ my ($lei, $docid, $ent) = @_;
+ require PublicInbox::Search;
+ $ent //= {};
+ my $xdb;
+ if ($xdb = delete $ent->{xdb}) { # from inspect_num
+ } elsif (defined(my $dir = $lei->{opt}->{dir})) {
+ no warnings 'once';
+ $xdb = $PublicInbox::Search::X{Database}->new($dir);
+ } else {
+ $xdb = $lei->{lse}->xdb;
+ }
+ $xdb or return $lei->fail('no Xapian DB');
+ my $doc = $xdb->get_document($docid); # raises
+ my $data = $doc->get_data;
+ $ent->{docid} = $docid;
+ $ent->{data_length} = length($data);
+ $ent->{description} => $doc->get_description;
+ $ent->{$_} = $doc->$_ for (qw(termlist_count values_count));
+ my $cur = $doc->termlist_begin;
+ my $end = $doc->termlist_end;
+ for (; $cur != $end; $cur++) {
+ my $tn = $cur->get_termname;
+ $tn =~ s/\A([A-Z]+)// or warn "$tn no prefix! (???)";
+ my $term = ($1 // '');
+ push @{$ent->{terms}->{$term}}, $tn;
+ }
+ @$_ = sort(@$_) for values %{$ent->{terms} // {}};
+ $cur = $doc->values_begin;
+ $end = $doc->values_end;
+ for (; $cur != $end; $cur++) {
+ my $n = $cur->get_valueno;
+ my $v = $cur->get_value;
+ my $iv = PublicInbox::Search::sortable_unserialise($v);
+ $v = $iv + 0 if defined $iv;
+ # not using ->[$n] since we may have large gaps in $n
+ $ent->{'values'}->{$n} = $v;
+ }
+ $ent;
+}
+
+sub inspect_num ($$) {
+ my ($lei, $num) = @_;
+ my ($docid, $ibx);
+ my $ent = { num => $num };
+ if (defined(my $dir = $lei->{opt}->{dir})) {
+ my $num2docid = $lei->{lse}->can('num2docid');
+ if (-f "$dir/ei.lock") {
+ require PublicInbox::ExtSearch;
+ $ibx = PublicInbox::ExtSearch->new($dir);
+ } elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
+ require PublicInbox::Inbox; # v2, v1
+ $ibx = bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+ }
+ $ent->{xdb} = $ibx->xdb //
+ return $lei->fail("no Xapian DB for $dir");
+ $docid = $num2docid->($ibx, $num);
+ } else {
+ $ibx = $lei->{lse};
+ $lei->{lse}->xdb; # set {nshard} for num2docid
+ $docid = $lei->{lse}->num2docid($num);
+ }
+ if ($ibx && $ibx->over) {
+ my $smsg = $ibx->over->get_art($num);
+ $ent->{smsg} = { %$smsg } if $smsg;
+ }
+ inspect_docid($lei, $docid, $ent);
+}
+
sub inspect1 ($$$) {
my ($lei, $item, $more) = @_;
my $ent;
@@ -72,6 +141,10 @@ sub inspect1 ($$$) {
}
} elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
+ } elsif ($item =~ m!\Adocid:([0-9]+)\z!) {
+ $ent = inspect_docid($lei, $1 + 0);
+ } elsif ($item =~ m!\Anum:([0-9]+)\z!) {
+ $ent = inspect_num($lei, $1 + 0);
} else { # TODO: more things
return $lei->fail("$item not understood");
}
^ permalink raw reply related [relevance 60%]
* [PATCH 0/2] lei: support keywords off a single Maildir file
@ 2021-06-13 18:12 71% Eric Wong
2021-06-13 18:12 59% ` [PATCH 2/2] lei index+import: reject keywords from R/O IMAP Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-06-13 18:12 UTC (permalink / raw)
To: meta
I noticed the bug fixed by 2/2 while working on 1/2
Eric Wong (2):
lei_input: allow keywords when importing 1 file from Maildir
lei index+import: reject keywords from R/O IMAP
lib/PublicInbox/LeiInput.pm | 27 ++++++++++++++++++++++++++-
lib/PublicInbox/LeiStore.pm | 3 ++-
lib/PublicInbox/NetReader.pm | 11 ++++++-----
t/lei-index.t | 22 ++++++++++++++++++++++
4 files changed, 56 insertions(+), 7 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei index+import: reject keywords from R/O IMAP
2021-06-13 18:12 71% [PATCH 0/2] lei: support keywords off a single Maildir file Eric Wong
@ 2021-06-13 18:12 59% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-13 18:12 UTC (permalink / raw)
To: meta
Since users can't set IMAP flags in read-only IMAP folders,
we won't clobber local flags when importing from IMAP. This
also enables the local_blob fallback used for lei-index to
be used for index deduplication.
---
lib/PublicInbox/LeiStore.pm | 3 ++-
lib/PublicInbox/NetReader.pm | 11 ++++++-----
t/lei-index.t | 13 +++++++++++++
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 5446873e..f978288a 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -151,7 +151,8 @@ sub _docids_for ($$) {
my $oid = $cur->{blob};
my $docid = $cur->{num};
my $bref = $im ? $im->cat_blob($oid) : undef;
- $bref //= $eidx->git->cat_file($oid) // do {
+ $bref //= $eidx->git->cat_file($oid) //
+ _lms_rw($self)->local_blob($oid, 1) // do {
warn "W: $oid (#$docid) <$mid> not found\n";
next;
};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 30784199..0c2288d8 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -420,7 +420,8 @@ sub _imap_do_msg ($$$$$) {
my ($self, $uri, $uid, $raw, $flags) = @_;
# our target audience expects LF-only, save storage
$$raw =~ s/\r\n/\n/sg;
- my $kw = flags2kw($self, $uri, $uid, $flags) // return;
+ my $kw = defined($flags) ?
+ (flags2kw($self, $uri, $uid, $flags) // return) : undef;
my ($eml_cb, @args) = @{$self->{eml_each}};
$eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args);
}
@@ -537,8 +538,8 @@ E: $uri strangely, UIDVALIDLITY matches ($l_uidval)
EOF
$mic->Uid(1); # the default, we hope
my $err;
- if (!defined($single_uid) && $self->{each_old} &&
- perm_fl_ok($perm_fl)) {
+ my $use_fl = perm_fl_ok($perm_fl);
+ if (!defined($single_uid) && $self->{each_old} && $use_fl) {
$err = each_old_flags($self, $mic, $uri, $l_uid);
return $err if $err;
}
@@ -593,8 +594,8 @@ EOF
# messages get deleted, so holes appear
my $per_uid = delete $r->{$uid} // next;
my $raw = delete($per_uid->{$key}) // next;
- _imap_do_msg($self, $uri, $uid, \$raw,
- $per_uid->{FLAGS});
+ my $fl = $use_fl ? $per_uid->{FLAGS} : undef;
+ _imap_do_msg($self, $uri, $uid, \$raw, $fl);
$last_uid = $uid;
last if $self->{quit};
}
diff --git a/t/lei-index.t b/t/lei-index.t
index c142e79c..eeda5196 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -80,6 +80,19 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('index', "nntp://$nntp_host_port/t.v2");
lei_ok('index', "imap://$imap_host_port/t.v2.0");
is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
+
+ lei_ok qw(q m:multipart-html-sucks@11);
+ $res_a = json_utf8->decode($lei_out)->[0];
+ is_deeply($res_a->{'kw'}, ['seen'],
+ 'keywords still set after NNTP + IMAP import');
+
+ # ensure import works after lms->local_blob fallback in lei/store
+ lei_ok('import', 't/mda-mime.eml');
+ lei_ok qw(q m:multipart-html-sucks@11);
+ $res_b = json_utf8->decode($lei_out)->[0];
+ my $t = xqx(['git', "--git-dir=$store_path/ALL.git",
+ qw(cat-file -t), $res_b->{blob}]);
+ is($t, "blob\n", 'got blob');
});
done_testing;
^ permalink raw reply related [relevance 59%]
* [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
2021-06-12 0:10 48% ` [PATCH 2/5] lei ls-mail-source: write through to URL folder cache Eric Wong
@ 2021-06-12 0:10 71% ` Eric Wong
2021-06-12 0:10 71% ` [PATCH 4/5] lei import: use url_folder_cache for completion Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
Leftover while writing the test.
---
t/lei-import-http.t | 2 --
1 file changed, 2 deletions(-)
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
index 6cb8a753..2104c778 100644
--- a/t/lei-import-http.t
+++ b/t/lei-import-http.t
@@ -42,7 +42,5 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(!lei(qw(import --mail-sync), "$url/x\@example.com/raw"),
'--mail-sync fails on HTTP');
- diag $lei_err;
-
});
done_testing;
^ permalink raw reply related [relevance 71%]
* [PATCH 4/5] lei import: use url_folder_cache for completion
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
` (2 preceding siblings ...)
2021-06-12 0:10 71% ` [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message Eric Wong
@ 2021-06-12 0:10 71% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
And fix "lei index" completion while we're at it.
---
lib/PublicInbox/LeiImport.pm | 11 ++++++++---
lib/PublicInbox/LeiIndex.pm | 2 +-
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 08794f71..7580e37e 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -118,10 +118,15 @@ sub lei_import { # the main "lei import" method
sub _complete_import {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
- map { $match_cb->($_) } $lms->folders;
+ my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
+ my %f = map { $_ => 1 } @m;
+ my $sto = $lei->_lei_store;
+ if (my $lms = $sto ? $sto->search->lms : undef) {
+ @m = map { $match_cb->($_) } $lms->folders;
+ @f{@m} = @m;
+ }
+ keys %f;
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 4be0c649..5b545998 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -39,7 +39,7 @@ for my $m (qw(pmdir_cb input_net_cb)) {
*$m = PublicInbox::LeiImport->can($m);
}
-*_complete_import = \&PublicInbox::LeiImport::_complete_import;
+*_complete_index = \&PublicInbox::LeiImport::_complete_import;
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
^ permalink raw reply related [relevance 71%]
* [PATCH 1/5] lei: stop pager early on exit
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
@ 2021-06-12 0:10 71% ` Eric Wong
2021-06-12 0:10 48% ` [PATCH 2/5] lei ls-mail-source: write through to URL folder cache Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
This is necessary when using "ls-mail-source" on an unreachable
IMAP server.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 833d9c4d..122045ee 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -446,6 +446,7 @@ sub x_it ($$) {
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
dump_and_clear_log();
+ stop_pager($self);
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('x_it', $code);
} elsif ($self->{sock}) { # to lei(1) client
^ permalink raw reply related [relevance 71%]
* [PATCH 0/5] lei ls-mail-source-related things
@ 2021-06-12 0:10 71% Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
A couple of minor usability things that'll make "lei import"
easier for bash completion users.
Eric Wong (5):
lei: stop pager early on exit
lei ls-mail-source: write through to URL folder cache
t/lei-import-http: quiet unnecessary diag message
lei import: use url_folder_cache for completion
net_reader: canonicalize URL args on add_url
lib/PublicInbox/LEI.pm | 10 ++++++-
lib/PublicInbox/LeiImport.pm | 11 +++++---
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 42 ++++++++++++++++++++++++++++--
lib/PublicInbox/NetReader.pm | 2 ++
lib/PublicInbox/SharedKV.pm | 13 +++++++--
t/lei-import-http.t | 2 --
7 files changed, 71 insertions(+), 11 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 2/5] lei ls-mail-source: write through to URL folder cache
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
@ 2021-06-12 0:10 48% ` Eric Wong
2021-06-12 0:10 71% ` [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message Eric Wong
2021-06-12 0:10 71% ` [PATCH 4/5] lei import: use url_folder_cache for completion Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-12 0:10 UTC (permalink / raw)
To: meta
We'll be able to use this for shell completion for
lei import, lcat, tag, etc..
This also adds --url support for scripting purposes.
---
lib/PublicInbox/LEI.pm | 9 ++++++-
lib/PublicInbox/LeiLsMailSource.pm | 42 ++++++++++++++++++++++++++++--
lib/PublicInbox/SharedKV.pm | 13 +++++++--
3 files changed, 59 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 122045ee..50a7fdad 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -116,6 +116,12 @@ sub cache_dir ($) {
.'/lei');
}
+sub url_folder_cache {
+ my ($self) = @_;
+ require PublicInbox::SharedKV; # URI => updated_at_sec_
+ PublicInbox::SharedKV->new(cache_dir($self).'/uri_folder');
+}
+
sub ale {
my ($self) = @_;
$self->{ale} //= do {
@@ -197,7 +203,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
- qw(z|0 ascii l), @c_opt ],
+ qw(z|0 ascii l url), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
@@ -384,6 +390,7 @@ my %OPTDESC = (
'listing output format' ],
'l ls-search' => 'long listing format',
'l ls-mail-source' => 'long listing format',
+'url ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
'format|f=s ls-external' => $ls_format,
'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index e95c0b34..cadc61ed 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -21,19 +21,27 @@ sub input_path_url { # overrides LeiInput version
} elsif ($self->{lei}->{opt}->{z}) {
$ORS = "\0";
}
+ my @f;
if ($url =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($url);
+ my $sec = $lei->{net}->can('uri_section')->($uri);
my $mic = $lei->{net}->mic_get($uri);
my $l = $mic->folders_hash($uri->path); # server-side filter
+ @f = map { "$sec/$_->{name}" } @$l;
if ($json) {
+ $_->{url} = "$sec/$_->{name}" for @$l;
$lei->puts($json->encode($l));
} else {
+ if ($self->{lei}->{opt}->{url}) {
+ $_->{name} = "$sec/$_->{name}" for @$l;
+ }
$lei->out(join($ORS, (map { $_->{name} } @$l), ''));
}
} elsif ($url =~ m!\A(?:nntps?|s?news)://!i) {
my $uri = PublicInbox::URInntps->new($url);
my $nn = $lei->{net}->nn_get($uri);
my $l = $nn->newsgroups($uri->group); # name => description
+ my $sec = $lei->{net}->can('uri_section')->($uri);
if ($json) {
my $all = $nn->list;
my @x;
@@ -46,15 +54,30 @@ sub input_path_url { # overrides LeiInput version
$desc =~ s/\r\z//;
my ($hwm, $lwm, $status) = @{$all->{$ng}};
- push @x, { name => $ng, lwm => $lwm + 0,
+ push @x, { name => $ng, url => "$sec/$ng",
+ lwm => $lwm + 0,
hwm => $hwm + 0, status => $status,
description => $desc };
}
+ @f = map { "$sec/$_" } keys %$all;
$lei->puts($json->encode(\@x));
} else {
- $lei->out(join($ORS, sort(keys %$l), ''));
+ @f = map { "$sec/$_" } keys %$l;
+ if ($self->{lei}->{opt}->{url}) {
+ $lei->out(join($ORS, sort(@f), ''));
+ } else {
+ $lei->out(join($ORS, sort(keys %$l), ''));
+ }
}
} else { die "BUG: $url not supported" }
+ if (@f) {
+ my $fc = $lei->url_folder_cache;
+ my $lk = $fc->lock_for_scope;
+ $fc->dbh->begin_work;
+ my $now = time;
+ $fc->set($_, $now) for @f;
+ $fc->dbh->commit;
+ }
}
sub lei_ls_mail_source {
@@ -62,6 +85,7 @@ sub lei_ls_mail_source {
$url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return
$lei->fail('only NNTP and IMAP URLs supported');
my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
+ $self->{cfg} = $lei->_lei_cfg; # may be undef
$self->prepare_inputs($lei, [ $url ]) or return;
$lei->start_pager if -t $lei->{1};
my $ops = {};
@@ -74,6 +98,20 @@ sub lei_ls_mail_source {
$lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
}
+sub _complete_ls_mail_source {
+ my ($lei, @argv) = @_;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
+ my %f = map { $_ => 1 } @m;
+ my $sto = $lei->_lei_store;
+ if (my $lms = $sto ? $sto->search->lms : undef) {
+ @m = map { $match_cb->($_) } grep(
+ m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
+ @f{@m} = @m;
+ }
+ keys %f;
+}
+
no warnings 'once';
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index d65c3158..8347b195 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -11,7 +11,7 @@ use parent qw(PublicInbox::Lock);
use File::Temp qw(tempdir);
use DBI ();
use PublicInbox::Spawn;
-use File::Path qw(rmtree);
+use File::Path qw(rmtree make_path);
sub dbh {
my ($self, $lock) = @_;
@@ -46,8 +46,8 @@ CREATE TABLE IF NOT EXISTS kv (
sub new {
my ($cls, $dir, $base, $opt) = @_;
my $self = bless { opt => $opt }, $cls;
+ make_path($dir) if defined($dir) && !-d $dir;
$dir //= $self->{"tmp$$.$self"} = tempdir("skv.$$-XXXX", TMPDIR => 1);
- -d $dir or mkdir($dir) or die "mkdir($dir): $!";
$base //= '';
my $f = $self->{filename} = "$dir/$base.sqlite3";
$self->{lock_path} = $opt->{lock_path} // "$dir/$base.flock";
@@ -83,6 +83,15 @@ SELECT k,v FROM kv
$sth
}
+sub keys {
+ my ($self) = @_;
+ my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
+SELECT k FROM kv
+
+ $sth->execute;
+ map { $_->[0] } @{$sth->fetchall_arrayref};
+}
+
sub delete_by_val {
my ($self, $val, $lock) = @_;
$lock //= $self->lock_for_scope_fast;
^ permalink raw reply related [relevance 48%]
* [PATCH] lei ls-mail-source: list IMAP folders and NNTP groups
@ 2021-06-11 9:42 41% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-11 9:42 UTC (permalink / raw)
To: meta
While other tools can provide the same functionality, having
integration with git-credential is convenient, here. Caching
and completion will be implemented separately.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 3 ++
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiLsMailSource.pm | 82 ++++++++++++++++++++++++++++++
lib/PublicInbox/NetReader.pm | 15 +++---
t/lei-import-imap.t | 6 +++
t/lei-import-nntp.t | 7 +++
7 files changed, 108 insertions(+), 8 deletions(-)
create mode 100644 lib/PublicInbox/LeiLsMailSource.pm
diff --git a/MANIFEST b/MANIFEST
index 3d4c6cbd..d4b3e75d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -215,6 +215,7 @@ lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
lib/PublicInbox/LeiLcat.pm
lib/PublicInbox/LeiLsLabel.pm
+lib/PublicInbox/LeiLsMailSource.pm
lib/PublicInbox/LeiLsMailSync.pm
lib/PublicInbox/LeiLsSearch.pm
lib/PublicInbox/LeiMailSync.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d34997fd..833d9c4d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -196,6 +196,8 @@ our %CMD = ( # sorted in order of importance/use:
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
+'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
+ qw(z|0 ascii l), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
@@ -381,6 +383,7 @@ my %OPTDESC = (
'format|f=s ls-search' => ['OUT|json|jsonl|concatjson',
'listing output format' ],
'l ls-search' => 'long listing format',
+'l ls-mail-source' => 'long listing format',
'format|f=s ls-external' => $ls_format,
'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 24211bf0..92d67715 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -229,7 +229,7 @@ sub prepare_inputs { # returns undef on error
if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) {
require PublicInbox::NetReader;
$net //= PublicInbox::NetReader->new;
- $net->add_url($input);
+ $net->add_url($input, $self->{-ls_ok});
push @{$sync->{ok}}, $input if $sync;
} elsif ($input_path =~ m!\Ahttps?://!i) { # mboxrd.gz
# TODO: how would we detect r/w JMAP?
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
new file mode 100644
index 00000000..e95c0b34
--- /dev/null
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -0,0 +1,82 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# command for listing NNTP groups and IMAP folders,
+# handy for users with git-credential-helper configured
+# TODO: list JMAP labels
+package PublicInbox::LeiLsMailSource;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+
+sub input_path_url { # overrides LeiInput version
+ my ($self, $url) = @_;
+ # TODO: support ndjson and other JSONs we support elsewhere
+ my $json;
+ my $lei = $self->{lei};
+ my $ORS = "\n";
+ if ($self->{lei}->{opt}->{l}) {
+ $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
+ $json->ascii(1) if $lei->{opt}->{ascii};
+ } elsif ($self->{lei}->{opt}->{z}) {
+ $ORS = "\0";
+ }
+ if ($url =~ m!\Aimaps?://!i) {
+ my $uri = PublicInbox::URIimap->new($url);
+ my $mic = $lei->{net}->mic_get($uri);
+ my $l = $mic->folders_hash($uri->path); # server-side filter
+ if ($json) {
+ $lei->puts($json->encode($l));
+ } else {
+ $lei->out(join($ORS, (map { $_->{name} } @$l), ''));
+ }
+ } elsif ($url =~ m!\A(?:nntps?|s?news)://!i) {
+ my $uri = PublicInbox::URInntps->new($url);
+ my $nn = $lei->{net}->nn_get($uri);
+ my $l = $nn->newsgroups($uri->group); # name => description
+ if ($json) {
+ my $all = $nn->list;
+ my @x;
+ for my $ng (sort keys %$l) {
+ my $desc = $l->{$ng};
+
+# we need to drop CR ourselves iff using IO::Socket::SSL since
+# Net::Cmd::getline doesn't get used by Net::NNTP if TLS is in play, noted in:
+# <https://rt.cpan.org/Ticket/Display.html?id=129966>
+ $desc =~ s/\r\z//;
+
+ my ($hwm, $lwm, $status) = @{$all->{$ng}};
+ push @x, { name => $ng, lwm => $lwm + 0,
+ hwm => $hwm + 0, status => $status,
+ description => $desc };
+ }
+ $lei->puts($json->encode(\@x));
+ } else {
+ $lei->out(join($ORS, sort(keys %$l), ''));
+ }
+ } else { die "BUG: $url not supported" }
+}
+
+sub lei_ls_mail_source {
+ my ($lei, $url, $pfx) = @_;
+ $url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return
+ $lei->fail('only NNTP and IMAP URLs supported');
+ my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
+ $self->prepare_inputs($lei, [ $url ]) or return;
+ $lei->start_pager if -t $lei->{1};
+ my $ops = {};
+ $lei->{auth}->op_merge($ops, $self);
+ my $j = $self->{-wq_nr_workers} = 1; # locked
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
+}
+
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+
+1;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 2795a9d4..bcc6cbf0 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -236,18 +236,19 @@ W: see https://rt.cpan.org/Ticket/Display.html?id=129967 for updates
}
sub imap_uri {
- my ($url) = @_;
+ my ($url, $ls_ok) = @_;
require PublicInbox::URIimap;
my $uri = PublicInbox::URIimap->new($url);
- $uri ? $uri->canonical : undef;
+ $uri && ($ls_ok || $uri->mailbox) ? $uri->canonical : undef;
}
my %IS_NNTP = (news => 1, snews => 1, nntp => 1, nntps => 1);
sub nntp_uri {
- my ($url) = @_;
+ my ($url, $ls_ok) = @_;
require PublicInbox::URInntps;
my $uri = PublicInbox::URInntps->new($url);
- $uri && $IS_NNTP{$uri->scheme} && $uri->group ? $uri->canonical : undef;
+ $uri && $IS_NNTP{$uri->scheme} && ($ls_ok || $uri->group) ?
+ $uri->canonical : undef;
}
sub cfg_intvl ($$$) {
@@ -367,11 +368,11 @@ sub nntp_common_init ($;$) {
}
sub add_url {
- my ($self, $arg) = @_;
+ my ($self, $arg, $ls_ok) = @_;
my $uri;
- if ($uri = imap_uri($arg)) {
+ if ($uri = imap_uri($arg, $ls_ok)) {
push @{$self->{imap_order}}, $uri;
- } elsif ($uri = nntp_uri($arg)) {
+ } elsif ($uri = nntp_uri($arg, $ls_ok)) {
push @{$self->{nntp_order}}, $uri;
} else {
push @{$self->{unsupported_url}}, $arg;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 34fd6cf9..12f6fad0 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -14,6 +14,12 @@ undef $sock;
test_lei({ tmpdir => $tmpdir }, sub {
my $url = "imap://$host_port/t.v2.0";
my $url_orig = $url;
+ lei_ok(qw(ls-mail-source), "imap://$host_port/");
+ like($lei_out, qr/^t\.v2\.0$/ms, 'shows mailbox');
+ lei_ok(qw(ls-mail-source), $url);
+ is($lei_out, "t.v2.0\n", 'shows only mailbox with filter');
+ lei_ok(qw(ls-mail-source -l), "imap://$host_port/");
+ is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON');
lei_ok(qw(q z:1..));
my $out = json_utf8->decode($lei_out);
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index 662da309..f2c35406 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -17,6 +17,13 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $out = json_utf8->decode($lei_out);
is_deeply($out, [ undef ], 'nothing imported, yet');
my $url = "nntp://$host_port/t.v2";
+ lei_ok(qw(ls-mail-source), "nntp://$host_port/");
+ like($lei_out, qr/^t\.v2$/ms, 'shows newsgroup');
+ lei_ok(qw(ls-mail-source), $url);
+ is($lei_out, "t.v2\n", 'shows only newsgroup with filter');
+ lei_ok(qw(ls-mail-source -l), "nntp://$host_port/");
+ is(ref(json_utf8->decode($lei_out)), 'ARRAY', 'ls-mail-source JSON');
+
lei_ok('import', $url);
lei_ok(qw(q z:1..));
$out = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 41%]
* [PATCH] lei tag: less confusing warning about unimported messages
@ 2021-06-09 23:27 58% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 23:27 UTC (permalink / raw)
To: meta
"unimported" is more meaningful than "missing", here. And
instead of having every worker spew about unimported messages,
we'll accumulate and only print one warning line. This
necessitated alterating ->DESTROY behavior and persisting
the client socket within the $lei object itself, not just
the PktOp consumer object.
---
lib/PublicInbox/LEI.pm | 21 ++++++++++++++++-----
lib/PublicInbox/LeiTag.pm | 12 ++++++------
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index beeb8b48..d34997fd 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -567,6 +567,11 @@ sub pkt_op_pair {
$end;
}
+sub incr {
+ my ($self, $field, $nr) = @_;
+ $self->{counters}->{$field} += $nr;
+}
+
sub workers_start {
my ($lei, $wq, $jobs, $ops, $flds) = @_;
$ops = {
@@ -574,6 +579,7 @@ sub workers_start {
'|' => [ \&sigpipe_handler, $lei ],
'x_it' => [ \&x_it, $lei ],
'child_error' => [ \&child_error, $lei ],
+ 'incr' => [ \&incr, $lei ],
($ops ? %$ops : ()),
};
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
@@ -583,8 +589,6 @@ sub workers_start {
$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
- # {-lei_sock} persists script/lei process until ops->{''} EOF callback
- $op_c->{-lei_sock} = $lei->{sock};
@$end = ();
$lei->event_step_init;
($op_c, $ops);
@@ -1092,10 +1096,11 @@ sub event_step {
sub event_step_init {
my ($self) = @_;
- return if $self->{-event_init_done}++;
- if (my $sock = $self->{sock}) { # using DS->EventLoop
+ my $sock = $self->{sock} or return;
+ $self->{-event_init_done} //= do { # persist til $ops done
$self->SUPER::new($sock, EPOLLIN|EPOLLET);
- }
+ $sock;
+ };
}
sub noop {}
@@ -1246,6 +1251,12 @@ sub busy { 1 } # prevent daemon-shutdown if client is connected
# can immediately reread it
sub DESTROY {
my ($self) = @_;
+ if (my $counters = delete $self->{counters}) {
+ for my $k (sort keys %$counters) {
+ my $nr = $counters->{$k};
+ $self->child_error(1 << 8, "$nr $k messages");
+ }
+ }
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
# preserve $? for ->fail or ->x_it code
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index e0532653..463fb921 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -15,7 +15,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
$self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
$self->{vmd_mod});
} else {
- ++$self->{missing};
+ ++$self->{unimported};
}
}
@@ -40,7 +40,7 @@ sub lei_tag { # the "lei tag" method
my ($lei, @argv) = @_;
my $sto = $lei->_lei_store(1);
$sto->write_prepare($lei);
- my $self = bless { missing => 0 }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$lei->ale; # refresh and prepare
my $vmd_mod = $self->vmd_mod_extract(\@argv);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
@@ -58,10 +58,10 @@ sub lei_tag { # the "lei tag" method
$lei->wait_wq_events($op_c, $ops);
}
-sub note_missing {
+sub note_unimported {
my ($self) = @_;
- my $n = $self->{missing} or return;
- $self->{lei}->child_error(1 << 8, "$n missed messages");
+ my $n = $self->{unimported} or return;
+ $self->{lei}->{pkt_op_p}->pkt_do('incr', 'unimported', $n);
}
sub ipc_atfork_child {
@@ -69,7 +69,7 @@ sub ipc_atfork_child {
PublicInbox::LeiInput::input_only_atfork_child($self);
$self->{lse} = $self->{lei}->{sto}->search;
# this goes out-of-scope at worker process exit:
- PublicInbox::OnDestroy->new($$, \¬e_missing, $self);
+ PublicInbox::OnDestroy->new($$, \¬e_unimported, $self);
}
# Workaround bash word-splitting s to ['kw', ':', 'keyword' ...]
^ permalink raw reply related [relevance 58%]
* [PATCH v2] lei import: support --new-only for IMAP
2021-06-03 5:00 67% [RFC] lei import: support --new-only Eric Wong
@ 2021-06-09 22:39 62% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 22:39 UTC (permalink / raw)
To: meta
Taking ~40s to synchronize a ~75K message IMAP folder is
still a lot of time, so support an option to only touch
new messages.
This is similar to "offlineimap -q" (quick) or "mbsync --new"
switches, but lei already accepts "-q" as a shortcut for
--quiet. "--new" could work, but "--new-only" might be more
descriptive (or "--only-new"?), since the default fetches
also fetches new messages.
v2: warn for non-IMAP sources, I'm not sure it's worth it for
Maildir or other sources, yet. It will also make sense
for MH and JMAP once we support them.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 7 ++++++-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 265b7047..beeb8b48 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -240,7 +240,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s
+ qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index e3cb69ca..08794f71 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -78,12 +78,14 @@ sub do_import_index ($$@) {
$j =~ /\A([0-9]+),[0-9]+\z/ and $j = $1 + 0;
$j ||= scalar(@{$self->{inputs}}) || 1;
my $ikw;
- if (my $net = $lei->{net}) {
+ my $net = $lei->{net};
+ if ($net) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
if ($self->{-import_kw} && $net->{-lms_ro} &&
+ !$lei->{opt}->{'new-only'} &&
$net->{imap_order}) {
require PublicInbox::LeiImportKw;
$ikw = PublicInbox::LeiImportKw->new($lei);
@@ -94,6 +96,9 @@ sub do_import_index ($$@) {
my $nproc = $self->detect_nproc;
$j = $nproc if $j > $nproc;
}
+ if ($lei->{opt}->{'new-only'} && (!$net || !$net->{imap_order})) {
+ $lei->err('# --new-only is only for IMAP');
+ }
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
^ permalink raw reply related [relevance 62%]
* [PATCH] lei/store: do eidx_init before creating R/W lms dbh
@ 2021-06-09 10:03 69% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 10:03 UTC (permalink / raw)
To: meta
Sharing lms->{dbh} with eidx shards appears to be the cause of
the "Issuing rollback() due to DESTROY without explicit
disconnect() of DBD::SQLite::db handle" messages I've been
seeing from "lei up".
---
lib/PublicInbox/LeiMailSync.pm | 5 ++---
lib/PublicInbox/LeiStore.pm | 1 +
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index ec05404a..558988f3 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -405,13 +405,12 @@ sub imap_oid {
$oidbin ? unpack('H*', $oidbin) : undef;
}
-# FIXME: something with "lei <up|q>" is causing uncommitted transaction
-# warnings, not sure what...
+# FIXED? something with "lei <up|q>" is causing uncommitted transaction
+# TODO: remove soon
sub DESTROY {
my ($self) = @_;
my $dbh = delete($self->{dbh}) or return;
return if $dbh->{ReadOnly};
- use Carp;
undef $dbh;
warn "BUG $$ $0 $self {dbh} OPEN ppid=".getppid.' '.Carp::longmess();
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 0b033e3e..5446873e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -195,6 +195,7 @@ sub remove_eml_vmd { # remove just the VMD
sub _lms_rw ($) {
my ($self) = @_;
+ my ($eidx, $tl) = eidx_init($self);
$self->{lms} //= do {
require PublicInbox::LeiMailSync;
my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
^ permalink raw reply related [relevance 69%]
* [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data
2021-06-09 7:47 71% [PATCH 0/5] lei Maildir stuff Eric Wong
2021-06-09 7:47 58% ` [PATCH 3/5] lei tag: parallelize Maildir access Eric Wong
@ 2021-06-09 7:47 45% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 7:47 UTC (permalink / raw)
To: meta
This will be invoked automatically by "lei import" eventually,
but it may make sense to expose as a separate command.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 2 +
lib/PublicInbox/LeiPruneMailSync.pm | 97 +++++++++++++++++++++++++++++
lib/PublicInbox/NetReader.pm | 19 ++++++
lib/PublicInbox/NetWriter.pm | 21 +------
5 files changed, 121 insertions(+), 19 deletions(-)
create mode 100644 lib/PublicInbox/LeiPruneMailSync.pm
diff --git a/MANIFEST b/MANIFEST
index 7bdbf252..3d4c6cbd 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -222,6 +222,7 @@ lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiPmdir.pm
+lib/PublicInbox/LeiPruneMailSync.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 77fc5b8f..265b7047 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -245,6 +245,8 @@ our %CMD = ( # sorted in order of importance/use:
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
'forget sync information for a mail folder', @c_opt ],
+'prune-mail-sync' => [ 'LOCATION...|--all',
+ 'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
qw(all:s mode=s), @c_opt ],
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
new file mode 100644
index 00000000..79f3325d
--- /dev/null
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -0,0 +1,97 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei prune-mail-sync" drops dangling sync information
+package PublicInbox::LeiPruneMailSync;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::LeiExportKw;
+use PublicInbox::InboxWritable qw(eml_from_path);
+
+sub eml_match ($$) {
+ my ($eml, $oidbin) = @_;
+ $oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
+}
+
+sub prune_mdir { # lms->each_src callback
+ my ($oidbin, $id, $self, $mdir) = @_;
+ my @try = $$id =~ /:2,[a-zA-Z]*\z/ ? qw(cur new) : qw(new cur);
+ for my $d (@try) {
+ my $src = "$mdir/$d/$$id";
+ if ($self->{verify}) {
+ my $eml = eml_from_path($src) or next;
+ return if eml_match($eml, $oidbin);
+ } elsif (-f $src) {
+ return;
+ }
+ }
+ # both tries failed
+ $self->{lei}->qerr("# maildir:$mdir $$id gone");
+ $self->{lei}->{sto}->ipc_do('lms_clear_src', "maildir:$mdir", $id);
+}
+
+sub prune_imap { # lms->each_src callback
+ my ($oidbin, $uid, $self, $uids, $url) = @_;
+ return if exists $uids->{$uid};
+ $self->{lei}->qerr("# $url $uid gone");
+ $self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
+}
+
+sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
+ my ($self, $input, @args) = @_;
+ my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
+ if ($input =~ /\Amaildir:(.+)/i) {
+ my $mdir = $1;
+ $lms->each_src($input, \&prune_mdir, $self, $mdir);
+ } elsif ($input =~ m!\Aimaps?://!i) {
+ my $uri = PublicInbox::URIimap->new($input);
+ my $mic = $self->{lei}->{net}->mic_for_folder($uri);
+ my $uids = $mic->search('UID 1:*');
+ $uids = +{ map { $_ => undef } @$uids };
+ $lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
+ } else { die "BUG: $input not supported" }
+ my $wait = $self->{lei}->{sto}->ipc_do('done');
+}
+
+sub lei_prune_mail_sync {
+ my ($lei, @folders) = @_;
+ my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
+lei/store uninitialized, see lei-import(1)
+EOM
+ my $lse = $sto->search;
+ my $lms = $lse->lms or return $lei->fail(<<EOM);
+lei mail_sync uninitialized, see lei-import(1)
+EOM
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
+ }
+ delete $lms->{dbh};
+ $sto->write_prepare($lei);
+ my $self = bless { lse => $lse }, __PACKAGE__;
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ $self->prepare_inputs($lei, \@folders) or return;
+ my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
+ undef $lms; # for fork
+ my $ops = {};
+ $sto->write_prepare($lei);
+ $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $self->{-wq_nr_workers} = $j // 1; # locked
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
+}
+
+no warnings 'once';
+*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
+1;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 058f4313..2795a9d4 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -747,4 +747,23 @@ sub nntp_each {
sub new { bless {}, shift };
+# updates $uri with UIDVALIDITY
+sub mic_for_folder {
+ my ($self, $uri) = @_;
+ my $mic = $self->mic_get($uri) or die "E: not connected: $@";
+ my $m = $self->isa('PublicInbox::NetWriter') ? 'select' : 'examine';
+ $mic->$m($uri->mailbox) or return;
+ my $uidval;
+ for ($mic->Results) {
+ /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
+ $uidval = $1;
+ last;
+ }
+ $uidval //= $mic->uidvalidity($uri->mailbox) or
+ die "E: failed to get uidvalidity from <$uri>: $@";
+ $uri->uidvalidity($uidval);
+ $mic;
+}
+
+
1;
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 8ec7f85c..82288e6b 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,26 +26,9 @@ sub imap_append {
die "APPEND $folder: $@";
}
-# updates $uri with UIDVALIDITY
-sub mic_for_folder {
- my ($self, $uri) = @_;
- my $mic = $self->mic_get($uri) or die "E: not connected: $@";
- $mic->select($uri->mailbox) or return;
- my $uidval;
- for ($mic->Results) {
- /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
- $uidval = $1;
- last;
- }
- $uidval //= $mic->uidvalidity($uri->mailbox) or
- die "E: failed to get uidvalidity from <$uri>: $@";
- $uri->uidvalidity($uidval);
- $mic;
-}
-
sub imap_delete_all {
my ($self, $uri) = @_;
- my $mic = mic_for_folder($self, $uri) or return;
+ my $mic = $self->mic_for_folder($uri) or return;
my $sec = $self->can('uri_section')->($uri);
local $0 = $uri->mailbox." $sec";
if ($mic->delete_message('1:*')) {
@@ -55,7 +38,7 @@ sub imap_delete_all {
sub imap_delete_1 {
my ($self, $uri, $uid, $delete_mic) = @_;
- $$delete_mic //= mic_for_folder($self, $uri) or return;
+ $$delete_mic //= $self->mic_for_folder($uri) or return;
$$delete_mic->delete_message($uid);
}
^ permalink raw reply related [relevance 45%]
* [PATCH 0/5] lei Maildir stuff
@ 2021-06-09 7:47 71% Eric Wong
2021-06-09 7:47 58% ` [PATCH 3/5] lei tag: parallelize Maildir access Eric Wong
2021-06-09 7:47 45% ` [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-06-09 7:47 UTC (permalink / raw)
To: meta
I'm not sure if "prune-mail-sync" needs to be exposed, but I
suppose it could be useful in some cases. It's certainly easier
to implement.
"lei tag" gets a nice speedup for Maildirs, IMAP speedup is
probably better off done after we get IMAP pipelining far into
the future.
Not sure if anybody is using InboxWritable->import_maildir...
Eric Wong (5):
inbox_writable: fix import_maildir
mdir_reader: maildir_each_file: pass flags, skip Trash
lei tag: parallelize Maildir access
lei_mail_sync: hoist out --all handling from export-kw
lei prune-mail-sync: new command to prune invalid sync data
MANIFEST | 1 +
lib/PublicInbox/InboxWritable.pm | 15 ++---
lib/PublicInbox/LEI.pm | 2 +
lib/PublicInbox/LeiExportKw.pm | 32 +---------
lib/PublicInbox/LeiImport.pm | 12 ++--
lib/PublicInbox/LeiMailSync.pm | 36 +++++++++++
lib/PublicInbox/LeiPmdir.pm | 18 ++----
lib/PublicInbox/LeiPruneMailSync.pm | 97 +++++++++++++++++++++++++++++
lib/PublicInbox/LeiTag.pm | 8 ++-
lib/PublicInbox/MdirReader.pm | 5 +-
lib/PublicInbox/NetReader.pm | 19 ++++++
lib/PublicInbox/NetWriter.pm | 21 +------
12 files changed, 184 insertions(+), 82 deletions(-)
create mode 100644 lib/PublicInbox/LeiPruneMailSync.pm
^ permalink raw reply [relevance 71%]
* [PATCH 3/5] lei tag: parallelize Maildir access
2021-06-09 7:47 71% [PATCH 0/5] lei Maildir stuff Eric Wong
@ 2021-06-09 7:47 58% ` Eric Wong
2021-06-09 7:47 45% ` [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 7:47 UTC (permalink / raw)
To: meta
Since Maildir isn't guaranteed to have any sort of order, we
can parallelize inputs, here. On a 4-core system, this reduced
one of my tag invocations from 5.5 to 1.4s.
---
lib/PublicInbox/LeiImport.pm | 8 ++++----
lib/PublicInbox/LeiPmdir.pm | 10 ++--------
lib/PublicInbox/LeiTag.pm | 8 +++++---
3 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index cddd5619..e3cb69ca 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -35,13 +35,13 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
die "BUG: $f was not from a Maildir?\n";
my $kw = PublicInbox::MdirReader::flags2kw($fl);
substr($folder, 0, 0) = 'maildir:'; # add prefix
- my $lms = $self->{-lms_ro};
+ my $lse = $self->{lse} //= $self->{lei}->{sto}->search;
+ my $lms = $self->{-lms_ro} //= $lse->lms; # may be 0 or undef
my $oidbin = $lms ? $lms->name_oidbin($folder, $bn) : undef;
- my @docids = defined($oidbin) ?
- $self->{over}->oidbin_exists($oidbin) : ();
+ my @docids = defined($oidbin) ? $lse->over->oidbin_exists($oidbin) : ();
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
if (scalar @docids) {
- $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ $lse->kw_changed(undef, $kw, \@docids) or return;
}
if (my $eml = eml_from_path($f)) {
$vmd->{sync_info} = [ $folder, \$bn ] if $self->{-mail_sync};
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index aa9ce713..760f276c 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -30,15 +30,9 @@ sub new {
sub ipc_atfork_child {
my ($self) = @_;
- my $lei = $self->{lei};
- $lei->_lei_atfork_child;
my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}';
- $ipt->{lei} = $lei;
- $ipt->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
- $ipt->{lse} = $ipt->{sto}->search;
- $ipt->{over} = $ipt->{lse}->over;
- $ipt->{-lms_ro} //= $ipt->{lse}->lms; # may be undef or '0'
- $self->SUPER::ipc_atfork_child;
+ $ipt->{lei} = $self->{lei};
+ $ipt->ipc_atfork_child;
}
sub each_mdir_fn { # maildir_each_file callback
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 4b3ce7d8..e0532653 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiTag;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::InboxWritable qw(eml_from_path);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
@@ -24,8 +25,9 @@ sub input_mbox_cb {
input_eml_cb($self, $eml);
}
-sub input_maildir_cb { # maildir_each_eml cb
- my ($f, $kw, $eml, $self) = @_;
+sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
+ my ($self, $f) = @_;
+ my $eml = eml_from_path($f) or return;
input_eml_cb($self, $eml);
}
@@ -42,12 +44,12 @@ sub lei_tag { # the "lei tag" method
$lei->ale; # refresh and prepare
my $vmd_mod = $self->vmd_mod_extract(\@argv);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
+ $self->{vmd_mod} = $vmd_mod; # before LeiPmdir->new in prepare_inputs
$self->prepare_inputs($lei, \@argv) or return;
grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or
return $lei->fail('no keywords or labels specified');
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{vmd_mod} = $vmd_mod;
my $j = $self->{-wq_nr_workers} = 1; # locked for now
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
^ permalink raw reply related [relevance 58%]
* [PATCH] lei edit-search: fix and add a (weak) test
@ 2021-06-09 0:11 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-09 0:11 UTC (permalink / raw)
To: meta
This broke recently and lacked an automated test, so rely on
EDITOR=cat to ensure we have some coverage.
Fixes: d2670108f71b1eff ("pkt_op: make pkt_do an OO method")
---
lib/PublicInbox/LeiEditSearch.pm | 2 +-
t/lei-q-save.t | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 13713d24..82dfbf63 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -19,7 +19,7 @@ sub lei_edit_search {
my ($op_c, $op_p) = PublicInbox::PktOp->pair;
# $op_p will EOF when $EDITOR is done
$op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
- $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p ], \@cmd, {});
+ $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], \@cmd, {});
}
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 694b33b2..6c592088 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -195,5 +195,8 @@ test_lei(sub {
glob("$v2s/git/0.git/objects/*/*")));
ok($shared < $orig, 'fewer bytes stored with --shared') or
diag "shared=$shared orig=$orig";
+
+ lei_ok([qw(edit-search), $v2s], { VISUAL => 'cat', EDITOR => 'cat' });
+ like($lei_out, qr/^\[lei/sm, 'edit-search can cat');
});
done_testing;
^ permalink raw reply related [relevance 70%]
* [PATCH] lei pmdir: fix nproc for <= 4 CPUs
@ 2021-06-08 23:56 70% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 23:56 UTC (permalink / raw)
To: meta
I forgot my FreeBSD VM has 8 cores, actually, and tweaked the
nproc detection on that machine before finalizing commit
10b523eb017162240b1ac3647f8dcbbf2be348a7
("lei import: speed up repeated Maildir imports")
Fixes: 10b523eb01716224 ("lei import: speed up repeated Maildir imports")
---
lib/PublicInbox/LeiPmdir.pm | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index 5efb012e..b71efe70 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -13,14 +13,14 @@ use parent qw(PublicInbox::IPC);
sub new {
my ($cls, $lei, $ipt) = @_;
my $self = bless { -wq_ident => 'lei Maildir worker' }, $cls;
- my $jobs = $lei->{opt}->{jobs};
+ my $jobs = $lei->{opt}->{jobs} // '';
$jobs =~ /\A[0-9]+,([0-9]+)\z/ and $jobs = $1;
- my $nproc = $jobs // do {
- # untested with >=4 CPUs, though I suspect I/O latency
+ my $nproc = $jobs || do {
+ # barely tested with >=4 CPUs, though I suspect I/O latency
# of SATA SSD storage will make >=4 processes unnecessary,
# here. NVMe users may wish to use '-j'
my $n = $self->detect_nproc;
- $n = 4 if $n > 4;
+ $n = $n > 4 ? 4 : $n;
};
my ($op_c, $ops) = $lei->workers_start($self, $nproc,
undef, { ipt => $ipt }); # LeiInput subclass
^ permalink raw reply related [relevance 70%]
* [PATCH 1/3] lei: safety fix for multiple WQ classes
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
@ 2021-06-08 9:50 71% ` Eric Wong
2021-06-08 9:50 48% ` [PATCH 2/3] lei: generalize auxiliary WQ handling Eric Wong
2021-06-08 9:50 32% ` [PATCH 3/3] lei import: speed up repeated Maildir imports Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
For commands utilizing multiple workers, this simple change
generalizes the persistence mechanism and and prevents
lei->dclose from causing script/lei to exit if there are
still in-flight workers.
This ougth to prevent read-after-write consistency problems that
occasionally manifest in scripts (e.g. test cases) but usually
go unnoticed in normal use.
---
lib/PublicInbox/LEI.pm | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8adf70fa..0cf4d10b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -580,6 +580,8 @@ sub workers_start {
$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
+ # {-lei_sock} persists script/lei process until ops->{''} EOF callback
+ $op_c->{-lei_sock} = $lei->{sock};
@$end = ();
$lei->event_step_init;
($op_c, $ops);
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] lei: generalize auxiliary WQ handling
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
@ 2021-06-08 9:50 48% ` Eric Wong
2021-06-08 9:50 32% ` [PATCH 3/3] lei import: speed up repeated Maildir imports Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
op_wait_event is now more lei-specific since we no longer have
to care about oneshot and use a synchronous loop.
{ikw} (import-keywords) started a trend, but LeiPmdir (parallel
Maildir) is an upcoming WQ class that will follow this idea.
Eventually, {l2m} usage may be updated to follow this, too.
---
lib/PublicInbox/LEI.pm | 9 +++++++++
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 3 +--
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
lib/PublicInbox/LeiP2q.pm | 4 ++--
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/PktOp.pm | 6 ------
13 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0cf4d10b..ed01e8de 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -587,6 +587,15 @@ sub workers_start {
($op_c, $ops);
}
+# call this when we're ready to wait on events and yield to other clients
+sub wait_wq_events {
+ my ($lei, $op_c, $ops) = @_;
+ for my $wq (grep(defined, @$lei{qw(ikw)})) { # auxiliary WQs
+ $wq->wq_close(1);
+ }
+ $op_c->{ops} = $ops;
+}
+
sub _help {
require PublicInbox::LeiHelp;
PublicInbox::LeiHelp::call($_[0], $_[1], \%CMD, \%OPTDESC);
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 8de86565..09217964 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -169,7 +169,7 @@ sub lei_blob {
$lei->{wq1} = $self;
$self->wq_io_do('do_solve_blob', []);
$self->wq_close(1);
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 395a80f8..6550c242 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -59,7 +59,7 @@ sub lei_convert { # the main "lei convert" method
$lei->{wq1} = $self;
$self->wq_io_do('process_inputs', []);
$self->wq_close(1);
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index b31b065f..f8579221 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -160,7 +160,7 @@ EOM
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops); # calls net_merge_all_done if $lei->{auth}
+ $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
}
sub _complete_export_kw {
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 2efd4935..222f75c8 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -91,9 +91,8 @@ sub do_import_index ($$@) {
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
- $ikw->wq_close(1) if $ikw;
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub lei_import { # the main "lei import" method
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 6cea6ae8..70136135 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -76,7 +76,7 @@ sub bg_worker ($$$) {
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
$self->wq_close(1);
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub lei_ls_search {
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index a37e1d5c..39671f90 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -282,11 +282,11 @@ sub start {
require PublicInbox::Inbox;
require PublicInbox::Admin;
require PublicInbox::InboxWritable;
- my ($op, $ops) = $lei->workers_start($self, 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_mirror', []);
$self->wq_close(1);
- $op->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index f381a31c..c0c4563d 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -188,11 +188,11 @@ sub lei_p2q { # the "lei patch-to-query" entry point
} else {
$self->{input} = $input;
}
- my ($op, $ops) = $lei->workers_start($self, 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_p2q', []);
$self->wq_close(1);
- $op->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index c8bd0dfb..7607b44f 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -227,7 +227,7 @@ sub lei_rediff {
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index c6d28045..578e9811 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -38,7 +38,7 @@ sub lei_rm {
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
no warnings 'once';
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index b6abd533..4b3ce7d8 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -53,7 +53,7 @@ sub lei_tag { # the "lei tag" method
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub note_missing {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 75e55d47..beb955bb 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -532,7 +532,7 @@ sub do_query {
start_query($self);
}
$lei->event_step_init; # wait for shutdowns
- $op_c->op_wait_event($ops);
+ $lei->wait_wq_events($op_c, $ops);
}
sub add_uri {
diff --git a/lib/PublicInbox/PktOp.pm b/lib/PublicInbox/PktOp.pm
index ca098d3c..92e150a4 100644
--- a/lib/PublicInbox/PktOp.pm
+++ b/lib/PublicInbox/PktOp.pm
@@ -63,10 +63,4 @@ sub event_step {
}
}
-# call this when we're ready to wait on events
-sub op_wait_event {
- my ($self, $ops) = @_;
- $self->{ops} = $ops;
-}
-
1;
^ permalink raw reply related [relevance 48%]
* [PATCH 3/3] lei import: speed up repeated Maildir imports
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
2021-06-08 9:50 48% ` [PATCH 2/3] lei: generalize auxiliary WQ handling Eric Wong
@ 2021-06-08 9:50 32% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
On a 4-core CPU, this speeds up "lei import" on a largish
Maildir inbox with 75K messages from ~8 minutes down to ~40s.
Parallelizing alone did not bring any improvement and may
even hurt performance slightly, depending on CPU availability.
However, creating the index on the "fid" and "name" columns in
blob2name yields us the same speedup we got.
Parallelizing IMAP makes more sense due to the fact most IMAP
stores are non-local and subject to network latency.
Followup-to: bdecd7ed8e0dcf0b45491b947cd737ba8cfe38a3 ("lei import: speed up kw updates for old IMAP messages")
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 11 +++---
lib/PublicInbox/LeiImport.pm | 36 ++++++++++++------
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 31 +++++++++++-----
lib/PublicInbox/LeiMailSync.pm | 14 +++++++
lib/PublicInbox/LeiPmdir.pm | 67 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/MdirReader.pm | 22 ++++++-----
t/lei-import-maildir.t | 2 +-
9 files changed, 148 insertions(+), 38 deletions(-)
create mode 100644 lib/PublicInbox/LeiPmdir.pm
diff --git a/MANIFEST b/MANIFEST
index 5a70a144..7bdbf252 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -221,6 +221,7 @@ lib/PublicInbox/LeiMailSync.pm
lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
+lib/PublicInbox/LeiPmdir.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ed01e8de..77fc5b8f 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -240,7 +240,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s
+ qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
@@ -421,7 +421,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m wq1 ikw); # internal workers
+my @WQ_KEYS = qw(lxs l2m ikw pmd wq1); # internal workers
sub _drop_wq {
my ($self) = @_;
@@ -566,7 +566,7 @@ sub pkt_op_pair {
}
sub workers_start {
- my ($lei, $wq, $jobs, $ops) = @_;
+ my ($lei, $wq, $jobs, $ops, $flds) = @_;
$ops = {
'!' => [ \&fail_handler, $lei ],
'|' => [ \&sigpipe_handler, $lei ],
@@ -577,7 +577,8 @@ sub workers_start {
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
- $wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
+ $flds->{lei} = $lei;
+ $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
# {-lei_sock} persists script/lei process until ops->{''} EOF callback
@@ -590,7 +591,7 @@ sub workers_start {
# call this when we're ready to wait on events and yield to other clients
sub wait_wq_events {
my ($lei, $op_c, $ops) = @_;
- for my $wq (grep(defined, @$lei{qw(ikw)})) { # auxiliary WQs
+ for my $wq (grep(defined, @$lei{qw(ikw pmd)})) { # auxiliary WQs
$wq->wq_close(1);
}
$op_c->{ops} = $ops;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 222f75c8..b0e7ba6b 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiImport;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::InboxWritable qw(eml_from_path);
# /^input_/ subs are used by (or override) PublicInbox::LeiInput superclass
@@ -28,17 +29,26 @@ sub input_mbox_cb { # MboxReader callback
input_eml_cb($self, $eml, $vmd);
}
-sub input_maildir_cb { # maildir_each_eml cb
- my ($f, $kw, $eml, $self) = @_;
+sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
+ my ($self, $f, @args) = @_;
+ my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+ die "BUG: $f was not from a Maildir?\n";
+ my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn);
+ return if index($fl, 'T') >= 0; # no Trashed messages
+ my $kw = PublicInbox::MdirReader::flags2kw($fl);
+ substr($folder, 0, 0) = 'maildir:'; # add prefix
+ my $lms = $self->{-lms_ro};
+ my $oidbin = $lms ? $lms->name_oidbin($folder, $bn) : undef;
+ my @docids = defined($oidbin) ?
+ $self->{over}->oidbin_exists($oidbin) : ();
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
- if ($self->{-mail_sync}) {
- if ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) { # ugh...
- $vmd->{sync_info} = [ "maildir:$1", \(my $n = $2) ];
- } else {
- warn "E: $f was not from a Maildir?\n";
- }
+ if (scalar @docids) {
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ }
+ if (my $eml = eml_from_path($f)) {
+ $vmd->{sync_info} = [ $folder, \$bn ] if $self->{-mail_sync};
+ $self->input_eml_cb($eml, $vmd);
}
- $self->input_eml_cb($eml, $vmd);
}
sub input_net_cb { # imap_each / nntp_each
@@ -62,11 +72,13 @@ sub do_import_index ($$@) {
my $vmd_mod = $self->vmd_mod_extract(\@inputs);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
$self->{all_vmd} = $vmd_mod if scalar keys %$vmd_mod;
- $self->prepare_inputs($lei, \@inputs) or return;
+ $lei->ale; # initialize for workers to read (before LeiPmdir->new)
$self->{-mail_sync} = $lei->{opt}->{'mail-sync'} // 1;
+ $self->prepare_inputs($lei, \@inputs) or return;
- $lei->ale; # initialize for workers to read
- my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ my $j = $lei->{opt}->{jobs} // 0;
+ $j =~ /\A([0-9]+),[0-9]+\z/ and $j = $1 + 0;
+ $j ||= scalar(@{$self->{inputs}}) || 1;
my $ikw;
if (my $net = $lei->{net}) {
# $j = $net->net_concurrency($j); TODO
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index cc3e83e7..4be0c649 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -35,7 +35,7 @@ sub lei_index {
no warnings 'once';
no strict 'refs';
-for my $m (qw(input_maildir_cb input_net_cb)) {
+for my $m (qw(pmdir_cb input_net_cb)) {
*$m = PublicInbox::LeiImport->can($m);
}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 4ff7a379..24211bf0 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -151,9 +151,16 @@ sub input_path_url {
return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
$input appears to be a maildir, not $ifmt
EOM
- PublicInbox::MdirReader->new->maildir_each_eml($input,
- $self->can('input_maildir_cb'),
- $self, @args);
+ my $mdr = PublicInbox::MdirReader->new;
+ if (my $pmd = $self->{pmd}) {
+ $mdr->maildir_each_file($input,
+ $pmd->can('each_mdir_fn'),
+ $pmd, @args);
+ } else {
+ $mdr->maildir_each_eml($input,
+ $self->can('input_maildir_cb'),
+ $self, @args);
+ }
} else {
$lei->fail("$input unsupported (TODO)");
}
@@ -215,7 +222,7 @@ sub prepare_inputs { # returns undef on error
push @{$sync->{no}}, '/dev/stdin' if $sync;
}
my $net = $lei->{net}; # NetWriter may be created by l2m
- my (@f, @d);
+ my (@f, @md);
# e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX
for my $input (@$inputs) {
my $input_path = $input;
@@ -247,11 +254,11 @@ sub prepare_inputs { # returns undef on error
PublicInbox::MboxReader->reads($ifmt) or return
$lei->fail("$ifmt not supported");
} elsif (-d $input_path) {
- require PublicInbox::MdirReader;
$ifmt eq 'maildir' or return
$lei->fail("$ifmt not supported");
$sync and $input = 'maildir:'.
$lei->abs_path($input_path);
+ push @md, $input;
} else {
return $lei->fail("Unable to handle $input");
}
@@ -266,21 +273,18 @@ $input is `eml', not --in-format=$in_fmt
if ($devfd >= 0 || -f $input || -p _) {
push @{$sync->{no}}, $input if $sync;
push @f, $input;
- } elsif (-d $input) {
+ } elsif (-d "$input/new" && -d "$input/cur") {
if ($sync) {
$input = $lei->abs_path($input);
push @{$sync->{ok}}, $input;
}
- push @d, $input;
+ push @md, $input;
} else {
return $lei->fail("Unable to handle $input")
}
}
}
if (@f) { check_input_format($lei, \@f) or return }
- if (@d) { # TODO: check for MH vs Maildir, here
- require PublicInbox::MdirReader;
- }
if ($sync && $sync->{no}) {
return $lei->fail(<<"") if !$sync->{ok};
--mail-sync specified but no inputs support it
@@ -299,6 +303,13 @@ $input is `eml', not --in-format=$in_fmt
$lei->{auth} //= PublicInbox::LeiAuth->new;
$lei->{net} //= $net;
}
+ if (scalar(@md)) {
+ require PublicInbox::MdirReader;
+ if ($self->can('pmdir_cb')) {
+ require PublicInbox::LeiPmdir;
+ $self->{pmd} = PublicInbox::LeiPmdir->new($lei, $self);
+ }
+ }
$self->{inputs} = $inputs;
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 75603d89..ec05404a 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -66,6 +66,10 @@ CREATE TABLE IF NOT EXISTS blob2name (
UNIQUE (oidbin, fid, name)
)
+ # speeds up LeiImport->pmdir_cb (for "lei import") by ~6x:
+ $dbh->do(<<'');
+CREATE INDEX IF NOT EXISTS idx_fid_name ON blob2name(fid,name)
+
}
sub fid_for {
@@ -375,6 +379,16 @@ EOM
$sth->fetchrow_array;
}
+sub name_oidbin ($$$) {
+ my ($self, $mdir, $nm) = @_;
+ my $fid = $self->{fmap}->{$mdir} //= fid_for($self, $mdir) // return;
+ my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2name WHERE fid = ? AND name = ?
+EOM
+ $sth->execute($fid, $nm);
+ $sth->fetchrow_array;
+}
+
sub imap_oid {
my ($self, $lei, $uid_uri) = @_;
my $mailbox_uri = $uid_uri->clone;
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
new file mode 100644
index 00000000..5efb012e
--- /dev/null
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -0,0 +1,67 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# WQ worker for dealing with parallel Maildir reads;
+# this does NOT use the {shard_info} field of LeiToMail
+# (and we may remove {shard_info})
+# WQ key: {pmd}
+package PublicInbox::LeiPmdir;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+sub new {
+ my ($cls, $lei, $ipt) = @_;
+ my $self = bless { -wq_ident => 'lei Maildir worker' }, $cls;
+ my $jobs = $lei->{opt}->{jobs};
+ $jobs =~ /\A[0-9]+,([0-9]+)\z/ and $jobs = $1;
+ my $nproc = $jobs // do {
+ # untested with >=4 CPUs, though I suspect I/O latency
+ # of SATA SSD storage will make >=4 processes unnecessary,
+ # here. NVMe users may wish to use '-j'
+ my $n = $self->detect_nproc;
+ $n = 4 if $n > 4;
+ };
+ my ($op_c, $ops) = $lei->workers_start($self, $nproc,
+ undef, { ipt => $ipt }); # LeiInput subclass
+ $op_c->{ops} = $ops; # for PktOp->event_step
+ $lei->{pmd} = $self;
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->_lei_atfork_child;
+ my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}';
+ $ipt->{lei} = $lei;
+ $ipt->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
+ $ipt->{lse} = $ipt->{sto}->search;
+ $ipt->{over} = $ipt->{lse}->over;
+ $ipt->{-lms_ro} //= $ipt->{lse}->lms; # may be undef or '0'
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub each_mdir_fn { # maildir_each_file callback
+ my ($f, $self, @args) = @_;
+ $self->wq_io_do('mdir_iter', [], $f, @args);
+}
+
+sub mdir_iter { # via wq_io_do
+ my ($self, $f, @args) = @_;
+ $self->{ipt}->pmdir_cb($f, @args);
+}
+
+sub pmd_done_wait {
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lei->can('wq_done_wait')->($arg, $pid);
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $pmd = delete $lei->{pmd} or return $lei->fail;
+ $pmd->wq_wait_old(\&pmd_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index 304be63d..484bf0a8 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -87,17 +87,21 @@ sub maildir_each_eml {
sub new { bless {}, __PACKAGE__ }
sub flags2kw ($) {
- my @unknown;
- my %kw;
- for (split(//, $_[0])) {
- my $k = $c2kw{$_};
- if (defined($k)) {
- $kw{$k} = 1;
- } else {
- push @unknown, $_;
+ if (wantarray) {
+ my @unknown;
+ my %kw;
+ for (split(//, $_[0])) {
+ my $k = $c2kw{$_};
+ if (defined($k)) {
+ $kw{$k} = 1;
+ } else {
+ push @unknown, $_;
+ }
}
+ (\%kw, \@unknown);
+ } else {
+ [ sort(map { $c2kw{$_} // () } split(//, $_[0])) ];
}
- (\%kw, \@unknown);
}
1;
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index 688b10ce..c81e7805 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -28,7 +28,7 @@ test_lei(sub {
is(scalar(keys %v), 1, 'inspect handles relative and absolute paths');
my $inspect = json_utf8->decode([ keys %v ]->[0]);
is_deeply($inspect, {"maildir:$md" => { 'name.count' => 1 }},
- 'inspect maildir: path had expected output');
+ 'inspect maildir: path had expected output') or xbail($inspect);
lei_ok(qw(q s:boolean));
my $res = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 32%]
* [PATCH 0/3] lei import: speedup repeated Maildir import
@ 2021-06-08 9:50 69% Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-06-08 9:50 UTC (permalink / raw)
To: meta
More importantly, this series starts us down the road of
generalizing the use of auxiliary WQs like {ikw} (import
keywords for IMAP) and {pmd} (parallel Maildir).
It took me a bit to figure out some consistency problems
which I reordered and split out into 1/3. 2/3 should
make future work easier, and 3/3 gives the final speedup
along the lines of what I already did with IMAP.
Eric Wong (3):
lei: safety fix for multiple WQ classes
lei: generalize auxiliary WQ handling
lei import: speed up repeated Maildir imports
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 20 ++++++++--
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 39 +++++++++++++-------
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 31 +++++++++++-----
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMailSync.pm | 14 +++++++
lib/PublicInbox/LeiMirror.pm | 4 +-
lib/PublicInbox/LeiP2q.pm | 4 +-
lib/PublicInbox/LeiPmdir.pm | 67 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/MdirReader.pm | 22 ++++++-----
lib/PublicInbox/PktOp.pm | 6 ---
t/lei-import-maildir.t | 2 +-
20 files changed, 171 insertions(+), 57 deletions(-)
create mode 100644 lib/PublicInbox/LeiPmdir.pm
^ permalink raw reply [relevance 69%]
* [PATCH] lei/store: checkpoint commits mail_sync.sqlite3
@ 2021-06-07 19:06 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-07 19:06 UTC (permalink / raw)
To: meta
We mainly rely on ->done with lei/store, but moving to
->checkpoint probably makes sense. Note: over, msgmap, and
mail_sync all have slightly different transacation behavior;
perhaps they can be unified in the future.
---
lib/PublicInbox/LeiStore.pm | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 821782b9..0b033e3e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -419,6 +419,9 @@ sub checkpoint {
if (my $im = $self->{im}) {
$wait ? $im->barrier : $im->checkpoint;
}
+ if (my $lms = delete $self->{lms}) {
+ $lms->lms_commit;
+ }
$self->{priv_eidx}->checkpoint($wait);
}
^ permalink raw reply related [relevance 71%]
* [PATCH] lei: don't drop WQ workers on normal exit
@ 2021-06-05 21:06 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-05 21:06 UTC (permalink / raw)
To: meta
This is dangerous and causes race conditions on commands
which utilize multiple workqueues.
---
lib/PublicInbox/LEI.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 2dd21fc6..8adf70fa 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -480,6 +480,7 @@ sub sigint_reap {
sub fail ($$;$) {
my ($self, $buf, $exit_code) = @_;
+ $self->{failed}++;
err($self, $buf) if defined $buf;
# calls fail_handler:
$self->{pkt_op_p}->pkt_do('!') if $self->{pkt_op_p};
@@ -1043,7 +1044,7 @@ sub accept_dispatch { # Listener {post_accept} callback
sub dclose {
my ($self) = @_;
delete $self->{-progress};
- _drop_wq($self);
+ _drop_wq($self) if $self->{failed};
close(delete $self->{1}) if $self->{1}; # may reap_compress
$self->close if $self->{-event_init_done}; # PublicInbox::DS::close
}
^ permalink raw reply related [relevance 71%]
* [PATCH] INSTALL: note about lei metadata storage
@ 2021-06-05 21:04 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-05 21:04 UTC (permalink / raw)
To: meta
Since lei is for personal mailboxes, I don't think lei needs to
keep keyword and label changes in history. And fix a minor
wording problem ("or" => "nor") while we're at it.
---
INSTALL | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/INSTALL b/INSTALL
index e8bfba97..e93731f1 100644
--- a/INSTALL
+++ b/INSTALL
@@ -174,9 +174,10 @@ Other installation notes
Debian 8.x (jessie) users, use Debian 8.5 or later if using Xapian:
https://bugs.debian.org/808610
-public-inbox will never store unregeneratable data in Xapian
-or any other search database we might use; Xapian corruption
-will not destroy critical data.
+public-inbox-* commands will never store unregeneratable data in
+Xapian nor any other search database we might use; Xapian
+corruption will not destroy critical data. Note: `lei' DOES store
+unregeneratable data in Xapian and SQLite.
See the public-inbox-overview(7) man page for the next steps once
the installation is complete.
^ permalink raw reply related [relevance 71%]
* [RFC] lei import: support --new-only
@ 2021-06-03 5:00 67% Eric Wong
2021-06-09 22:39 62% ` [PATCH v2] lei import: support --new-only for IMAP Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-06-03 5:00 UTC (permalink / raw)
To: meta
Taking ~40s to synchronize a ~75K message IMAP folder is
still a lot of time, so support an option to only touch
new messages.
This is similar to "offlineimap -q" (quick) or "mbsync --new"
switches, but lei already accepts "-q" as a shortcut for
--quiet. "--new" could work, but "--new-only" might be more
descriptive (or "--only-new"?), since the default fetches
also fetches new messages.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7bda9408..9a2a323c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -240,7 +240,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s
+ qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 2efd4935..60f3241b 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -74,6 +74,7 @@ sub do_import_index ($$@) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
if ($self->{-import_kw} && $net->{-lms_ro} &&
+ !$lei->{opt}->{'new-only'} &&
$net->{imap_order}) {
require PublicInbox::LeiImportKw;
$ikw = PublicInbox::LeiImportKw->new($lei);
^ permalink raw reply related [relevance 67%]
* [PATCH v2] lei import: speed up kw updates for old IMAP messages
2021-06-03 0:17 40% [PATCH] lei import: speed up kw updates for old IMAP messages Eric Wong
@ 2021-06-03 1:05 38% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-03 1:05 UTC (permalink / raw)
To: meta
On a 4-core CPU, this speeds up "lei import" on a largish IMAP
inbox with 75K messages from ~21 minutes down to 40s.
Parallelizing with the new LeiImportKw WQ worker class gives a
near-linear speedup and brought the runtime down to ~5:40.
The new idx_fid_uid index on the "fid" and "uid" columns of
blob2num in mail_sync.sqlite3 brought us the final speedup.
An additional index on over.sqlite3#xref3(oidbin) did not help,
since idx_nntp already exists and speeds up the new ->oidbin_exists
internal API.
I initially experimented with a separate "lei import-kw" command
but decided against it since it's useless outside of IMAP+JMAP
and would require extra cognitive overhead for both users and
hackers. So LeiImportKw is just a WQ worker used by "lei import"
and not its own user-visible command.
v2: fix ikw_done_wait arg handling (ugh, confusing API :x)
---
Interdiff against v1:
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index e13dce07..2878cbdf 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -40,9 +40,10 @@ sub ck_update_kw { # via wq_io_do
}
sub ikw_done_wait {
- my ($lei) = @_;
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
my $wait = $lei->{sto}->ipc_do('done');
- $lei->wq_done_wait;
+ $lei->can('wq_done_wait')->($arg, $pid);
}
sub _lei_wq_eof { # EOF callback for main lei daemon
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 25 ++++++++--------
lib/PublicInbox/LeiImportKw.pm | 55 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 17 ++++++-----
lib/PublicInbox/NetReader.pm | 1 +
lib/PublicInbox/Over.pm | 10 ++++---
7 files changed, 86 insertions(+), 25 deletions(-)
create mode 100644 lib/PublicInbox/LeiImportKw.pm
diff --git a/MANIFEST b/MANIFEST
index 0b4bb380..5a70a144 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
+lib/PublicInbox/LeiImportKw.pm
lib/PublicInbox/LeiIndex.pm
lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 30f90798..7bda9408 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -421,7 +421,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1 ikw); # internal workers
sub _drop_wq {
my ($self) = @_;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 860a2c98..2efd4935 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -43,18 +43,14 @@ sub input_maildir_cb { # maildir_each_eml cb
sub input_net_cb { # imap_each / nntp_each
my ($uri, $uid, $kw, $eml, $self) = @_;
- my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
- $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
if (defined $eml) {
+ my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
+ $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
$self->input_eml_cb($eml, $vmd);
- } elsif ($vmd) { # old message, kw only
- my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
- my @docids = $self->{lse}->over->blob_exists($oid) or return;
- $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
- my $lei = $self->{lei};
- $lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
- $self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
- $vmd, \@docids);
+ } elsif (my $ikw = $self->{lei}->{ikw}) { # old message, kw only
+ # we send $uri as a bare SCALAR and not a URIimap ref to
+ # reduce socket traffic:
+ $ikw->wq_io_do('ck_update_kw', [], $$uri, $uid, $kw);
}
}
@@ -71,15 +67,17 @@ sub do_import_index ($$@) {
$lei->ale; # initialize for workers to read
my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ my $ikw;
if (my $net = $lei->{net}) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
- if ($self->{-import_kw}) {
+ if ($self->{-import_kw} && $net->{-lms_ro} &&
+ $net->{imap_order}) {
+ require PublicInbox::LeiImportKw;
+ $ikw = PublicInbox::LeiImportKw->new($lei);
$net->{each_old} = 1;
- $self->{-lms_ro} = $net->{-lms_ro};
- $self->{lse} = $sto->search;
}
}
} else {
@@ -93,6 +91,7 @@ sub do_import_index ($$@) {
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
+ $ikw->wq_close(1) if $ikw;
net_merge_all_done($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
new file mode 100644
index 00000000..2878cbdf
--- /dev/null
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -0,0 +1,55 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# WQ worker for dealing with LeiImport IMAP flags on already-imported messages
+# WQ key: {ikw}
+package PublicInbox::LeiImportKw;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+sub new {
+ my ($cls, $lei) = @_;
+ my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
+ my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
+ $op_c->{ops} = $ops; # for PktOp->event_step
+ $lei->{ikw} = $self;
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->_lei_atfork_child;
+ my $net = delete $lei->{net} // die 'BUG: no lei->{net}';
+ $self->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
+ $self->{verbose} = $lei->{opt}->{verbose};
+ $self->{lse} = $self->{sto}->search;
+ $self->{over} = $self->{lse}->over;
+ $self->{-lms_ro} = $net->{-lms_ro} || die 'BUG: net->{-lms_ro} FALSE';
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub ck_update_kw { # via wq_io_do
+ my ($self, $url, $uid, $kw) = @_;
+ my $oidbin = $self->{-lms_ro}->imap_oidbin($url, $uid) // return;
+ my @docids = $self->{over}->oidbin_exists($oidbin) or return;
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ $self->{verbose} and
+ $self->{lei}->qerr('# '.unpack('H*', $oidbin)." => @$kw\n");
+ $self->{sto}->ipc_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
+}
+
+sub ikw_done_wait {
+ my ($arg, $pid) = @_;
+ my ($self, $lei) = @$arg;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lei->can('wq_done_wait')->($arg, $pid);
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $ikw = delete $lei->{ikw} or return $lei->fail;
+ $ikw->wq_wait_old(\&ikw_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 22ee1109..75603d89 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS blob2num (
UNIQUE (oidbin, fid, uid)
)
+ # speeds up LeiImport->ck_update_kw (for "lei import") by 5-6x:
+ $dbh->do(<<'');
+CREATE INDEX IF NOT EXISTS idx_fid_uid ON blob2num(fid,uid)
+
$dbh->do(<<'');
CREATE TABLE IF NOT EXISTS blob2name (
oidbin VARBINARY NOT NULL,
@@ -361,15 +365,14 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
-sub imap_oid2 ($$$) {
- my ($self, $uri, $uid) = @_; # $uri MUST have UIDVALIDITY
- my $fid = $self->{fmap}->{"$uri"} //= fid_for($self, "$uri") // return;
+sub imap_oidbin ($$$) {
+ my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY
+ my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return;
my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
EOM
$sth->execute($fid, $uid);
- my ($oidbin) = $sth->fetchrow_array;
- $oidbin ? unpack('H*', $oidbin) : undef;
+ $sth->fetchrow_array;
}
sub imap_oid {
@@ -384,10 +387,10 @@ sub imap_oid {
}
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
- imap_oid2($self, $folders->[0], $uid_uri->uid);
+ my $oidbin = imap_oidbin($self, $folders->[0], $uid_uri->uid);
+ $oidbin ? unpack('H*', $oidbin) : undef;
}
-
# FIXME: something with "lei <up|q>" is causing uncommitted transaction
# warnings, not sure what...
sub DESTROY {
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 39a8f7fc..058f4313 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -471,6 +471,7 @@ sub each_old_flags ($$$$) {
while (my ($uid, $per_uid) = each %$r) {
my $kw = flags2kw($self, $uri, $uid, $per_uid->{FLAGS})
// next;
+ # LeiImport->input_net_cb
$eml_cb->($uri, $uid, $kw, undef, @args);
}
}
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 0e191c47..58fdea0e 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -349,13 +349,13 @@ sub check_inodes {
}
}
-sub blob_exists {
- my ($self, $oidhex) = @_;
+sub oidbin_exists {
+ my ($self, $oidbin) = @_;
if (wantarray) {
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
my $tmp = $sth->fetchall_arrayref;
map { $_->[0] } @$tmp;
@@ -363,10 +363,12 @@ SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT COUNT(*) FROM xref3 WHERE oidbin = ?
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
$sth->fetchrow_array;
}
}
+sub blob_exists { oidbin_exists($_[0], pack('H*', $_[1])) }
+
1;
^ permalink raw reply related [relevance 38%]
* [PATCH] lei import: speed up kw updates for old IMAP messages
@ 2021-06-03 0:17 40% Eric Wong
2021-06-03 1:05 38% ` [PATCH v2] " Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-06-03 0:17 UTC (permalink / raw)
To: meta
On a 4-core CPU, this speeds up "lei import" on a largish IMAP
inbox with 75K messages from ~21 minutes down to 40s.
Parallelizing with the new LeiImportKw WQ worker class gives a
near-linear speedup and brought the runtime down to ~5:40.
The new idx_fid_uid index on the "fid" and "uid" columns of
blob2num in mail_sync.sqlite3 brought us the final speedup.
An additional index on over.sqlite3#xref3(oidbin) did not help,
since idx_nntp already exists and speeds up the new ->oidbin_exists
internal API.
I initially experimented with a separate "lei import-kw" command
but decided against it since it's useless outside of IMAP+JMAP
and would require extra cognitive overhead for both users and
hackers. So LeiImportKw is just a WQ worker used by "lei import"
and not its own user-visible command.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 25 ++++++++--------
lib/PublicInbox/LeiImportKw.pm | 54 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 17 ++++++-----
lib/PublicInbox/NetReader.pm | 1 +
lib/PublicInbox/Over.pm | 10 ++++---
7 files changed, 85 insertions(+), 25 deletions(-)
create mode 100644 lib/PublicInbox/LeiImportKw.pm
diff --git a/MANIFEST b/MANIFEST
index 0b4bb380..5a70a144 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
+lib/PublicInbox/LeiImportKw.pm
lib/PublicInbox/LeiIndex.pm
lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 30f90798..7bda9408 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -421,7 +421,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1 ikw); # internal workers
sub _drop_wq {
my ($self) = @_;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 860a2c98..2efd4935 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -43,18 +43,14 @@ sub input_maildir_cb { # maildir_each_eml cb
sub input_net_cb { # imap_each / nntp_each
my ($uri, $uid, $kw, $eml, $self) = @_;
- my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
- $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
if (defined $eml) {
+ my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
+ $vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
$self->input_eml_cb($eml, $vmd);
- } elsif ($vmd) { # old message, kw only
- my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
- my @docids = $self->{lse}->over->blob_exists($oid) or return;
- $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
- my $lei = $self->{lei};
- $lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
- $self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
- $vmd, \@docids);
+ } elsif (my $ikw = $self->{lei}->{ikw}) { # old message, kw only
+ # we send $uri as a bare SCALAR and not a URIimap ref to
+ # reduce socket traffic:
+ $ikw->wq_io_do('ck_update_kw', [], $$uri, $uid, $kw);
}
}
@@ -71,15 +67,17 @@ sub do_import_index ($$@) {
$lei->ale; # initialize for workers to read
my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ my $ikw;
if (my $net = $lei->{net}) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
$net->{-lms_ro} = $sto->search->lms // 0;
- if ($self->{-import_kw}) {
+ if ($self->{-import_kw} && $net->{-lms_ro} &&
+ $net->{imap_order}) {
+ require PublicInbox::LeiImportKw;
+ $ikw = PublicInbox::LeiImportKw->new($lei);
$net->{each_old} = 1;
- $self->{-lms_ro} = $net->{-lms_ro};
- $self->{lse} = $sto->search;
}
}
} else {
@@ -93,6 +91,7 @@ sub do_import_index ($$@) {
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
+ $ikw->wq_close(1) if $ikw;
net_merge_all_done($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
new file mode 100644
index 00000000..e13dce07
--- /dev/null
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -0,0 +1,54 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# WQ worker for dealing with LeiImport IMAP flags on already-imported messages
+# WQ key: {ikw}
+package PublicInbox::LeiImportKw;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+
+sub new {
+ my ($cls, $lei) = @_;
+ my $self = bless { -wq_ident => 'lei import_kw worker' }, $cls;
+ my ($op_c, $ops) = $lei->workers_start($self, $self->detect_nproc);
+ $op_c->{ops} = $ops; # for PktOp->event_step
+ $lei->{ikw} = $self;
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->_lei_atfork_child;
+ my $net = delete $lei->{net} // die 'BUG: no lei->{net}';
+ $self->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
+ $self->{verbose} = $lei->{opt}->{verbose};
+ $self->{lse} = $self->{sto}->search;
+ $self->{over} = $self->{lse}->over;
+ $self->{-lms_ro} = $net->{-lms_ro} || die 'BUG: net->{-lms_ro} FALSE';
+ $self->SUPER::ipc_atfork_child;
+}
+
+sub ck_update_kw { # via wq_io_do
+ my ($self, $url, $uid, $kw) = @_;
+ my $oidbin = $self->{-lms_ro}->imap_oidbin($url, $uid) // return;
+ my @docids = $self->{over}->oidbin_exists($oidbin) or return;
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+ $self->{verbose} and
+ $self->{lei}->qerr('# '.unpack('H*', $oidbin)." => @$kw\n");
+ $self->{sto}->ipc_do('set_eml_vmd', undef, { kw => $kw }, \@docids);
+}
+
+sub ikw_done_wait {
+ my ($lei) = @_;
+ my $wait = $lei->{sto}->ipc_do('done');
+ $lei->wq_done_wait;
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+ my ($lei) = @_;
+ my $ikw = delete $lei->{ikw} or return $lei->fail;
+ $ikw->wq_wait_old(\&ikw_done_wait, $lei);
+}
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 22ee1109..75603d89 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS blob2num (
UNIQUE (oidbin, fid, uid)
)
+ # speeds up LeiImport->ck_update_kw (for "lei import") by 5-6x:
+ $dbh->do(<<'');
+CREATE INDEX IF NOT EXISTS idx_fid_uid ON blob2num(fid,uid)
+
$dbh->do(<<'');
CREATE TABLE IF NOT EXISTS blob2name (
oidbin VARBINARY NOT NULL,
@@ -361,15 +365,14 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
-sub imap_oid2 ($$$) {
- my ($self, $uri, $uid) = @_; # $uri MUST have UIDVALIDITY
- my $fid = $self->{fmap}->{"$uri"} //= fid_for($self, "$uri") // return;
+sub imap_oidbin ($$$) {
+ my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY
+ my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return;
my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
EOM
$sth->execute($fid, $uid);
- my ($oidbin) = $sth->fetchrow_array;
- $oidbin ? unpack('H*', $oidbin) : undef;
+ $sth->fetchrow_array;
}
sub imap_oid {
@@ -384,10 +387,10 @@ sub imap_oid {
}
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
- imap_oid2($self, $folders->[0], $uid_uri->uid);
+ my $oidbin = imap_oidbin($self, $folders->[0], $uid_uri->uid);
+ $oidbin ? unpack('H*', $oidbin) : undef;
}
-
# FIXME: something with "lei <up|q>" is causing uncommitted transaction
# warnings, not sure what...
sub DESTROY {
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 39a8f7fc..058f4313 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -471,6 +471,7 @@ sub each_old_flags ($$$$) {
while (my ($uid, $per_uid) = each %$r) {
my $kw = flags2kw($self, $uri, $uid, $per_uid->{FLAGS})
// next;
+ # LeiImport->input_net_cb
$eml_cb->($uri, $uid, $kw, undef, @args);
}
}
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 0e191c47..58fdea0e 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -349,13 +349,13 @@ sub check_inodes {
}
}
-sub blob_exists {
- my ($self, $oidhex) = @_;
+sub oidbin_exists {
+ my ($self, $oidbin) = @_;
if (wantarray) {
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
my $tmp = $sth->fetchall_arrayref;
map { $_->[0] } @$tmp;
@@ -363,10 +363,12 @@ SELECT docid FROM xref3 WHERE oidbin = ? ORDER BY docid ASC
my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
SELECT COUNT(*) FROM xref3 WHERE oidbin = ?
- $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
$sth->execute;
$sth->fetchrow_array;
}
}
+sub blob_exists { oidbin_exists($_[0], pack('H*', $_[1])) }
+
1;
^ permalink raw reply related [relevance 40%]
* [PATCH] lei export-kw: do not write directly to mail_sync.sqlite3
@ 2021-06-02 10:03 58% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-02 10:03 UTC (permalink / raw)
To: meta
Only the lei/store process should be writing to files/DBs
in lei/store.
---
lib/PublicInbox/LeiExportKw.pm | 21 +++++++++++----------
lib/PublicInbox/LeiStore.pm | 23 +++++++++++++++++++----
2 files changed, 30 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 92c3aa43..b31b065f 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -28,6 +28,7 @@ sub export_kw_md { # LeiMailSync->each_src callback
PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown);
my $dst = "$mdir/cur/$bn";
my @fail;
+ my $lei = $self->{lei};
for my $d (@try) {
my $src = "$mdir/$d/$$id";
next if $src eq $dst;
@@ -39,15 +40,15 @@ sub export_kw_md { # LeiMailSync->each_src callback
# unlink(2) may ENOENT from parallel invocation,
# ignore it, but not other serious errors
if (!unlink($src) and $! != ENOENT) {
- $self->{lei}->child_error(1,
- "E: unlink($src): $!");
+ $lei->child_error(1, "E: unlink($src): $!");
}
- $self->{lms}->mv_src("maildir:$mdir",
- $oidbin, $id, $bn) or die;
+ $lei->{sto}->ipc_do('lms_mv_src', "maildir:$mdir",
+ $oidbin, $id, $bn);
return; # success anyways if link(2) worked
}
if ($! == ENOENT && !-e $src) { # some other process moved it
- $self->{lms}->clear_src("maildir:$mdir", $id);
+ $lei->{sto}->ipc_do('lms_clear_src',
+ "maildir:$mdir", $id);
next;
}
push @fail, $src if $! != EEXIST;
@@ -56,7 +57,7 @@ sub export_kw_md { # LeiMailSync->each_src callback
# both tries failed
my $e = $!;
my $orig = '['.join('|', @fail).']';
- $self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
+ $lei->child_error(1, "link($orig, $dst) ($oidhex): $e");
}
sub export_kw_imap { # LeiMailSync->each_src callback
@@ -69,8 +70,7 @@ sub export_kw_imap { # LeiMailSync->each_src callback
# overrides PublicInbox::LeiInput::input_path_url
sub input_path_url {
my ($self, $input, @args) = @_;
- my $lms = $self->{lms} //= $self->{lse}->lms;
- $lms->lms_begin;
+ my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
@@ -81,7 +81,7 @@ sub input_path_url {
$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
$mic->expunge;
} else { die "BUG: $input not supported" }
- $lms->lms_commit;
+ my $wait = $self->{lei}->{sto}->ipc_do('done');
}
sub lei_export_kw {
@@ -151,8 +151,9 @@ EOM
$self->{imap_mod_kw} = $net->can($self->{-merge_kw} ?
'imap_add_kw' : 'imap_set_kw');
}
- undef $lms;
+ undef $lms; # for fork
my $ops = {};
+ $sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 6888afb4..821782b9 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -193,15 +193,30 @@ sub remove_eml_vmd { # remove just the VMD
\@docids;
}
-sub set_sync_info {
- my ($self, $oidhex, $folder, $id) = @_;
- ($self->{lms} //= do {
+sub _lms_rw ($) {
+ my ($self) = @_;
+ $self->{lms} //= do {
require PublicInbox::LeiMailSync;
my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
my $lms = PublicInbox::LeiMailSync->new($f);
$lms->lms_begin;
$lms;
- })->set_src($oidhex, $folder, $id);
+ };
+}
+
+sub lms_clear_src {
+ my ($self, $folder, $id) = @_;
+ _lms_rw($self)->clear_src($folder, $id);
+}
+
+sub lms_mv_src {
+ my ($self, $folder, $oidbin, $id, $newbn) = @_;
+ _lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn);
+}
+
+sub set_sync_info {
+ my ($self, $oidhex, $folder, $id) = @_;
+ _lms_rw($self)->set_src($oidhex, $folder, $id);
}
sub _remove_if_local { # git->cat_async arg
^ permalink raw reply related [relevance 58%]
* [PATCH] lei: remove "forget" (old name for "rm")
@ 2021-06-01 19:20 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-06-01 19:20 UTC (permalink / raw)
To: meta
"rm" is probably the better name for it, since it matches
"public-inbox-learn rm"
---
lib/PublicInbox/LEI.pm | 3 ---
1 file changed, 3 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3527cf09..30f90798 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -218,9 +218,6 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt,
pass_through('-kw:foo for delete') ],
-'forget' => [ '[--stdin|--oid=OID|--by-mid=MID]',
- "exclude message(s) on stdin from `q' search results",
- qw(stdin| oid=s exact by-mid|mid:s), @c_opt ],
'purge-mailsource' => [ 'LOCATION|--all',
'remove imported messages from IMAP, Maildirs, and MH',
^ permalink raw reply related [relevance 71%]
* [PATCH] lei import: reduce writes to lei/store on IMAP sync
@ 2021-05-31 10:20 86% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-31 10:20 UTC (permalink / raw)
To: meta
We don't need to write VMD changes to lei/store if local
keywords are unchanged.
---
lib/PublicInbox/LeiImport.pm | 1 +
lib/PublicInbox/LeiSearch.pm | 8 +++++---
lib/PublicInbox/NetReader.pm | 2 +-
3 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index f9a46ec5..860a2c98 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -50,6 +50,7 @@ sub input_net_cb { # imap_each / nntp_each
} elsif ($vmd) { # old message, kw only
my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
my @docids = $self->{lse}->over->blob_exists($oid) or return;
+ $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
my $lei = $self->{lei};
$lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
$self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index b09d1e45..d0963e92 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -133,9 +133,11 @@ sub xoids_for {
# returns true if $eml is indexed by lei/store and keywords don't match
sub kw_changed {
my ($self, $eml, $new_kw_sorted, $docids) = @_;
- my $xoids = xoids_for($self, $eml) // return;
- $docids //= [];
- @$docids = sort { $a <=> $b } values %$xoids;
+ if ($eml) {
+ my $xoids = xoids_for($self, $eml) // return;
+ $docids //= [];
+ @$docids = sort { $a <=> $b } values %$xoids;
+ }
my $cur_kw = msg_keywords($self, $docids->[0]);
# RFC 5550 sec 5.9 on the $Forwarded keyword states:
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index b97444fd..39a8f7fc 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -409,7 +409,7 @@ sub flags2kw ($$$$) {
warn "# unknown IMAP flag $f <$uri/;UID=$uid>\n";
}
}
- @$kw = sort @$kw; # for all UI/UX purposes
+ @$kw = sort @$kw; # for LeiSearch->kw_changed and UI/UX purposes
$kw;
}
^ permalink raw reply related [relevance 86%]
* Re: [PATCH 0/4] lei lcat usability things
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
` (3 preceding siblings ...)
2021-05-30 6:33 61% ` [PATCH 4/4] lei: support implicit stdin by default Eric Wong
@ 2021-05-30 11:52 71% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 11:52 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> Nothing major, just some things to make lcat nicer for the
> inevitable network outages to come
>
> Eric Wong (4):
> lei lcat+inspect: start wiring up completion
> lei lcat: allow IMAP folder URLs w/o UIDVALIDITY
Btw, "lei lcat $URL" is great in the case adding a label/keyword
with "lei import" was forgotten:
lei import $URL
# oops, I forgot to add +L:somelabel to the above import
# invocation
lei lcat $URL -f mboxrd | lei tag -f mboxrd - +L:somelabel
Running "lei tag $URL +L:somelabel" directly works, too,
but that's extra network traffic + latency.
^ permalink raw reply [relevance 71%]
* [PATCH] lei import: import IMAP flag changes from old messages
@ 2021-05-30 11:45 42% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 11:45 UTC (permalink / raw)
To: meta
This makes "lei import" behavior with IMAP folders more
consistent with that with Maildir.
Opening IMAP folders read-write with "SELECT" (instead of
read-only with "EXAMINE") was necessary, since it lets an IMAP
server communicate to us as to whether or not it's worth
refetching IMAP flags of previously imported messages.
Fetching UID+FLAGS only is one of the fastest IMAP operations
with dovecot, our -imapd and presumably other common IMAP servers.
It is issued by common MUAs such as mutt after every SELECT.
Users may now rely on "lei import" exclusively to merge mail and
keywords into lei/store, and "lei export-kw" to propagate
keyword changes back to IMAP servers.
A sticks-and-stones workflow for personal mailboxes is currently:
lei import imaps://$MY_PERSONAL_INBOX
lei q --mua=$MUA -o /tmp/results SEARCH TERMS...
# do stuff from within $MUA to /tmp/results
lei import /tmp/results # read keyword changes from MUA
lei export-kw imaps://$MY_PERSONAL_INBOX
# repeat when new stuff shows up in personal inbox
The next goal is to automate repeated imports + export-kw
commands with with inotify and IMAP IDLE.
---
lib/PublicInbox/LeiImport.pm | 18 ++++++++-
lib/PublicInbox/LeiMailSync.pm | 21 +++++++----
lib/PublicInbox/NetReader.pm | 68 +++++++++++++++++++++++++++++-----
3 files changed, 88 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 01e6c93c..f9a46ec5 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -45,7 +45,16 @@ sub input_net_cb { # imap_each / nntp_each
my ($uri, $uid, $kw, $eml, $self) = @_;
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
$vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
- $self->input_eml_cb($eml, $vmd);
+ if (defined $eml) {
+ $self->input_eml_cb($eml, $vmd);
+ } elsif ($vmd) { # old message, kw only
+ my $oid = $self->{-lms_ro}->imap_oid2($uri, $uid) // return;
+ my @docids = $self->{lse}->over->blob_exists($oid) or return;
+ my $lei = $self->{lei};
+ $lei->qerr("# $oid => @$kw\n") if $lei->{opt}->{verbose};
+ $self->{lei}->{sto}->ipc_do('set_eml_vmd', undef,
+ $vmd, \@docids);
+ }
}
sub do_import_index ($$@) {
@@ -65,7 +74,12 @@ sub do_import_index ($$@) {
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
- $net->{-lms_ro} = $lei->_lei_store->search->lms // 0;
+ $net->{-lms_ro} = $sto->search->lms // 0;
+ if ($self->{-import_kw}) {
+ $net->{each_old} = 1;
+ $self->{-lms_ro} = $net->{-lms_ro};
+ $self->{lse} = $sto->search;
+ }
}
} else {
my $nproc = $self->detect_nproc;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index c7f78239..36cd564c 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -361,6 +361,17 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
+sub imap_oid2 ($$$) {
+ my ($self, $uri, $uid) = @_; # $uri MUST have UIDVALIDITY
+ my $fid = $self->{fmap}->{"$uri"} //= fid_for($self, "$uri") // return;
+ my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
+EOM
+ $sth->execute($fid, $uid);
+ my ($oidbin) = $sth->fetchrow_array;
+ $oidbin ? unpack('H*', $oidbin) : undef;
+}
+
sub imap_oid {
my ($self, $lei, $uid_uri) = @_;
my $mailbox_uri = $uid_uri->clone;
@@ -373,16 +384,10 @@ sub imap_oid {
}
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
- my $fid = $self->{fmap}->{$folders->[0]} //=
- fid_for($self, $folders->[0]) // return;
- my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
-SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
-EOM
- $sth->execute($fid, $uid_uri->uid);
- my ($oidbin) = $sth->fetchrow_array;
- $oidbin ? unpack('H*', $oidbin) : undef;
+ imap_oid2($self, $folders->[0], $uid_uri->uid);
}
+
# FIXME: something with "lei <up|q>" is causing uncommitted transaction
# warnings, not sure what...
sub DESTROY {
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 54c6b082..b97444fd 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -396,10 +396,8 @@ sub errors {
undef;
}
-sub _imap_do_msg ($$$$$) {
- my ($self, $uri, $uid, $raw, $flags) = @_;
- # our target audience expects LF-only, save storage
- $$raw =~ s/\r\n/\n/sg;
+sub flags2kw ($$$$) {
+ my ($self, $uri, $uid, $flags) = @_;
my $kw = [];
for my $f (split(/ /, $flags)) {
if (my $k = $IMAPflags2kw{$f}) {
@@ -412,6 +410,14 @@ sub _imap_do_msg ($$$$$) {
}
}
@$kw = sort @$kw; # for all UI/UX purposes
+ $kw;
+}
+
+sub _imap_do_msg ($$$$$) {
+ my ($self, $uri, $uid, $raw, $flags) = @_;
+ # our target audience expects LF-only, save storage
+ $$raw =~ s/\r\n/\n/sg;
+ my $kw = flags2kw($self, $uri, $uid, $flags) // return;
my ($eml_cb, @args) = @{$self->{eml_each}};
$eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args);
}
@@ -447,17 +453,56 @@ sub itrk_last ($$;$$) {
($itrk, $l_uid, $l_uidval //= $r_uidval);
}
+# import flags of already-seen messages
+sub each_old_flags ($$$$) {
+ my ($self, $mic, $uri, $l_uid) = @_;
+ $l_uid ||= 1;
+ my $sec = uri_section($uri);
+ my $bs = $self->{imap_opt}->{$sec}->{batch_size} // 10000;
+ my ($eml_cb, @args) = @{$self->{eml_each}};
+ for (my $n = 1; $n <= $l_uid; $n += $bs) {
+ my $end = $n + $bs;
+ $end = $l_uid if $end > $l_uid;
+ my $r = $mic->fetch_hash("$n:$end", 'FLAGS');
+ if (!$r) {
+ return if $!{EINTR} && $self->{quit};
+ return "E: $uri UID FETCH $n:$end error: $!";
+ }
+ while (my ($uid, $per_uid) = each %$r) {
+ my $kw = flags2kw($self, $uri, $uid, $per_uid->{FLAGS})
+ // next;
+ $eml_cb->($uri, $uid, $kw, undef, @args);
+ }
+ }
+}
+
+# returns true if PERMANENTFLAGS indicates FLAGS of already imported
+# messages are meaningful
+sub perm_fl_ok ($) {
+ my ($perm_fl) = @_;
+ return if !defined($perm_fl);
+ for my $f (split(/[ \t]+/, $perm_fl)) {
+ return 1 if $IMAPflags2kw{$f};
+ }
+ undef;
+}
+
sub _imap_fetch_all ($$$) {
my ($self, $mic, $orig_uri) = @_;
my $sec = uri_section($orig_uri);
my $mbx = $orig_uri->mailbox;
$mic->Clear(1); # trim results history
- $mic->examine($mbx) or return "E: EXAMINE $mbx ($sec) failed: $!";
- my ($r_uidval, $r_uidnext);
+
+ # we need to check for mailbox writability to see if we care about
+ # FLAGS from already-imported messages.
+ my $cmd = $self->{each_old} ? 'select' : 'examine';
+ $mic->$cmd($mbx) or return "E: \U$cmd\E $mbx ($sec) failed: $!";
+
+ my ($r_uidval, $r_uidnext, $perm_fl);
for ($mic->Results) {
+ /^\* OK \[PERMANENTFLAGS \(([^\)]*)\)\].*/ and $perm_fl = $1;
/^\* OK \[UIDVALIDITY ([0-9]+)\].*/ and $r_uidval = $1;
/^\* OK \[UIDNEXT ([0-9]+)\].*/ and $r_uidnext = $1;
- last if $r_uidval && $r_uidnext;
}
$r_uidval //= $mic->uidvalidity($mbx) //
return "E: $orig_uri cannot get UIDVALIDITY";
@@ -486,6 +531,13 @@ EOF
E: $uri local UID exceeds remote ($l_uid > $r_uid)
E: $uri strangely, UIDVALIDLITY matches ($l_uidval)
EOF
+ $mic->Uid(1); # the default, we hope
+ my $err;
+ if (!defined($single_uid) && $self->{each_old} &&
+ perm_fl_ok($perm_fl)) {
+ $err = each_old_flags($self, $mic, $uri, $l_uid);
+ return $err if $err;
+ }
return if $l_uid >= $r_uid; # nothing to do
$l_uid ||= 1;
my ($mod, $shard) = @{$self->{shard_info} // []};
@@ -493,13 +545,11 @@ EOF
my $m = $mod ? " [(UID % $mod) == $shard]" : '';
warn "# $uri fetching UID $l_uid:$r_uid$m\n";
}
- $mic->Uid(1); # the default, we hope
my $bs = $self->{imap_opt}->{$sec}->{batch_size} // 1;
my $req = $mic->imap4rev1 ? 'BODY.PEEK[]' : 'RFC822.PEEK';
my $key = $req;
$key =~ s/\.PEEK//;
my ($uids, $batch);
- my $err;
do {
# I wish "UID FETCH $START:*" could work, but:
# 1) servers do not need to return results in any order
^ permalink raw reply related [relevance 42%]
* [PATCH 3/4] lei lcat: support maildir: paths, too
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
2021-05-30 6:33 68% ` [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY Eric Wong
@ 2021-05-30 6:33 63% ` Eric Wong
2021-05-30 6:33 61% ` [PATCH 4/4] lei: support implicit stdin by default Eric Wong
2021-05-30 11:52 71% ` [PATCH 0/4] lei lcat usability things Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
This could be helpful in case when a Maildir is on a slow
or unmounted filesystem and lei/store is on fast storage.
---
lib/PublicInbox/LeiLcat.pm | 37 +++++++++++++++++++++++--------------
1 file changed, 23 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 5bd20c1f..cb5eb5b4 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -11,10 +11,26 @@ use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
use PublicInbox::MID qw($MID_EXTRACT);
+sub lcat_folder ($$$) {
+ my ($lei, $lms, $folder) = @_;
+ $lms //= $lei->{lse}->lms // return;
+ my $folders = [ $folder];
+ my $err = $lms->arg2folder($lei, $folders);
+ $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
+ if ($err && $err->{fail}) {
+ $lei->child_error(1 << 8, "# unknown folder: $folder");
+ } else {
+ for my $f (@$folders) {
+ my $fid = $lms->fid_for($f);
+ push @{$lei->{lcat_fid}}, $fid;
+ }
+ }
+}
+
sub lcat_imap_uri ($$) {
my ($lei, $uri) = @_;
my $lms = $lei->{lse}->lms or return;
- # cf. LeiToMail->wq_atexit_child
+ # cf. LeiXsearch->lcat_dump
if (defined $uri->uid) {
my $oidhex = $lms->imap_oid($lei, $uri);
if (ref(my $err = $oidhex)) { # art2folder error
@@ -24,17 +40,7 @@ sub lcat_imap_uri ($$) {
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
push @{$lei->{lcat_fid}}, $fid;
} else {
- my $folders = [ $$uri ];
- my $err = $lms->arg2folder($lei, $folders);
- $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
- if ($err && $err->{fail}) {
- $lei->child_error(1 << 8, "# unknown folder: $uri");
- } else {
- for my $f (@$folders) {
- my $fid = $lms->fid_for($f);
- push @{$lei->{lcat_fid}}, $fid;
- }
- }
+ lcat_folder($lei, $lms, $$uri);
}
}
@@ -44,7 +50,10 @@ sub extract_1 ($$) {
my $u = $1;
require PublicInbox::URIimap;
lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
- '""'; # blank query, using {lcat_blob} or {lcat
+ '""'; # blank query, using {lcat_blob} or {lcat_fid}
+ } elsif ($x =~ m!\b(maildir:.+)!i) {
+ lcat_folder($lei, undef, $1);
+ '""'; # blank query, using {lcat_blob} or {lcat_fid}
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
@@ -147,7 +156,7 @@ sub _complete_lcat {
my $sto = $lei->_lei_store or return;
my $lms = $sto->search->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
- grep(m!\A[a-z]+://!, map { $match_cb->($_) } $lms->folders);
+ map { $match_cb->($_) } $lms->folders;
}
1;
^ permalink raw reply related [relevance 63%]
* [PATCH 4/4] lei: support implicit stdin by default
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
` (2 preceding siblings ...)
2021-05-30 6:33 63% ` [PATCH 3/4] lei lcat: support maildir: paths, too Eric Wong
@ 2021-05-30 6:33 61% ` Eric Wong
2021-05-30 11:52 71% ` [PATCH 0/4] lei lcat usability things Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
This adds implicit stdin suppport for p2q and lcat,
while rm and rediff no longer need explicit support
for it.
---
lib/PublicInbox/LEI.pm | 12 ++++++++++--
lib/PublicInbox/LeiRediff.pm | 1 -
lib/PublicInbox/LeiRm.pm | 1 -
t/lei-p2q.t | 4 ++++
4 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f2dfc320..3527cf09 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -638,7 +638,15 @@ sub optparse ($$$) {
my $ok;
for my $o (@or) {
if ($o =~ /\A--([a-z0-9\-]+)/) {
- $ok = defined($OPT->{$1});
+ my $sw = $1;
+ # assume pipe/regular file on stdin
+ # w/o args means stdin
+ if ($sw eq 'stdin' && !@$argv &&
+ (-p $self->{0} ||
+ -f _) && -r _) {
+ $OPT->{stdin} //= 1;
+ }
+ $ok = defined($OPT->{$sw});
last if $ok;
} elsif (defined($argv->[$i])) {
$ok = 1;
@@ -906,7 +914,7 @@ sub start_mua {
}
push @cmd, $mfolder unless defined($replaced);
if ($self->{sock}) { # lei(1) client process runs it
- # restore terminal: echo $query | lei q -stdin --mua=...
+ # restore terminal: echo $query | lei q --stdin --mua=...
my $io = [];
$io->[0] = $self->{1} if $self->{opt}->{stdin} && -t $self->{1};
send_exec_cmd($self, $io, \@cmd, {});
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 2e793df5..c8bd0dfb 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -201,7 +201,6 @@ sub input_eml_cb { # callback for all emails
sub lei_rediff {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
- $lei->{opt}->{stdin} = 1 if !@inputs;
$lei->{opt}->{'in-format'} //= 'eml';
# maybe it's a non-email (code) blob from a coderepo
my $git_dirs = $lei->{opt}->{'git-dir'} //= [];
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 185b6a15..c6d28045 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -31,7 +31,6 @@ sub input_maildir_cb {
sub lei_rm {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
- $lei->{opt}->{stdin} = 1 if !@inputs;
$lei->{opt}->{'in-format'} //= 'eml';
my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
$self->prepare_inputs($lei, \@inputs) or return;
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index f8b073cf..58506f94 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -14,6 +14,10 @@ test_lei(sub {
lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin') or diag $lei_err;
+ sysseek($fh, 0, 0) or xbail "lseek: $!";
+ lei_ok([qw(p2q -w dfpost)], undef, { %$lei_opt, 0 => $fh });
+ is($lei_out, "dfpost:6e006fd73b1d\n", 'implicit --stdin');
+
lei_ok(qw(p2q --uri t/data/0001.patch -w), 'dfpost,dfn');
is($lei_out, "dfpost%3A6e006fd73b1d+".
"dfn%3Alib%2FPublicInbox%2FSearch.pm\n",
^ permalink raw reply related [relevance 61%]
* [PATCH 0/4] lei lcat usability things
@ 2021-05-30 6:33 71% Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
Nothing major, just some things to make lcat nicer for the
inevitable network outages to come
Eric Wong (4):
lei lcat+inspect: start wiring up completion
lei lcat: allow IMAP folder URLs w/o UIDVALIDITY
lei lcat: support maildir:... paths, too
lei: support implicit stdin by default
lib/PublicInbox/LEI.pm | 12 ++++++++++--
lib/PublicInbox/LeiInspect.pm | 8 ++++++++
lib/PublicInbox/LeiLcat.pm | 33 ++++++++++++++++++++++++++++++---
lib/PublicInbox/LeiRediff.pm | 1 -
lib/PublicInbox/LeiRm.pm | 1 -
t/lei-import-imap.t | 3 +++
t/lei-p2q.t | 4 ++++
7 files changed, 55 insertions(+), 7 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/4] lei lcat+inspect: start wiring up completion
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
@ 2021-05-30 6:33 71% ` Eric Wong
2021-05-30 6:33 68% ` [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
Colons and other delimiters still cause problems for our bash
completion, but some completion is better than no completion.
---
lib/PublicInbox/LeiInspect.pm | 8 ++++++++
lib/PublicInbox/LeiLcat.pm | 8 ++++++++
2 files changed, 16 insertions(+)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 7205979e..eb2634b4 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -99,4 +99,12 @@ sub lei_inspect {
$lei->out(']') if $multi;
}
+sub _complete_inspect {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ map { $match_cb->($_) } $lms->folders;
+}
+
1;
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index effc3682..81ab1e36 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -132,4 +132,12 @@ no args allowed on command-line with --stdin
$lei->_start_query;
}
+sub _complete_lcat {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ grep(m!\A[a-z]+://!, map { $match_cb->($_) } $lms->folders);
+}
+
1;
^ permalink raw reply related [relevance 71%]
* [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
@ 2021-05-30 6:33 68% ` Eric Wong
2021-05-30 6:33 63% ` [PATCH 3/4] lei lcat: support maildir: paths, too Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-30 6:33 UTC (permalink / raw)
To: meta
Requiring UIDVALIDITY on the command-line is of course
unreasonable.
---
lib/PublicInbox/LeiLcat.pm | 12 +++++++++++-
t/lei-import-imap.t | 3 +++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 81ab1e36..5bd20c1f 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -24,7 +24,17 @@ sub lcat_imap_uri ($$) {
} elsif (defined(my $fid = $lms->fid_for($$uri))) {
push @{$lei->{lcat_fid}}, $fid;
} else {
- $lei->child_error(1 << 8, "# unknown folder: $uri");
+ my $folders = [ $$uri ];
+ my $err = $lms->arg2folder($lei, $folders);
+ $lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
+ if ($err && $err->{fail}) {
+ $lei->child_error(1 << 8, "# unknown folder: $uri");
+ } else {
+ for my $f (@$folders) {
+ my $fid = $lms->fid_for($f);
+ push @{$lei->{lcat_fid}}, $fid;
+ }
+ }
}
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 895b19ff..34fd6cf9 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -13,6 +13,7 @@ my $host_port = tcp_host_port($sock);
undef $sock;
test_lei({ tmpdir => $tmpdir }, sub {
my $url = "imap://$host_port/t.v2.0";
+ my $url_orig = $url;
lei_ok(qw(q z:1..));
my $out = json_utf8->decode($lei_out);
@@ -100,6 +101,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok qw(lcat -f json), $uid_url;
$out = json_utf8->decode($lei_out);
is(scalar(@$out), 2, 'got JSON') or diag explain($out);
+ lei_ok qw(lcat), $url_orig;
+ is($lei_out, $orig, 'lcat w/o UID works');
});
done_testing;
^ permalink raw reply related [relevance 68%]
* [PATCH 0/3] lei: v2 and lcat/import/IMAP things
@ 2021-05-29 20:20 71% Eric Wong
2021-05-29 20:20 36% ` [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support Eric Wong
2021-05-29 20:20 61% ` [PATCH 3/3] lei q: --sort and --save|v2 are incompatible Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-29 20:20 UTC (permalink / raw)
To: meta
Been using lei more and more for IMAP things.
Eric Wong (3):
TODO: add pipelining note
lei import|lcat: improve+fix single message IMAP support
lei q: --sort and --save|v2 are incompatible
TODO | 2 ++
lib/PublicInbox/LeiLcat.pm | 25 ++++++++------
lib/PublicInbox/LeiMailSync.pm | 23 ++++++++-----
lib/PublicInbox/LeiQuery.pm | 5 ++-
lib/PublicInbox/LeiToMail.pm | 18 +++-------
lib/PublicInbox/LeiXSearch.pm | 60 +++++++++++++++++++++++++++++++---
lib/PublicInbox/NetReader.pm | 6 ++--
t/lei-import-imap.t | 13 +++++++-
8 files changed, 112 insertions(+), 40 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei q: --sort and --save|v2 are incompatible
2021-05-29 20:20 71% [PATCH 0/3] lei: v2 and lcat/import/IMAP things Eric Wong
2021-05-29 20:20 36% ` [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support Eric Wong
@ 2021-05-29 20:20 61% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-29 20:20 UTC (permalink / raw)
To: meta
Saved searches rely on (reverse) docid ordering for efficient
incremental results, and sorting any other way prevents that.
Update comment description in LeiQuery while we're at it:
"ls-query" and "rm-query" are "ls-search" and "forget-search",
respectively, and "mv-query" is implicit with "edit-search"
---
lib/PublicInbox/LeiQuery.pm | 5 ++++-
lib/PublicInbox/LeiToMail.pm | 12 +++++-------
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 0435a516..eb7b98d4 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -1,7 +1,8 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# handles lei <q|ls-query|rm-query|mv-query> commands
+# handles "lei q" command and provides internals for
+# several other sub-commands (up, lcat, ...)
package PublicInbox::LeiQuery;
use strict;
use v5.10.1;
@@ -130,6 +131,8 @@ sub lei_q {
} else {
die "unrecognized --sort=$sort\n";
}
+ $opt->{save} and return
+ $self->fail('--save and --sort are incompatible');
}
$self->{mset_opt} = \%mset_opt;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 997ce599..5b5caee7 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -384,6 +384,7 @@ sub new {
my $fmt = $lei->{ovv}->{fmt};
my $dst = $lei->{ovv}->{dst};
my $self = bless {}, $cls;
+ my @conflict;
if ($fmt eq 'maildir') {
require PublicInbox::MdirReader;
$self->{base_type} = 'maildir';
@@ -412,13 +413,14 @@ sub new {
require PublicInbox::LeiViewText;
$lei->{lvt} = PublicInbox::LeiViewText->new($lei);
$self->{base_type} = 'text';
+ @conflict = qw(mua save);
} elsif ($fmt eq 'v2') {
die "--dedupe=oid and v2 are incompatible\n" if
($lei->{opt}->{dedupe}//'') eq 'oid';
$self->{base_type} = 'v2';
$lei->{opt}->{save} = \1;
- die "--mua incompatible with v2\n" if $lei->{opt}->{mua};
$dst = $lei->{ovv}->{dst} = $lei->abs_path($dst);
+ @conflict = qw(mua sort);
} else {
die "bad mail --format=$fmt\n";
}
@@ -426,12 +428,8 @@ sub new {
(-d $dst || (-e _ && !-w _)) and die
"$dst exists and is not a writable file\n";
}
- if ($self->{base_type} eq 'text') {
- my @err = map {
- defined($lei->{opt}->{$_}) ? "--$_" : ();
- } (qw(mua save));
- die "@err incompatible with $fmt\n" if @err;
- }
+ my @err = map { defined($lei->{opt}->{$_}) ? "--$_" : () } @conflict;
+ die "@err incompatible with $fmt\n" if @err;
$self->{dst} = $dst;
$lei->{dedupe} = $lei->{lss} // do {
my $dd_cls = 'PublicInbox::'.
^ permalink raw reply related [relevance 61%]
* [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support
2021-05-29 20:20 71% [PATCH 0/3] lei: v2 and lcat/import/IMAP things Eric Wong
@ 2021-05-29 20:20 36% ` Eric Wong
2021-05-29 20:20 61% ` [PATCH 3/3] lei q: --sort and --save|v2 are incompatible Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-29 20:20 UTC (permalink / raw)
To: meta
lcat can now dump the memoized contents of entire IMAP folders,
not just a single UID. It's now parallelized and pipelined for
multiple lei2mail workers.
Furthemore, various forms of JSON output work consistently
with blob-only output, now.
While working on this, I noticed NetReader was passing UID URLs
to imap_each callbacks, which was causing mail_sync.sqlite3 to
store UIDs in `folders' and clearly wrong so it's now fixed.
---
lib/PublicInbox/LeiLcat.pm | 25 ++++++++------
lib/PublicInbox/LeiMailSync.pm | 23 ++++++++-----
lib/PublicInbox/LeiToMail.pm | 6 ----
lib/PublicInbox/LeiXSearch.pm | 60 +++++++++++++++++++++++++++++++---
lib/PublicInbox/NetReader.pm | 6 ++--
t/lei-import-imap.t | 13 +++++++-
6 files changed, 101 insertions(+), 32 deletions(-)
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index f9d9633a..effc3682 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -11,14 +11,21 @@ use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_imap_uid_uri ($$) {
- my ($lei, $uid_uri) = @_;
+sub lcat_imap_uri ($$) {
+ my ($lei, $uri) = @_;
my $lms = $lei->{lse}->lms or return;
- my $oidhex = $lms->imap_oid($lei, $uid_uri);
- if (ref(my $err = $oidhex)) { # art2folder error
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ # cf. LeiToMail->wq_atexit_child
+ if (defined $uri->uid) {
+ my $oidhex = $lms->imap_oid($lei, $uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ push @{$lei->{lcat_blob}}, $oidhex;
+ } elsif (defined(my $fid = $lms->fid_for($$uri))) {
+ push @{$lei->{lcat_fid}}, $fid;
+ } else {
+ $lei->child_error(1 << 8, "# unknown folder: $uri");
}
- push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
}
sub extract_1 ($$) {
@@ -26,10 +33,8 @@ sub extract_1 ($$) {
if ($x =~ m!\b(imaps?://[^>]+)!i) {
my $u = $1;
require PublicInbox::URIimap;
- $u = PublicInbox::URIimap->new($u);
- defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
- $lei->child_error(1 << 8, "# no UID= in $u");
- '""'; # blank query, using {lcat_blob}
+ lcat_imap_uri($lei, PublicInbox::URIimap->new($u));
+ '""'; # blank query, using {lcat_blob} or {lcat
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 5c0988b5..c7f78239 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -64,9 +64,9 @@ CREATE TABLE IF NOT EXISTS blob2name (
}
-sub _fid_for {
+sub fid_for {
my ($self, $folder, $rw) = @_;
- my $dbh = $self->{dbh};
+ my $dbh = $self->{dbh} //= dbh_new($self, $rw);
my $sel = 'SELECT fid FROM folders WHERE loc = ? LIMIT 1';
my ($fid) = $dbh->selectrow_array($sel, undef, $folder);
return $fid if defined $fid;
@@ -111,7 +111,7 @@ EOM
sub set_src {
my ($self, $oidhex, $folder, $id) = @_;
- my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
my $sth;
if (ref($id)) { # scalar name
$id = $$id;
@@ -128,7 +128,7 @@ INSERT OR IGNORE INTO blob2num (oidbin, fid, uid) VALUES (?, ?, ?)
sub clear_src {
my ($self, $folder, $id) = @_;
- my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
my $sth;
if (ref($id)) { # scalar name
$id = $$id;
@@ -146,7 +146,7 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ?
# Maildir-only
sub mv_src {
my ($self, $folder, $oidbin, $id, $newbn) = @_;
- my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
my $sth = $self->{dbh}->prepare_cached(<<'');
UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
@@ -158,7 +158,12 @@ sub each_src {
my ($self, $folder, $cb, @args) = @_;
my $dbh = $self->{dbh} //= dbh_new($self);
my ($fid, $sth);
- $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder) // return;
+ if (ref($folder) eq 'HASH') {
+ $fid = $folder->{fid} // die "BUG: no `fid'";
+ } else {
+ $fid = $self->{fmap}->{$folder} //=
+ fid_for($self, $folder) // return;
+ }
$sth = $dbh->prepare('SELECT oidbin,uid FROM blob2num WHERE fid = ?');
$sth->execute($fid);
while (my ($oidbin, $id) = $sth->fetchrow_array) {
@@ -176,7 +181,7 @@ sub location_stats {
my $dbh = $self->{dbh} //= dbh_new($self);
my $fid;
my $ret = {};
- $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder) // return;
+ $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder) // return;
my ($row) = $dbh->selectrow_array(<<"", undef, $fid);
SELECT COUNT(name) FROM blob2name WHERE fid = ?
@@ -349,7 +354,7 @@ sub forget_folder {
my ($self, $folder) = @_;
my ($fid, $sth);
$fid = delete($self->{fmap}->{$folder}) //
- _fid_for($self, $folder) // return;
+ fid_for($self, $folder) // return;
my $dbh = $self->{dbh};
$dbh->do('DELETE FROM blob2name WHERE fid = ?', undef, $fid);
$dbh->do('DELETE FROM blob2num WHERE fid = ?', undef, $fid);
@@ -369,7 +374,7 @@ sub imap_oid {
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
my $fid = $self->{fmap}->{$folders->[0]} //=
- _fid_for($self, $folders->[0]) // return;
+ fid_for($self, $folders->[0]) // return;
my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
EOM
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 078f2551..997ce599 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -756,12 +756,6 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
my $lei = $self->{lei};
- if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
- for my $oid (@{$lei->{lcat_blob}}) {
- my $smsg = { blob => $oid, pct => 100 };
- write_mail($self, $smsg);
- }
- }
delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
my $nr = delete($lei->{-nr_write}) or return;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d6d42a01..f7c1e559 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -256,6 +256,14 @@ sub query_combined_mset { # non-parallel for non-"--threads" users
$lei->{ovv}->ovv_atexit_child($lei);
}
+sub _smsg_fill ($$) {
+ my ($smsg, $eml) = @_;
+ $smsg->populate($eml);
+ $smsg->parse_references($eml, mids($eml));
+ $smsg->{$_} //= '' for qw(from to cc ds subject references mid);
+ delete @$smsg{qw(From Subject -ds -ts)};
+}
+
sub each_remote_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
my $xoids = $lei->{ale}->xoids_for($eml, 1);
@@ -265,10 +273,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
my $smsg = bless {}, 'PublicInbox::Smsg';
$smsg->{blob} = $xoids ? (keys(%$xoids))[0]
: git_sha(1, $eml)->hexdigest;
- $smsg->populate($eml);
- $smsg->parse_references($eml, mids($eml));
- $smsg->{$_} //= '' for qw(from to cc ds subject references mid);
- delete @$smsg{qw(From Subject -ds -ts)};
+ _smsg_fill($smsg, $eml);
wait_startq($lei);
if ($lei->{-progress}) {
++$lei->{-nr_remote_eml};
@@ -453,6 +458,9 @@ sub start_query ($;$) { # always runs in main (lei-daemon) process
for my $uris (@$q) {
$self->wq_io_do('query_remote_mboxrd', [], $uris);
}
+ if ($self->{-do_lcat}) {
+ $self->wq_io_do('lcat_dump', []);
+ }
$self->wq_close(1); # lei_xsearch workers stop when done
}
@@ -518,6 +526,7 @@ sub do_query {
@$end = ();
$self->{opt_threads} = $lei->{opt}->{threads};
$self->{opt_sort} = $lei->{opt}->{'sort'};
+ $self->{-do_lcat} = $lei->{lcat_blob} // $lei->{lcat_fid};
if ($l2m) {
$l2m->net_merge_all_done unless $lei->{auth};
} else {
@@ -561,5 +570,48 @@ sub prepare_external {
push @{$self->{locals}}, $loc;
}
+sub _lcat_i { # LeiMailSync->each_src iterator callback
+ my ($oidbin, $id, $each_smsg) = @_;
+ $each_smsg->({blob => unpack('H*', $oidbin), pct => 100});
+}
+
+sub _lcat2smsg { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $smsg) = @_;
+ if ($bref) {
+ my $eml = PublicInbox::Eml->new($bref);
+ my $json_dump = delete $smsg->{-json_dump};
+ bless $smsg, 'PublicInbox::Smsg';
+ _smsg_fill($smsg, $eml);
+ $json_dump->($smsg, undef, $eml);
+ }
+}
+
+sub lcat_dump {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
+ my $git = $lei->{ale}->git;
+ if (!$lei->{l2m}) {
+ my $json_dump = $each_smsg;
+ $each_smsg = sub {
+ my ($smsg) = @_;
+ use Data::Dumper;
+ $smsg->{-json_dump} = $json_dump;
+ $git->cat_async($smsg->{blob}, \&_lcat2smsg, $smsg);
+ };
+ }
+ for my $oid (@{$lei->{lcat_blob} // []}) {
+ $each_smsg->({ blob => $oid, pct => 100 });
+ }
+ if (my $fids = delete $lei->{lcat_fid}) {
+ my $lms = $lei->{lse}->lms;
+ for my $fid (@$fids) {
+ $lms->each_src({fid => $fid}, \&_lcat_i, $each_smsg);
+ }
+ }
+ $git->async_wait_all;
+ undef $each_smsg; # may commit
+ $lei->{ovv}->ovv_atexit_child($lei);
+}
1;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 76d2fe62..54c6b082 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -471,8 +471,10 @@ EOF
my $uri = $orig_uri->clone;
my $single_uid = $uri->uid;
my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
- $itrk = $l_uid = undef if defined($single_uid);
-
+ if (defined($single_uid)) {
+ $itrk = $l_uid = undef;
+ $uri->uid(undef); # for eml_cb
+ }
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 59d481d5..895b19ff 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -48,6 +48,13 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok($stats->{'uid.min'} < $stats->{'uid.max'}, 'min < max');
ok($stats->{'uid.count'} > 0, 'count > 0');
+ lei_ok('lcat', $url);
+ is(scalar(grep(/^# blob:/, split(/\n/ms, $lei_out))),
+ $stats->{'uid.count'}, 'lcat on URL dumps folder');
+ lei_ok qw(lcat -f json), $url;
+ $out = json_utf8->decode($lei_out);
+ is(scalar(@$out) - 1, $stats->{'uid.count'}, 'lcat JSON dumps folder');
+
lei_ok(qw(q z:1..));
$out = json_utf8->decode($lei_out);
ok(scalar(@$out) > 1, 'got imported messages');
@@ -77,6 +84,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
my $uid_url = "$url/;UID=".$stats->{'uid.max'};
lei_ok 'import', $uid_url;
+ lei_ok 'ls-mail-sync';
+ is($lei_out, "$url\n", 'ls-mail-sync added URL w/o UID');
lei_ok 'inspect', $uid_url;
$lei_out =~ /([a-f0-9]{40,})/ or
xbail 'inspect missed blob with UID URL';
@@ -88,7 +97,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $orig = $lei_out;
lei_ok 'lcat', "blob:$blob";
is($lei_out, $orig, 'lcat understands blob:...');
- ok(!lei('lcat', $url), "lcat doesn't work on IMAP URL w/o UID");
+ lei_ok qw(lcat -f json), $uid_url;
+ $out = json_utf8->decode($lei_out);
+ is(scalar(@$out), 2, 'got JSON') or diag explain($out);
});
done_testing;
^ permalink raw reply related [relevance 36%]
* [PATCH] lei q|up: support v2:/path/to/inboxdir destination
@ 2021-05-28 22:39 39% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 22:39 UTC (permalink / raw)
To: meta
This allows "lei-managed pseudo mailing lists" as described
by Konstantin.
Alternates use is optional and can be enables via --shared.
This doesn't manage or edit ~/.public-inbox/config; presumably
there'll need to be some tweaking of search parameters before
finalizing and making the inbox publicly accessible via HTTP/NNTP.
Link: https://public-inbox.org/meta/20210426164454.5zd5kgugfhfwfkpo@nitro.local/T/
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiOverview.pm | 2 ++
lib/PublicInbox/LeiSavedSearch.pm | 8 ++++-
lib/PublicInbox/LeiToMail.pm | 59 +++++++++++++++++++++++++++++--
lib/PublicInbox/LeiXSearch.pm | 1 +
lib/PublicInbox/V2Writable.pm | 11 +++++-
t/lei-q-save.t | 30 ++++++++++++++++
7 files changed, 107 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e5ff9e5d..f2dfc320 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -163,7 +163,7 @@ our %CMD = ( # sorted in order of importance/use:
qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
- color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
+ shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 28891460..e4242d9b 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -108,6 +108,8 @@ sub new {
$opt->{alert} //= [ ':WINCH,:bell' ] if -t $lei->{1};
}
}
+ return $lei->fail('--shared is only for v2 inbox output') if
+ $self->{fmt} ne 'v2' && $lei->{opt}->{shared};
$self;
}
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 48d252f1..929380ed 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -14,7 +14,7 @@ use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::MID qw(mids_for_index);
use Digest::SHA qw(sha256_hex);
-my $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf):!i; # TODO: put in LeiToMail?
+my $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
# move this to PublicInbox::Config if other things use it:
my %cquote = ("\n" => '\\n', "\t" => '\\t', "\b" => '\\b');
@@ -290,6 +290,12 @@ EOM
my $dir_old = lss_dir_for($lei, \$old_path, 1);
my $dir_new = lss_dir_for($lei, \$new_path);
return if $dir_new eq $dir_old; # no change, likely
+
+ ($old_out =~ m!\Av2:!i || $new_out =~ m!\Av2:!) and
+ return $lei->fail(<<EOM);
+conversions from/to v2 inboxes not supported at this time
+EOM
+
return $lei->fail(<<EOM) if -e $dir_new;
lei.q.output changed from `$old_out' to `$new_out'
However, $dir_new exists
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b3aec50b..a7382169 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -353,16 +353,28 @@ sub _text_write_cb ($$) {
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$lse->xsmsg_vmd($smsg) if $lse;
- $eml //= PublicInbox::Eml->new($bref); # copy bref
+ $eml //= PublicInbox::Eml->new($bref);
return if $dedupe && $dedupe->is_dup($eml, $smsg);
my $lk = $ovv->lock_for_scope;
$lei->out(${$lvt->eml_to_text($smsg, $eml)}, "\n");
}
}
+sub _v2_write_cb ($$) {
+ my ($self, $lei) = @_;
+ my $dedupe = $lei->{dedupe};
+ $dedupe->prepare_dedupe if $dedupe;
+ sub { # for git_to_mail
+ my ($bref, $smsg, $eml) = @_;
+ $eml //= PublicInbox::Eml->new($bref);
+ return if $dedupe && $dedupe->is_dup($eml, $smsg);
+ $lei->{v2w}->ipc_do('add', $eml); # V2Writable->add
+ }
+}
+
sub write_cb { # returns a callback for git_to_mail
my ($self, $lei) = @_;
- # _mbox_write_cb, _maildir_write_cb or _imap_write_cb
+ # _mbox_write_cb, _maildir_write_cb, _imap_write_cb, _v2_write_cb
my $m = "_$self->{base_type}_write_cb";
$self->$m($lei);
}
@@ -400,6 +412,13 @@ sub new {
require PublicInbox::LeiViewText;
$lei->{lvt} = PublicInbox::LeiViewText->new($lei);
$self->{base_type} = 'text';
+ } elsif ($fmt eq 'v2') {
+ die "--dedupe=oid and v2 are incompatible\n" if
+ ($lei->{opt}->{dedupe}//'') eq 'oid';
+ $self->{base_type} = 'v2';
+ $lei->{opt}->{save} = \1;
+ die "--mua incompatible with v2\n" if $lei->{opt}->{mua};
+ $dst = $lei->{ovv}->{dst} = $lei->abs_path($dst);
} else {
die "bad mail --format=$fmt\n";
}
@@ -599,9 +618,43 @@ sub _do_augment_mbox {
$dedupe->pause_dedupe if $dedupe;
}
+sub _pre_augment_v2 {
+ my ($self, $lei) = @_;
+ my $dir = $self->{dst};
+ require PublicInbox::InboxWritable;
+ my ($ibx, @creat);
+ if (-d $dir) {
+ my $opt = { -min_inbox_version => 2 };
+ require PublicInbox::Admin;
+ my @ibx = PublicInbox::Admin::resolve_inboxes([ $dir ], $opt);
+ $ibx = $ibx[0] or die "$dir is not a v2 inbox\n";
+ } else {
+ $creat[0] = {};
+ $ibx = PublicInbox::Inbox->new({
+ name => 'lei-result', # XXX configurable
+ inboxdir => $dir,
+ version => 2,
+ address => [ 'lei@example.com' ],
+ });
+ }
+ PublicInbox::InboxWritable->new($ibx, @creat);
+ $ibx->init_inbox if @creat;
+ my $v2w = $lei->{v2w} = $ibx->importer;
+ $v2w->ipc_lock_init("$dir/ipc.lock");
+ $v2w->ipc_worker_spawn("lei/v2w $dir", $lei->oldset, { lei => $lei });
+ return if !$lei->{opt}->{shared};
+ my $d = "$lei->{ale}->{git}->{git_dir}/objects";
+ my $al = "$dir/git/0.git/objects/info/alternates";
+ open my $fh, '+>>', $al or die "open($al): $!";
+ seek($fh, 0, SEEK_SET) or die "seek($al): $!";
+ grep(/\A\Q$d\E\n/, <$fh>) and return;
+ print $fh "$d\n" or die "print($al): $!";
+ close $fh or die "close($al): $!";
+}
+
sub pre_augment { # fast (1 disk seek), runs in same process as post_augment
my ($self, $lei) = @_;
- # _pre_augment_maildir, _pre_augment_mbox
+ # _pre_augment_maildir, _pre_augment_mbox, _pre_augment_v2
my $m = $self->can("_pre_augment_$self->{base_type}") or return;
$m->($self, $lei);
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 2e548a7a..d6d42a01 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -368,6 +368,7 @@ sub query_done { # EOF callback for main daemon
warn "BUG: {sto} missing with --mail-sync";
}
my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
+ $wait = $lei->{v2w} ? $lei->{v2w}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 0461257f..573d9c6f 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -6,7 +6,7 @@
package PublicInbox::V2Writable;
use strict;
use v5.10.1;
-use parent qw(PublicInbox::Lock);
+use parent qw(PublicInbox::Lock PublicInbox::IPC);
use PublicInbox::SearchIdxShard;
use PublicInbox::IPC;
use PublicInbox::Eml;
@@ -1431,4 +1431,13 @@ W: interrupted, --xapian-only --reindex required upon restart
EOF
}
+sub ipc_atfork_child {
+ my ($self) = @_;
+ if (my $lei = delete $self->{lei}) {
+ $lei->_lei_atfork_child;
+ close(delete $lei->{pkt_op_p});
+ }
+ $self->SUPER::ipc_atfork_child;
+}
+
1;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index bea65133..694b33b2 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -3,6 +3,8 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
use PublicInbox::Smsg;
+use List::Util qw(sum);
+
my $doc1 = eml_load('t/plack-qp.eml');
$doc1->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 5)}));
my $doc2 = eml_load('t/utf8.eml');
@@ -165,5 +167,33 @@ test_lei(sub {
skip "symlinks not supported in $home?: $!", 1;
lei_ok('up', "$home/ln -s");
};
+
+ my $v2 = "$home/v2"; # v2: as an output destination
+ my (@before, @after);
+ require PublicInbox::MboxReader;
+ lei_ok(qw(q z:0.. -o), "v2:$v2");
+ lei_ok(qw(q z:0.. -o), "mboxrd:$home/before", '--only', $v2, '-j1,1');
+ open my $fh, '<', "$home/before";
+ PublicInbox::MboxReader->mboxrd($fh, sub { push @before, $_[0] });
+ isnt(scalar(@before), 0, 'initial v2 written');
+ my $orig = sum(map { -f $_ ? -s _ : () } (
+ glob("$v2/git/0.git/objects/*/*")));
+ lei_ok(qw(import t/data/0001.patch));
+ lei_ok 'up', $v2;
+ lei_ok(qw(q z:0.. -o), "mboxrd:$home/after", '--only', $v2, '-j1,1');
+ open $fh, '<', "$home/after";
+ PublicInbox::MboxReader->mboxrd($fh, sub { push @after, $_[0] });
+
+ my $last = shift @after;
+ $last->header_set('Status');
+ is_deeply($last, eml_load('t/data/0001.patch'), 'lei up worked on v2');
+ is_deeply(\@before, \@after, 'got same results');
+
+ my $v2s = "$home/v2s";
+ lei_ok(qw(q --shared z:0.. -o), "v2:$v2s");
+ my $shared = sum(map { -f $_ ? -s _ : () } (
+ glob("$v2s/git/0.git/objects/*/*")));
+ ok($shared < $orig, 'fewer bytes stored with --shared') or
+ diag "shared=$shared orig=$orig";
});
done_testing;
^ permalink raw reply related [relevance 39%]
* [PATCH] lei: retry_reopen on read-only Xapian access
@ 2021-05-28 19:47 49% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 19:47 UTC (permalink / raw)
To: meta
Xapian DBs may be modified by a parallel process while we're
reading it, and Xapian's MVCC model places the burden on readers
to retry operations.
We'll also have retry_reopen croak instead of die on errors,
which ought to help us track down some "Document not found"
errors I've occasionally seen when using "lei <q|up>".
---
lib/PublicInbox/LeiSearch.pm | 33 ++++++++++++++++++++++++---------
lib/PublicInbox/LeiXSearch.pm | 21 ++++++++++++++-------
lib/PublicInbox/Search.pm | 7 ++++---
3 files changed, 42 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 9297d060..b09d1e45 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -18,7 +18,7 @@ sub num2docid ($$) {
($num - 1) * $nshard + $num % $nshard + 1;
}
-sub msg_keywords {
+sub _msg_kw { # retry_reopen callback
my ($self, $num) = @_; # num_or_mitem
my $xdb = $self->xdb; # set {nshard};
my $docid = ref($num) ? $num->get_docid : num2docid($self, $num);
@@ -27,13 +27,16 @@ sub msg_keywords {
wantarray ? sort(keys(%$kw)) : $kw;
}
-# returns undef if blob is unknown
-sub oid_keywords {
- my ($self, $oidhex) = @_;
- my @num = $self->over->blob_exists($oidhex) or return;
+sub msg_keywords {
+ my ($self, $num) = @_; # num_or_mitem
+ $self->retry_reopen(\&_msg_kw, $num);
+}
+
+sub _oid_kw { # retry_reopen callback
+ my ($self, $nums) = @_;
my $xdb = $self->xdb; # set {nshard};
my %kw;
- for my $num (@num) { # there should only be one...
+ for my $num (@$nums) { # there should only be one...
my $doc = $xdb->get_document(num2docid($self, $num));
my $x = xap_terms('K', $doc);
%kw = (%kw, %$x);
@@ -41,10 +44,15 @@ sub oid_keywords {
\%kw;
}
-# lookup keywords+labels for external messages
-sub xsmsg_vmd {
+# returns undef if blob is unknown
+sub oid_keywords {
+ my ($self, $oidhex) = @_;
+ my @num = $self->over->blob_exists($oidhex) or return;
+ $self->retry_reopen(\&_oid_kw, \@num);
+}
+
+sub _xsmsg_vmd { # retry_reopen
my ($self, $smsg, $want_label) = @_;
- return if $smsg->{kw};
my $xdb = $self->xdb; # set {nshard};
my (%kw, %L, $doc, $x);
$kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged});
@@ -62,6 +70,13 @@ sub xsmsg_vmd {
$smsg->{L} = [ sort keys %L ] if scalar(keys(%L));
}
+# lookup keywords+labels for external messages
+sub xsmsg_vmd {
+ my ($self, $smsg, $want_label) = @_;
+ return if $smsg->{kw}; # already set by LeiXSearch->mitem_kw
+ $self->retry_reopen(\&_xsmsg_vmd, $smsg, $want_label);
+}
+
# when a message has no Message-IDs at all, this is needed for
# unsent Draft messages, at least
sub content_key ($) {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 760f9718..2e548a7a 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -67,17 +67,23 @@ sub remotes { @{$_[0]->{remotes} // []} }
# called by PublicInbox::Search::xdb (usually via ->mset)
sub xdb_shards_flat { @{$_[0]->{shards_flat} // []} }
-sub mitem_kw ($$;$) {
- my ($smsg, $mitem, $flagged) = @_;
- my $kw = xap_terms('K', my $doc = $mitem->get_document);
+sub _mitem_kw { # retry_reopen callback
+ my ($srch, $smsg, $mitem, $flagged) = @_;
+ my $doc = $mitem->get_document;
+ my $kw = xap_terms('K', $doc);
$kw->{flagged} = 1 if $flagged;
+ my $L = xap_terms('L', $doc);
# we keep the empty {kw} array here to prevent expensive work in
# ->xsmsg_vmd, _unbless_smsg will clobber it iff it's empty
$smsg->{kw} = [ sort keys %$kw ];
- my $L = xap_terms('L', $doc);
$smsg->{L} = [ sort keys %$L ] if scalar(keys %$L);
}
+sub mitem_kw ($$$;$) {
+ my ($srch, $smsg, $mitem, $flagged) = @_;
+ $srch->retry_reopen(\&_mitem_kw, $smsg, $mitem, $flagged);
+}
+
# like over->get_art
sub smsg_for {
my ($self, $mitem) = @_;
@@ -90,7 +96,7 @@ sub smsg_for {
my $smsg = $ibx->over->get_art($num);
return if $smsg->{bytes} == 0; # external message
if ($ibx->can('msg_keywords')) {
- mitem_kw($smsg, $mitem);
+ mitem_kw($self, $smsg, $mitem);
}
$smsg;
}
@@ -194,7 +200,8 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $mitem = delete $n2item{$n};
next if $smsg->{bytes} == 0;
if ($mitem && $can_kw) {
- mitem_kw($smsg, $mitem, $fl);
+ mitem_kw($srch, $smsg, $mitem,
+ $fl);
} elsif ($mitem && $fl) {
# call ->xsmsg_vmd, later
$smsg->{lei_q_tt_flagged} = 1;
@@ -210,7 +217,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
my $mitem = $items[$i++];
my $smsg = $over->get_art($n) or next;
next if $smsg->{bytes} == 0;
- mitem_kw($smsg, $mitem, $fl) if $can_kw;
+ mitem_kw($srch, $smsg, $mitem, $fl) if $can_kw;
$each_smsg->($smsg, $mitem);
}
}
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index fbcff2c3..59a5a3b0 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -9,6 +9,7 @@ use parent qw(Exporter);
our @EXPORT_OK = qw(retry_reopen int_val get_pct xap_terms);
use List::Util qw(max);
use POSIX qw(strftime);
+use Carp ();
# values for searching, changing the numeric value breaks
# compatibility with old indices (so don't change them it)
@@ -405,16 +406,16 @@ sub retry_reopen {
# Exception: The revision being read has been discarded -
# you should call Xapian::Database::reopen()
if (ref($@) =~ /\bDatabaseModifiedError\b/) {
- warn "reopen try #$i on $@\n";
+ warn "# reopen try #$i on $@\n";
reopen($self);
} else {
# let caller decide how to spew, because ExtMsg queries
# get wonky and trigger:
# "something terrible happened at .../Xapian/Enquire.pm"
- die;
+ Carp::croak($@);
}
}
- die "Too many Xapian database modifications in progress\n";
+ Carp::croak("Too many Xapian database modifications in progress\n");
}
sub _do_enquire {
^ permalink raw reply related [relevance 49%]
* [PATCH 0/3] lei: diagnostic and reliability fix
@ 2021-05-28 9:45 71% Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
1/3 may no longer be needed, but doesn't hurt, either.
It was made while tracking down occasional failures for
v3 of "lei: handle a single IMAP message in most places":
https://public-inbox.org/meta/20210528093740.GB18768@dcvr/
2/3 is necessary for parallel invocations from different dirs,
and in multi-tenant situations; and I just noticed 3/3 because
I had a typo somewhere else :x
Eric Wong (3):
t/lei-*: better diagnostics for occasional failures
lei: restore working directory in more places
script/lei: drop leftover message about fallback
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiLcat.pm | 4 +---
lib/PublicInbox/LeiQuery.pm | 4 +---
lib/PublicInbox/LeiXSearch.pm | 1 +
script/lei | 1 -
t/lei-q-kw.t | 2 +-
t/lei-q-remote-import.t | 6 +++---
t/lei-q-save.t | 6 ++++--
8 files changed, 17 insertions(+), 13 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] script/lei: drop leftover message about fallback
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
2021-05-28 9:45 62% ` [PATCH 2/3] lei: restore working directory in more places Eric Wong
@ 2021-05-28 9:45 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
Non-daemon lei isn't implemented, anymore.
---
script/lei | 1 -
1 file changed, 1 deletion(-)
diff --git a/script/lei b/script/lei
index 4d752fd8..fce8124a 100755
--- a/script/lei
+++ b/script/lei
@@ -98,7 +98,6 @@ lei-daemon could not start, exited with \$?=$?
# try connecting again anyways, unlink+bind may be racy
connect($sock, $addr) or die <<"";
connect($path): $! (after attempted daemon start)
-Falling back to (slow) one-shot mode
}
# (Socket::MsgHdr|Inline::C), $sock are all available:
^ permalink raw reply related [relevance 71%]
* [PATCH 2/3] lei: restore working directory in more places
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
@ 2021-05-28 9:45 62% ` Eric Wong
2021-05-28 9:45 71% ` [PATCH 3/3] script/lei: drop leftover message about fallback Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
Every tick of the event loop can change the working directory,
so we need to restore it for every client if they operate
in different directories.
This would be easier if we had openat(2) and friends in Perl;
but Inline::C is practically required for lei, now.
---
lib/PublicInbox/LEI.pm | 6 ++++++
lib/PublicInbox/LeiLcat.pm | 4 +---
lib/PublicInbox/LeiQuery.pm | 4 +---
lib/PublicInbox/LeiXSearch.pm | 1 +
4 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 07378ca7..e5ff9e5d 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1239,6 +1239,12 @@ sub wq_done_wait { # dwaitpid callback
$lei->dclose;
}
+sub fchdir {
+ my ($lei) = @_;
+ my $dh = $lei->{3} // die 'BUG: lei->{3} (CWD) gone';
+ chdir($dh) || $lei->fail("fchdir: $!");
+}
+
sub wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 0f585ff5..f9d9633a 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -89,9 +89,7 @@ sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
my ($lei) = @_; # $_[1] = $rbuf
if (defined($_[1])) {
$_[1] eq '' and return eval {
- if (my $dfd = $lei->{3}) {
- chdir($dfd) or return $lei->fail("fchdir: $!");
- }
+ $lei->fchdir or return;
my @argv = split(/\s+/, $lei->{mset_opt}->{qstr});
$lei->{mset_opt}->{qstr} = extract_all($lei, @argv)
or return;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 1999a534..0435a516 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -51,9 +51,7 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
my ($self) = @_; # $_[1] = $rbuf
if (defined($_[1])) {
$_[1] eq '' and return eval {
- if (my $dfd = $self->{3}) {
- chdir($dfd) or return $self->fail("fchdir: $!");
- }
+ $self->fchdir or return;
$self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr};
$self->{lse}->query_approxidate($self->{lse}->git,
$self->{mset_opt}->{qstr});
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index e2a8e8e3..760f9718 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -394,6 +394,7 @@ Error closing $lei->{ovv}->{dst}: $!
sub do_post_augment {
my ($lei) = @_;
my $l2m = $lei->{l2m} or return; # client disconnected
+ $lei->fchdir or return;
my $err;
eval { $l2m->post_augment($lei) };
$err = $@;
^ permalink raw reply related [relevance 62%]
* [PATCH 1/3] t/lei-*: better diagnostics for occasional failures
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
@ 2021-05-28 9:45 61% ` Eric Wong
2021-05-28 9:45 62% ` [PATCH 2/3] lei: restore working directory in more places Eric Wong
2021-05-28 9:45 71% ` [PATCH 3/3] script/lei: drop leftover message about fallback Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:45 UTC (permalink / raw)
To: meta
Some of these have been failing occasionally, not sure
how, yet...
---
t/lei-q-kw.t | 2 +-
t/lei-q-remote-import.t | 6 +++---
t/lei-q-save.t | 6 ++++--
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 074c573d..528751b4 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -116,7 +116,7 @@ for my $sfx ('', '.gz') {
}
$res{$mid} = $eml;
});
- is_deeply(\%res, $exp, '--augment worked');
+ is_deeply(\%res, $exp, '--augment worked') or diag $lei_err;
lei_ok(qw(q -o), "mboxrd:/dev/stdout", qw(m:qp@example.com)) or
diag $lei_err;
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 80067061..7db684d9 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -49,10 +49,10 @@ test_lei({ tmpdir => $tmpdir }, sub {
open my $fh, '>', "$o.lock";
$cmd[-1] = 'm:qp@example.com';
- unlink $o or BAIL_OUT $!;
+ unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
lei_ok(@cmd, '--lock=none');
- ok(-f $o && -s _, '--lock=none respected');
- unlink $o or BAIL_OUT $!;
+ ok(-f $o && -s _, '--lock=none respected') or diag $lei_err;
+ unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
ok(!lei(@cmd, '--lock=dotlock,timeout=0.000001'), 'dotlock fails');
ok(-f $o && !-s _, 'nothing output on lock failure');
unlink "$o.lock" or BAIL_OUT $!;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index aed38a51..bea65133 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -25,7 +25,8 @@ test_lei(sub {
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
lei_ok qw(q -q --save z:0.. d:last.week..), '-o', "MAILDIR:$home/md/";
my %before = map { $_ => 1 } glob("$home/md/cur/*");
- is_deeply(eml_load((keys %before)[0]), $doc1, 'doc1 matches');
+ my $f = (keys %before)[0] or xbail({before => \%before});
+ is_deeply(eml_load($f), $doc1, 'doc1 matches');
lei_ok qw(ls-mail-sync);
is($lei_out, "maildir:$home/md\n", 'canonicalized mail sync name');
@@ -45,7 +46,8 @@ test_lei(sub {
my %after = map { $_ => 1 } glob("$home/md/{new,cur}/*");
is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
is(scalar(keys %after), 1, 'one new message added');
- is_deeply(eml_load((keys %after)[0]), $doc2, 'doc2 matches');
+ $f = (keys %after)[0] or xbail({after => \%after});
+ is_deeply(eml_load($f), $doc2, 'doc2 matches');
# check stdin
lei_ok [qw(q --save - -o), "mboxcl2:mbcl2" ],
^ permalink raw reply related [relevance 61%]
* Re: [PATCH 5/6] lei: handle a single IMAP message in most places
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
@ 2021-05-28 9:37 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 9:37 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> v2: enforce UID in URL, fail without
Pushed v3 as commit 9b3cd5e254fafa08c774a24f85c2b2eac12a9de5
v3: fix error reporting (s/fail/child_error/)
Using ->fail was causing errors to cascade elsewhere, not 100%
sure why...
interdiff:
diff -u b/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
--- b/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -28,7 +28,7 @@
require PublicInbox::URIimap;
$u = PublicInbox::URIimap->new($u);
defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
- $lei->fail("# no UID= in $u");
+ $lei->child_error(1 << 8, "# no UID= in $u");
'""'; # blank query, using {lcat_blob}
} elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
^ permalink raw reply [relevance 71%]
* [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
@ 2021-05-28 0:07 71% ` Eric Wong
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
2021-05-28 0:07 71% ` [PATCH 6/6] lei: add TODO item for FUSE mount Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
This is low priority, for now.
---
lib/PublicInbox/LEI.pm | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7acc05bf..ad5f06be 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -274,13 +274,11 @@ our %CMD = ( # sorted in order of importance/use:
'daemon-pid' => [ '', 'show the PID of the lei-daemon' ],
'help' => [ '[SUBCOMMAND]', 'show help' ],
-# XXX do we need this?
-# 'git' => [ '[ANYTHING...]', 'git(1) wrapper', pass_through('git') ],
-
-'reorder-local-store-and-break-history' => [ '[REFNAME]',
- 'rewrite git history in an attempt to improve compression',
- qw(gc!), @c_opt ],
-
+# TODO
+#'reorder-local-store-and-break-history' => [ '[REFNAME]',
+# 'rewrite git history in an attempt to improve compression',
+# qw(gc!), @c_opt ],
+#
# internal commands are prefixed with '_'
'_complete' => [ '[...]', 'internal shell completion helper',
pass_through('everything') ],
^ permalink raw reply related [relevance 71%]
* [PATCH 6/6] lei: add TODO item for FUSE mount
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
@ 2021-05-28 0:07 71% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
It seems possible and natural to allow browsing lei/store as
a Maildir (as well as read-write JMAP/IMAP store).
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ad5f06be..07378ca7 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -278,6 +278,7 @@ our %CMD = ( # sorted in order of importance/use:
#'reorder-local-store-and-break-history' => [ '[REFNAME]',
# 'rewrite git history in an attempt to improve compression',
# qw(gc!), @c_opt ],
+#'fuse-mount' => [ 'PATHNAME', 'expose lei/store as Maildir(s)', @c_opt ],
#
# internal commands are prefixed with '_'
'_complete' => [ '[...]', 'internal shell completion helper',
^ permalink raw reply related [relevance 71%]
* [PATCH 0/6] lei: odds and ends and a resend
2021-05-27 10:49 44% [PATCH] lei: handle a single IMAP message in most places Eric Wong
@ 2021-05-28 0:07 70% ` Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
"lei: handle a single IMAP message in most places" didn't actually
apply, since I was still running 3/6 in this series. I've been
having trouble reproducing the uncommitted transaction error
messages I've been seeing from lms, but hopefully 3/6 can
eventually find it.
1/6 and 2/6 were somethings I noticed randomly. Been looking
into FUSE support for exposing lei/store as a Maildir, too.
Eric Wong (6):
viewdiff: make $UNSAFE a variable
viewdiff: escape '{' and '}' for regexp
lei_mail_sync: debug code for uncommitted txn
lei: mark reorder-and-rewrite-local-history as a TODO item
lei: handle a single IMAP message in most places
lei: add TODO item for FUSE mount
lib/PublicInbox/LEI.pm | 13 ++++++-------
lib/PublicInbox/LeiInspect.pm | 24 ++++++++++++++++++++++--
lib/PublicInbox/LeiLcat.pm | 33 ++++++++++++++++++++-------------
lib/PublicInbox/LeiMailSync.pm | 33 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiToMail.pm | 8 +++++++-
lib/PublicInbox/LeiXSearch.pm | 3 +++
lib/PublicInbox/NetReader.pm | 9 +++++++--
lib/PublicInbox/ViewDiff.pm | 11 +++++------
t/lei-import-imap.t | 15 +++++++++++++++
9 files changed, 118 insertions(+), 31 deletions(-)
^ permalink raw reply [relevance 70%]
* [PATCH 5/6] lei: handle a single IMAP message in most places
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
@ 2021-05-28 0:07 46% ` Eric Wong
2021-05-28 9:37 71% ` Eric Wong
2021-05-28 0:07 71% ` [PATCH 6/6] lei: add TODO item for FUSE mount Eric Wong
2 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-28 0:07 UTC (permalink / raw)
To: meta
"lei import" can now import a single IMAP message via
<imaps://example.com/MAILBOX/;UID=$UID>
Likewise, "lei inspect" can show the blob information for UID
URLs and "lei lcat" can display the blob without network access
if imported.
"lei lcat" also gets rid of some unused code and supports
"blob:$OIDHEX" syntax as described in the comments (and used by
our "text" output format).
v2: enforce UID in URL, fail without
---
lib/PublicInbox/LeiInspect.pm | 24 ++++++++++++++++++++++--
lib/PublicInbox/LeiLcat.pm | 33 ++++++++++++++++++++-------------
lib/PublicInbox/LeiToMail.pm | 8 +++++++-
lib/PublicInbox/NetReader.pm | 9 +++++++--
t/lei-import-imap.t | 15 +++++++++++++++
5 files changed, 71 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 46b9197f..7205979e 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -24,6 +24,19 @@ sub inspect_blob ($$) {
$ent;
}
+sub inspect_imap_uid ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $ent = {};
+ my $lse = $lei->{lse} or return $ent;
+ my $lms = $lse->lms or return $ent;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ $ent->{$$uid_uri} = $oidhex;
+ $ent;
+}
+
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
@@ -49,8 +62,15 @@ sub inspect1 ($$$) {
my $ent;
if ($item =~ /\Ablob:(.+)/) {
$ent = inspect_blob($lei, $1);
- } elsif ($item =~ m!\Aimaps?://!i ||
- $item =~ m!\A(?:maildir|mh):!i || -d $item) {
+ } elsif ($item =~ m!\Aimaps?://!i) {
+ require PublicInbox::URIimap;
+ my $uri = PublicInbox::URIimap->new($item);
+ if (defined($uri->uid)) {
+ $ent = inspect_imap_uid($lei, $uri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ } elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
} else { # TODO: more things
return $lei->fail("$item not understood");
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 87729acf..c4712662 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -9,27 +9,31 @@ use strict;
use v5.10.1;
use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
-use URI;
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_redispatch {
- my ($lei, $out, $op_p) = @_;
- my $l = bless { %$lei }, ref($lei);
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
- eval {
- $l->qerr("# updating $out");
- up1($l, $out);
- $l->qerr("# $out done");
- };
- $l->err($@) if $@;
+sub lcat_imap_uid_uri ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $lms = $lei->{lse}->lms or return;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
}
sub extract_1 ($$) {
my ($lei, $x) = @_;
- if ($x =~ m!\b([a-z]+?://\S+)!i) {
+ if ($x =~ m!\b(imaps?://[^>]+)!i) {
+ my $u = $1;
+ require PublicInbox::URIimap;
+ $u = PublicInbox::URIimap->new($u);
+ defined($u->uid) ? lcat_imap_uid_uri($lei, $u) :
+ $lei->fail("# no UID= in $u");
+ '""'; # blank query, using {lcat_blob}
+ } elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
+ require URI;
$u = URI->new($u);
my $p = $u->path;
my $term;
@@ -57,6 +61,9 @@ sub extract_1 ($$) {
$1;
} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
"mid:$1";
+ } elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
+ push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+ '""'; # blank query
} else {
undef;
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ad6b9439..b3aec50b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -702,8 +702,14 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
- delete $self->{wcb};
my $lei = $self->{lei};
+ if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
+ for my $oid (@{$lei->{lcat_blob}}) {
+ my $smsg = { blob => $oid, pct => 100 };
+ write_mail($self, $smsg);
+ }
+ }
+ delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
my $nr = delete($lei->{-nr_write}) or return;
return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 73b8b1cd..76d2fe62 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -469,7 +469,10 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
EOF
my $uri = $orig_uri->clone;
+ my $single_uid = $uri->uid;
my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
+ $itrk = $l_uid = undef if defined($single_uid);
+
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
@@ -499,7 +502,9 @@ EOF
# I wish "UID FETCH $START:*" could work, but:
# 1) servers do not need to return results in any order
# 2) Mail::IMAPClient doesn't offer a streaming API
- unless ($uids = $mic->search("UID $l_uid:*")) {
+ if (defined $single_uid) {
+ $uids = [ $single_uid ];
+ } elsif (!($uids = $mic->search("UID $l_uid:*"))) {
return if $!{EINTR} && $self->{quit};
return "E: $uri UID SEARCH $l_uid:* error: $!";
}
@@ -541,7 +546,7 @@ EOF
}
run_commit_cb($self);
$itrk->update_last($r_uidval, $last_uid) if $itrk;
- } until ($err || $self->{quit});
+ } until ($err || $self->{quit} || defined($single_uid));
$err;
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 5283cc23..59d481d5 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -75,5 +75,20 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok 'forget-mail-sync', $url;
lei_ok 'ls-mail-sync';
unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
+ my $uid_url = "$url/;UID=".$stats->{'uid.max'};
+ lei_ok 'import', $uid_url;
+ lei_ok 'inspect', $uid_url;
+ $lei_out =~ /([a-f0-9]{40,})/ or
+ xbail 'inspect missed blob with UID URL';
+ my $blob = $1;
+ lei_ok 'lcat', $uid_url;
+ like $lei_out, qr/^Subject: /sm,
+ 'lcat shows mail text with UID URL';
+ like $lei_out, qr/\bblob:$blob\b/, 'lcat showed blob';
+ my $orig = $lei_out;
+ lei_ok 'lcat', "blob:$blob";
+ is($lei_out, $orig, 'lcat understands blob:...');
+ ok(!lei('lcat', $url), "lcat doesn't work on IMAP URL w/o UID");
});
+
done_testing;
^ permalink raw reply related [relevance 46%]
* [PATCH] lei: handle a single IMAP message in most places
@ 2021-05-27 10:49 44% Eric Wong
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-27 10:49 UTC (permalink / raw)
To: meta
"lei import" can now import a single IMAP message via
<imaps://example.com/MAILBOX/;UID=$UID>
Likewise, "lei inspect" can show the blob information for UID
URLs and "lei lcat" can display the blob without network access
if imported.
"lei lcat" also gets rid of some unused code and supports
"blob:$OIDHEX" syntax as described in the comments (and used by
our "text" output format).
---
lib/PublicInbox/LeiInspect.pm | 24 +++++++++++++++++++++--
lib/PublicInbox/LeiLcat.pm | 35 +++++++++++++++++++++-------------
lib/PublicInbox/LeiMailSync.pm | 23 ++++++++++++++++++++++
lib/PublicInbox/LeiToMail.pm | 8 +++++++-
lib/PublicInbox/NetReader.pm | 9 +++++++--
t/lei-import-imap.t | 14 ++++++++++++++
6 files changed, 95 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 46b9197f..7205979e 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -24,6 +24,19 @@ sub inspect_blob ($$) {
$ent;
}
+sub inspect_imap_uid ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $ent = {};
+ my $lse = $lei->{lse} or return $ent;
+ my $lms = $lse->lms or return $ent;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ $ent->{$$uid_uri} = $oidhex;
+ $ent;
+}
+
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
@@ -49,8 +62,15 @@ sub inspect1 ($$$) {
my $ent;
if ($item =~ /\Ablob:(.+)/) {
$ent = inspect_blob($lei, $1);
- } elsif ($item =~ m!\Aimaps?://!i ||
- $item =~ m!\A(?:maildir|mh):!i || -d $item) {
+ } elsif ($item =~ m!\Aimaps?://!i) {
+ require PublicInbox::URIimap;
+ my $uri = PublicInbox::URIimap->new($item);
+ if (defined($uri->uid)) {
+ $ent = inspect_imap_uid($lei, $uri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ } elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
} else { # TODO: more things
return $lei->fail("$item not understood");
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 87729acf..e5d8e805 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -9,27 +9,33 @@ use strict;
use v5.10.1;
use PublicInbox::LeiViewText;
use URI::Escape qw(uri_unescape);
-use URI;
use PublicInbox::MID qw($MID_EXTRACT);
-sub lcat_redispatch {
- my ($lei, $out, $op_p) = @_;
- my $l = bless { %$lei }, ref($lei);
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
- eval {
- $l->qerr("# updating $out");
- up1($l, $out);
- $l->qerr("# $out done");
- };
- $l->err($@) if $@;
+sub lcat_imap_uid_uri ($$) {
+ my ($lei, $uid_uri) = @_;
+ my $lms = $lei->{lse}->lms or return;
+ my $oidhex = $lms->imap_oid($lei, $uid_uri);
+ if (ref(my $err = $oidhex)) { # art2folder error
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ push @{$lei->{lcat_blob}}, $oidhex; # cf. LeiToMail->wq_atexit_child
+ '""'; # blank query
}
sub extract_1 ($$) {
my ($lei, $x) = @_;
- if ($x =~ m!\b([a-z]+?://\S+)!i) {
+ if ($x =~ m!\b(imaps?://[^>]+)!i) {
+ my $u = $1;
+ require PublicInbox::URIimap;
+ $u = PublicInbox::URIimap->new($u);
+ if (!defined($u->uid)) {
+ $lei->qerr("# no UID= in $u");
+ }
+ lcat_imap_uid_uri($lei, $u);
+ } elsif ($x =~ m!\b([a-z]+?://\S+)!i) {
my $u = $1;
$u =~ s/[\>\]\)\,\.\;]+\z//;
+ require URI;
$u = URI->new($u);
my $p = $u->path;
my $term;
@@ -57,6 +63,9 @@ sub extract_1 ($$) {
$1;
} elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
"mid:$1";
+ } elsif ($x =~ /\bblob:([0-9a-f]{7,})\b/) {
+ push @{$lei->{lcat_blob}}, $1; # cf. LeiToMail->wq_atexit_child
+ '""'; # blank query
} else {
undef;
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index b3e5e035..eb2b0c0f 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -356,6 +356,29 @@ sub forget_folder {
$dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
}
+sub imap_oid {
+ my ($self, $lei, $uid_uri) = @_;
+ my $mailbox_uri = $uid_uri->clone;
+ $mailbox_uri->uid(undef);
+ warn "$uid_uri $mailbox_uri";
+ my $folders = [ $$mailbox_uri ];
+ if (my $err = $self->arg2folder($lei, $folders)) {
+ if ($err->{fail}) {
+ $lei->qerr("# no sync information for $mailbox_uri");
+ return;
+ }
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ my $fid = $self->{fmap}->{$folders->[0]} //=
+ _fid_for($self, $folders->[0]) // return;
+ my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
+SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ?
+EOM
+ $sth->execute($fid, $uid_uri->uid);
+ my ($oidbin) = $sth->fetchrow_array;
+ $oidbin ? unpack('H*', $oidbin) : undef;
+}
+
sub DESTROY {
my ($self) = @_;
my $dbh = $self->{dbh} or return;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ad6b9439..b3aec50b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -702,8 +702,14 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
- delete $self->{wcb};
my $lei = $self->{lei};
+ if (!$self->{-wq_worker_nr} && $lei->{lcat_blob}) {
+ for my $oid (@{$lei->{lcat_blob}}) {
+ my $smsg = { blob => $oid, pct => 100 };
+ write_mail($self, $smsg);
+ }
+ }
+ delete $self->{wcb};
$lei->{ale}->git->async_wait_all;
my $nr = delete($lei->{-nr_write}) or return;
return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 73b8b1cd..76d2fe62 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -469,7 +469,10 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
EOF
my $uri = $orig_uri->clone;
+ my $single_uid = $uri->uid;
my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
+ $itrk = $l_uid = undef if defined($single_uid);
+
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
@@ -499,7 +502,9 @@ EOF
# I wish "UID FETCH $START:*" could work, but:
# 1) servers do not need to return results in any order
# 2) Mail::IMAPClient doesn't offer a streaming API
- unless ($uids = $mic->search("UID $l_uid:*")) {
+ if (defined $single_uid) {
+ $uids = [ $single_uid ];
+ } elsif (!($uids = $mic->search("UID $l_uid:*"))) {
return if $!{EINTR} && $self->{quit};
return "E: $uri UID SEARCH $l_uid:* error: $!";
}
@@ -541,7 +546,7 @@ EOF
}
run_commit_cb($self);
$itrk->update_last($r_uidval, $last_uid) if $itrk;
- } until ($err || $self->{quit});
+ } until ($err || $self->{quit} || defined($single_uid));
$err;
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 5283cc23..2f928a2e 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -75,5 +75,19 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok 'forget-mail-sync', $url;
lei_ok 'ls-mail-sync';
unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
+ my $uid_url = "$url/;UID=".$stats->{'uid.max'};
+ lei_ok 'import', $uid_url;
+ lei_ok 'inspect', $uid_url;
+ $lei_out =~ /([a-f0-9]{40,})/ or
+ xbail 'inspect missed blob with UID URL';
+ my $blob = $1;
+ lei_ok 'lcat', $uid_url;
+ like $lei_out, qr/^Subject: /sm,
+ 'lcat shows mail text with UID URL';
+ like $lei_out, qr/\bblob:$blob\b/, 'lcat showed blob';
+ my $orig = $lei_out;
+ lei_ok 'lcat', "blob:$blob";
+ is($lei_out, $orig, 'lcat understands blob:...');
});
+
done_testing;
^ permalink raw reply related [relevance 44%]
* [PATCH] lei rm: new command to remove messages from index
@ 2021-05-26 23:50 50% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-26 23:50 UTC (permalink / raw)
To: meta
This is similar to "public-inbox-learn rm", but it's
possible to point an entire Maildir/IMAP/mbox*/newsgroup
at it.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 5 +++-
lib/PublicInbox/LeiRm.pm | 50 +++++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiStore.pm | 29 ++++++++++++++++++++-
t/lei-import-maildir.t | 7 ++++++
5 files changed, 90 insertions(+), 2 deletions(-)
create mode 100644 lib/PublicInbox/LeiRm.pm
diff --git a/MANIFEST b/MANIFEST
index 23423e0b..0b4bb380 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -223,6 +223,7 @@ lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
+lib/PublicInbox/LeiRm.pm
lib/PublicInbox/LeiSavedSearch.pm
lib/PublicInbox/LeiSearch.pm
lib/PublicInbox/LeiStore.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6ff249d0..7acc05bf 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -206,7 +206,10 @@ our %CMD = ( # sorted in order of importance/use:
qw(verbose|v+), @c_opt ],
'edit-search' => [ 'OUTPUT', "edit saved search via `git config --edit'",
@c_opt ],
-
+'rm' => [ '--stdin|LOCATION...',
+ 'remove a message from the index and prevent reindexing',
+ 'stdin|', # /|\z/ must be first for lone dash
+ @c_opt ],
'plonk' => [ '--threads|--from=IDENT',
'exclude mail matching From: or threads from non-Message-ID searches',
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
new file mode 100644
index 00000000..185b6a15
--- /dev/null
+++ b/lib/PublicInbox/LeiRm.pm
@@ -0,0 +1,50 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# implements the "lei rm" command, you can point this at
+# an entire spam mailbox or read a message from stdin
+package PublicInbox::LeiRm;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+
+sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
+ my ($self, $eml) = @_;
+ $self->{lei}->{sto}->ipc_do('remove_eml', $eml);
+}
+
+sub input_mbox_cb { # MboxReader callback
+ my ($eml, $self) = @_;
+ input_eml_cb($self, $eml);
+}
+
+sub input_net_cb { # callback for ->imap_each, ->nntp_each
+ my (undef, undef, $kw, $eml, $self) = @_; # @_[0,1]: url + uid ignored
+ input_eml_cb($self, $eml);
+}
+
+sub input_maildir_cb {
+ my (undef, $kw, $eml, $self) = @_; # $_[0] $filename ignored
+ input_eml_cb($self, $eml);
+}
+
+sub lei_rm {
+ my ($lei, @inputs) = @_;
+ $lei->_lei_store(1)->write_prepare($lei);
+ $lei->{opt}->{stdin} = 1 if !@inputs;
+ $lei->{opt}->{'in-format'} //= 'eml';
+ my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
+ $self->prepare_inputs($lei, \@inputs) or return;
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $op_c->op_wait_event($ops);
+}
+
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+
+1;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index af5edbc2..6888afb4 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -183,7 +183,7 @@ sub add_eml_vmd {
\@docids;
}
-sub remove_eml_vmd {
+sub remove_eml_vmd { # remove just the VMD
my ($self, $eml, $vmd) = @_;
my ($eidx, $tl) = eidx_init($self);
my @docids = _docids_for($self, $eml);
@@ -204,6 +204,33 @@ sub set_sync_info {
})->set_src($oidhex, $folder, $id);
}
+sub _remove_if_local { # git->cat_async arg
+ my ($bref, $oidhex, $type, $size, $self) = @_;
+ $self->{im}->remove($bref) if $bref;
+}
+
+# remove the entire message from the index, does not touch mail_sync.sqlite3
+sub remove_eml {
+ my ($self, $eml) = @_;
+ my $im = $self->importer; # may create new epoch
+ my ($eidx, $tl) = eidx_init($self);
+ my $oidx = $eidx->{oidx};
+ my @docids = _docids_for($self, $eml);
+ my $git = $eidx->git;
+ for my $docid (@docids) {
+ my $xr3 = $oidx->get_xref3($docid, 1);
+ for my $row (@$xr3) {
+ my (undef, undef, $oidbin) = @$row;
+ my $oidhex = unpack('H*', $oidbin);
+ $git->cat_async($oidhex, \&_remove_if_local, $self);
+ }
+ $eidx->idx_shard($docid)->ipc_do('xdb_remove', $docid);
+ $oidx->delete_by_num($docid);
+ }
+ $git->cat_async_wait;
+ \@docids;
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->{-fake_im} // $self->importer; # may create new epoch
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index f813440a..688b10ce 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -68,5 +68,12 @@ test_lei(sub {
$res = json_utf8->decode($lei_out);
is_deeply($res, [ undef ], 'trashed message not imported')
or diag explain($imp_err, $res);
+
+ lei_ok qw(rm t/data/0001.patch);
+ lei_ok(qw(q s:boolean));
+ is($lei_out, "[null]\n", 'removed message gone from results');
+ my $g0 = "$ENV{HOME}/.local/share/lei/store/local/0.git";
+ my $x = xqx(['git', "--git-dir=$g0", qw(cat-file blob HEAD:d)]);
+ is($?, 0, "git cat-file shows file is `d'");
});
done_testing;
^ permalink raw reply related [relevance 50%]
* [PATCH] lei: require Socket::MsgHdr or Inline::C, drop oneshot
@ 2021-05-26 18:08 28% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-26 18:08 UTC (permalink / raw)
To: meta
The cost of supporting separate code paths between oneshot and
daemon isn't worth the trouble; especially if there are more
users to support. The test suite time nearly doubles with
oneshot, so that's hurting developer productivity.
FD passing is currently required to work efficiently with
remote HTTP(S) queries which return large messages, as seen in
commit 708b182a57373172f5523f3dc297659d58e03b58
("ipc: wq: handle >MAX_ARG_STRLEN && <EMSGSIZE case").
Additionally, upcoming support for IMAP IDLE and inotify-based
monitoring of Maildirs cannot work properly without a background
daemon.
---
lib/PublicInbox/GitCredential.pm | 2 +-
lib/PublicInbox/LEI.pm | 58 +------------
lib/PublicInbox/LeiEditSearch.pm | 19 ++---
lib/PublicInbox/LeiStore.pm | 13 +--
lib/PublicInbox/LeiSucks.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 26 +++---
lib/PublicInbox/TestCommon.pm | 15 +---
script/lei | 137 +++++++++++++++----------------
t/lei-daemon.t | 9 --
9 files changed, 97 insertions(+), 185 deletions(-)
diff --git a/lib/PublicInbox/GitCredential.pm b/lib/PublicInbox/GitCredential.pm
index 2d81817c..b29780d6 100644
--- a/lib/PublicInbox/GitCredential.pm
+++ b/lib/PublicInbox/GitCredential.pm
@@ -9,7 +9,7 @@ sub run ($$;$) {
my ($in_r, $in_w, $out_r);
my $cmd = [ qw(git credential), $op ];
pipe($in_r, $in_w) or die "pipe: $!";
- if ($lei && !$lei->{oneshot}) { # we'll die if disconnected:
+ if ($lei) { # we'll die if disconnected:
pipe($out_r, my $out_w) or die "pipe: $!";
$lei->send_exec_cmd([ $in_r, $out_w ], $cmd, {});
} else {
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index c8d2f315..6ff249d0 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -444,19 +444,6 @@ sub x_it ($$) {
dump_and_clear_log();
if (my $s = $self->{pkt_op_p} // $self->{sock}) {
send($s, "x_it $code", MSG_EOR);
- } elsif ($self->{oneshot}) {
- # don't want to end up using $? from child processes
- _drop_wq($self);
- # cleanup anything that has tempfiles or open file handles
- %PATH2CFG = ();
- delete @$self{qw(ovv dedupe sto cfg)};
- if (my $signum = ($code & 127)) { # usually SIGPIPE (13)
- $SIG{PIPE} = 'DEFAULT'; # $SIG{$signum} doesn't work
- kill $signum, $$;
- sleep(1) while 1; # wait for signal
- } else {
- $quit->($code >> 8);
- }
} # else ignore if client disconnected
}
@@ -921,17 +908,6 @@ sub start_mua {
my $io = [];
$io->[0] = $self->{1} if $self->{opt}->{stdin} && -t $self->{1};
send_exec_cmd($self, $io, \@cmd, {});
- } elsif ($self->{oneshot}) {
- my $pid = fork // die "fork: $!";
- if ($pid > 0) { # original process
- if ($self->{opt}->{stdin} && -t STDOUT) {
- open STDIN, '+<&', \*STDOUT or die "dup2: $!";
- }
- exec(@cmd);
- warn "exec @cmd: $!\n";
- POSIX::_exit(1);
- }
- POSIX::setsid() > 0 or die "setsid: $!";
}
if ($self->{lxs} && $self->{au_done}) { # kick wait_startq
syswrite($self->{au_done}, 'q' x ($self->{lxs}->{jobs} // 0));
@@ -952,14 +928,11 @@ sub send_exec_cmd { # tell script/lei to execute a command
sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
my ($self) = @_;
my $alerts = $self->{opt}->{alert} // return;
+ my $sock = $self->{sock};
while (my $op = shift(@$alerts)) {
if ($op eq ':WINCH') {
# hit the process group that started the MUA
- if ($self->{sock}) {
- send($self->{sock}, '-WINCH', MSG_EOR);
- } elsif ($self->{oneshot}) {
- kill('-WINCH', $$);
- }
+ send($sock, '-WINCH', MSG_EOR) if $sock;
} elsif ($op eq ':bell') {
out($self, "\a");
} elsif ($op =~ /(?<!\\),/) { # bare ',' (not ',,')
@@ -968,11 +941,7 @@ sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
my $cmd = $1; # run an arbitrary command
require Text::ParseWords;
$cmd = [ Text::ParseWords::shellwords($cmd) ];
- if (my $s = $self->{sock}) {
- send($s, exec_buf($cmd, {}), MSG_EOR);
- } elsif ($self->{oneshot}) {
- $self->{"pid.$self.$$"}->{spawn($cmd)} = $cmd;
- }
+ send($sock, exec_buf($cmd, {}), MSG_EOR) if $sock;
} else {
err($self, "W: unsupported --alert=$op"); # non-fatal
}
@@ -1009,9 +978,6 @@ sub start_pager {
if ($self->{sock}) { # lei(1) process runs it
delete @$new_env{keys %$env}; # only set iff unset
send_exec_cmd($self, [ @$rdr{0..2} ], [$pager], $new_env);
- } elsif ($self->{oneshot}) {
- my $cmd = [$pager];
- $self->{"pid.$self.$$"}->{spawn($cmd, $new_env, $rdr)} = $cmd;
} else {
die 'BUG: start_pager w/o socket';
}
@@ -1253,29 +1219,13 @@ sub lazy_start {
sub busy { 1 } # prevent daemon-shutdown if client is connected
-# for users w/o Socket::Msghdr installed or Inline::C enabled
-sub oneshot {
- my ($main_pkg) = @_;
- my $exit = $main_pkg->can('exit'); # caller may override exit()
- local $quit = $exit if $exit;
- local %PATH2CFG;
- umask(077) // die("umask(077): $!");
- my $self = bless { oneshot => 1, env => \%ENV }, __PACKAGE__;
- for (0..2) { open($self->{$_}, '+<&=', $_) or die "open fd=$_: $!" }
- dispatch($self, @ARGV);
- x_it($self, $self->{child_error}) if $self->{child_error};
-}
-
# ensures stdout hits the FS before sock disconnects so a client
# can immediately reread it
sub DESTROY {
my ($self) = @_;
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
- my $err = $?;
- my $oneshot_pids = delete $self->{"pid.$self.$$"} or return;
- waitpid($_, 0) for keys %$oneshot_pids;
- $? = $err if $err; # preserve ->fail or ->x_it code
+ # preserve $? for ->fail or ->x_it code
}
sub wq_done_wait { # dwaitpid callback
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index 30ac65bd..13713d24 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -14,19 +14,12 @@ sub lei_edit_search {
my @cmd = (qw(git config --edit -f), $lss->{'-f'});
$lei->qerr("# spawning @cmd");
$lss->edit_begin($lei);
- if ($lei->{oneshot}) {
- require PublicInbox::Spawn;
- waitpid(PublicInbox::Spawn::spawn(\@cmd), 0);
- # non-fatal, editor could fail after successful write
- $lei->child_error($?) if $?;
- $lss->edit_done($lei);
- } else { # run in script/lei foreground
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- # $op_p will EOF when $EDITOR is done
- $op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
- $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p ], \@cmd, {});
- }
+ # run in script/lei foreground
+ require PublicInbox::PktOp;
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ # $op_p will EOF when $EDITOR is done
+ $op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
+ $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p ], \@cmd, {});
}
*_complete_edit_search = \&PublicInbox::LeiUp::_complete_up;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index a7a0ebef..af5edbc2 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -431,20 +431,15 @@ sub write_prepare {
my $d = $lei->store_path;
$self->ipc_lock_init("$d/ipc.lock");
substr($d, -length('/lei/store'), 10, '');
- my $err_pipe;
- unless ($lei->{oneshot}) {
- pipe(my ($r, $w)) or die "pipe: $!";
- $err_pipe = [ $r, $w ];
- }
+ pipe(my ($r, $w)) or die "pipe: $!";
+ my $err_pipe = [ $r, $w ];
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
$self->ipc_worker_spawn("lei/store $d", $lei->oldset,
{ lei => $lei, err_pipe => $err_pipe });
- if ($err_pipe) {
- require PublicInbox::LeiStoreErr;
- PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
- }
+ require PublicInbox::LeiStoreErr;
+ PublicInbox::LeiStoreErr->new($err_pipe->[0], $lei);
}
$lei->{sto} = $self;
}
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index 2ce64d62..a71158f3 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -23,8 +23,7 @@ sub lei_sucks {
}
eval { require PublicInbox };
my $pi_ver = eval('$PublicInbox::VERSION') // '(???)';
- my $daemon = $lei->{oneshot} ? 'oneshot' : 'daemon';
- my @out = ("lei $pi_ver mode=$daemon\n",
+ my @out = ("lei $pi_ver\n",
"perl $Config{version} / $os $rel / $mac ".
"ptrsize=$Config{ptrsize}\n");
chomp(my $gv = `git --version` || "git missing");
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 4399c4fb..9069232b 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -76,22 +76,18 @@ sub lei_up {
my @all = PublicInbox::LeiSavedSearch::list($lei);
my @local = grep(!m!\Aimaps?://!i, @all);
$lei->_lei_store->write_prepare($lei); # share early
- if ($lei->{oneshot}) { # synchronous
- up1_redispatch($lei, $_) for @local;
- } else {
- # daemon mode, re-dispatch into our event loop w/o
- # creating an extra fork-level
- require PublicInbox::DS;
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@local) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
- }
- $lei->event_step_init;
- $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
+ # daemon mode, re-dispatch into our event loop w/o
+ # creating an extra fork-level
+ require PublicInbox::DS;
+ require PublicInbox::PktOp;
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ for my $o (@local) {
+ PublicInbox::DS::requeue(sub {
+ up1_redispatch($lei, $o, $op_p);
+ });
}
+ $lei->event_step_init;
+ $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
} else {
up1($lei, $out);
}
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 460c9da0..83dcf650 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -544,7 +544,8 @@ EOM
($tmpdir, $for_destroy) = tmpdir unless $tmpdir;
state $persist_xrd = $ENV{TEST_LEI_DAEMON_PERSIST_DIR};
SKIP: {
- skip 'TEST_LEI_ONESHOT set', 1 if $ENV{TEST_LEI_ONESHOT};
+ $ENV{TEST_LEI_ONESHOT} and
+ xbail 'TEST_LEI_ONESHOT no longer supported';
my $home = "$tmpdir/lei-daemon";
mkdir($home, 0700) or BAIL_OUT "mkdir: $!";
local $ENV{HOME} = $home;
@@ -568,22 +569,12 @@ EOM
lei_ok(qw(daemon-kill), \"daemon-kill after $t");
}
}; # SKIP for lei_daemon
- unless ($test_opt->{daemon_only}) {
- $ENV{TEST_LEI_DAEMON_ONLY} and
- skip 'TEST_LEI_DAEMON_ONLY set', 1;
- require_ok 'PublicInbox::LEI';
- my $home = "$tmpdir/lei-oneshot";
- mkdir($home, 0700) or BAIL_OUT "mkdir: $!";
- local $ENV{HOME} = $home;
- local $ENV{XDG_RUNTIME_DIR} = '/dev/null';
- $cb->();
- }
if ($daemon_pid) {
for (0..10) {
kill(0, $daemon_pid) or last;
tick;
}
- ok(!kill(0, $daemon_pid), "$t daemon stopped after oneshot");
+ ok(!kill(0, $daemon_pid), "$t daemon stopped");
my $f = "$daemon_xrd/lei/errors.log";
open my $fh, '<', $f or BAIL_OUT "$f: $!";
my @l = <$fh>;
diff --git a/script/lei b/script/lei
index bec6b001..4d752fd8 100755
--- a/script/lei
+++ b/script/lei
@@ -9,10 +9,18 @@ my $narg = 5;
my $sock;
my $recv_cmd = PublicInbox::CmdIPC4->can('recv_cmd4');
my $send_cmd = PublicInbox::CmdIPC4->can('send_cmd4') // do {
+ my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //= (
+ $ENV{XDG_CACHE_HOME} //
+ ( ($ENV{HOME} // '/nonexistent').'/.cache' )
+ ).'/public-inbox/inline-c';
+ if (!-d $inline_dir) {
+ require File::Path;
+ File::Path::make_path($inline_dir);
+ }
require PublicInbox::Spawn; # takes ~50ms even if built *sigh*
$recv_cmd = PublicInbox::Spawn->can('recv_cmd4');
PublicInbox::Spawn->can('send_cmd4');
-};
+} // die 'please install Inline::C or Socket::MsgHdr';
my %pids;
my $sigchld = sub {
@@ -66,80 +74,69 @@ my $exec_cmd = sub {
}
};
-if ($send_cmd && eval {
- my $path = do {
- my $runtime_dir = ($ENV{XDG_RUNTIME_DIR} // '') . '/lei';
- die \0 if $runtime_dir eq '/dev/null/lei'; # oneshot forced
- if ($runtime_dir eq '/lei') {
- require File::Spec;
- $runtime_dir = File::Spec->tmpdir."/lei-$<";
- }
- unless (-d $runtime_dir) {
- require File::Path;
- File::Path::mkpath($runtime_dir, 0, 0700);
- }
- "$runtime_dir/$narg.seq.sock";
- };
- my $addr = pack_sockaddr_un($path);
- socket($sock, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
- unless (connect($sock, $addr)) { # start the daemon if not started
- local $ENV{PERL5LIB} = join(':', @INC);
- open(my $daemon, '-|', $^X, qw[-MPublicInbox::LEI
- -E PublicInbox::LEI::lazy_start(@ARGV)],
- $path, $! + 0, $narg) or die "popen: $!";
- while (<$daemon>) { warn $_ } # EOF when STDERR is redirected
- close($daemon) or warn <<"";
+my $runtime_dir = ($ENV{XDG_RUNTIME_DIR} // '') . '/lei';
+if ($runtime_dir eq '/lei') {
+ require File::Spec;
+ $runtime_dir = File::Spec->tmpdir."/lei-$<";
+}
+unless (-d $runtime_dir) {
+ require File::Path;
+ File::Path::make_path($runtime_dir, { mode => 0700 });
+}
+my $path = "$runtime_dir/$narg.seq.sock";
+my $addr = pack_sockaddr_un($path);
+socket($sock, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
+unless (connect($sock, $addr)) { # start the daemon if not started
+ local $ENV{PERL5LIB} = join(':', @INC);
+ open(my $daemon, '-|', $^X, qw[-MPublicInbox::LEI
+ -E PublicInbox::LEI::lazy_start(@ARGV)],
+ $path, $! + 0, $narg) or die "popen: $!";
+ while (<$daemon>) { warn $_ } # EOF when STDERR is redirected
+ close($daemon) or warn <<"";
lei-daemon could not start, exited with \$?=$?
- # try connecting again anyways, unlink+bind may be racy
- connect($sock, $addr) or die <<"";
+ # try connecting again anyways, unlink+bind may be racy
+ connect($sock, $addr) or die <<"";
connect($path): $! (after attempted daemon start)
Falling back to (slow) one-shot mode
+}
+# (Socket::MsgHdr|Inline::C), $sock are all available:
+open my $dh, '<', '.' or die "open(.) $!";
+my $buf = join("\0", scalar(@ARGV), @ARGV);
+while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
+$buf .= "\0\0";
+my $n = $send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR);
+if (!$n) {
+ die "sendmsg: $! (check RLIMIT_NOFILE)\n" if $!{ETOOMANYREFS};
+ die "sendmsg: $!\n";
+}
+my $x_it_code = 0;
+while (1) {
+ my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
+ if (scalar(@fds) == 1 && !defined($fds[0])) {
+ next if $!{EINTR};
+ last if $!{ECONNRESET};
+ die "recvmsg: $!";
}
- # (Socket::MsgHdr|Inline::C), $sock are all available:
- open my $dh, '<', '.' or die "open(.) $!";
- my $buf = join("\0", scalar(@ARGV), @ARGV);
- while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
- $buf .= "\0\0";
- my $n = $send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR);
- if (!$n) {
- die "sendmsg: $! (check RLIMIT_NOFILE)\n" if $!{ETOOMANYREFS};
- die "sendmsg: $!\n";
- }
- 1;
-}) { # connected and request sent to lei-daemon, wait for responses or EOF
- my $x_it_code = 0;
- while (1) {
- my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
- if (scalar(@fds) == 1 && !defined($fds[0])) {
- next if $!{EINTR};
- last if $!{ECONNRESET};
- die "recvmsg: $!";
- }
- last if $buf eq '';
- if ($buf =~ /\Aexec (.+)\z/) {
- $exec_cmd->(\@fds, split(/\0/, $1));
- } elsif ($buf eq '-WINCH') {
- kill($buf, @parent); # for MUA
- } elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
- $x_it_code ||= $1 + 0;
- last;
- } elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
- $x_it_code ||= $1 + 0;
- } else {
- $sigchld->();
- die $buf;
- }
- }
- $sigchld->();
- if (my $sig = ($x_it_code & 127)) {
- kill $sig, $$;
- sleep(1) while 1;
+ last if $buf eq '';
+ if ($buf =~ /\Aexec (.+)\z/) {
+ $exec_cmd->(\@fds, split(/\0/, $1));
+ } elsif ($buf eq '-WINCH') {
+ kill($buf, @parent); # for MUA
+ } elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
+ $x_it_code ||= $1 + 0;
+ last;
+ } elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
+ $x_it_code ||= $1 + 0;
+ } else {
+ $sigchld->();
+ die $buf;
}
- exit($x_it_code >> 8);
-} else { # for systems lacking Socket::MsgHdr or Inline::C
- warn $@ if $@ && !ref($@);
- require PublicInbox::LEI;
- PublicInbox::LEI::oneshot(__PACKAGE__);
}
+$sigchld->();
+if (my $sig = ($x_it_code & 127)) {
+ kill $sig, $$;
+ sleep(1) while 1;
+}
+exit($x_it_code >> 8);
diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index 84e2791d..a7c4b799 100644
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -73,15 +73,6 @@ test_lei({ daemon_only => 1 }, sub {
lei_ok('daemon-pid');
chomp $lei_out;
is($lei_out, $new_pid, 'PID unchanged after -0/-CHLD');
-
- SKIP: { # socket inaccessible
- skip "cannot test connect EPERM as root", 3 if $> == 0;
- chmod 0000, $sock or BAIL_OUT "chmod 0000: $!";
- lei_ok('help', \'connect fail, one-shot fallback works');
- like($lei_err, qr/\bconnect\(/, 'connect error noted');
- like($lei_out, qr/^usage: /, 'help output works');
- chmod 0700, $sock or BAIL_OUT "chmod 0700: $!";
- }
unlink $sock or BAIL_OUT "unlink($sock) $!";
for (0..100) {
kill('CHLD', $new_pid) or last;
^ permalink raw reply related [relevance 28%]
* [PATCH 0/2] ipc: fix "lei q" w/ HTTP(S) externals
@ 2021-05-25 22:19 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-25 22:19 UTC (permalink / raw)
To: meta
1/2 was found while making 2/2
Eric Wong (2):
ipc: avoid potential stack-not-refcounted bug
ipc: wq: handle >MAX_ARG_STRLEN && <EMSGSIZE case
lib/PublicInbox/IPC.pm | 47 +++++++++++++++++++++++--------------
lib/PublicInbox/WQWorker.pm | 2 +-
t/ipc.t | 11 ++++++---
3 files changed, 38 insertions(+), 22 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs
2021-05-25 11:01 71% [PATCH 0/3] lei forget-mail-sync: drop sync information Eric Wong
@ 2021-05-25 11:01 90% ` Eric Wong
2021-05-25 11:01 55% ` [PATCH 3/3] lei forget-mail-sync: new command to drop sync information Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-25 11:01 UTC (permalink / raw)
To: meta
This is to match LeiInput behavior when mail-sync is
enabled, since rel2abs won't resolve symlinks.
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d8ba8fd5..fabc01f8 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -130,7 +130,7 @@ EOM
for (@folders) {
next if $all{$_}; # ok
if (-d "$_/new" && -d "$_/cur") {
- my $d = 'maildir:'.$lei->rel2abs($_);
+ my $d = 'maildir:'.$lei->abs_path($_);
push(@no, $_) unless $all{$d};
$_ = $d;
} elsif (m!\Aimaps?://!i) {
^ permalink raw reply related [relevance 90%]
* [PATCH 0/3] lei forget-mail-sync: drop sync information
@ 2021-05-25 11:01 71% Eric Wong
2021-05-25 11:01 90% ` [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs Eric Wong
2021-05-25 11:01 55% ` [PATCH 3/3] lei forget-mail-sync: new command to drop sync information Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-25 11:01 UTC (permalink / raw)
To: meta
Sometimes users lose interest in a Maildir or IMAP folder, etc;
this drops sync information to not clutter up disk space,
"ls-mail-sync" output, etc.
Eric Wong (3):
lei export-kw: use lei->abs_path instead of rel2abs
lei_mail_sync: args2folder: common folder lookup sub
lei forget-mail-sync: new command to drop sync information
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 3 +-
lib/PublicInbox/LeiExportKw.pm | 34 +++---------------
lib/PublicInbox/LeiForgetMailSync.pm | 30 ++++++++++++++++
lib/PublicInbox/LeiInspect.pm | 21 +++++------
lib/PublicInbox/LeiMailSync.pm | 54 ++++++++++++++++++++++++++++
t/lei-import-imap.t | 4 +++
7 files changed, 104 insertions(+), 43 deletions(-)
create mode 100644 lib/PublicInbox/LeiForgetMailSync.pm
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei forget-mail-sync: new command to drop sync information
2021-05-25 11:01 71% [PATCH 0/3] lei forget-mail-sync: drop sync information Eric Wong
2021-05-25 11:01 90% ` [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs Eric Wong
@ 2021-05-25 11:01 55% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-25 11:01 UTC (permalink / raw)
To: meta
Sometimes a user stops caring to sync an IMAP or Maildir
folder, or wants to force a resync. Let them run this
command to have lei forget all the sync information about
the mail folder.
This won't delete any stored messages in git, but will
leave "lei index" users with dangling references.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 3 ++-
lib/PublicInbox/LeiForgetMailSync.pm | 30 ++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 11 ++++++++++
t/lei-import-imap.t | 4 ++++
5 files changed, 48 insertions(+), 1 deletion(-)
create mode 100644 lib/PublicInbox/LeiForgetMailSync.pm
diff --git a/MANIFEST b/MANIFEST
index 2d1ad5c3..23423e0b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -204,6 +204,7 @@ lib/PublicInbox/LeiDedupe.pm
lib/PublicInbox/LeiEditSearch.pm
lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
+lib/PublicInbox/LeiForgetMailSync.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 628908b5..c8d2f315 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -243,7 +243,8 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| offset=i recursive|r exclude=s include|I=s
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
-
+'forget-mail-sync' => [ 'LOCATION...',
+ 'forget sync information for a mail folder', @c_opt ],
'export-kw' => [ 'LOCATION...|--all',
'one-time export of keywords of sync sources',
qw(all:s mode=s), @c_opt ],
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
new file mode 100644
index 00000000..46dde1a7
--- /dev/null
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -0,0 +1,30 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei forget-mail-sync" drop synchronization information
+# TODO: figure out what to do about "lei index" users having
+# dangling references. Perhaps just documenting "lei index"
+# use being incompatible with "forget-mail-sync" use is
+# sufficient.
+
+package PublicInbox::LeiForgetMailSync;
+use strict;
+use v5.10.1;
+use PublicInbox::LeiExportKw;
+
+sub lei_forget_mail_sync {
+ my ($lei, @folders) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
+ delete $lms->{dbh};
+ $lms->lms_begin;
+ $lms->forget_folder($_) for @folders;
+ $lms->lms_commit;
+}
+
+*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 094cf1fd..d9c30580 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -345,4 +345,15 @@ EOF
$err;
}
+sub forget_folder {
+ my ($self, $folder) = @_;
+ my ($fid, $sth);
+ $fid = delete($self->{fmap}->{$folder}) //
+ _fid_for($self, $folder) // return;
+ my $dbh = $self->{dbh};
+ $dbh->do('DELETE FROM blob2name WHERE fid = ?', undef, $fid);
+ $dbh->do('DELETE FROM blob2num WHERE fid = ?', undef, $fid);
+ $dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid);
+}
+
1;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index d3935c82..5283cc23 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -71,5 +71,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
like($lei_out, qr!^Content-Type: multipart/mixed;!sm, 'got full blob');
lei_ok('blob', "$psgi_attach:2");
is($lei_out, "b64\xde\xad\xbe\xef\n", 'got attachment');
+
+ lei_ok 'forget-mail-sync', $url;
+ lei_ok 'ls-mail-sync';
+ unlike($lei_out, qr!\Q$host_port\E!, 'sync info gone after forget');
});
done_testing;
^ permalink raw reply related [relevance 55%]
* [PATCH] lei export-kw: fix case-insensitivity typo for IMAP URLs
@ 2021-05-24 22:48 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-24 22:48 UTC (permalink / raw)
To: meta
Fixes: a9bfba8eb31dbb4c ("lei inspect: use LeiMailSync->match_imap_url")
---
lib/PublicInbox/LeiExportKw.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 404570c5..d8ba8fd5 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -75,7 +75,7 @@ sub input_path_url {
my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
$lms->each_src($input, \&export_kw_md, $self, $mdir);
- } elsif ($input =~ m!\Aimaps?://i!) {
+ } elsif ($input =~ m!\Aimaps?://!i) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{nwr}->mic_for_folder($uri);
$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
^ permalink raw reply related [relevance 71%]
* [PATCH 0/2] lei IMAP usability stuff
@ 2021-05-23 21:36 71% Eric Wong
2021-05-23 21:36 46% ` [PATCH 1/2] lei inspect: use LeiMailSync->match_imap_url Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-23 21:36 UTC (permalink / raw)
To: meta
Slowly making IMAP usability less bad.
Eric Wong (2):
lei inspect: use LeiMailSync->match_imap_url
lei_mail_sync: reject IMAP URLs w/o UIDVALIDITY
lib/PublicInbox/LeiExportKw.pm | 43 ++++++----------------------------
lib/PublicInbox/LeiInspect.pm | 13 +---------
lib/PublicInbox/LeiMailSync.pm | 37 +++++++++++++++++++++++++++++
t/lei-import-imap.t | 10 ++++++--
4 files changed, 53 insertions(+), 50 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei inspect: use LeiMailSync->match_imap_url
2021-05-23 21:36 71% [PATCH 0/2] lei IMAP usability stuff Eric Wong
@ 2021-05-23 21:36 46% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 21:36 UTC (permalink / raw)
To: meta
Move match_imap_url into LeiMailSync so it can be used in more
places, such as "lei inspect". Upcoming commands such as
"lei forget-mail-sync" and {add,forget,pause,resume}-watch will
also support relaxed IMAP matching rules since there's
no reasonable way to expect users use ";UIDVALIDITY=" on the
command-line.
---
lib/PublicInbox/LeiExportKw.pm | 43 ++++++----------------------------
lib/PublicInbox/LeiInspect.pm | 13 +---------
lib/PublicInbox/LeiMailSync.pm | 32 +++++++++++++++++++++++++
t/lei-import-imap.t | 10 ++++++--
4 files changed, 48 insertions(+), 50 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 82a4db04..404570c5 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -75,7 +75,7 @@ sub input_path_url {
my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
$lms->each_src($input, \&export_kw_md, $self, $mdir);
- } elsif ($input =~ m!\Aimaps?://!) {
+ } elsif ($input =~ m!\Aimaps?://i!) {
my $uri = PublicInbox::URIimap->new($input);
my $mic = $self->{nwr}->mic_for_folder($uri);
$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
@@ -84,37 +84,6 @@ sub input_path_url {
$lms->lms_commit;
}
-sub match_imap_url ($$) {
- my ($all, $url) = @_; # $all = [ $lms->folders ];
- require PublicInbox::URIimap;
- my $cli = PublicInbox::URIimap->new($url)->canonical;
- my ($s, $h, $mb) = ($cli->scheme, $cli->host, $cli->mailbox);
- my @uri = map { PublicInbox::URIimap->new($_)->canonical }
- grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$mb\E\b!, @$all);
- my @match;
- for my $x (@uri) {
- next if $x->mailbox ne $cli->mailbox;
- next if $x->host ne $cli->host;
- next if $x->port != $cli->port;
- my $x_uidval = $x->uidvalidity;
- next if ($cli->uidvalidity // $x_uidval) != $x_uidval;
-
- # allow nothing in CLI to possibly match ";AUTH=ANONYMOUS"
- if (defined($x->auth) && !defined($cli->auth) &&
- !defined($cli->user)) {
- push @match, $x;
- # or maybe user was forgotten on CLI:
- } elsif (defined($x->user) && !defined($cli->user)) {
- push @match, $x;
- } elsif (($x->user//"\0") eq ($cli->user//"\0")) {
- push @match, $x;
- }
- }
- return $match[0] if scalar(@match) <= 1;
- warn "E: `$url' is ambiguous:\n\t", join("\n\t", @match), "\n";
- undef;
-}
-
sub lei_export_kw {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
@@ -166,12 +135,14 @@ EOM
$_ = $d;
} elsif (m!\Aimaps?://!i) {
my $orig = $_;
- if (my $canon = match_imap_url(\@all, $orig)) {
+ my $res = $lms->match_imap_url($orig, $all);
+ if (ref $res) {
+ $_ = $$res;
$lei->qerr(<<EOM);
-# using `$canon' instead of `$orig'
+# using `$res' instead of `$orig'
EOM
- $_ = $canon;
} else {
+ $lei->err($res) if defined $res;
push @no, $orig;
}
} else {
@@ -188,7 +159,7 @@ EOF
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $opt->{jobs} // scalar(@{$self->{inputs}}) || 1;
- if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
+ if (my @ro = grep(!/\A(?:maildir|imaps?):/i, @folders)) {
return $lei->fail("cannot export to read-only folders: @ro");
}
my $m = $opt->{mode} // 'merge';
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index f79ebc9a..7fd33289 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -31,18 +31,7 @@ sub inspect_sync_folder ($$) {
my $lms = $lse->lms or return $ent;
my @folders;
if ($folder =~ m!\Aimaps?://!i) {
- require PublicInbox::URIimap;
- my $uri = PublicInbox::URIimap->new($folder)->canonical;
- if (defined($uri->uidvalidity)) {
- $folders[0] = $$uri;
- } else {
- my @maybe = $lms->folders($$uri);
- @folders = grep {
- my $u = PublicInbox::URIimap->new($_);
- $uri->uidvalidity($u->uidvalidity);
- $$uri eq $$u;
- } @maybe;
- }
+ @folders = map { $_->as_string } $lms->match_imap_url($folder);
} elsif ($folder =~ m!\A(maildir|mh):(.+)!i) {
my $type = lc $1;
$folders[0] = "$type:".$lei->abs_path($2);
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 32e17c65..b2986686 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -265,4 +265,36 @@ WHERE b.oidbin = ?
undef;
}
+sub match_imap_url {
+ my ($self, $url, $all) = @_; # $all = [ $lms->folders ];
+ $all //= [ $self->folders ];
+ require PublicInbox::URIimap;
+ my $want = PublicInbox::URIimap->new($url)->canonical;
+ my ($s, $h, $mb) = ($want->scheme, $want->host, $want->mailbox);
+ my @uri = map { PublicInbox::URIimap->new($_)->canonical }
+ grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$mb\E\b!, @$all);
+ my @match;
+ for my $x (@uri) {
+ next if $x->mailbox ne $want->mailbox;
+ next if $x->host ne $want->host;
+ next if $x->port != $want->port;
+ my $x_uidval = $x->uidvalidity;
+ next if ($want->uidvalidity // $x_uidval) != $x_uidval;
+
+ # allow nothing in want to possibly match ";AUTH=ANONYMOUS"
+ if (defined($x->auth) && !defined($want->auth) &&
+ !defined($want->user)) {
+ push @match, $x;
+ # or maybe user was forgotten on CLI:
+ } elsif (defined($x->user) && !defined($want->user)) {
+ push @match, $x;
+ } elsif (($x->user//"\0") eq ($want->user//"\0")) {
+ push @match, $x;
+ }
+ }
+ return @match if wantarray;
+ scalar(@match) <= 1 ? $match[0] :
+ "E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
+}
+
1;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index d424ebb1..d3935c82 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -22,9 +22,15 @@ test_lei({ tmpdir => $tmpdir }, sub {
is_deeply(json_utf8->decode($lei_out), {}, 'no inspect stats, yet');
lei_ok('import', $url);
+ lei_ok('inspect', $url);
+ my $res = json_utf8->decode($lei_out);
+ is(scalar keys %$res, 1, 'got one key in inspect URL');
+ my $re = qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E
+ /t\.v2\.0;UIDVALIDITY=\d+!x;
+ like((keys %$res)[0], qr/$re\z/, 'got expanded key');
+
lei_ok 'ls-mail-sync';
- like($lei_out, qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E
- /t\.v2\.0;UIDVALIDITY=\d+\n\z!x, 'ls-mail-sync');
+ like($lei_out, qr!$re\n\z!, 'ls-mail-sync');
chomp(my $u = $lei_out);
lei_ok('import', $u, \'UIDVALIDITY match in URL');
$url = $u;
^ permalink raw reply related [relevance 46%]
* [PATCH] lei <q|up>: set \Recent on non-empty mbox and Maildir
@ 2021-05-23 8:01 35% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 8:01 UTC (permalink / raw)
To: meta
Despite JMAP not supporting the equivalent of the IMAP \Recent
flag, it is useful for "lei q --augment", and "lei up" users to
be able to distinguish new results from old-but-unread messages
in an mbox or Maildir.
For mbox family messages, we'll drop the "O" status flag when
appending to mboxes, and we'll write to the "new" subdirectory
of Maildirs.
Behavior when writing to initially empty Maildirs and mboxes
remains unchanged since there's no need to distinguish between
new and old results in the initial case. Having users wait
for a rename(2) storm or complete mbox rewrite hurts UX.
With IMAP mailboxes, \Recent is already enforced by the IMAP
server and IMAP clients have no way of changing it(*)
(*) mutt uses the "Old" IMAP flag which isn't part of RFC 3501,
other MUAs may do similar things.
---
lib/PublicInbox/LeiDedupe.pm | 6 +++++
lib/PublicInbox/LeiSavedSearch.pm | 7 ++++++
lib/PublicInbox/LeiToMail.pm | 42 +++++++++++++++++++++----------
lib/PublicInbox/LeiXSearch.pm | 15 +++++++++--
t/lei-q-kw.t | 13 ++++++++--
t/lei-q-save.t | 4 +--
t/lei_to_mail.t | 16 ++++++------
7 files changed, 75 insertions(+), 28 deletions(-)
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index 378f748e..ed52e417 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -127,4 +127,10 @@ sub pause_dedupe {
delete($skv->{dbh}) if $skv;
}
+sub dedupe_nr {
+ my $skv = $_[0]->[0] or return undef;
+ my @n = $skv->count;
+ $n[0];
+}
+
1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 01b987d1..48d252f1 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -309,6 +309,13 @@ E: rename($dir_old, $dir_new) error: $!
EOM
}
+# cf. LeiDedupe->dedupe_nr
+sub dedupe_nr {
+ my $oidx = $_[0]->{oidx} // die 'BUG: no {oidx}';
+ my @n = $oidx->{dbh}->selectrow_array('SELECT COUNT(*) FROM over');
+ $n[0];
+}
+
no warnings 'once';
*nntp_url = \&cloneurl;
*base_url = \&PublicInbox::Inbox::base_url;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index f3c03969..ad6b9439 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -41,11 +41,14 @@ sub _mbox_hdr_buf ($$$) {
warn "# keyword `$k' not supported for mbox\n";
}
}
- # Messages are always 'O' (non-\Recent in IMAP), it saves
- # MUAs the trouble of rewriting the mbox if no other
- # changes are made. We put 'O' at the end (e.g. "Status: RO")
- # to match mutt(1) output.
- $eml->header_set('Status', join('', sort(@{$hdr{Status}})). 'O');
+ # When writing to empty mboxes, messages are always 'O'
+ # (not-\Recent in IMAP), it saves MUAs the trouble of
+ # rewriting the mbox if no other changes are made.
+ # We put 'O' at the end (e.g. "Status: RO") to match mutt(1) output.
+ # We only set smsg->{-recent} if augmenting existing stores.
+ my $status = join('', sort(@{$hdr{Status}}));
+ $status .= 'O' unless $smsg->{-recent};
+ $eml->header_set('Status', $status) if $status;
if (my $chars = delete $hdr{'X-Status'}) {
$eml->header_set('X-Status', join('', sort(@$chars)));
}
@@ -196,11 +199,13 @@ sub _mbox_write_cb ($$) {
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe;
my $lse = $lei->{lse}; # may be undef
+ my $set_recent = $dedupe->dedupe_nr;
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($buf);
return if $dedupe->is_dup($eml, $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
+ $smsg->{-recent} = 1 if $set_recent;
$buf = $eml2mbox->($eml, $smsg);
if ($atomic_append) {
atomic_append($lei, $buf);
@@ -248,8 +253,8 @@ sub kw2suffix ($;@) {
join('', sort(map { $kw2char{$_} // () } @$kw, @_));
}
-sub _buf2maildir {
- my ($dst, $buf, $smsg) = @_;
+sub _buf2maildir ($$$$) {
+ my ($dst, $buf, $smsg, $dir) = @_;
my $kw = $smsg->{kw} // [];
my $rand = ''; # chosen by die roll :P
my ($tmp, $fh, $base, $ok);
@@ -260,11 +265,7 @@ sub _buf2maildir {
} while (!($ok = sysopen($fh, $tmp, O_CREAT|O_EXCL|O_WRONLY)) &&
$!{EEXIST} && ($rand = _rand.','));
if ($ok && print $fh $$buf and close($fh)) {
- # ignore new/ and write only to cur/, otherwise MUAs
- # with R/W access to the Maildir will end up doing
- # a mass rename which can take a while with thousands
- # of messages.
- $dst .= 'cur/';
+ $dst .= $dir; # 'new/' or 'cur/'
$rand = '';
do {
$base = $rand.$common.':2,'.kw2suffix($kw);
@@ -289,6 +290,11 @@ sub _maildir_write_cb ($$) {
my $lse = $lei->{lse}; # may be undef
my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
my $out = $sto ? 'maildir:'.$lei->rel2abs($dst) : undef;
+
+ # Favor cur/ and only write to new/ when augmenting. This
+ # saves MUAs from having to do a mass rename when the initial
+ # search result set is huge.
+ my $dir = $dedupe && $dedupe->dedupe_nr ? 'new/' : 'cur/';
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
@@ -296,7 +302,8 @@ sub _maildir_write_cb ($$) {
PublicInbox::Eml->new($$bref),
$smsg);
$lse->xsmsg_vmd($smsg) if $lse;
- my $n = _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+ my $n = _buf2maildir($dst, $bref // \($eml->as_string),
+ $smsg, $dir);
$sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
++$lei->{-nr_write};
}
@@ -648,7 +655,16 @@ sub do_post_auth {
$lei->{1} = $zpipe->[1];
close $zpipe->[0];
}
+ my $au_peers = delete $self->{au_peers};
+ if ($au_peers) { # wait for peer l2m to finish augmenting:
+ $au_peers->[1] = undef;
+ sysread($au_peers->[0], my $barrier1, 1);
+ }
$self->{wcb} = $self->write_cb($lei);
+ if ($au_peers) { # wait for peer l2m to set write_cb
+ $au_peers->[3] = undef;
+ sysread($au_peers->[2], my $barrier2, 1);
+ }
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index e69a4edd..3482082d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -482,11 +482,22 @@ sub do_query {
if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {
$lei->start_mua;
}
+ my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
+ if ($l2m->{-wq_nr_workers} > 1 &&
+ $l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
+ # setup two barriers to coordinate dedupe_nr
+ # between l2m workers
+ pipe(my ($a_r, $a_w)) or die "pipe: $!";
+ fcntl($a_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
+ pipe(my ($b_r, $b_w)) or die "pipe: $!";
+ fcntl($b_r, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
+ $l2m->{au_peers} = [ $a_r, $a_w, $b_r, $b_w ];
+ }
$l2m->wq_workers_start('lei2mail', undef,
$lei->oldset, { lei => $lei });
pipe($lei->{startq}, $lei->{au_done}) or die "pipe: $!";
- # 1031: F_SETPIPE_SZ
- fcntl($lei->{startq}, 1031, 4096) if $^O eq 'linux';
+ fcntl($lei->{startq}, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
+ delete $l2m->{au_peers};
}
$self->wq_workers_start('lei_xsearch', undef,
$lei->oldset, { lei => $lei });
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index c00a0a43..074c573d 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -14,7 +14,6 @@ my $exp = {
'<testmessage@example.com>' => eml_load('t/utf8.eml'),
};
$exp->{'<qp@example.com>'}->header_set('Status', 'RO');
-$exp->{'<testmessage@example.com>'}->header_set('Status', 'O');
test_lei(sub {
lei_ok(qw(import -F eml t/plack-qp.eml));
@@ -105,7 +104,17 @@ for my $sfx ('', '.gz') {
my %res;
PublicInbox::MboxReader->mboxrd($fh, sub {
my ($eml) = @_;
- $res{$eml->header_raw('Message-ID')} = $eml;
+ my $mid = $eml->header_raw('Message-ID');
+ if ($mid eq '<testmessage@example.com>') {
+ is_deeply([$eml->header('Status')], [],
+ "no status $sfx");
+ $eml->header_set('Status');
+ } elsif ($mid eq '<qp@example.com>') {
+ is($eml->header('Status'), 'RO', 'status preserved');
+ } else {
+ fail("unknown mid $mid");
+ }
+ $res{$mid} = $eml;
});
is_deeply(\%res, $exp, '--augment worked');
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 753d5b20..aed38a51 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -42,7 +42,7 @@ test_lei(sub {
lei_ok qw(up -q md -C), $home;
lei_ok qw(up -q . -C), "$home/md";
lei_ok qw(up -q), "/$home/md";
- my %after = map { $_ => 1 } glob("$home/md/cur/*");
+ my %after = map { $_ => 1 } glob("$home/md/{new,cur}/*");
is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
is(scalar(keys %after), 1, 'one new message added');
is_deeply(eml_load((keys %after)[0]), $doc2, 'doc2 matches');
@@ -155,7 +155,7 @@ test_lei(sub {
$im->add(PublicInbox::Eml->new($diff));
$im->done;
lei_ok('up', $o);
- @m = glob("$o/cur/*");
+ @m = glob("$o/{new,cur}/*");
is(scalar(@m), 2, 'got 2nd result due to different OID');
SKIP: {
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 32532a98..35904706 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -90,7 +90,7 @@ my $fn = "$tmpdir/x.mbox";
my ($mbox) = shuffle(@MBOX); # pick one, shouldn't matter
my $wcb_get = sub {
my ($fmt, $dst) = @_;
- delete $lei->{dedupe};
+ delete $lei->{dedupe}; # to be recreated
$lei->{ovv} = bless {
fmt => $fmt,
dst => $dst
@@ -119,13 +119,12 @@ my $orig = do {
like($raw, qr/^blah\n/sm, 'wrote content');
unlink $fn or BAIL_OUT $!;
- local $lei->{opt} = { jobs => 2 };
$wcb = $wcb_get->($mbox, $fn);
ok(-f $fn && !-s _, 'truncated mbox destination');
$wcb->(\($dup = $buf), $deadbeef);
$commit->($wcb);
open $fh, '<', $fn or BAIL_OUT $!;
- is(do { local $/; <$fh> }, $raw, 'jobs > 1');
+ is(do { local $/; <$fh> }, $raw, 'wrote identical content');
$raw;
};
@@ -158,21 +157,20 @@ for my $zsfx (qw(gz bz2 xz)) {
ok($dc_cmd, "decompressor for .$zsfx");
my $f = "$fn.$zsfx";
my $wcb = $wcb_get->($mbox, $f);
- $wcb->(\(my $dup = $buf), $deadbeef);
+ $wcb->(\(my $dup = $buf), { %$deadbeef });
$commit->($wcb);
my $uncompressed = xqx([@$dc_cmd, $f]);
is($uncompressed, $orig, "$zsfx works unlocked");
- local $lei->{opt} = { jobs => 2 }; # for atomic writes
unlink $f or BAIL_OUT "unlink $!";
$wcb = $wcb_get->($mbox, $f);
- $wcb->(\($dup = $buf), $deadbeef);
+ $wcb->(\($dup = $buf), { %$deadbeef });
$commit->($wcb);
is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock");
local $lei->{opt} = { augment => 1 };
$wcb = $wcb_get->($mbox, $f);
- $wcb->(\($dup = $buf . "\nx\n"), $deadbeef);
+ $wcb->(\($dup = $buf . "\nx\n"), { %$deadbeef });
$commit->($wcb);
my $cat = popen_rd([@$dc_cmd, $f]);
@@ -182,9 +180,9 @@ for my $zsfx (qw(gz bz2 xz)) {
like($raw[1], qr/\nblah\n\nx\n\z/s, "augmented $zsfx");
like($raw[0], qr/\nblah\n\z/s, "original preserved $zsfx");
- local $lei->{opt} = { augment => 1, jobs => 2 };
+ local $lei->{opt} = { augment => 1 };
$wcb = $wcb_get->($mbox, $f);
- $wcb->(\($dup = $buf . "\ny\n"), $deadbeef);
+ $wcb->(\($dup = $buf . "\ny\n"), { %$deadbeef });
$commit->($wcb);
my @raw3;
^ permalink raw reply related [relevance 35%]
* [PATCH 2/3] lei export-kw: support exporting keywords to IMAP
2021-05-23 1:38 71% [PATCH 0/3] lei export-kw: IMAP support Eric Wong
@ 2021-05-23 1:38 49% ` Eric Wong
2021-05-23 1:38 47% ` [PATCH 3/3] lei export-kw: relax IMAP URL matching Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 1:38 UTC (permalink / raw)
To: meta
We support writing to IMAP stores in other places (just like
Maildir), and it's actually less complex for us to write to
IMAP. Neither usability nor performance is ideal, but usability
will be addressed in the next commit to relax CLI argument
checking.
Performance is poor due to the synchronous Mail::IMAPClient
API and will need to be addressed with pipelining sometime
further in the future.
---
lib/PublicInbox/LeiExportKw.pm | 31 +++++++++++++++++++++++--------
lib/PublicInbox/LeiToMail.pm | 9 +++++----
lib/PublicInbox/NetWriter.pm | 10 ++++++++++
xt/net_writer-imap.t | 18 +++++++++++++-----
4 files changed, 51 insertions(+), 17 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index db4f7441..5ad33959 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -59,15 +59,28 @@ sub export_kw_md { # LeiMailSync->each_src callback
$self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
}
+sub export_kw_imap { # LeiMailSync->each_src callback
+ my ($oidbin, $id, $self, $mic) = @_;
+ my $oidhex = unpack('H*', $oidbin);
+ my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return;
+ $self->{imap_mod_kw}->($self->{nwr}, $mic, $id, [ keys %$sto_kw ]);
+}
+
# overrides PublicInbox::LeiInput::input_path_url
sub input_path_url {
my ($self, $input, @args) = @_;
my $lms = $self->{lms} //= $self->{lse}->lms;
$lms->lms_begin;
- if ($input =~ s/\Amaildir://i) {
+ if ($input =~ /\Amaildir:(.+)/i) {
+ my $mdir = $1;
require PublicInbox::LeiToMail; # kw2suffix
- $lms->each_src("maildir:$input", \&export_kw_md, $self, $input);
- }
+ $lms->each_src($input, \&export_kw_md, $self, $mdir);
+ } elsif ($input =~ m!\Aimaps?://!) {
+ my $uri = PublicInbox::URIimap->new($input);
+ my $mic = $self->{nwr}->mic_for_folder($uri);
+ $lms->each_src($$uri, \&export_kw_imap, $self, $mic);
+ $mic->expunge;
+ } else { die "BUG: $input not supported" }
$lms->lms_commit;
}
@@ -137,11 +150,6 @@ EOF
if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
return $lei->fail("cannot export to read-only folders: @ro");
}
- if (my $net = $lei->{net}) {
- require PublicInbox::NetWriter;
- bless $net, 'PublicInbox::NetWriter';
- }
- undef $lms;
my $m = $opt->{mode} // 'merge';
if ($m eq 'merge') { # default
$self->{-merge_kw} = 1;
@@ -151,6 +159,13 @@ EOF
--mode=$m not supported (`set' or `merge')
EOM
}
+ if (my $net = $lei->{net}) {
+ require PublicInbox::NetWriter;
+ $self->{nwr} = bless $net, 'PublicInbox::NetWriter';
+ $self->{imap_mod_kw} = $net->can($self->{-merge_kw} ?
+ 'imap_add_kw' : 'imap_set_kw');
+ }
+ undef $lms;
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9d4c856..f3c03969 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -307,11 +307,12 @@ sub _imap_write_cb ($$) {
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
my $append = $lei->{net}->can('imap_append');
- my $mic = $lei->{net}->mic_get($self->{uri});
- my $folder = $self->{uri}->mailbox;
+ my $uri = $self->{uri};
+ my $mic = $lei->{net}->mic_get($uri);
+ my $folder = $uri->mailbox;
+ $uri->uidvalidity($mic->uidvalidity($folder));
my $lse = $lei->{lse}; # may be undef
my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
- my $out = $lei->{ovv}->{dst};
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
@@ -327,7 +328,7 @@ sub _imap_write_cb ($$) {
# imap_append returns UID if IMAP server has UIDPLUS extension
($sto && $uid =~ /\A[0-9]+\z/) and
$sto->ipc_do('set_sync_info',
- $smsg->{blob}, $out, $uid + 0);
+ $smsg->{blob}, $$uri, $uid + 0);
++$lei->{-nr_write};
}
}
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 2032a1fd..8ec7f85c 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,10 +26,20 @@ sub imap_append {
die "APPEND $folder: $@";
}
+# updates $uri with UIDVALIDITY
sub mic_for_folder {
my ($self, $uri) = @_;
my $mic = $self->mic_get($uri) or die "E: not connected: $@";
$mic->select($uri->mailbox) or return;
+ my $uidval;
+ for ($mic->Results) {
+ /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
+ $uidval = $1;
+ last;
+ }
+ $uidval //= $mic->uidvalidity($uri->mailbox) or
+ die "E: failed to get uidvalidity from <$uri>: $@";
+ $uri->uidvalidity($uidval);
$mic;
}
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 1298b958..0e6d4831 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -157,12 +157,20 @@ test_lei(sub {
lei_ok qw(import -F eml), $f, \'import local copy w/o keywords';
+ lei_ok 'ls-mail-sync'; diag $lei_out;
+ lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3
+ lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f;
+ lei_ok 'ls-mail-sync'; diag $lei_out;
+ chomp(my $uri_val = $lei_out);
+ lei_ok 'export-kw', $uri_val;
$mic = $nwr->mic_for_folder($folder_uri);
- # dummy set to ensure second set_kw clobbers
- $nwr->imap_set_kw($mic, $uid[0], [ qw(seen answered flagged) ]
- )->expunge or BAIL_OUT "expunge $@";
- $nwr->imap_set_kw($mic, $uid[0], [ 'seen' ]
- )->expunge or BAIL_OUT "expunge $@";
+ my $flags = $mic->flags($uid[0]);
+ is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ],
+ 'IMAP flags set by export-kw') or diag explain($flags);
+
+ # ensure this imap_set_kw clobbers
+ $nwr->imap_set_kw($mic, $uid[0], [ 'seen' ])->expunge or
+ BAIL_OUT "expunge $@";
$mic = undef;
@res = ();
$nwr->imap_each($folder_uri, $imap_slurp_all, \@res);
^ permalink raw reply related [relevance 49%]
* [PATCH 3/3] lei export-kw: relax IMAP URL matching
2021-05-23 1:38 71% [PATCH 0/3] lei export-kw: IMAP support Eric Wong
2021-05-23 1:38 49% ` [PATCH 2/3] lei export-kw: support exporting keywords to IMAP Eric Wong
@ 2021-05-23 1:38 47% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-23 1:38 UTC (permalink / raw)
To: meta
It's unreasonable to expect UIDVALIDITY= to be specified in
command-line arguments. We'll also check for cases without
"$USER@" or ";AUTH=", since we accept those forms on the
command-line.
---
lib/PublicInbox/LeiExportKw.pm | 41 ++++++++++++++++++++++++++++++++++
xt/net_writer-imap.t | 30 ++++++++++++++-----------
2 files changed, 58 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 5ad33959..82a4db04 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -84,6 +84,37 @@ sub input_path_url {
$lms->lms_commit;
}
+sub match_imap_url ($$) {
+ my ($all, $url) = @_; # $all = [ $lms->folders ];
+ require PublicInbox::URIimap;
+ my $cli = PublicInbox::URIimap->new($url)->canonical;
+ my ($s, $h, $mb) = ($cli->scheme, $cli->host, $cli->mailbox);
+ my @uri = map { PublicInbox::URIimap->new($_)->canonical }
+ grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$mb\E\b!, @$all);
+ my @match;
+ for my $x (@uri) {
+ next if $x->mailbox ne $cli->mailbox;
+ next if $x->host ne $cli->host;
+ next if $x->port != $cli->port;
+ my $x_uidval = $x->uidvalidity;
+ next if ($cli->uidvalidity // $x_uidval) != $x_uidval;
+
+ # allow nothing in CLI to possibly match ";AUTH=ANONYMOUS"
+ if (defined($x->auth) && !defined($cli->auth) &&
+ !defined($cli->user)) {
+ push @match, $x;
+ # or maybe user was forgotten on CLI:
+ } elsif (defined($x->user) && !defined($cli->user)) {
+ push @match, $x;
+ } elsif (($x->user//"\0") eq ($cli->user//"\0")) {
+ push @match, $x;
+ }
+ }
+ return $match[0] if scalar(@match) <= 1;
+ warn "E: `$url' is ambiguous:\n\t", join("\n\t", @match), "\n";
+ undef;
+}
+
sub lei_export_kw {
my ($lei, @folders) = @_;
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
@@ -133,6 +164,16 @@ EOM
my $d = 'maildir:'.$lei->rel2abs($_);
push(@no, $_) unless $all{$d};
$_ = $d;
+ } elsif (m!\Aimaps?://!i) {
+ my $orig = $_;
+ if (my $canon = match_imap_url(\@all, $orig)) {
+ $lei->qerr(<<EOM);
+# using `$canon' instead of `$orig'
+EOM
+ $_ = $canon;
+ } else {
+ push @no, $orig;
+ }
} else {
push @no, $_;
}
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 0e6d4831..ec8f80d1 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -23,7 +23,8 @@ my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)).
"-$$-".sprintf('%x', int(rand(0xffffffff)));
my $nwr = PublicInbox::NetWriter->new;
chop($imap_url) if substr($imap_url, -1) eq '/';
-my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder");
+my $folder_url = "$imap_url/$folder";
+my $folder_uri = PublicInbox::URIimap->new($folder_url);
is($folder_uri->mailbox, $folder, 'folder correct') or
BAIL_OUT "BUG: bad $$uri";
$nwr->add_url($$folder_uri);
@@ -120,13 +121,13 @@ test_lei(sub {
}
$set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) if $cred_set;
- lei_ok qw(q f:qp@example.com -o), $$folder_uri;
+ lei_ok qw(q f:qp@example.com -o), $folder_url;
$nwr->imap_each($folder_uri, $imap_slurp_all, my $res = []);
is(scalar(@$res), 1, 'got one deduped result') or diag explain($res);
is_deeply($res->[0]->[1], $plack_qp_eml,
'lei q wrote expected result');
- lei_ok qw(q f:matz -a -o), $$folder_uri;
+ lei_ok qw(q f:matz -a -o), $folder_url;
$nwr->imap_each($folder_uri, $imap_slurp_all, my $aug = []);
is(scalar(@$aug), 2, '2 results after augment') or diag explain($aug);
my $exp = $res->[0]->[1]->as_string;
@@ -136,13 +137,13 @@ test_lei(sub {
is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1,
'new result shown after augment');
- lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string;
+ lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_url;
$nwr->imap_each($folder_uri, $imap_slurp_all, my $empty = []);
is(scalar(@$empty), 0, 'no results w/o augment');
my $f = 't/utf8.eml'; # <testmessage@example.com>
$exp = eml_load($f);
- lei_ok qw(convert -F eml -o), $$folder_uri, $f;
+ lei_ok qw(convert -F eml -o), $folder_url, $f;
my (@uid, @res);
$nwr->imap_each($folder_uri, sub {
my ($u, $uid, $kw, $eml) = @_;
@@ -157,12 +158,15 @@ test_lei(sub {
lei_ok qw(import -F eml), $f, \'import local copy w/o keywords';
- lei_ok 'ls-mail-sync'; diag $lei_out;
- lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3
+ lei_ok 'import', $folder_url; # populate mail_sync.sqlite3
lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f;
- lei_ok 'ls-mail-sync'; diag $lei_out;
- chomp(my $uri_val = $lei_out);
- lei_ok 'export-kw', $uri_val;
+ lei_ok 'ls-mail-sync';
+ my @ls = split(/\n/, $lei_out);
+ is(scalar(@ls), 1, 'only one folder in ls-mail-sync') or xbail(\@ls);
+ for my $l (@ls) {
+ like($l, qr/;UIDVALIDITY=\d+\z/, 'UIDVALIDITY');
+ }
+ lei_ok 'export-kw', $folder_url;
$mic = $nwr->mic_for_folder($folder_uri);
my $flags = $mic->flags($uid[0]);
is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ],
@@ -177,7 +181,7 @@ test_lei(sub {
is_deeply(\@res, [ [ ['seen'], $exp ] ], 'seen flag set') or
diag explain(\@res);
- lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string,
+ lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_url,
\'clobber folder but import flag';
$nwr->imap_each($folder_uri, $imap_slurp_all, $empty = []);
is_deeply($empty, [], 'clobbered folder');
@@ -206,7 +210,7 @@ EOM
run_script(\@cmd) or BAIL_OUT "init wtest";
xsys(qw(git config), "--file=$ENV{HOME}/.public-inbox/config",
'publicinbox.wtest.watch',
- $$folder_uri) == 0 or BAIL_OUT "git config $?";
+ $folder_url) == 0 or BAIL_OUT "git config $?";
my $watcherr = "$ENV{HOME}/watch.err";
open my $err_wr, '>>', $watcherr or BAIL_OUT $!;
my $pub_cfg = PublicInbox::Config->new;
@@ -231,7 +235,7 @@ EOM
ok(defined($mm->num_for('forwarded@test.example.com')),
'-watch takes forwarded message');
undef $w; # done with watch
- lei_ok qw(import), $$folder_uri;
+ lei_ok qw(import), $folder_url;
lei_ok qw(q m:forwarded@test.example.com);
is_deeply(json_utf8->decode($lei_out)->[0]->{kw}, ['forwarded'],
'forwarded kw imported from IMAP');
^ permalink raw reply related [relevance 47%]
* [PATCH 0/3] lei export-kw: IMAP support
@ 2021-05-23 1:38 71% Eric Wong
2021-05-23 1:38 49% ` [PATCH 2/3] lei export-kw: support exporting keywords to IMAP Eric Wong
2021-05-23 1:38 47% ` [PATCH 3/3] lei export-kw: relax IMAP URL matching Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-23 1:38 UTC (permalink / raw)
To: meta
Dealing with high-latency connections to be addressed at some
point, but probably not right away...
Eric Wong (3):
net_reader|net_writer: pass URI refs deeper into callbacks
lei export-kw: support exporting keywords to IMAP
lei export-kw: relax IMAP URL matching
lib/PublicInbox/LeiExportKw.pm | 72 ++++++++++++++++++++++++++++++----
lib/PublicInbox/LeiImport.pm | 4 +-
lib/PublicInbox/LeiToMail.pm | 13 +++---
lib/PublicInbox/NetReader.pm | 11 +++---
lib/PublicInbox/NetWriter.pm | 22 +++++++----
xt/net_writer-imap.t | 38 ++++++++++++------
6 files changed, 117 insertions(+), 43 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 5/8] lei index: support command-line options
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
2021-05-21 10:28 53% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
@ 2021-05-21 10:28 71% ` Eric Wong
2021-05-21 10:28 32% ` [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs Eric Wong
2021-05-21 10:28 46% ` [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
This mostly takes after "lei import", and at least --quiet needs
to be supported.
---
lib/PublicInbox/LEI.pm | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d7768426..15680fe3 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -233,6 +233,11 @@ our %CMD = ( # sorted in order of importance/use:
'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
qw(prune), @c_opt ],
+'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
+ qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
+ verbose|v+ incremental!),
+ PublicInbox::LeiQuery::curl_opt(), # mainly for --proxy=
+ @c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s
^ permalink raw reply related [relevance 71%]
* [PATCH 2/8] lei: drop EOFpipe in favor of PktOp
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
@ 2021-05-21 10:28 66% ` Eric Wong
2021-05-21 10:28 53% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
lei already uses PktOp and SOCK_SEQPACKET throughout; whereas
EOFpipe had one single use in lei. Since PktOp is a strict
superset of EOFpipe functionality, we may be able to get rid of
EOFpipe entirely.
However, lei is considered a portability canary and I'm not sure
if the stable public-inbox-* code can drop EOFpipe just yet.
---
lib/PublicInbox/LEI.pm | 9 ++++-----
lib/PublicInbox/LeiOverview.pm | 2 +-
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 98e79a76..d7768426 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1151,7 +1151,7 @@ sub lazy_start {
(Socket::MsgHdr || Inline::C) missing/unconfigured (narg=$narg);
require PublicInbox::Listener;
- require PublicInbox::EOFpipe;
+ require PublicInbox::PktOp;
(-p STDOUT) or die "E: stdout must be a pipe\n";
open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
STDIN->autoflush(1);
@@ -1165,13 +1165,12 @@ sub lazy_start {
my $exit_code;
my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
local $quit = do {
- pipe(my ($eof_r, $eof_w)) or die "pipe: $!";
- PublicInbox::EOFpipe->new($eof_r, \&noop, undef);
+ my (undef, $eof_p) = PublicInbox::PktOp->pair;
sub {
$exit_code //= shift;
my $lis = $pil or exit($exit_code);
- # closing eof_w triggers \&noop wakeup
- $listener = $eof_w = $pil = $path = undef;
+ # closing eof_p triggers \&noop wakeup
+ $listener = $eof_p = $pil = $path = undef;
$lis->close; # DS::close
PublicInbox::DS->SetLoopTimeout(1000);
};
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index bfb8b143..28891460 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -119,7 +119,7 @@ sub ovv_begin {
} # TODO HTML/Atom/...
}
-# called once by parent (via PublicInbox::EOFpipe)
+# called once by parent (via PublicInbox::PktOp '' => query_done)
sub ovv_end {
my ($self, $lei) = @_;
if ($self->{fmt} eq 'json') {
^ permalink raw reply related [relevance 66%]
* [PATCH 3/8] lei tag: support tagging index-only messages
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
@ 2021-05-21 10:28 53% ` Eric Wong
2021-05-21 10:28 71% ` [PATCH 5/8] lei index: support command-line options Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
This will make some of our tests faster and allow users to try
more features of lei without high storage requirements.
---
lib/PublicInbox/LeiSearch.pm | 8 ++++++--
lib/PublicInbox/LeiTag.pm | 10 ++++++++--
lib/PublicInbox/LeiToMail.pm | 4 +++-
t/lei-index.t | 12 +++++++++++-
4 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index c2b12146..fb19229f 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -63,7 +63,9 @@ sub content_key ($) {
}
sub _cmp_1st { # git->cat_async callback
- my ($bref, $oid, $type, $size, $cmp) = @_; # cmp: [chash, xoids, smsg]
+ my ($bref, $oid, $type, $size, $cmp) = @_;
+ # cmp: [chash, xoids, smsg, lms]
+ $bref //= $cmp->[3] ? $cmp->[3]->local_blob($oid, 1) : undef;
if ($bref && content_hash(PublicInbox::Eml->new($bref)) eq $cmp->[0]) {
$cmp->[1]->{$oid} = $cmp->[2]->{num};
}
@@ -78,6 +80,8 @@ sub xoids_for {
my @overs = ($self->over // $self->overs_all);
my $git = $self->git;
my $xoids = {};
+ # no lms when used via {ale}:
+ my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir});
for my $mid (@$mids) {
for my $o (@overs) {
my ($id, $prev);
@@ -85,7 +89,7 @@ sub xoids_for {
next if $cur->{bytes} == 0 ||
$xoids->{$cur->{blob}};
$git->cat_async($cur->{blob}, \&_cmp_1st,
- [ $chash, $xoids, $cur ]);
+ [$chash, $xoids, $cur, $lms]);
if ($min && scalar(keys %$xoids) >= $min) {
$git->cat_async_wait;
return $xoids;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index c650e886..b6abd533 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -9,7 +9,8 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
- if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) {
+ if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
+ $self->{lei}->{ale}->xoids_for($eml)) {
$self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
$self->{vmd_mod});
} else {
@@ -17,7 +18,11 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
}
}
-sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
+sub input_mbox_cb {
+ my ($eml, $self) = @_;
+ $eml->header_set($_) for (qw(X-Status Status));
+ input_eml_cb($self, $eml);
+}
sub input_maildir_cb { # maildir_each_eml cb
my ($f, $kw, $eml, $self) = @_;
@@ -60,6 +65,7 @@ sub note_missing {
sub ipc_atfork_child {
my ($self) = @_;
PublicInbox::LeiInput::input_only_atfork_child($self);
+ $self->{lse} = $self->{lei}->{sto}->search;
# this goes out-of-scope at worker process exit:
PublicInbox::OnDestroy->new($$, \¬e_missing, $self);
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index da3a95d2..0cbdff8b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -650,7 +650,9 @@ sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
$lei->_lei_atfork_child;
- $self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse};
+ if (my $lse = $lei->{lse}) {
+ $self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
+ }
$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
diff --git a/t/lei-index.t b/t/lei-index.t
index b7dafb71..9a45d885 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -40,7 +40,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $res_a = json_utf8->decode($lei_out);
my $blob = $res_a->[0]->{'blob'};
like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example');
- lei_ok('blob', $blob);
+ lei_ok(qw(-C / blob), $blob);
is($lei_out, $expect, 'got expected blob via Maildir');
lei_ok(qw(q mid:qp@example.com -f text));
like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
@@ -58,6 +58,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $res_b = json_utf8->decode($lei_out);
is_deeply($res_b, $res_a, 'no extra DB entries');
+ # ensure tag works on index-only messages:
+ lei_ok(qw(tag +kw:seen t/utf8.eml));
+ lei_ok(qw(q mid:testmessage@example.com));
+ is_deeply(json_utf8->decode($lei_out)->[0]->{kw},
+ ['seen'], 'seen kw can be set on index-only message');
+
+ lei_ok(qw(q z:0.. -o), "$tmpdir/all-results") for (1..2);
+ is_deeply([xqx($all_obj)], \@objs,
+ 'no new objects after 2x q to trigger implicit import');
+
lei_ok('index', "nntp://$nntp_host_port/t.v2");
lei_ok('index', "imap://$imap_host_port/t.v2.0");
is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
^ permalink raw reply related [relevance 53%]
* [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
` (3 preceding siblings ...)
2021-05-21 10:28 32% ` [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs Eric Wong
@ 2021-05-21 10:28 46% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
Just having UIDVALIDITY in the URI isn't enough, since a single
lei user may have multiple IMAP logins on the same server.
This leads to compatibility problems and forces a reimport for
the few users already using this lei functionality, but it's not
stable nor released, yet.
---
lib/PublicInbox/NetReader.pm | 42 ++++++++++++++++++++++--------------
t/lei-import-imap.t | 9 +++++---
2 files changed, 32 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index fd0d1682..a532b218 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -58,12 +58,10 @@ sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
# mic_for may prompt the user and store auth info, prepares mic_get
sub mic_for ($$$$) { # mic = Mail::IMAPClient
- my ($self, $url, $mic_args, $lei) = @_;
- require PublicInbox::URIimap;
- my $uri = PublicInbox::URIimap->new($url);
+ my ($self, $uri, $mic_args, $lei) = @_;
require PublicInbox::GitCredential;
my $cred = bless {
- url => $url,
+ url => "$uri",
protocol => $uri->scheme,
host => $uri->host,
username => $uri->user,
@@ -83,13 +81,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
};
require PublicInbox::IMAPClient;
my $mic = mic_new($self, $mic_arg, $sec, $uri) or
- die "E: <$url> new: $@\n";
+ die "E: <$uri> new: $@\n";
# default to using STARTTLS if it's available, but allow
# it to be disabled since I usually connect to localhost
if (!$mic_arg->{Ssl} && !defined($mic_arg->{Starttls}) &&
$mic->has_capability('STARTTLS') &&
$mic->can('starttls')) {
- $mic->starttls or die "E: <$url> STARTTLS: $@\n";
+ $mic->starttls or die "E: <$uri> STARTTLS: $@\n";
}
# do we even need credentials?
@@ -111,8 +109,13 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
if ($mic->login && $mic->IsAuthenticated) {
# success! keep IMAPClient->new arg in case we get disconnected
$self->{mic_arg}->{$sec} = $mic_arg;
+ if ($cred) {
+ $uri->user($cred->{username}) if !defined($uri->user);
+ } elsif ($mic_arg->{Authmechanism} eq 'ANONYMOUS') {
+ $uri->auth('ANONYMOUS') if !defined($uri->auth);
+ }
} else {
- $err = "E: <$url> LOGIN: $@\n";
+ $err = "E: <$uri> LOGIN: $@\n";
if ($cred && defined($cred->{password})) {
$err =~ s/\Q$cred->{password}\E/*******/g;
}
@@ -304,15 +307,16 @@ sub imap_common_init ($;$) {
# make sure we can connect and cache the credentials in memory
$self->{mic_arg} = {}; # schema://authority => IMAPClient->new args
my $mics = {}; # schema://authority => IMAPClient obj
- for my $uri (@{$self->{imap_order}}) {
- my $sec = uri_section($uri);
+ for my $orig_uri (@{$self->{imap_order}}) {
+ my $sec = uri_section($orig_uri);
+ my $uri = PublicInbox::URIimap->new("$sec/");
my $mic = $mics->{$sec} //=
- mic_for($self, "$sec/", $mic_args, $lei) //
+ mic_for($self, $uri, $mic_args, $lei) //
die "Unable to continue\n";
next unless $self->isa('PublicInbox::NetWriter');
- my $dst = $uri->mailbox // next;
+ my $dst = $orig_uri->mailbox // next;
next if $mic->exists($dst); # already exists
- $mic->create($dst) or die "CREATE $dst failed <$uri>: $@";
+ $mic->create($dst) or die "CREATE $dst failed <$orig_uri>: $@";
}
$mics;
}
@@ -419,12 +423,18 @@ sub run_commit_cb ($) {
$cb->(@args);
}
-sub _itrk_last ($$;$) {
- my ($self, $uri, $r_uidval) = @_;
+sub itrk_last ($$;$$) {
+ my ($self, $uri, $r_uidval, $mic) = @_;
return (undef, undef, $r_uidval) unless $self->{incremental};
my ($itrk, $l_uid, $l_uidval);
if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0
$uri->uidvalidity($r_uidval) if defined $r_uidval;
+ if ($mic) {
+ my $auth = $mic->Authmechanism // '';
+ $uri->auth($auth) if $auth eq 'ANONYMOUS';
+ my $user = $mic->User;
+ $uri->user($user) if defined($user);
+ }
my $x;
$l_uid = ($lms && ($x = $lms->location_stats($$uri))) ?
$x->{'uid.max'} : undef;
@@ -459,7 +469,7 @@ E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
EOF
my $uri = $orig_uri->clone;
- my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval);
+ my ($itrk, $l_uid, $l_uidval) = itrk_last($self, $uri, $r_uidval, $mic);
return <<EOF if $l_uidval != $r_uidval;
E: $uri UIDVALIDITY mismatch
E: local=$l_uidval != remote=$r_uidval
@@ -612,7 +622,7 @@ sub _nntp_fetch_all ($$$) {
# IMAPTracker is also used for tracking NNTP, UID == article number
# LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
# expensive. So we assume newsgroups don't change:
- my ($itrk, $l_art) = _itrk_last($self, $uri);
+ my ($itrk, $l_art) = itrk_last($self, $uri);
# allow users to specify articles to refetch
# cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index fd15ef4f..d424ebb1 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -23,9 +23,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('import', $url);
lei_ok 'ls-mail-sync';
- like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-mail-sync');
+ like($lei_out, qr!\Aimap://;AUTH=ANONYMOUS\@\Q$host_port\E
+ /t\.v2\.0;UIDVALIDITY=\d+\n\z!x, 'ls-mail-sync');
chomp(my $u = $lei_out);
lei_ok('import', $u, \'UIDVALIDITY match in URL');
+ $url = $u;
$u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s;
ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected');
@@ -33,7 +35,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $inspect = json_utf8->decode($lei_out);
my @k = keys %$inspect;
is(scalar(@k), 1, 'one URL resolved');
- like($k[0], qr!\A\Q$url\E;UIDVALIDITY=\d+\z!, 'inspect URL matches');
+ is($k[0], $url, 'inspect URL matches');
my $stats = $inspect->{$k[0]};
is_deeply([ sort keys %$stats ],
[ qw(uid.count uid.max uid.min) ], 'keys match');
@@ -55,7 +57,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $x = json_utf8->decode($lei_out);
is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect');
is(ref($x->{'mail-sync'}), 'HASH', 'sync in inspect');
- is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect');
+ is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect')
+ or diag explain($x);
my $psgi_attach = 'cfa3622cbeffc9bd6b0fc66c4d60d420ba74f60d';
lei_ok('blob', $psgi_attach);
^ permalink raw reply related [relevance 46%]
* [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
` (2 preceding siblings ...)
2021-05-21 10:28 71% ` [PATCH 5/8] lei index: support command-line options Eric Wong
@ 2021-05-21 10:28 32% ` Eric Wong
2021-05-21 10:28 46% ` [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
IMAP will eventually be supported.
---
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 4 +
lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiMailSync.pm | 10 ++
lib/PublicInbox/LeiSearch.pm | 14 +++
lib/PublicInbox/LeiToMail.pm | 8 +-
lib/PublicInbox/MdirReader.pm | 14 +++
t/lei-export-kw.t | 35 +++++++
t/mdir_reader.t | 5 +
9 files changed, 270 insertions(+), 2 deletions(-)
create mode 100644 lib/PublicInbox/LeiExportKw.pm
create mode 100644 t/lei-export-kw.t
diff --git a/MANIFEST b/MANIFEST
index 684128aa..2d1ad5c3 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -202,6 +202,7 @@ lib/PublicInbox/LeiConvert.pm
lib/PublicInbox/LeiCurl.pm
lib/PublicInbox/LeiDedupe.pm
lib/PublicInbox/LeiEditSearch.pm
+lib/PublicInbox/LeiExportKw.pm
lib/PublicInbox/LeiExternal.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
@@ -408,6 +409,7 @@ t/iso-2202-jp.eml
t/kqnotify.t
t/lei-convert.t
t/lei-daemon.t
+t/lei-export-kw.t
t/lei-externals.t
t/lei-import-http.t
t/lei-import-imap.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 15680fe3..628908b5 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -243,6 +243,10 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| offset=i recursive|r exclude=s include|I=s
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
+
+'export-kw' => [ 'LOCATION...|--all',
+ 'one-time export of keywords of sync sources',
+ qw(all:s mode=s), @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
new file mode 100644
index 00000000..db4f7441
--- /dev/null
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -0,0 +1,180 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# front-end for the "lei export-kw" sub-command
+package PublicInbox::LeiExportKw;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use Errno qw(EEXIST ENOENT);
+
+sub export_kw_md { # LeiMailSync->each_src callback
+ my ($oidbin, $id, $self, $mdir) = @_;
+ my $oidhex = unpack('H*', $oidbin);
+ my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return;
+ my $bn = $$id;
+ my ($md_kw, $unknown, @try);
+ if ($bn =~ s/:2,([a-zA-Z]*)\z//) {
+ ($md_kw, $unknown) = PublicInbox::MdirReader::flags2kw($1);
+ @try = qw(cur new);
+ } else {
+ $unknown = [];
+ @try = qw(new cur);
+ }
+ if ($self->{-merge_kw} && $md_kw) { # merging keywords is the default
+ @$sto_kw{keys %$md_kw} = values(%$md_kw);
+ }
+ $bn .= ':2,'.
+ PublicInbox::LeiToMail::kw2suffix([keys %$sto_kw], @$unknown);
+ my $dst = "$mdir/cur/$bn";
+ my @fail;
+ for my $d (@try) {
+ my $src = "$mdir/$d/$$id";
+ next if $src eq $dst;
+
+ # we use link(2) + unlink(2) since rename(2) may
+ # inadvertently clobber if the "uniquefilename" part wasn't
+ # actually unique.
+ if (link($src, $dst)) { # success
+ # unlink(2) may ENOENT from parallel invocation,
+ # ignore it, but not other serious errors
+ if (!unlink($src) and $! != ENOENT) {
+ $self->{lei}->child_error(1,
+ "E: unlink($src): $!");
+ }
+ $self->{lms}->mv_src("maildir:$mdir",
+ $oidbin, $id, $bn) or die;
+ return; # success anyways if link(2) worked
+ }
+ if ($! == ENOENT && !-e $src) { # some other process moved it
+ $self->{lms}->clear_src("maildir:$mdir", $id);
+ next;
+ }
+ push @fail, $src if $! != EEXIST;
+ }
+ return unless @fail;
+ # both tries failed
+ my $e = $!;
+ my $orig = '['.join('|', @fail).']';
+ $self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
+}
+
+# overrides PublicInbox::LeiInput::input_path_url
+sub input_path_url {
+ my ($self, $input, @args) = @_;
+ my $lms = $self->{lms} //= $self->{lse}->lms;
+ $lms->lms_begin;
+ if ($input =~ s/\Amaildir://i) {
+ require PublicInbox::LeiToMail; # kw2suffix
+ $lms->each_src("maildir:$input", \&export_kw_md, $self, $input);
+ }
+ $lms->lms_commit;
+}
+
+sub lei_export_kw {
+ my ($lei, @folders) = @_;
+ my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
+lei/store uninitialized, see lei-import(1)
+EOM
+ my $lse = $sto->search;
+ my $lms = $lse->lms or return $lei->fail(<<EOM);
+lei mail_sync uninitialized, see lei-import(1)
+EOM
+ my $opt = $lei->{opt};
+ my $all = $opt->{all};
+ my @all = $lms->folders;
+ if (defined $all) { # --all=<local|remote>
+ my %x = map { $_ => $_ } split(/,/, $all);
+ my @ok = grep(defined, delete(@x{qw(local remote), ''}));
+ my @no = keys %x;
+ if (@no) {
+ @no = (join(',', @no));
+ return $lei->fail(<<EOM);
+--all=@no not accepted (must be `local' and/or `remote')
+EOM
+ }
+ my (%seen, @inc);
+ for my $ok (@ok) {
+ if ($ok eq 'local') {
+ @inc = grep(!m!\A[a-z0-9\+]+://!i, @all);
+ } elsif ($ok eq 'remote') {
+ @inc = grep(m!\A[a-z0-9\+]+://!i, @all);
+ } elsif ($ok ne '') {
+ return $lei->fail("--all=$all not understood");
+ } else {
+ @inc = @all;
+ }
+ for (@inc) {
+ push(@folders, $_) unless $seen{$_}++;
+ }
+ }
+ return $lei->fail(<<EOM) if !@folders;
+no --mail-sync folders known to lei
+EOM
+ } else {
+ my %all = map { $_ => 1 } @all;
+ my @no;
+ for (@folders) {
+ next if $all{$_}; # ok
+ if (-d "$_/new" && -d "$_/cur") {
+ my $d = 'maildir:'.$lei->rel2abs($_);
+ push(@no, $_) unless $all{$d};
+ $_ = $d;
+ } else {
+ push @no, $_;
+ }
+ }
+ my $no = join("\n\t", @no);
+ return $lei->fail(<<EOF) if @no;
+No sync information for: $no
+Run `lei ls-mail-sync' to display valid choices
+EOF
+ }
+ my $self = bless { lse => $lse }, __PACKAGE__;
+ $lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
+ $self->prepare_inputs($lei, \@folders) or return;
+ my $j = $opt->{jobs} // scalar(@{$self->{inputs}}) || 1;
+ if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
+ return $lei->fail("cannot export to read-only folders: @ro");
+ }
+ if (my $net = $lei->{net}) {
+ require PublicInbox::NetWriter;
+ bless $net, 'PublicInbox::NetWriter';
+ }
+ undef $lms;
+ my $m = $opt->{mode} // 'merge';
+ if ($m eq 'merge') { # default
+ $self->{-merge_kw} = 1;
+ } elsif ($m eq 'set') {
+ } else {
+ return $lei->fail(<<EOM);
+--mode=$m not supported (`set' or `merge')
+EOM
+ }
+ my $ops = {};
+ $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
+ $self->{-wq_nr_workers} = $j // 1; # locked
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
+ net_merge_all_done($self) unless $lei->{auth};
+ $op_c->op_wait_event($ops); # calls net_merge_all_done if $lei->{auth}
+}
+
+sub _complete_export_kw {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ map { $match_cb->($_) } $lms->folders;
+}
+
+no warnings 'once';
+
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
+# the following works even when LeiAuth is lazy-loaded
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+
+1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 3bada42d..32e17c65 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -138,6 +138,16 @@ DELETE FROM blob2num WHERE fid = ? AND uid = ?
$sth->execute($fid, $id);
}
+# Maildir-only
+sub mv_src {
+ my ($self, $folder, $oidbin, $id, $newbn) = @_;
+ my $fid = $self->{fmap}->{$folder} //= _fid_for($self, $folder, 1);
+ my $sth = $self->{dbh}->prepare_cached(<<'');
+UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
+
+ $sth->execute($newbn, $fid, $oidbin, $$id);
+}
+
# read-only, iterates every oidbin + UID or name for a given folder
sub each_src {
my ($self, $folder, $cb, @args) = @_;
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index fb19229f..9297d060 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -27,6 +27,20 @@ sub msg_keywords {
wantarray ? sort(keys(%$kw)) : $kw;
}
+# returns undef if blob is unknown
+sub oid_keywords {
+ my ($self, $oidhex) = @_;
+ my @num = $self->over->blob_exists($oidhex) or return;
+ my $xdb = $self->xdb; # set {nshard};
+ my %kw;
+ for my $num (@num) { # there should only be one...
+ my $doc = $xdb->get_document(num2docid($self, $num));
+ my $x = xap_terms('K', $doc);
+ %kw = (%kw, %$x);
+ }
+ \%kw;
+}
+
# lookup keywords+labels for external messages
sub xsmsg_vmd {
my ($self, $smsg, $want_label) = @_;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 0cbdff8b..96a1f881 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -243,10 +243,14 @@ sub _rand () {
sprintf('%x,%x,%x,%x', rand(0xffffffff), time, $$, ++$seq);
}
+sub kw2suffix ($;@) {
+ my $kw = shift;
+ join('', sort(map { $kw2char{$_} // () } @$kw, @_));
+}
+
sub _buf2maildir {
my ($dst, $buf, $smsg) = @_;
my $kw = $smsg->{kw} // [];
- my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw));
my $rand = ''; # chosen by die roll :P
my ($tmp, $fh, $base, $ok);
my $common = $smsg->{blob} // _rand;
@@ -263,7 +267,7 @@ sub _buf2maildir {
$dst .= 'cur/';
$rand = '';
do {
- $base = $rand.$common.':2,'.$sfx
+ $base = $rand.$common.':2,'.kw2suffix($kw);
} while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} &&
($rand = _rand.','));
die "link($tmp, $dst$base): $!" unless $ok;
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index 7a0641fb..304be63d 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -86,4 +86,18 @@ sub maildir_each_eml {
sub new { bless {}, __PACKAGE__ }
+sub flags2kw ($) {
+ my @unknown;
+ my %kw;
+ for (split(//, $_[0])) {
+ my $k = $c2kw{$_};
+ if (defined($k)) {
+ $kw{$k} = 1;
+ } else {
+ push @unknown, $_;
+ }
+ }
+ (\%kw, \@unknown);
+}
+
1;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
new file mode 100644
index 00000000..9531949a
--- /dev/null
+++ b/t/lei-export-kw.t
@@ -0,0 +1,35 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Copy qw(cp);
+use File::Path qw(make_path);
+require_mods(qw(lei -imapd Mail::IMAPClient));
+my ($tmpdir, $for_destroy) = tmpdir;
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my $expect = eml_load('t/data/0001.patch');
+test_lei({ tmpdir => $tmpdir }, sub {
+ my $home = $ENV{HOME};
+ my $md = "$home/md";
+ make_path("$md/new", "$md/cur", "$md/tmp");
+ cp('t/data/0001.patch', "$md/new/y") or xbail "cp $md $!";
+ cp('t/data/message_embed.eml', "$md/cur/x:2,S") or xbail "cp $md $!";
+ lei_ok qw(index -q), $md;
+ lei_ok qw(tag t/data/0001.patch +kw:seen);
+ lei_ok qw(export-kw --all=local);
+ ok(!-e "$md/new/y", 'original gone');
+ is_deeply(eml_load("$md/cur/y:2,S"), $expect,
+ "`seen' kw exported");
+
+ lei_ok qw(tag t/data/0001.patch +kw:answered);
+ lei_ok qw(export-kw --all=local);
+ ok(!-e "$md/cur/y:2,S", 'seen-only file gone');
+ is_deeply(eml_load("$md/cur/y:2,RS"), $expect, "`R' added");
+
+ lei_ok qw(tag t/data/0001.patch -kw:answered -kw:seen);
+ lei_ok qw(export-kw --mode=set --all=local);
+ ok(!-e "$md/cur/y:2,RS", 'seen+answered file gone');
+ is_deeply(eml_load("$md/cur/y:2,"), $expect, 'no keywords left');
+});
+
+done_testing;
diff --git a/t/mdir_reader.t b/t/mdir_reader.t
index 51b38af4..c927e1a7 100644
--- a/t/mdir_reader.t
+++ b/t/mdir_reader.t
@@ -19,4 +19,9 @@ is(maildir_path_flags('/path/to/foo:2,'), '', 'no flags in path');
use_ok 'PublicInbox::InboxWritable', qw(eml_from_path);
is(eml_from_path('.'), undef, 'eml_from_path fails on directory');
+is_deeply([PublicInbox::MdirReader::flags2kw('S')], [{ 'seen' => 1 }, []],
+ "`seen' kw set from flag");
+is_deeply([PublicInbox::MdirReader::flags2kw('Su')], [{ 'seen' => 1 }, ['u']],
+ 'unknown flag ignored');
+
done_testing;
^ permalink raw reply related [relevance 32%]
* [PATCH 0/8] lei: export-kw, IMAP import incompatibility
@ 2021-05-21 10:28 64% Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
"lei export-kw" is a new command. I'm not sure exactly how
it'll be used but it's probably more of a plumbing command,
for now. My brain hurts thinking about synchronization
and merge/conflict resolution when it comes to propagating
keywords assignments/clearing.
(I frequently mark messages as Unread in my MUA so I know to
reread them in the future, and I suspect it's a common thing).
mail_sync.sqlite3 now tracks AUTH=ANONYMOUS or username in the
folder name to account for lei(Unix) users having multiple IMAP
accounts on the same host with the same folders+UIDVALIDITY.
"lei import imap(s)://" users will waste a bit of bandwidth
resyncing as a result.
Eric Wong (8):
treewide: favor open(..., '+<&=', $fd)
lei: drop EOFpipe in favor of PktOp
lei tag: support tagging index-only messages
lei_input: fix canonicalization of Maildirs for sync
lei index: support command-line options
lei export-kw: new command to export keywords to Maildirs
uri_imap: support uid/auth/user as full accessors
lei import: store IMAP user+auth in mail_sync folder URI
MANIFEST | 2 +
examples/unsubscribe.milter | 3 +-
lib/PublicInbox/DS.pm | 3 +-
lib/PublicInbox/Daemon.pm | 2 +-
lib/PublicInbox/LEI.pm | 18 +++-
lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiInput.pm | 3 +-
lib/PublicInbox/LeiMailSync.pm | 10 ++
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiSearch.pm | 22 +++-
lib/PublicInbox/LeiTag.pm | 10 +-
lib/PublicInbox/LeiToMail.pm | 12 ++-
lib/PublicInbox/MdirReader.pm | 14 +++
lib/PublicInbox/NetReader.pm | 42 +++++---
lib/PublicInbox/Sigfd.pm | 3 +-
lib/PublicInbox/URIimap.pm | 82 +++++++++++----
t/epoll.t | 7 +-
t/lei-export-kw.t | 35 +++++++
t/lei-import-imap.t | 9 +-
t/lei-index.t | 12 ++-
t/mdir_reader.t | 5 +
t/uri_imap.t | 60 ++++++++---
22 files changed, 461 insertions(+), 75 deletions(-)
create mode 100644 lib/PublicInbox/LeiExportKw.pm
create mode 100644 t/lei-export-kw.t
^ permalink raw reply [relevance 64%]
* Re: [PATCH] lei rediff: fix construction of git-diff options
2021-05-21 4:38 70% [PATCH] lei rediff: fix construction of git-diff options Kyle Meyer
@ 2021-05-21 9:52 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-21 9:52 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> When generating git-diff options, lei-rediff extracts the single
> character option from the lei option spec. However, there's no check
> that the regular expression actually matches, leading to an
> unintentional git-diff option when there isn't a short option (e.g.,
> --inter-hunk-context=1 maps to the invalid `git diff --color -w1').
>
> Check for a match before trying to extract the single character
> option.
>
> Fixes: cf0c7ce3ce81b5c3 (lei rediff: regenerate diffs from stdin)
Oops :x Spending years in another language corrupted my
mind to believe $1 gets clobbered on failed matches;
but that's not the case for Perl.
An unrelated patch I'm working on had the same problem :x
Anyways, pushed as commit 3c1d0da4d0608b5a87371e602a911964d7c1498c
Thanks.
^ permalink raw reply [relevance 71%]
* [PATCH] lei rediff: fix construction of git-diff options
@ 2021-05-21 4:38 70% Kyle Meyer
2021-05-21 9:52 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-05-21 4:38 UTC (permalink / raw)
To: meta
When generating git-diff options, lei-rediff extracts the single
character option from the lei option spec. However, there's no check
that the regular expression actually matches, leading to an
unintentional git-diff option when there isn't a short option (e.g.,
--inter-hunk-context=1 maps to the invalid `git diff --color -w1').
Check for a match before trying to extract the single character
option.
Fixes: cf0c7ce3ce81b5c3 (lei rediff: regenerate diffs from stdin)
---
lib/PublicInbox/LeiRediff.pm | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 3c8ebe41..2e793df5 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -108,8 +108,9 @@ EOM
push @cmd, '--'.($opt->{color} && !$opt->{'no-color'} ? '' : 'no-').
'color';
for my $o (@PublicInbox::LEI::diff_opt) {
- $o =~ s/\|([a-z0-9])\b//i; # remove single char short option
- my $c = $1;
+ my $c = '';
+ # remove single char short option
+ $o =~ s/\|([a-z0-9])\b//i and $c = $1;
if ($o =~ s/=[is]@\z//) {
my $v = $opt->{$o} or next;
push @cmd, map { $c ? "-$c$_" : "--$o=$_" } @$v;
base-commit: 2f720902ed702b64d918165ba21a96dabbeeca26
--
2.31.1
^ permalink raw reply related [relevance 70%]
* [PATCH] lei: relax rules for "new" in Maildir
@ 2021-05-19 8:54 57% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-19 8:54 UTC (permalink / raw)
To: meta
mbsync and offlineimap both use ":2," suffixes for filenames in
"new/", however my interpretation of the Maildir spec at
<https://cr.yp.to/proto/maildir.html> is that ":2," is only for
files in "cur/". My interpretation also matches that of
doveecot, but we'll allow what mbsync and offlineimap do given
their popularity.
---
lib/PublicInbox/LeiMailSync.pm | 13 ++++++++++---
lib/PublicInbox/MdirReader.pm | 5 ++++-
t/lei-index.t | 10 +++++++++-
3 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 803de48c..3bada42d 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -233,9 +233,16 @@ WHERE b.oidbin = ?
$b2n->execute(pack('H*', $oidhex));
while (my ($d, $n) = $b2n->fetchrow_array) {
substr($d, 0, length('maildir:')) = '';
- my $f = "$d/" . ($n =~ /:2,[a-zA-Z]*\z/ ? "cur/$n" : "new/$n");
- open my $fh, '<', $f or next;
- if (-s $fh) {
+ # n.b. both mbsync and offlineimap use ":2," as a suffix
+ # in "new/", despite (from what I understand of reading
+ # <https://cr.yp.to/proto/maildir.html>), the ":2," only
+ # applies to files in "cur/".
+ my @try = $n =~ /:2,[a-zA-Z]+\z/ ? qw(cur new) : qw(new cur);
+ for my $x (@try) {
+ my $f = "$d/$x/$n";
+ open my $fh, '<', $f or next;
+ # some (buggy) Maildir writers are non-atomic:
+ next unless -s $fh;
local $/;
my $raw = <$fh>;
if ($vrfy && git_sha(1, \$raw)->hexdigest ne $oidhex) {
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index b49c8ceb..7a0641fb 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -61,7 +61,10 @@ sub maildir_each_eml {
while (defined(my $bn = readdir($dh))) {
next if substr($bn, 0, 1) eq '.';
my @f = split(/:/, $bn, -1);
- next if scalar(@f) != 1;
+
+ # mbsync and offlineimap both use "2," in "new/"
+ next if ($f[1] // '2,') ne '2,' || defined($f[2]);
+
next if defined($mod) && !shard_ok($bn, $mod, $shard);
my $f = $pfx.$bn;
my $eml = eml_from_path($f) or next;
diff --git a/t/lei-index.t b/t/lei-index.t
index 3382d42b..b7dafb71 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -27,6 +27,11 @@ my $expect = do {
local $/;
<$fh>;
};
+
+# mbsync and offlineimap both put ":2," in "new/" files:
+symlink(File::Spec->rel2abs('t/utf8.eml'), "$tmpdir/md/new/u:2,") or
+ xbail "symlink $!";
+
test_lei({ tmpdir => $tmpdir }, sub {
my $store_path = "$ENV{HOME}/.local/share/lei/store/";
@@ -40,13 +45,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok(qw(q mid:qp@example.com -f text));
like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
+ lei_ok(qw(q mid:testmessage@example.com -f text));
+ lei_ok(qw(-C / blob --mail 9bf1002c49eb075df47247b74d69bcd555e23422));
+
my $all_obj = ['git', "--git-dir=$store_path/ALL.git",
qw(cat-file --batch-check --batch-all-objects)];
is_deeply([xqx($all_obj)], [], 'no git objects');
lei_ok('import', 't/plack-qp.eml');
ok(grep(/\A$blob blob /, my @objs = xqx($all_obj)),
'imported blob');
- lei_ok(qw(q z:0.. --dedupe=none));
+ lei_ok(qw(q m:qp@example.com --dedupe=none));
my $res_b = json_utf8->decode($lei_out);
is_deeply($res_b, $res_a, 'no extra DB entries');
^ permalink raw reply related [relevance 57%]
* Re: [PATCH 8/9] doc lei: add manpages for new commands
2021-05-17 8:09 71% ` Eric Wong
@ 2021-05-17 23:24 71% ` Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 23:24 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> I squashed this fix in, though ('make check' revealed it):
Oy, sorry about that. Thanks for the fixup.
^ permalink raw reply [relevance 71%]
* Re: [PATCH 8/9] doc lei: add manpages for new commands
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
@ 2021-05-17 8:09 71% ` Eric Wong
2021-05-17 23:24 71% ` Kyle Meyer
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-17 8:09 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
Thanks, pushed this series out and your lcat fix as
commit 94be0536ce5eaf1d5a67f217dce112607624389c
("lei lcat: fix handling of multiple MSGID_OR_URL arguments")
> diff --git a/MANIFEST b/MANIFEST
> index 29e62880..c7084671 100644
> --- a/MANIFEST
> +++ b/MANIFEST
> @@ -23,21 +23,28 @@ Documentation/hosted.txt
> Documentation/include.mk
> Documentation/lei-add-external.pod
> Documentation/lei-blob.pod
> +Documentation/lei-cat.pod
I squashed this fix in, though ('make check' revealed it):
diff -u b/MANIFEST b/MANIFEST
--- b/MANIFEST
+++ b/MANIFEST
@@ -23,7 +23,6 @@
Documentation/include.mk
Documentation/lei-add-external.pod
Documentation/lei-blob.pod
-Documentation/lei-cat.pod
Documentation/lei-config.pod
Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
@@ -33,6 +32,7 @@
Documentation/lei-forget-search.pod
Documentation/lei-import.pod
Documentation/lei-init.pod
+Documentation/lei-lcat.pod
Documentation/lei-ls-external.pod
Documentation/lei-ls-label.pod
Documentation/lei-ls-mail-sync.pod
^ permalink raw reply [relevance 71%]
* [PATCH] lei lcat: fix handling of multiple MSGID_OR_URL arguments
@ 2021-05-17 3:37 71% Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:37 UTC (permalink / raw)
To: meta
`lei lcat' is documented as being able to display multiple messages,
but this works only with --stdin because the positional argument
MSGID_OR_URL is missing a period.
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b82fb003..98e79a76 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -168,7 +168,7 @@ our %CMD = ( # sorted in order of importance/use:
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
-'lcat' => [ '--stdin|MSGID_OR_URL..', 'display local copy of message(s)',
+'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
'stdin|', # /|\z/ must be first for lone dash
# some of these options are ridiculous for lcat
@lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
base-commit: 8cc23ac6f7a3847977ec57c2a3e9e391fdb94da6
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 9/9] doc lei: update manpages with new options
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (6 preceding siblings ...)
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
@ 2021-05-17 3:35 66% ` Kyle Meyer
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-blob.pod | 2 ++
Documentation/lei-import.pod | 16 ++++++++++++++++
Documentation/lei-q.pod | 10 ++++++++++
3 files changed, 28 insertions(+)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index 36c75d53..5b611d11 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -80,6 +80,8 @@ L<lei-q(1)>.
=item --no-torsocks
+=item --proxy=PROTO://HOST[:PORT]
+
=back
=head1 CONTACT
diff --git a/Documentation/lei-import.pod b/Documentation/lei-import.pod
index 1fe2dac3..c29a085b 100644
--- a/Documentation/lei-import.pod
+++ b/Documentation/lei-import.pod
@@ -46,6 +46,22 @@ Default: fcntl,dotlock
Don't import message keywords (or "flags" in IMAP terminology).
+=item --no-incremental
+
+Import already seen IMAP and NNTP articles.
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
+
+Default: C<auto>
+
+=item --proxy=PROTO://HOST[:PORT]
+
+Use the specified proxy (e.g., C<socks5h://0:9050>).
+
=back
=head1 CONTACT
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index c4a7eea3..fbe61920 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -26,6 +26,10 @@ TODO: mention curl options?
Read search terms from stdin.
+=item --save
+
+Save a search for L<lei-up(1)>.
+
=item --output=MFOLDER
=item -o MFOLDER
@@ -66,6 +70,10 @@ default destination (stdout).
Using a C<format:> prefix with the C<--output> destination is
preferred when not writing to stdout.
+=item --no-color
+
+Disable color (for C<--format=text>).
+
=item --pretty
Pretty print C<json> or C<concatjson> output. If stdout is opened to
@@ -229,6 +237,8 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
Default: C<auto>
+=item --proxy=PROTO://HOST[:PORT]
+
=back
=head1 CONTACT
--
2.31.1
^ permalink raw reply related [relevance 66%]
* [PATCH 8/9] doc lei: add manpages for new commands
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (5 preceding siblings ...)
2021-05-17 3:35 54% ` [PATCH 7/9] doc lei: add manpage for convert Kyle Meyer
@ 2021-05-17 3:35 27% ` Kyle Meyer
2021-05-17 8:09 71% ` Eric Wong
2021-05-17 3:35 66% ` [PATCH 9/9] doc lei: update manpages with new options Kyle Meyer
7 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-edit-search.pod | 28 ++++++++++
Documentation/lei-forget-search.pod | 28 ++++++++++
Documentation/lei-lcat.pod | 79 +++++++++++++++++++++++++++++
Documentation/lei-ls-mail-sync.pod | 55 ++++++++++++++++++++
Documentation/lei-ls-search.pod | 65 ++++++++++++++++++++++++
Documentation/lei-overview.pod | 12 +++++
Documentation/lei-q.pod | 2 +-
Documentation/lei-rediff.pod | 79 +++++++++++++++++++++++++++++
Documentation/lei-up.pod | 48 ++++++++++++++++++
Documentation/lei.pod | 16 ++++++
Documentation/txt2pre | 8 +++
MANIFEST | 7 +++
Makefile.PL | 8 +--
13 files changed, 431 insertions(+), 4 deletions(-)
create mode 100644 Documentation/lei-edit-search.pod
create mode 100644 Documentation/lei-forget-search.pod
create mode 100644 Documentation/lei-lcat.pod
create mode 100644 Documentation/lei-ls-mail-sync.pod
create mode 100644 Documentation/lei-ls-search.pod
create mode 100644 Documentation/lei-rediff.pod
create mode 100644 Documentation/lei-up.pod
diff --git a/Documentation/lei-edit-search.pod b/Documentation/lei-edit-search.pod
new file mode 100644
index 00000000..7908b5a2
--- /dev/null
+++ b/Documentation/lei-edit-search.pod
@@ -0,0 +1,28 @@
+=head1 NAME
+
+lei-edit-search - edit saved search
+
+=head1 SYNOPSIS
+
+lei edit-search [OPTIONS] OUTPUT
+
+=head1 DESCRIPTION
+
+Invoke C<git config --edit> to edit the saved search at C<OUTPUT>.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-ls-search(1)>, L<lei-forget-search(1)>, L<lei-up(1)>, L<lei-q(1)>
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
new file mode 100644
index 00000000..49bc1d68
--- /dev/null
+++ b/Documentation/lei-forget-search.pod
@@ -0,0 +1,28 @@
+=head1 NAME
+
+lei-forget-search - forget saved search
+
+=head1 SYNOPSIS
+
+lei forget-search [OPTIONS] OUTPUT
+
+=head1 DESCRIPTION
+
+Forget a saved search at C<OUTPUT>.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-edit-search(1)>, L<lei-ls-search(1)>, L<lei-up(1)>, L<lei-q(1)>
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
new file mode 100644
index 00000000..5a2bdb5a
--- /dev/null
+++ b/Documentation/lei-lcat.pod
@@ -0,0 +1,79 @@
+=head1 NAME
+
+lei-lcat - display local copy of messages(s)
+
+=head1 SYNOPSIS
+
+lei lcat [OPTIONS] MSGID_OR_URL [MSGID_OR_URL...]
+
+lei lcat [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+lcat (local cat) is a wrapper around L<lei-q(1)> that displays local
+messages by Message-ID. It is able to extract Message-IDs from URLs
+as well as from common formats such as C<E<lt>$MSGIDE<gt>> and
+C<id:$MSGID>. When reading from stdin, input that isn't understood is
+discarded, so the caller doesn't have to bother extracting the
+Message-ID or link from surrounding text (e.g., a "Link: $URL" line).
+
+=head1 OPTIONS
+
+The following options, described in L<lei-q(1)>, are supported.
+
+=over
+
+=item --[no-]remote
+
+=item --no-local
+
+=item --no-external
+
+=item --no-import-remote
+
+=item --torsocks=auto|no|yes, --no-torsocks
+
+=item --proxy=PROTO://HOST[:PORT]
+
+=item -o MFOLDER, --output=MFOLDER
+
+=item -d STRATEGY, --dedupe=STRATEGY
+
+=item -t, --threads
+
+=item -s KEY, --sort=KEY
+
+=item -r, --reverse
+
+=item --offset=NUMBER
+
+=item -g, --globoff
+
+=item -a, --augment
+
+=item --lock=METHOD
+
+=item --alert=COMMAND
+
+=item --mua=COMMAND
+
+=item --no-color
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-blob(1)>
diff --git a/Documentation/lei-ls-mail-sync.pod b/Documentation/lei-ls-mail-sync.pod
new file mode 100644
index 00000000..37aa910f
--- /dev/null
+++ b/Documentation/lei-ls-mail-sync.pod
@@ -0,0 +1,55 @@
+=head1 NAME
+
+lei-ls-mail-sync - list mail sync folders
+
+=head1 SYNOPSIS
+
+lei mail-sync [OPTIONS] [FILTER]
+
+=head1 DESCRIPTION
+
+List mail sync folders. If C<FILTER> is given, restrict the output to
+matching entries.
+
+=head1 OPTIONS
+
+=over
+
+=item -g
+
+=item --globoff
+
+Do not match C<FILTER> using C<*?> wildcards and C<[]> ranges.
+
+=item --local
+
+Limit operations to the local filesystem.
+
+=item --remote
+
+Limit operations to those requiring network access.
+
+=item -z
+
+=item -0
+
+Use C<\0> (NUL) instead of newline (CR) to delimit lines.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-up(1)>
diff --git a/Documentation/lei-ls-search.pod b/Documentation/lei-ls-search.pod
new file mode 100644
index 00000000..138dbbff
--- /dev/null
+++ b/Documentation/lei-ls-search.pod
@@ -0,0 +1,65 @@
+=head1 NAME
+
+lei-ls-search - list saved search queries
+
+=head1 SYNOPSIS
+
+lei ls-search [OPTIONS] [PREFIX]
+
+=head1 DESCRIPTION
+
+List saved search queries. If C<PREFIX> is given, restrict the output
+to entries that start with the specified value.
+
+=head1 OPTIONS
+
+=over
+
+=item -f FORMAT
+
+=item --format=FORMAT
+
+Display JSON output rather than default short output that includes
+only the saved search location. Possible values are C<json>,
+C<jsonl>, or C<concatjson>.
+
+=item --pretty
+
+Pretty print C<json> or C<concatjson> output. If stdout is opened to
+a tty and used as the C<--output> destination, C<--pretty> is enabled
+by default.
+
+=item -l
+
+Long listing format (shortcut for C<--format=json>).
+
+=item --ascii
+
+Escape non-ASCII characters.
+
+=item -z
+
+=item -0
+
+Use C<\0> (NUL) instead of newline (CR) to delimit lines. This option
+is incompatible with C<--format>.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-up(1)>, L<lei-edit-search(1)>,
+L<lei-forget-search(1)>
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index 4a34bc16..e80cb094 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -98,6 +98,18 @@ Search for messages that have post-image git blob IDs that match those
of the current repository's HEAD commit, writing them to the Maildir
directory "mdir" and flagging the messages that were an exact match.
+=item $ git show -s HEAD | lei lcat -
+
+Display a local message for the public-inbox link contained in a
+commit message.
+
+=item $ lei q -f text m:MID | lei rediff -U5 -
+
+Feed a message containing a diff to L<lei-rediff(1)> to regenerate its
+diff with five context lines. Unless C<--git-dir> is specified, this
+requires the current working directory to be within the associated
+code repository.
+
=back
=head1 PERFORMANCE NOTES
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 39199849..c4a7eea3 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -246,5 +246,5 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-add-external(1)>,
+L<lei-add-external(1)>, L<lei-lcat(1)>, L<lei-up(1)>,
L<Xapian::QueryParser Syntax|https://xapian.org/docs/queryparser.html>
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
new file mode 100644
index 00000000..5fdde230
--- /dev/null
+++ b/Documentation/lei-rediff.pod
@@ -0,0 +1,79 @@
+=head1 NAME
+
+lei-rediff - regenerate a diff with different options
+
+=head1 SYNOPSIS
+
+lei rediff [OPTIONS] LOCATION [LOCATION...]
+
+lei rediff [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+Read a message from C<LOCATION> or stdin and regenerate its diff with
+the specified L<git-diff(1)> options. This is useful if you want to
+change the display of the original patch (e.g., increasing context,
+coloring moved lines differently, or using an external diff viewer).
+
+=head1 OPTIONS
+
+In addition to many L<git-diff(1)>, the following options are
+supported.
+
+=over
+
+=item --stdin
+
+Read message from stdin.
+
+=item --git-dir=DIR
+
+Specify an additional .git/ directory to scan. This option may be
+given multiple times.
+
+=item --no-cwd
+
+Do not look in the git repository of the current working directory.
+
+=item -v
+
+=item --verbose
+
+Provide more feedback on stderr.
+
+=back
+
+The options below, described in L<lei-q(1)>, are also supported.
+
+=over
+
+=item --[no-]remote
+
+=item --no-local
+
+=item --no-external
+
+=item --no-import-remote
+
+=item --torsocks=auto|no|yes, --no-torsocks
+
+=item --proxy=PROTO://HOST[:PORT]
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-blob(1)>, L<lei-p2q(1)>
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
new file mode 100644
index 00000000..cea0f619
--- /dev/null
+++ b/Documentation/lei-up.pod
@@ -0,0 +1,48 @@
+=head1 NAME
+
+lei-up - update a saved search
+
+=head1 SYNOPSIS
+
+lei up [OPTIONS] OUTPUT
+
+lei up [OPTIONS] --all=TYPE
+
+=head1 DESCRIPTION
+
+Update the saved search at C<OUTPUT> or all saved searches of C<TYPE>
+(currently C<local> is the only supported value).
+
+=head1 OPTIONS
+
+The following options, described in L<lei-q(1)>, are supported.
+
+=over
+
+=item --lock=METHOD
+
+=item --alert=CMD
+
+=item --mua=CMD
+
+This option is incompatible with C<--all>.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-q(1)>, L<lei-ls-search(1)>, L<lei-edit-search(1)>,
+L<lei-forget-search(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 78e1fb1e..3af9e2ee 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -72,10 +72,24 @@ store and configured externals are
=item * L<lei-blob(1)>
+=item * L<lei-config(1)>
+
+=item * L<lei-edit-search(1)>
+
+=item * L<lei-forget-search(1)>
+
+=item * L<lei-lcat(1)>
+
+=item * L<lei-ls-search(1)>
+
=item * L<lei-p2q(1)>
=item * L<lei-q(1)>
+=item * L<lei-rediff(1)>
+
+=item * L<lei-up(1)>
+
=back
Other subcommands include
@@ -92,6 +106,8 @@ Other subcommands include
=item * L<lei-ls-label(1)>
+=item * L<lei-ls-mail-sync(1)>
+
=back
=head1 FILES
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 201566ee..d98dfdd3 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -17,15 +17,22 @@ for (qw[lei(1)
lei-convert(1)
lei-daemon-kill(1)
lei-daemon-pid(1)
+ lei-edit-search(1)
lei-forget-external(1)
+ lei-forget-search(1)
lei-import(1)
lei-init(1)
+ lei-lcat(1)
lei-ls-external(1)
lei-ls-label(1)
+ lei-ls-mail-sync(1)
+ lei-ls-search(1)
lei-overview(7)
lei-p2q(1)
lei-q(1)
+ lei-rediff(1)
lei-tag(1)
+ lei-up(1)
public-inbox.cgi(1)
public-inbox-compact(1)
public-inbox-config(5)
@@ -68,6 +75,7 @@ for (qw[git(1)
git-config(1)
git-credential(1)
git-daemon(1)
+ git-diff(1)
git-fast-import(1)
git-fetch(1)
git-filter-branch(1)
diff --git a/MANIFEST b/MANIFEST
index 29e62880..c7084671 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -23,21 +23,28 @@ Documentation/hosted.txt
Documentation/include.mk
Documentation/lei-add-external.pod
Documentation/lei-blob.pod
+Documentation/lei-cat.pod
Documentation/lei-config.pod
Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
+Documentation/lei-edit-search.pod
Documentation/lei-forget-external.pod
+Documentation/lei-forget-search.pod
Documentation/lei-import.pod
Documentation/lei-init.pod
Documentation/lei-ls-external.pod
Documentation/lei-ls-label.pod
+Documentation/lei-ls-mail-sync.pod
+Documentation/lei-ls-search.pod
Documentation/lei-mail-formats.pod
Documentation/lei-overview.pod
Documentation/lei-p2q.pod
Documentation/lei-q.pod
+Documentation/lei-rediff.pod
Documentation/lei-store-format.pod
Documentation/lei-tag.pod
+Documentation/lei-up.pod
Documentation/lei.pod
Documentation/lei_design_notes.txt
Documentation/marketing.txt
diff --git a/Makefile.PL b/Makefile.PL
index d149a164..2af8c2f1 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -45,9 +45,11 @@ $v->{-m1} = [ map {
}
} @EXE_FILES,
qw(
- lei-add-external lei-blob lei-config lei-convert lei-daemon-kill
- lei-daemon-pid lei-forget-external lei-import lei-init lei-ls-external
- lei-ls-label lei-p2q lei-q lei-tag)];
+ lei-add-external lei-blob lei-config lei-convert lei-edit-search
+ lei-daemon-kill lei-daemon-pid lei-forget-external lei-forget-search
+ lei-import lei-init lei-lcat lei-ls-external lei-ls-label
+ lei-ls-mail-sync lei-ls-search lei-p2q lei-q lei-rediff lei-tag
+ lei-up)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
--
2.31.1
^ permalink raw reply related [relevance 27%]
* [PATCH 7/9] doc lei: add manpage for convert
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (4 preceding siblings ...)
2021-05-17 3:35 71% ` [PATCH 6/9] doc lei q: add missing value for --lock Kyle Meyer
@ 2021-05-17 3:35 54% ` Kyle Meyer
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
2021-05-17 3:35 66% ` [PATCH 9/9] doc lei: update manpages with new options Kyle Meyer
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
Link: https://public-inbox.org/meta/20210429015738.GA30172@dcvr/
---
Documentation/lei-convert.pod | 73 +++++++++++++++++++++++++++++++++++
Documentation/lei.pod | 2 +
Documentation/txt2pre | 1 +
MANIFEST | 1 +
Makefile.PL | 6 +--
5 files changed, 80 insertions(+), 3 deletions(-)
create mode 100644 Documentation/lei-convert.pod
diff --git a/Documentation/lei-convert.pod b/Documentation/lei-convert.pod
new file mode 100644
index 00000000..e8a71393
--- /dev/null
+++ b/Documentation/lei-convert.pod
@@ -0,0 +1,73 @@
+=head1 NAME
+
+lei-convert - one-time conversion from URL or filesystem to another format
+
+=head1 SYNOPSIS
+
+lei convert [OPTIONS] LOCATION
+
+lei convert [OPTIONS] (--stdin|-)
+
+=head1 DESCRIPTION
+
+Convert messages to another format. C<LOCATION> is a source of
+messages: a directory (Maildir), a file, or a URL (C<imap://>,
+C<imaps://>, C<nntp://>, or C<nntps://>). URLs requiring
+authentication must use L<netrc(5)> and/or L<git-credential(1)> to
+fill in the username and password.
+
+For a regular file, the location must have a C<E<lt>formatE<gt>:>
+prefix specifying one of the following formats: C<eml>, C<mboxrd>,
+C<mboxcl2>, C<mboxcl>, or C<mboxo>.
+
+=head1 OPTIONS
+
+=over
+
+=item -F MAIL_FORMAT
+
+=item --in-format=MAIL_FORMAT
+
+Message input format. Unless messages are given on stdin, using a
+format prefix with C<LOCATION> is preferred.
+
+=back
+
+The following options are also supported and are described in
+L<lei-q(1)>.
+
+=over
+
+=item -o MFOLDER, --output=MFOLDER
+
+=item --lock METHOD
+
+=item --no-kw
+
+=item --no-import-remote
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+=item --proxy=PROTO://HOST[:PORT]
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+
+=head1 SEE ALSO
+
+L<lei-add-external(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 58646577..78e1fb1e 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -84,6 +84,8 @@ Other subcommands include
=item * L<lei-config(1)>
+=item * L<lei-convert(1)>
+
=item * L<lei-daemon-kill(1)>
=item * L<lei-daemon-pid(1)>
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 25d81cf9..201566ee 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -14,6 +14,7 @@ for (qw[lei(1)
lei-add-external(1)
lei-blob(1)
lei-config(1)
+ lei-convert(1)
lei-daemon-kill(1)
lei-daemon-pid(1)
lei-forget-external(1)
diff --git a/MANIFEST b/MANIFEST
index a6d94c77..29e62880 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -24,6 +24,7 @@ Documentation/include.mk
Documentation/lei-add-external.pod
Documentation/lei-blob.pod
Documentation/lei-config.pod
+Documentation/lei-convert.pod
Documentation/lei-daemon-kill.pod
Documentation/lei-daemon-pid.pod
Documentation/lei-forget-external.pod
diff --git a/Makefile.PL b/Makefile.PL
index b06b6ab8..d149a164 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -45,9 +45,9 @@ $v->{-m1} = [ map {
}
} @EXE_FILES,
qw(
- lei-add-external lei-blob lei-config lei-daemon-kill lei-daemon-pid
- lei-forget-external lei-import lei-init lei-ls-external lei-ls-label
- lei-p2q lei-q lei-tag)];
+ lei-add-external lei-blob lei-config lei-convert lei-daemon-kill
+ lei-daemon-pid lei-forget-external lei-import lei-init lei-ls-external
+ lei-ls-label lei-p2q lei-q lei-tag)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
--
2.31.1
^ permalink raw reply related [relevance 54%]
* [PATCH 6/9] doc lei q: add missing value for --lock
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (3 preceding siblings ...)
2021-05-17 3:35 71% ` [PATCH 5/9] doc lei q: fix a typo Kyle Meyer
@ 2021-05-17 3:35 71% ` Kyle Meyer
2021-05-17 3:35 54% ` [PATCH 7/9] doc lei: add manpage for convert Kyle Meyer
` (2 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-q.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index e1c20a0d..39199849 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -171,7 +171,7 @@ option applies to C<--include>, C<--exclude>, and C<--only>.
Disable the default behavior of memoizing remote messages into the
local store.
-=item --lock
+=item --lock=METHOD
L<mbox(5)> locking method(s) to use: C<dotlock>, C<fcntl>, C<flock> or
C<none>.
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 4/9] doc lei: resort lei-tag entries
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
2021-05-17 3:35 82% ` [PATCH 3/9] doc lei blob: point to lei-q for shared options Kyle Meyer
@ 2021-05-17 3:35 90% ` Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 5/9] doc lei q: fix a typo Kyle Meyer
` (4 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
The command was renamed in 54da988cfb049ea2 (lei tag: rename from "lei
mark", 2021-03-30). Relocate its entries in txt2pre and Makefile.PL
to restore alphabetical sorting.
---
Documentation/txt2pre | 2 +-
Makefile.PL | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 7b9d7853..25d81cf9 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -21,10 +21,10 @@ for (qw[lei(1)
lei-init(1)
lei-ls-external(1)
lei-ls-label(1)
- lei-tag(1)
lei-overview(7)
lei-p2q(1)
lei-q(1)
+ lei-tag(1)
public-inbox.cgi(1)
public-inbox-compact(1)
public-inbox-config(5)
diff --git a/Makefile.PL b/Makefile.PL
index 401c033e..b06b6ab8 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -47,7 +47,7 @@ $v->{-m1} = [ map {
qw(
lei-add-external lei-blob lei-config lei-daemon-kill lei-daemon-pid
lei-forget-external lei-import lei-init lei-ls-external lei-ls-label
- lei-tag lei-p2q lei-q)];
+ lei-p2q lei-q lei-tag)];
$v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
public-inbox-v2-format public-inbox-extindex-format
lei-mail-formats lei-store-format
--
2.31.1
^ permalink raw reply related [relevance 90%]
* [PATCH 3/9] doc lei blob: point to lei-q for shared options
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
@ 2021-05-17 3:35 82% ` Kyle Meyer
2021-05-17 3:35 90% ` [PATCH 4/9] doc lei: resort lei-tag entries Kyle Meyer
` (5 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
lei-blob shares a good number of options with lei-q. Refer to lei-q's
manpage rather than repeating the descriptions.
Also, add lei-q to lei-blob's "see also" section.
Link: https://public-inbox.org/meta/YGFImEcWX1mCJJwv@dcvr/
---
Documentation/lei-blob.pod | 47 ++++++++++----------------------------
1 file changed, 12 insertions(+), 35 deletions(-)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index 76918324..36c75d53 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -49,60 +49,37 @@ Provide pre-image pathname as a hint for reconstructing C<OID>.
Provide post-image pathname as a hint for reconstructing C<OID>.
-=for comment
-TODO: The below options are shared with lei-q. Any good approaches to
-not repeating the text?
+=item -v
-=item --[no-]remote
+=item --verbose
-Whether to include results requiring network access. When local
-externals are configured, C<--remote> must be explicitly passed to
-enable reporting of results from remote externals.
+Provide more feedback on stderr.
-=item --no-local
+=back
-Limit operations to those requiring network access.
+The following options are also supported and are described in
+L<lei-q(1)>.
-=item --no-external
+=over
-Don't include results from externals.
+=item --[no-]remote
-=item -I LOCATION
+=item --no-local
-=item --include=LOCATION
+=item --no-external
-Include specified external in search. This option may be given
-multiple times.
+=item -I LOCATION, --include=LOCATION
=item --exclude=LOCATION
-Exclude specified external from search. This option may be given
-multiple times.
-
=item --only=LOCATION
-Use only the specified external for search. This option may be given
-multiple times, in which case the search uses only the specified set.
-
=item --no-import-remote
-Disable the default behavior of memoizing remote messages into the
-local store.
-
-=item -v
-
-=item --verbose
-
-Provide more feedback on stderr.
-
=item --torsocks=auto|no|yes
=item --no-torsocks
-Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
-
-Default: C<auto>
-
=back
=head1 CONTACT
@@ -121,4 +98,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
=head1 SEE ALSO
-L<lei-add-external(1)>
+L<lei-add-external(1)>, L<lei-q(1)>
--
2.31.1
^ permalink raw reply related [relevance 82%]
* [PATCH 5/9] doc lei q: fix a typo
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
` (2 preceding siblings ...)
2021-05-17 3:35 90% ` [PATCH 4/9] doc lei: resort lei-tag entries Kyle Meyer
@ 2021-05-17 3:35 71% ` Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 6/9] doc lei q: add missing value for --lock Kyle Meyer
` (3 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
---
Documentation/lei-q.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 46884aa5..e1c20a0d 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -99,7 +99,7 @@ Augment output destination instead of clobbering it.
=item --no-import-before
-Do not importing keywords before writing to an existing output
+Do not import keywords before writing to an existing output
destination.
=item --threads
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 1/9] doc lei blob: avoid combined description of separate options
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
@ 2021-05-17 3:35 68% ` Kyle Meyer
2021-05-17 3:35 82% ` [PATCH 3/9] doc lei blob: point to lei-q for shared options Kyle Meyer
` (6 subsequent siblings)
7 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
The next commit will update the manpages to split each option's
variants into separate items. This change won't mix well with
--oid-a, --path-a, and --path-b. These different options all share a
single description, and, if each form is on its own line, the link
between the variants of each option would no longer be clear.
Use a dedicated description for each option to avoid confusion.
---
Documentation/lei-blob.pod | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index bb316e71..341b5505 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -33,12 +33,15 @@ C<--oid-a>, C<--path-a>, and C<--path-b>.
=item -A OID-A, --oid-a=OID-A
+Provide pre-image object ID as a hint for reconstructing C<OID>.
+
=item -a PATH-A, --path-a=PATH-A
+Provide pre-image pathname as a hint for reconstructing C<OID>.
+
=item -b PATH-B, --path-b=PATH-B
-Provide pre-image object ID, pre-image pathname, or post-image
-pathname as a hint for reconstructing C<OID>.
+Provide post-image pathname as a hint for reconstructing C<OID>.
=for comment
TODO: The below options are shared with lei-q. Any good approaches to
--
2.31.1
^ permalink raw reply related [relevance 68%]
* [PATCH 0/9] doc: lei manpages, round 5
@ 2021-05-17 3:35 61% Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
` (7 more replies)
0 siblings, 8 replies; 200+ results
From: Kyle Meyer @ 2021-05-17 3:35 UTC (permalink / raw)
To: meta
This series updates the lei manpages, continuing from
<20210329031117.28516-1-kyle@kyleam.com>. It covers changes up to the
current tip of master (236831da32b1240d..8cc23ac6f7a38479).
The second patch touches outside of lei manpages as a followup to
e226f18934eb7291 (doc: lei q: split =item aliases onto separate lines,
2021-04-28).
[1/9] doc lei blob: avoid combined description of separate options
[2/9] doc: split option variants into separate items
[3/9] doc lei blob: point to lei-q for shared options
[4/9] doc lei: resort lei-tag entries
[5/9] doc lei q: fix a typo
[6/9] doc lei q: add missing value for --lock
[7/9] doc lei: add manpage for convert
[8/9] doc lei: add manpages for new commands
[9/9] doc lei: update manpages with new options
Documentation/lei-add-external.pod | 12 ++-
Documentation/lei-blob.pod | 62 +++++++--------
.../{lei-import.pod => lei-convert.pod} | 40 ++++++----
Documentation/lei-edit-search.pod | 28 +++++++
Documentation/lei-forget-external.pod | 4 +-
Documentation/lei-forget-search.pod | 28 +++++++
Documentation/lei-import.pod | 20 ++++-
Documentation/lei-init.pod | 4 +-
Documentation/lei-lcat.pod | 79 +++++++++++++++++++
Documentation/lei-ls-external.pod | 8 +-
Documentation/lei-ls-label.pod | 8 +-
...i-ls-external.pod => lei-ls-mail-sync.pod} | 22 +++---
Documentation/lei-ls-search.pod | 65 +++++++++++++++
Documentation/lei-overview.pod | 12 +++
Documentation/lei-p2q.pod | 8 +-
Documentation/lei-q.pod | 16 +++-
Documentation/lei-rediff.pod | 79 +++++++++++++++++++
Documentation/lei-tag.pod | 8 +-
Documentation/lei-up.pod | 48 +++++++++++
Documentation/lei.pod | 18 +++++
Documentation/public-inbox-compact.pod | 6 +-
Documentation/public-inbox-convert.pod | 4 +-
Documentation/public-inbox-daemon.pod | 16 +++-
Documentation/public-inbox-extindex.pod | 4 +-
Documentation/public-inbox-imapd.pod | 4 +-
Documentation/public-inbox-index.pod | 12 ++-
Documentation/public-inbox-init.pod | 20 +++--
Documentation/public-inbox-nntpd.pod | 4 +-
Documentation/public-inbox-xcpdb.pod | 14 +++-
Documentation/txt2pre | 11 ++-
MANIFEST | 8 ++
Makefile.PL | 8 +-
32 files changed, 580 insertions(+), 100 deletions(-)
copy Documentation/{lei-import.pod => lei-convert.pod} (54%)
create mode 100644 Documentation/lei-edit-search.pod
create mode 100644 Documentation/lei-forget-search.pod
create mode 100644 Documentation/lei-lcat.pod
copy Documentation/{lei-ls-external.pod => lei-ls-mail-sync.pod} (64%)
create mode 100644 Documentation/lei-ls-search.pod
create mode 100644 Documentation/lei-rediff.pod
create mode 100644 Documentation/lei-up.pod
base-commit: 8cc23ac6f7a3847977ec57c2a3e9e391fdb94da6
--
2.31.1
^ permalink raw reply [relevance 61%]
* Re: [PATCH] lei rediff: handle stdin like other commands
2021-05-16 2:42 71% [PATCH] lei rediff: handle stdin like other commands Kyle Meyer
@ 2021-05-16 15:23 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-16 15:23 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Thanks, pushed your 3 patches as:
8cc23ac6 lei rediff: handle stdin like other commands
2c1cbdc7 lei rediff: mark --color-moved value as optional
1110da3e lei_input: drop misplaced word from error message
^ permalink raw reply [relevance 71%]
* [PATCH] lei rediff: handle stdin like other commands
@ 2021-05-16 2:42 71% Kyle Meyer
2021-05-16 15:23 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-05-16 2:42 UTC (permalink / raw)
To: meta
`lei rediff' reads from stdin when no argument is specified, but this
is likely unintentional given that other lei commands instead have a
--stdin|- option and that `lei rediff --help' includes --stdin.
Align rediff's handling with the other commands'.
---
lib/PublicInbox/LEI.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5f178418..2144e390 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -180,8 +180,9 @@ our %CMD = ( # sorted in order of importance/use:
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@lxs_opt, @c_opt ],
-'rediff' => [ '[--stdin|LOCATION...]',
+'rediff' => [ '--stdin|LOCATION...',
'regenerate a diff with different options',
+ 'stdin|', # /|\z/ must be first for lone dash
qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
@diff_opt, @lxs_opt, @c_opt ],
base-commit: 79e274b293a71fb41dd8bf6e9598a8e8a24fed4f
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH] lei rediff: mark --color-moved value as optional
@ 2021-05-16 2:41 71% Kyle Meyer
0 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-16 2:41 UTC (permalink / raw)
To: meta
`git-diff --color-moved' can be called without a mode, in which case
it defaults to "zebra".
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5f178418..3e17a614 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -141,7 +141,7 @@ my @lxs_opt = (qw(remote! local! external! include|I=s@ exclude=s@ only=s@
our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
output-indicator-context=s indent-heuristic!
minimal patience histogram anchored=s@ diff-algorithm=s
- color-moved=s color-moved-ws=s no-color-moved no-color-moved-ws
+ color-moved:s color-moved-ws=s no-color-moved no-color-moved-ws
word-diff:s word-diff-regex=s color-words:s no-renames
rename-empty! check ws-error-highlight=s full-index binary
abbrev:i break-rewrites|B:s find-renames|M:s find-copies:s
base-commit: 79e274b293a71fb41dd8bf6e9598a8e8a24fed4f
--
2.31.1
^ permalink raw reply related [relevance 71%]
* lei timestamp resolution for mail synchronization
@ 2021-05-07 0:13 71% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-07 0:13 UTC (permalink / raw)
To: meta
Is millisecond resolution "good enough" for mail synchronization?
Or maybe even seconds is sufficient...
When syncing mail across machines/devices/reboots, using the
system clock seems required (anybody syncing mail also uses
NTP, right?).
While Time::HiRes allows access to .tv_nsec for some APIs;
Time::HiRes::stat converts it to a double-precision float
(NV/Numeric Value in perlguts-speak). That's not enough for
microsecond accuracy, even; and of course I don't think most
kernels or HW is actually accurate down to the nanosecond level.
^ permalink raw reply [relevance 71%]
* [PATCH 3/3] lei rediff: do not automatically store patches/mails
2021-05-05 17:49 71% [PATCH 0/3] lei rediff fixes Eric Wong
2021-05-05 17:49 55% ` [PATCH 2/3] lei rediff: capture and regenerate file modes Eric Wong
@ 2021-05-05 17:49 65% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-05 17:49 UTC (permalink / raw)
To: meta
We can use a temporary lei/store to avoid cluttering up
future search results. This is especially useful since
we expect "lei rediff" to be useful for non-email diffs
and individual attachments, too.
---
lib/PublicInbox/LeiRediff.pm | 9 +++++----
t/solver_git.t | 3 +++
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index d73e3e28..deb6c3ae 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -70,7 +70,6 @@ EOM
my $tb = $ta;
$tb =~ tr!A!B!;
my $lei = $self->{lei};
- my $wait = delete($self->{-do_done}) ? $lei->{sto}->ipc_do('done') : 0;
while (my ($oid_a, $oid_b, $pa, $pb, $ma, $mb) = splice(@$ctxq, 0, 6)) {
my $xa = $blob->{$oid_a} //= solve_1($self, $oid_a,
{ path_b => $pa });
@@ -193,8 +192,8 @@ sub extract_oids { # Eml each_part callback
sub input_eml_cb { # callback for all emails
my ($self, $eml) = @_;
- $self->{lei}->{sto}->ipc_do('add_eml', $eml);
- $self->{-do_done} = 1;
+ $self->{tmp_sto}->add_eml($eml);
+ $self->{tmp_sto}->done;
$eml->each_part(\&extract_oids, $self, 1);
}
@@ -239,7 +238,9 @@ sub ipc_atfork_child {
binmode $lei->{1}, ':utf8';
$self->{blobs} = {}; # oidhex => filename
$self->{rdtmp} = File::Temp->newdir('lei-rediff-XXXX', TMPDIR => 1);
- $self->{rmt} = [ map {
+ $self->{tmp_sto} = PublicInbox::LeiStore->new(
+ "$self->{rdtmp}/tmp.store", { creat => 1 });
+ $self->{rmt} = [ $self->{tmp_sto}->search, map {
PublicInbox::LeiRemote->new($lei, $_)
} $self->{lxs}->remotes ];
$self->{gits} = [ map {
diff --git a/t/solver_git.t b/t/solver_git.t
index ad0c295b..13e478b3 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -86,6 +86,9 @@ index 15ac20eb..771486c4
EOM
ok(index($lei_out, $exp) >= 0,
'preserve mode, regen header + context from -U0 patch');
+ my $e = { GIT_DIR => "$ENV{HOME}/.local/share/lei/store/ALL.git" };
+ my @x = xqx([qw(git cat-file --batch-all-objects --batch-check)], $e);
+ is_deeply(\@x, [], 'no objects stored') or diag explain(\@x);
});
test_lei({tmpdir => "$tmpdir/index-eml-only"}, sub {
^ permalink raw reply related [relevance 65%]
* [PATCH 2/3] lei rediff: capture and regenerate file modes
2021-05-05 17:49 71% [PATCH 0/3] lei rediff fixes Eric Wong
@ 2021-05-05 17:49 55% ` Eric Wong
2021-05-05 17:49 65% ` [PATCH 3/3] lei rediff: do not automatically store patches/mails Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-05 17:49 UTC (permalink / raw)
To: meta
Don't lose file mode information when regenerating a diff.
---
MANIFEST | 1 +
lib/PublicInbox/LeiRediff.pm | 19 +++++++++++++++----
t/solve/bare.patch | 8 ++++++++
t/solver_git.t | 13 +++++++++++++
4 files changed, 37 insertions(+), 4 deletions(-)
create mode 100644 t/solve/bare.patch
diff --git a/MANIFEST b/MANIFEST
index 7be07aa5..a6d94c77 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -480,6 +480,7 @@ t/shared_kv.t
t/sigfd.t
t/solve/0001-simple-mod.patch
t/solve/0002-rename-with-modifications.patch
+t/solve/bare.patch
t/solver_git.t
t/spamcheck_spamc.t
t/spawn.t
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 6c734bef..d73e3e28 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -16,6 +16,8 @@ use PublicInbox::Import;
use PublicInbox::LEI;
use PublicInbox::SolverGit;
+my $MODE = '(100644|120000|100755|160000)';
+
sub rediff_user_cb { # called by solver when done
my ($res, $self) = @_;
my $lei = $self->{lei};
@@ -69,7 +71,7 @@ EOM
$tb =~ tr!A!B!;
my $lei = $self->{lei};
my $wait = delete($self->{-do_done}) ? $lei->{sto}->ipc_do('done') : 0;
- while (my ($oid_a, $oid_b, $pa, $pb) = splice(@$ctxq, 0, 4)) {
+ while (my ($oid_a, $oid_b, $pa, $pb, $ma, $mb) = splice(@$ctxq, 0, 6)) {
my $xa = $blob->{$oid_a} //= solve_1($self, $oid_a,
{ path_b => $pa });
my $xb = $blob->{$oid_b} //= solve_1($self, $oid_b, {
@@ -77,8 +79,8 @@ EOM
path_a => $pa,
path_b => $pb
});
- $ta .= "M 100644 $xa ".git_quote($pa)."\n" if $xa;
- $tb .= "M 100644 $xb ".git_quote($pb)."\n" if $xb;
+ $ta .= "M $ma $xa ".git_quote($pa)."\n" if $xa;
+ $tb .= "M $mb $xb ".git_quote($pb)."\n" if $xb;
}
my $rw = $self->{gits}->[-1]; # has all known alternates
if (!$rw->{-tmp}) {
@@ -148,6 +150,15 @@ sub extract_oids { # Eml each_part callback
if (scalar(@top) >= 4 &&
$top[1] =~ $PublicInbox::ViewDiff::IS_OID &&
$top[0] =~ $PublicInbox::ViewDiff::IS_OID) {
+ my ($ma, $mb);
+ $x =~ /^old mode $MODE/sm and $ma = $1;
+ $x =~ /^new mode $MODE/sm and $mb = $1;
+ if (!defined($ma) && $x =~
+ /^index [a-z0-9]+\.\.[a-z0-9]+ $MODE/sm) {
+ $ma = $mb = $1;
+ }
+ $ma //= '100644';
+ $mb //= $ma;
my ($oid_a, $oid_b, $pa, $pb) = splice(@top, 0, 4);
$pa eq '/dev/null' or
$pa = (split(m'/', git_unquote($pa), 2))[1];
@@ -155,7 +166,7 @@ sub extract_oids { # Eml each_part callback
$pb = (split(m'/', git_unquote($pb), 2))[1];
$blobs->{$oid_a} //= undef;
$blobs->{$oid_b} //= undef;
- push @$ctxq, $oid_a, $oid_b, $pa, $pb;
+ push @$ctxq, $oid_a, $oid_b, $pa, $pb, $ma, $mb;
} elsif ($ctxq) {
my @out;
for (split(/^/sm, $x)) {
diff --git a/t/solve/bare.patch b/t/solve/bare.patch
new file mode 100644
index 00000000..25d7f344
--- /dev/null
+++ b/t/solve/bare.patch
@@ -0,0 +1,8 @@
+diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
+old mode 100644
+new mode 100755
+index 15ac20eb..771486c4
+--- a/script/public-inbox-extindex
++++ b/script/public-inbox-extindex
+@@ -4 +3,0 @@
+-# Basic tool to create a Xapian search index for a public-inbox.
diff --git a/t/solver_git.t b/t/solver_git.t
index 44cbbfdb..ad0c295b 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -73,6 +73,19 @@ test_lei({tmpdir => "$tmpdir/rediff"}, sub {
lei_ok(qw(rediff -q -U9 t/solve/0001-simple-mod.patch));
like($lei_out, qr!^\Q+++\E b/TODO\n@@ -103,9 \+103,11 @@!sm,
'got more context with -U9');
+ lei_ok(qw(rediff -q -U9 t/solve/bare.patch));
+ my $exp = <<'EOM';
+diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
+old mode 100644
+new mode 100755
+index 15ac20eb..771486c4
+--- a/script/public-inbox-extindex
++++ b/script/public-inbox-extindex
+@@ -1,13 +1,12 @@
+ #!perl -w
+EOM
+ ok(index($lei_out, $exp) >= 0,
+ 'preserve mode, regen header + context from -U0 patch');
});
test_lei({tmpdir => "$tmpdir/index-eml-only"}, sub {
^ permalink raw reply related [relevance 55%]
* [PATCH 0/3] lei rediff fixes
@ 2021-05-05 17:49 71% Eric Wong
2021-05-05 17:49 55% ` [PATCH 2/3] lei rediff: capture and regenerate file modes Eric Wong
2021-05-05 17:49 65% ` [PATCH 3/3] lei rediff: do not automatically store patches/mails Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-05 17:49 UTC (permalink / raw)
To: meta
1/3 was necessary anyways, and also provided a test case for
2/3. 3/3 makes things less surprising, and thankfully wasn't
difficult at all.
Eric Wong (3):
script/public-inbox-extindex: chmod +x
lei rediff: capture and regenerate file modes
lei rediff: do not automatically store patches/mails
MANIFEST | 1 +
lib/PublicInbox/LeiRediff.pm | 28 ++++++++++++++++++++--------
script/public-inbox-extindex | 1 -
t/solve/bare.patch | 8 ++++++++
t/solver_git.t | 16 ++++++++++++++++
5 files changed, 45 insertions(+), 9 deletions(-)
mode change 100644 => 100755 script/public-inbox-extindex
create mode 100644 t/solve/bare.patch
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei blob: support "lei index"-ed mail
2021-05-05 10:46 71% [PATCH 0/2] lei rediff + solver-related fix Eric Wong
2021-05-05 10:46 30% ` [PATCH 1/2] lei rediff: regenerate diffs from stdin Eric Wong
@ 2021-05-05 10:46 87% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-05 10:46 UTC (permalink / raw)
To: meta
Normal git retrieval don't work for Maildir blobs indexed using
"lei index". Fortunately, this oddness is limited to the
LeiStore class and we can override smsg_eml with a fallback
to read blobs from Maildirs.
---
lib/PublicInbox/LeiSearch.pm | 10 ++++++++++
t/solver_git.t | 10 ++++++++++
2 files changed, 20 insertions(+)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index cd28a700..c2b12146 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -144,4 +144,14 @@ sub lms {
-f $f ? PublicInbox::LeiMailSync->new($f) : undef;
}
+# allow SolverGit->resolve_patch to work with "lei index"
+sub smsg_eml {
+ my ($self, $smsg) = @_;
+ PublicInbox::Inbox::smsg_eml($self, $smsg) // do {
+ my $lms = lms($self);
+ my $bref = $lms ? $lms->local_blob($smsg->{blob}, 1) : undef;
+ $bref ? PublicInbox::Eml->new($bref) : undef;
+ };
+}
+
1;
diff --git a/t/solver_git.t b/t/solver_git.t
index e566efb3..44cbbfdb 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -27,6 +27,11 @@ my $ibx = create_inbox 'v2', version => 2,
$im->add(eml_load 't/solve/0001-simple-mod.patch') or BAIL_OUT;
$im->add($patch2) or BAIL_OUT;
};
+my $md = "$tmpdir/md";
+File::Path::mkpath([map { $md.$_ } (qw(/ /cur /new /tmp))]);
+symlink(abs_path('t/solve/0001-simple-mod.patch'), "$md/cur/foo:2,") or
+ xbail "symlink: $!";
+
my $v1_0_0_tag = 'cb7c42b1e15577ed2215356a2bf925aef59cdd8d';
my $v1_0_0_tag_short = substr($v1_0_0_tag, 0, 16);
my $expect = '69df7d565d49fbaaeb0a067910f03dc22cd52bd0';
@@ -70,6 +75,11 @@ test_lei({tmpdir => "$tmpdir/rediff"}, sub {
'got more context with -U9');
});
+test_lei({tmpdir => "$tmpdir/index-eml-only"}, sub {
+ lei_ok(qw(index), $md);
+ lei_ok(qw(blob 69df7d5)); # hits LeiSearch->smsg_eml -> lms->local_blob
+});
+
my $git = PublicInbox::Git->new($git_dir);
$ibx->{-repo_objs} = [ $git ];
my $res;
^ permalink raw reply related [relevance 87%]
* [PATCH 0/2] lei rediff + solver-related fix
@ 2021-05-05 10:46 71% Eric Wong
2021-05-05 10:46 30% ` [PATCH 1/2] lei rediff: regenerate diffs from stdin Eric Wong
2021-05-05 10:46 87% ` [PATCH 2/2] lei blob: support "lei index"-ed mail Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-05-05 10:46 UTC (permalink / raw)
To: meta
"rediff" is something I've wanted in the WWW UI for several
years, however:
1) the amount of git-diff options is staggering and
I find HTML <form> elements difficult-to-use
2) git-diff can be monumentally expensive, on top of
being difficult/impossible to cache
..so I couldn't figure out a good way to support it
On a local system, we can feed "git diff" command-line options
and not worry about 10K users trying to regenerate diffs at
once.
Eric Wong (2):
lei rediff: regenerate diffs from stdin
lei blob: support "lei index"-ed mail
MANIFEST | 1 +
lib/PublicInbox/Inbox.pm | 2 +-
lib/PublicInbox/LEI.pm | 22 ++++
lib/PublicInbox/LeiInput.pm | 6 +
lib/PublicInbox/LeiRediff.pm | 245 +++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSearch.pm | 10 ++
t/solver_git.t | 18 ++-
7 files changed, 302 insertions(+), 2 deletions(-)
create mode 100644 lib/PublicInbox/LeiRediff.pm
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei rediff: regenerate diffs from stdin
2021-05-05 10:46 71% [PATCH 0/2] lei rediff + solver-related fix Eric Wong
@ 2021-05-05 10:46 30% ` Eric Wong
2021-05-05 10:46 87% ` [PATCH 2/2] lei blob: support "lei index"-ed mail Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-05-05 10:46 UTC (permalink / raw)
To: meta
Sometimes a mailed patch is generated with non-ideal output,
(lacking context, noisy whitespace changes, etc.), or a user
wants to use the same external diff viewer they've configured
git to use.
Since we have SolverGit to regenerate arbitrary blobs from
patches; this new command allows us to regenerate a diff with
different options using the blobs SolverGit gives us.
The amount of git-diff(1) options is mind numbing, so it's
likely I missed some favorites or botched the getopt spec
translation.
This also fixes Inbox::base_url to check psgi.url_scheme
before attempting to generate URLs and avoid uninitialized
variable warnings. Oddly, the "lei blob" tests did not
trigger these uninitialized warnings.
Note: this will automatically import+index the message(s)
it's regenerating, because solver relies on being able
to lookup pre/postimage OIDs and read blobs.
---
MANIFEST | 1 +
lib/PublicInbox/Inbox.pm | 2 +-
lib/PublicInbox/LEI.pm | 22 ++++
lib/PublicInbox/LeiInput.pm | 6 +
lib/PublicInbox/LeiRediff.pm | 245 +++++++++++++++++++++++++++++++++++
t/solver_git.t | 8 +-
6 files changed, 282 insertions(+), 2 deletions(-)
create mode 100644 lib/PublicInbox/LeiRediff.pm
diff --git a/MANIFEST b/MANIFEST
index b40147b0..7be07aa5 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -211,6 +211,7 @@ lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiQuery.pm
+lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRemote.pm
lib/PublicInbox/LeiSavedSearch.pm
lib/PublicInbox/LeiSearch.pm
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index da7ea75f..b94ffdb0 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -241,7 +241,7 @@ sub cloneurl {
sub base_url {
my ($self, $env) = @_; # env - PSGI env
- if ($env) {
+ if ($env && $env->{'psgi.url_scheme'}) {
my $url = PublicInbox::Git::host_prefix_url($env, '');
# for mount in Plack::Builder
$url .= '/' if $url !~ m!/\z!;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index c5fdfeb8..9dbbeba9 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -135,6 +135,23 @@ my @lxs_opt = (qw(remote! local! external! include|I=s@ exclude=s@ only=s@
import-remote! no-torsocks torsocks=s),
PublicInbox::LeiQuery::curl_opt());
+# we don't support -C as an alias for --find-copies since it's already
+# used for chdir
+our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
+ output-indicator-context=s indent-heuristic!
+ minimal patience histogram anchored=s@ diff-algorithm=s
+ color-moved=s color-moved-ws=s no-color-moved no-color-moved-ws
+ word-diff:s word-diff-regex=s color-words:s no-renames
+ rename-empty! check ws-error-highlight=s full-index binary
+ abbrev:i break-rewrites|B:s find-renames|M:s find-copies:s
+ find-copies-harder irreversible-delete|D l=i diff-filter=s
+ S=s G=s find-object=s pickaxe-all pickaxe-regex O=s R
+ relative:s text|a ignore-cr-at-eol ignore-space-at-eol
+ ignore-space-change|b ignore-all-space|w ignore-blank-lines
+ inter-hunk-context=i function-context|W exit-code ext-diff
+ no-ext-diff textconv! src-prefix=s dst-prefix=s no-prefix
+ line-prefix=s);
+
# we generate shell completion + help using %CMD and %OPTDESC,
# see lei__complete() and PublicInbox::LeiHelp
# command => [ positional_args, 1-line description, Getopt::Long option spec ]
@@ -162,6 +179,11 @@ our %CMD = ( # sorted in order of importance/use:
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@lxs_opt, @c_opt ],
+'rediff' => [ '[--stdin|LOCATION...]',
+ 'regenerate a diff with different options',
+ qw(git-dir=s@ cwd! verbose|v+ color:s no-color),
+ @diff_opt, @lxs_opt, @c_opt ],
+
'add-external' => [ 'LOCATION',
'add/set priority of a publicinbox|extindex for extra matches',
qw(boost=i mirror=s no-torsocks torsocks=s inbox-version=i
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 46eea111..87083564 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -69,6 +69,12 @@ error reading $name: $!
# but no Content-Length or "From " escaping.
# "git format-patch" also generates such files by default.
$buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+
+ # a user may feed just a body: git diff | lei rediff -U9
+ if ($self->{-force_eml}) {
+ my $eml = PublicInbox::Eml->new($buf);
+ substr($buf, 0, 0) = "\n\n" if !$eml->{bdy};
+ }
$self->input_eml_cb(PublicInbox::Eml->new(\$buf), @args);
} else {
# prepare_inputs already validated $ifmt
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
new file mode 100644
index 00000000..6c734bef
--- /dev/null
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -0,0 +1,245 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# The "lei rediff" sub-command, regenerates diffs with new options
+package PublicInbox::LeiRediff;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use File::Temp 0.19 (); # 0.19 for ->newdir
+use PublicInbox::Spawn qw(spawn which);
+use PublicInbox::MsgIter qw(msg_part_text);
+use PublicInbox::ViewDiff;
+use PublicInbox::LeiBlob;
+use PublicInbox::Git qw(git_quote git_unquote);
+use PublicInbox::Import;
+use PublicInbox::LEI;
+use PublicInbox::SolverGit;
+
+sub rediff_user_cb { # called by solver when done
+ my ($res, $self) = @_;
+ my $lei = $self->{lei};
+ my $log_buf = delete $lei->{log_buf};
+ $$log_buf =~ s/^/# /sgm;
+ ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+ $lei->qerr($$log_buf);
+ my ($git, $oid, $type, $size, $di) = @$res;
+ my $oid_want = delete $self->{cur_oid_want};
+
+ # don't try to support all the git-show(1) options for non-blob,
+ # this is just a convenience:
+ $type ne 'blob' and return $lei->err(<<EOF);
+# $oid is a $type of $size bytes in:
+# $git->{git_dir} (wanted: $oid_want)
+EOF
+ $self->{blob}->{$oid_want} = $oid;
+ push @{$self->{gits}}, $git if $git->{-tmp};
+}
+
+# returns a full blob for oid_want
+sub solve_1 ($$$) {
+ my ($self, $oid_want, $hints) = @_;
+ return if $oid_want =~ /\A0+\z/;
+ $self->{cur_oid_want} = $oid_want;
+ my $solver = bless {
+ gits => $self->{gits},
+ user_cb => \&rediff_user_cb,
+ uarg => $self,
+ inboxes => [ $self->{lxs}->locals, @{$self->{rmt}} ],
+ }, 'PublicInbox::SolverGit';
+ open my $log, '+>', \(my $log_buf = '') or die "PerlIO::scalar: $!";
+ $self->{lei}->{log_buf} = \$log_buf;
+ local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
+ $solver->solve($self->{lei}->{env}, $log, $oid_want, $hints);
+ $self->{blob}->{$oid_want}; # full OID
+}
+
+sub diff_ctxq ($$) {
+ my ($self, $ctxq) = @_;
+ return unless $ctxq;
+ my $blob = $self->{blob};
+ my $ta = <<'EOM';
+reset refs/heads/A
+commit refs/heads/A
+author <a@s> 0 +0000
+committer <c@s> 0 +0000
+data 0
+EOM
+ my $tb = $ta;
+ $tb =~ tr!A!B!;
+ my $lei = $self->{lei};
+ my $wait = delete($self->{-do_done}) ? $lei->{sto}->ipc_do('done') : 0;
+ while (my ($oid_a, $oid_b, $pa, $pb) = splice(@$ctxq, 0, 4)) {
+ my $xa = $blob->{$oid_a} //= solve_1($self, $oid_a,
+ { path_b => $pa });
+ my $xb = $blob->{$oid_b} //= solve_1($self, $oid_b, {
+ oid_a => $oid_a,
+ path_a => $pa,
+ path_b => $pb
+ });
+ $ta .= "M 100644 $xa ".git_quote($pa)."\n" if $xa;
+ $tb .= "M 100644 $xb ".git_quote($pb)."\n" if $xb;
+ }
+ my $rw = $self->{gits}->[-1]; # has all known alternates
+ if (!$rw->{-tmp}) {
+ my $d = "$self->{rdtmp}/for_tree.git";
+ -d $d or PublicInbox::Import::init_bare($d);
+ my $f = "$d/objects/info/alternates"; # always overwrite
+ open my $fh, '>', $f or die "open $f: $!";
+ for my $git (@{$self->{gits}}) {
+ print $fh $git->git_path('objects'),"\n";
+ }
+ close $fh or die "close $f: $!";
+ $rw = PublicInbox::Git->new($d);
+ }
+ pipe(my ($r, $w)) or die "pipe: $!";
+ my $pid = spawn(['git', "--git-dir=$rw->{git_dir}",
+ qw(fast-import --quiet --done --date-format=raw)],
+ $lei->{env}, { 2 => $lei->{2}, 0 => $r });
+ close $r or die "close r fast-import: $!";
+ print $w $ta, "\n", $tb, "\ndone\n" or die "print fast-import: $!";
+ close $w or die "close w fast-import: $!";
+ waitpid($pid, 0);
+ die "fast-import failed: \$?=$?" if $?;
+
+ my @cmd = qw(diff);
+ my $opt = $lei->{opt};
+ push @cmd, '--'.($opt->{color} && !$opt->{'no-color'} ? '' : 'no-').
+ 'color';
+ for my $o (@PublicInbox::LEI::diff_opt) {
+ $o =~ s/\|([a-z0-9])\b//i; # remove single char short option
+ my $c = $1;
+ if ($o =~ s/=[is]@\z//) {
+ my $v = $opt->{$o} or next;
+ push @cmd, map { $c ? "-$c$_" : "--$o=$_" } @$v;
+ } elsif ($o =~ s/=[is]\z//) {
+ my $v = $opt->{$o} // next;
+ push @cmd, $c ? "-$c$v" : "--$o=$v";
+ } elsif ($o =~ s/:[is]\z//) {
+ my $v = $opt->{$o} // next;
+ push @cmd, $c ? "-$c$v" :
+ ($v eq '' ? "--$o" : "--$o=$v");
+ } elsif ($o =~ s/!\z//) {
+ my $v = $opt->{$o} // next;
+ push @cmd, $v ? "--$o" : "--no-$o";
+ } elsif ($opt->{$o}) {
+ push @cmd, $c ? "-$c" : "--$o";
+ }
+ }
+ $lei->qerr("# git @cmd");
+ push @cmd, qw(A B);
+ unshift @cmd, 'git', "--git-dir=$rw->{git_dir}";
+ $pid = spawn(\@cmd, $lei->{env}, { 2 => $lei->{2}, 1 => $lei->{1} });
+ waitpid($pid, 0);
+ $lei->child_error($?) if $?; # for git diff --exit-code
+}
+
+sub extract_oids { # Eml each_part callback
+ my ($ary, $self) = @_;
+ my ($p, undef, $idx) = @$ary;
+ $self->{lei}->out($p->header_obj->as_string, "\n");
+ my ($s, undef) = msg_part_text($p, $p->content_type || 'text/plain');
+ defined $s or return;
+ my @top = split($PublicInbox::ViewDiff::EXTRACT_DIFFS, $s);
+ undef $s;
+ my $blobs = $self->{blobs}; # blobs to resolve
+ my $ctxq;
+ while (defined(my $x = shift @top)) {
+ if (scalar(@top) >= 4 &&
+ $top[1] =~ $PublicInbox::ViewDiff::IS_OID &&
+ $top[0] =~ $PublicInbox::ViewDiff::IS_OID) {
+ my ($oid_a, $oid_b, $pa, $pb) = splice(@top, 0, 4);
+ $pa eq '/dev/null' or
+ $pa = (split(m'/', git_unquote($pa), 2))[1];
+ $pb eq '/dev/null' or
+ $pb = (split(m'/', git_unquote($pb), 2))[1];
+ $blobs->{$oid_a} //= undef;
+ $blobs->{$oid_b} //= undef;
+ push @$ctxq, $oid_a, $oid_b, $pa, $pb;
+ } elsif ($ctxq) {
+ my @out;
+ for (split(/^/sm, $x)) {
+ if (/\A-- \r?\n/s) { # email sig starts
+ push @out, $_;
+ $ctxq = diff_ctxq($self, $ctxq);
+ } elsif ($ctxq && (/\A[\+\- ]/ || /\A@@ / ||
+ # allow totally blank lines w/o leading
+ # SP, git-apply does:
+ /\A\r?\n/s)) {
+ next;
+ } else {
+ push @out, $_;
+ }
+ }
+ $self->{lei}->out(@out) if @out;
+ } else {
+ $ctxq = diff_ctxq($self, $ctxq);
+ $self->{lei}->out($x);
+ }
+ }
+ $ctxq = diff_ctxq($self, $ctxq);
+}
+
+sub input_eml_cb { # callback for all emails
+ my ($self, $eml) = @_;
+ $self->{lei}->{sto}->ipc_do('add_eml', $eml);
+ $self->{-do_done} = 1;
+ $eml->each_part(\&extract_oids, $self, 1);
+}
+
+sub lei_rediff {
+ my ($lei, @inputs) = @_;
+ $lei->_lei_store(1)->write_prepare($lei);
+ $lei->{opt}->{stdin} = 1 if !@inputs;
+ $lei->{opt}->{'in-format'} //= 'eml';
+ # maybe it's a non-email (code) blob from a coderepo
+ my $git_dirs = $lei->{opt}->{'git-dir'} //= [];
+ if ($lei->{opt}->{cwd} // 1) {
+ my $cgd = PublicInbox::LeiBlob::get_git_dir($lei, '.');
+ unshift(@$git_dirs, $cgd) if defined $cgd;
+ }
+ return $lei->fail('no --git-dir to try') unless @$git_dirs;
+ my $lxs = $lei->lxs_prepare;
+ if ($lxs->remotes) {
+ require PublicInbox::LeiRemote;
+ $lei->{curl} //= which('curl') or return
+ $lei->fail('curl needed for', $lxs->remotes);
+ }
+ $lei->ale->refresh_externals($lxs);
+ my $self = bless {
+ -force_eml => 1, # for LeiInput->input_fh
+ lxs => $lxs,
+ }, __PACKAGE__;
+ $self->prepare_inputs($lei, \@inputs) or return;
+ my $isatty = -t $lei->{1};
+ $lei->{opt}->{color} //= $isatty;
+ $lei->start_pager if $isatty;
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
+ $lei->{wq1} = $self;
+ net_merge_all_done($self) unless $lei->{auth};
+ $op_c->op_wait_event($ops);
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ PublicInbox::LeiInput::input_only_atfork_child(@_);
+ my $lei = $self->{lei};
+ $lei->{1}->autoflush(1);
+ binmode $lei->{1}, ':utf8';
+ $self->{blobs} = {}; # oidhex => filename
+ $self->{rdtmp} = File::Temp->newdir('lei-rediff-XXXX', TMPDIR => 1);
+ $self->{rmt} = [ map {
+ PublicInbox::LeiRemote->new($lei, $_)
+ } $self->{lxs}->remotes ];
+ $self->{gits} = [ map {
+ PublicInbox::Git->new($lei->rel2abs($_))
+ } @{$self->{lei}->{opt}->{'git-dir'}} ];
+ $lei->{env}->{'psgi.errors'} = $lei->{2}; # ugh...
+ $lei->{env}->{TMPDIR} = $self->{rdtmp}->dirname;
+ undef;
+}
+
+no warnings 'once';
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+1;
diff --git a/t/solver_git.t b/t/solver_git.t
index 75387b2a..e566efb3 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -32,7 +32,7 @@ my $v1_0_0_tag_short = substr($v1_0_0_tag, 0, 16);
my $expect = '69df7d565d49fbaaeb0a067910f03dc22cd52bd0';
my $non_existent = 'ee5e32211bf62ab6531bdf39b84b6920d0b6775a';
-test_lei({tmpdir => $tmpdir}, sub {
+test_lei({tmpdir => "$tmpdir/blob"}, sub {
lei_ok('blob', '--mail', $patch2_oid, '-I', $ibx->{inboxdir},
\'--mail works for existing oid');
is($lei_out, $patch2->as_string, 'blob matches');
@@ -64,6 +64,12 @@ test_lei({tmpdir => $tmpdir}, sub {
lei_ok('blob', $v1_0_0_tag_short, '-I', $ibx->{inboxdir});
});
+test_lei({tmpdir => "$tmpdir/rediff"}, sub {
+ lei_ok(qw(rediff -q -U9 t/solve/0001-simple-mod.patch));
+ like($lei_out, qr!^\Q+++\E b/TODO\n@@ -103,9 \+103,11 @@!sm,
+ 'got more context with -U9');
+});
+
my $git = PublicInbox::Git->new($git_dir);
$ibx->{-repo_objs} = [ $git ];
my $res;
^ permalink raw reply related [relevance 30%]
* [PATCH] lei index: new command to index mail w/o git storage
@ 2021-05-04 9:49 26% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-04 9:49 UTC (permalink / raw)
To: meta
Since completely purging blobs from git is slow, users may wish
to index messages in Maildirs (and eventually other local
storage) without storing data in git.
Much code from LeiImport and LeiInput is reused, and a new dummy
FakeImport class supplies a non-storing $im->add and minimize
changes to LeiStore.
The tricky part of this command is to support "lei import"
after a message has gone through "lei index". Relying on
$smsg->{bytes} == 0 (as we do for external-only vmd storage)
does not work here, since it would break searching for "z:"
byte-ranges when not using externals.
This eventually required PublicInbox::Import::add to use a
SharedKV to keep track of imported blobs and prevent
duplication.
---
MANIFEST | 3 ++
lib/PublicInbox/FakeImport.pm | 23 ++++++++++++++
lib/PublicInbox/Import.pm | 26 +++++++--------
lib/PublicInbox/LeiBlob.pm | 38 +++++++++++++++-------
lib/PublicInbox/LeiImport.pm | 15 ++++++---
lib/PublicInbox/LeiIndex.pm | 48 ++++++++++++++++++++++++++++
lib/PublicInbox/LeiInput.pm | 30 ++++++------------
lib/PublicInbox/LeiMailSync.pm | 27 ++++++++++++++++
lib/PublicInbox/LeiStore.pm | 10 +++++-
lib/PublicInbox/LeiToMail.pm | 10 +++++-
lib/PublicInbox/OverIdx.pm | 18 +++++++++++
t/lei-index.t | 58 ++++++++++++++++++++++++++++++++++
12 files changed, 253 insertions(+), 53 deletions(-)
create mode 100644 lib/PublicInbox/FakeImport.pm
create mode 100644 lib/PublicInbox/LeiIndex.pm
create mode 100644 t/lei-index.t
diff --git a/MANIFEST b/MANIFEST
index e23297fa..42729b9c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -148,6 +148,7 @@ lib/PublicInbox/EmlContentFoo.pm
lib/PublicInbox/ExtMsg.pm
lib/PublicInbox/ExtSearch.pm
lib/PublicInbox/ExtSearchIdx.pm
+lib/PublicInbox/FakeImport.pm
lib/PublicInbox/FakeInotify.pm
lib/PublicInbox/Feed.pm
lib/PublicInbox/Filter/Base.pm
@@ -198,6 +199,7 @@ lib/PublicInbox/LeiExternal.pm
lib/PublicInbox/LeiForgetSearch.pm
lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
+lib/PublicInbox/LeiIndex.pm
lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
@@ -404,6 +406,7 @@ t/lei-import-imap.t
t/lei-import-maildir.t
t/lei-import-nntp.t
t/lei-import.t
+t/lei-index.t
t/lei-lcat.t
t/lei-mirror.t
t/lei-p2q.t
diff --git a/lib/PublicInbox/FakeImport.pm b/lib/PublicInbox/FakeImport.pm
new file mode 100644
index 00000000..dea25cbe
--- /dev/null
+++ b/lib/PublicInbox/FakeImport.pm
@@ -0,0 +1,23 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# pretend to do PublicInbox::Import::add for "lei index"
+package PublicInbox::FakeImport;
+use strict;
+use PublicInbox::ContentHash qw(git_sha);
+
+sub new { bless { bytes_added => 0 }, __PACKAGE__ }
+
+sub add {
+ my ($self, $eml, $check_cb, $smsg) = @_;
+ $smsg->populate($eml);
+ my $raw = $eml->as_string;
+ $smsg->{blob} = git_sha(1, \$raw)->hexdigest;
+ $smsg->set_bytes($raw, length($raw));
+ if (my $oidx = delete $smsg->{-oidx}) { # used by LeiStore
+ $oidx->vivify_xvmd($smsg) or return;
+ }
+ 1;
+}
+
+1;
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 3adf9dec..362cdc47 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -413,19 +413,19 @@ sub add {
$smsg->{blob} = $self->get_mark(":$blob");
$smsg->set_bytes($raw_email, $n);
if (my $oidx = delete $smsg->{-oidx}) { # used by LeiStore
- my @docids = $oidx->blob_exists($smsg->{blob});
- my @vivify_xvmd;
- for my $id (@docids) {
- if (my $cur = $oidx->get_art($id)) {
- # already imported if bytes > 0
- return if $cur->{bytes} > 0;
- push @vivify_xvmd, $id;
- } else {
- warn "W: $smsg->{blob} ",
- "#$id gone (bug?)\n";
- }
- }
- $smsg->{-vivify_xvmd} = \@vivify_xvmd;
+ my $eidx_git = delete $smsg->{-eidx_git};
+
+ # we need this sharedkv to dedupe blobs added in the
+ # same fast-import transaction
+ my $u = $self->{uniq_skv} //= do {
+ require PublicInbox::SharedKV;
+ my $x = PublicInbox::SharedKV->new;
+ $x->dbh;
+ $x;
+ };
+ return if !$u->set_maybe(pack('H*', $smsg->{blob}), 1);
+ return if (!$oidx->vivify_xvmd($smsg) &&
+ $eidx_git->check($smsg->{blob}));
}
}
my $ref = $self->{ref};
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 710430a2..8de86565 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -87,6 +87,16 @@ sub cat_attach_i { # Eml->each_part callback
$lei->out($part->body);
}
+sub extract_attach ($$$) {
+ my ($lei, $blob, $bref) = @_;
+ my $eml = PublicInbox::Eml->new($bref);
+ $eml->each_part(\&cat_attach_i, $lei, 1);
+ my $idx = delete $lei->{-attach_idx};
+ defined($idx) and return $lei->fail(<<EOM);
+E: attachment $idx not found in $blob
+EOM
+}
+
sub lei_blob {
my ($lei, $blob) = @_;
$lei->start_pager if -t $lei->{1};
@@ -106,7 +116,7 @@ sub lei_blob {
}
my $rdr = {};
if ($opt->{mail}) {
- $rdr->{2} = $lei->{2};
+ open $rdr->{2}, '+>', undef or die "open: $!";
} else {
open $rdr->{2}, '>', '/dev/null' or die "open: $!";
}
@@ -115,21 +125,25 @@ sub lei_blob {
if (defined $lei->{-attach_idx}) {
my $fh = popen_rd($cmd, $lei->{env}, $rdr);
require PublicInbox::Eml;
- my $str = do { local $/; <$fh> };
- if (close $fh) {
- my $eml = PublicInbox::Eml->new(\$str);
- $eml->each_part(\&cat_attach_i, $lei, 1);
- my $idx = delete $lei->{-attach_idx};
- defined($idx) and return $lei->fail(<<EOM);
-E: attachment $idx not found in $blob
-EOM
- }
+ my $buf = do { local $/; <$fh> };
+ return extract_attach($lei, $blob, \$buf) if close($fh);
} else {
$rdr->{1} = $lei->{1};
waitpid(spawn($cmd, $lei->{env}, $rdr), 0);
}
- return if $? == 0;
- return $lei->child_error($?) if $opt->{mail};
+ my $ce = $?;
+ return if $ce == 0;
+ my $sto = $lei->_lei_store;
+ my $lms = $sto ? $sto->search->lms : undef;
+ if (my $bref = $lms ? $lms->local_blob($blob, 1) : undef) {
+ defined($lei->{-attach_idx}) and
+ return extract_attach($lei, $blob, $bref);
+ return $lei->out($$bref);
+ } elsif ($opt->{mail}) {
+ my $eh = $rdr->{2};
+ seek($eh, 0, 0);
+ return $lei->child_error($ce, do { local $/; <$eh> });
+ } # else: fall through to solver below
}
# maybe it's a non-email (code) blob from a coderepo
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 6a57df47..55925cc5 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -38,21 +38,20 @@ sub input_maildir_cb { # maildir_each_eml cb
warn "E: $f was not from a Maildir?\n";
}
}
- input_eml_cb($self, $eml, $vmd);
+ $self->input_eml_cb($eml, $vmd);
}
sub input_net_cb { # imap_each / nntp_each
my ($url, $uid, $kw, $eml, $self) = @_;
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
$vmd->{sync_info} = [ $url, $uid ] if $self->{-mail_sync};
- input_eml_cb($self, $eml, $vmd);
+ $self->input_eml_cb($eml, $vmd);
}
-sub lei_import { # the main "lei import" method
- my ($lei, @inputs) = @_;
+sub do_import_index ($$@) {
+ my ($self, $lei, @inputs) = @_;
my $sto = $lei->_lei_store(1);
$sto->write_prepare($lei);
- my $self = bless {}, __PACKAGE__;
$self->{-import_kw} = $lei->{opt}->{kw} // 1;
my $vmd_mod = $self->vmd_mod_extract(\@inputs);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
@@ -83,6 +82,12 @@ sub lei_import { # the main "lei import" method
$op_c->op_wait_event($ops);
}
+sub lei_import { # the main "lei import" method
+ my ($lei, @inputs) = @_;
+ my $self = bless {}, __PACKAGE__;
+ do_import_index($self, $lei, @inputs);
+}
+
sub _complete_import {
my ($lei, @argv) = @_;
my $sto = $lei->_lei_store or return;
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
new file mode 100644
index 00000000..cc3e83e7
--- /dev/null
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -0,0 +1,48 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# front-end for the "lei index" sub-command, this is similar to
+# "lei import" but doesn't put a git blob into ~/.local/share/lei/store
+package PublicInbox::LeiIndex;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::LeiImport;
+
+# /^input_/ subs are used by (or override) PublicInbox::LeiInput superclass
+sub input_eml_cb { # used by input_maildir_cb and input_net_cb
+ my ($self, $eml, $vmd) = @_;
+ my $xoids = $self->{lei}->{ale}->xoids_for($eml);
+ if (my $all_vmd = $self->{all_vmd}) {
+ @$vmd{keys %$all_vmd} = values %$all_vmd;
+ }
+ $self->{lei}->{sto}->ipc_do('index_eml_only', $eml, $vmd, $xoids);
+}
+
+sub input_fh { # overrides PublicInbox::LeiInput::input_fh
+ my ($self, $ifmt, $fh, $input, @args) = @_;
+ $self->{lei}->child_error(1<<8, <<EOM);
+$input ($ifmt) not yet supported, try `lei import'
+EOM
+}
+
+sub lei_index {
+ my ($lei, @argv) = @_;
+ $lei->{opt}->{'mail-sync'} = 1;
+ my $self = bless {}, __PACKAGE__;
+ PublicInbox::LeiImport::do_import_index($self, $lei, @argv);
+}
+
+no warnings 'once';
+no strict 'refs';
+for my $m (qw(input_maildir_cb input_net_cb)) {
+ *$m = PublicInbox::LeiImport->can($m);
+}
+
+*_complete_import = \&PublicInbox::LeiImport::_complete_import;
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
+
+# the following works even when LeiAuth is lazy-loaded
+*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+1;
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 85caac35..46eea111 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -1,7 +1,7 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# parent class for LeiImport, LeiConvert
+# parent class for LeiImport, LeiConvert, LeiIndex
package PublicInbox::LeiInput;
use strict;
use v5.10.1;
@@ -93,11 +93,7 @@ sub handle_http_input ($$@) {
my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
grep(/\A--compressed\z/, @$curl) or
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
- eval {
- PublicInbox::MboxReader->mboxrd($fh,
- $self->can('input_mbox_cb'),
- $self, @args);
- };
+ eval { $self->input_fh('mboxrd', $fh, $url, @args) };
my $err = $@;
waitpid($pid, 0);
$? || $err and
@@ -221,14 +217,8 @@ sub prepare_inputs { # returns undef on error
require PublicInbox::NetReader;
$net //= PublicInbox::NetReader->new;
$net->add_url($input);
- if ($sync) {
- if ($input =~ m!\Aimaps?://!) {
- push @{$sync->{ok}}, $input;
- } else {
- push @{$sync->{no}}, $input;
- }
- }
- } elsif ($input_path =~ m!\Ahttps?://!i) {
+ push @{$sync->{ok}}, $input if $sync;
+ } elsif ($input_path =~ m!\Ahttps?://!i) { # mboxrd.gz
# TODO: how would we detect r/w JMAP?
push @{$sync->{no}}, $input if $sync;
prepare_http_input($self, $lei, $input_path) or return;
@@ -239,12 +229,10 @@ sub prepare_inputs { # returns undef on error
--in-format=$in_fmt and `$ifmt:' conflict
}
- if ($sync) {
- if ($ifmt =~ /\A(?:maildir|mh)\z/i) {
- push @{$sync->{ok}}, $input;
- } else {
- push @{$sync->{no}}, $input;
- }
+ if ($ifmt =~ /\A(?:maildir|mh)\z/i) {
+ push @{$sync->{ok}}, $input if $sync;
+ } else {
+ push @{$sync->{no}}, $input if $sync;
}
my $devfd = $lei->path_to_fd($input_path) // return;
if ($devfd >= 0 || (-f $input_path || -p _)) {
@@ -260,7 +248,7 @@ sub prepare_inputs { # returns undef on error
} else {
return $lei->fail("Unable to handle $input");
}
- } elsif ($input =~ /\.(eml|patch)\z/i && -f $input) {
+ } elsif ($input =~ /\.(?:eml|patch)\z/i && -f $input) {
lc($in_fmt//'eml') eq 'eml' or return $lei->fail(<<"");
$input is `eml', not --in-format=$in_fmt
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 2ce189fa..2e74e433 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiMailSync;
use strict;
use v5.10.1;
use DBI;
+use PublicInbox::ContentHash qw(git_sha);
sub dbh_new {
my ($self, $rw) = @_;
@@ -208,4 +209,30 @@ sub folders {
map { $_->[0] } @{$dbh->selectall_arrayref($sql, undef, @pfx)};
}
+sub local_blob {
+ my ($self, $oidhex, $vrfy) = @_;
+ my $dbh = $self->{dbh} //= dbh_new($self);
+ my $b2n = $dbh->prepare(<<'');
+SELECT f.loc,b.name FROM blob2name b
+LEFT JOIN folders f ON b.fid = f.fid
+WHERE b.oidbin = ?
+
+ $b2n->execute(pack('H*', $oidhex));
+ while (my ($d, $n) = $b2n->fetchrow_array) {
+ substr($d, 0, length('maildir:')) = '';
+ my $f = "$d/" . ($n =~ /:2,[a-zA-Z]*\z/ ? "cur/$n" : "new/$n");
+ open my $fh, '<', $f or next;
+ if (-s $fh) {
+ local $/;
+ my $raw = <$fh>;
+ if ($vrfy && git_sha(1, \$raw)->hexdigest ne $oidhex) {
+ warn "$f changed $oidhex\n";
+ next;
+ }
+ return \$raw;
+ }
+ }
+ undef;
+}
+
1;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 29362b2e..a7a0ebef 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -206,10 +206,11 @@ sub set_sync_info {
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
- my $im = $self->importer; # may create new epoch
+ my $im = $self->{-fake_im} // $self->importer; # may create new epoch
my ($eidx, $tl) = eidx_init($self);
my $oidx = $eidx->{oidx}; # PublicInbox::Import::add checks this
my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
+ $smsg->{-eidx_git} = $eidx->git if !$self->{-fake_im};
my $im_mark = $im->add($eml, undef, $smsg);
if ($vmd && $vmd->{sync_info}) {
set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
@@ -276,6 +277,13 @@ sub set_eml {
set_eml_vmd($self, $eml, $vmd);
}
+sub index_eml_only {
+ my ($self, $eml, $vmd, $xoids) = @_;
+ require PublicInbox::FakeImport;
+ local $self->{-fake_im} = PublicInbox::FakeImport->new;
+ set_eml($self, $eml, $vmd, $xoids);
+}
+
sub _external_only ($$$) {
my ($self, $xoids, $eml) = @_;
my $eidx = $self->{priv_eidx};
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 64061788..da3a95d2 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -137,9 +137,15 @@ sub eml2mboxcl2 {
sub git_to_mail { # git->cat_async callback
my ($bref, $oid, $type, $size, $arg) = @_;
+ my ($write_cb, $smsg) = @$arg;
+ if ($type eq 'missing' && $smsg->{-lms_ro}) {
+ if ($bref = $smsg->{-lms_ro}->local_blob($oid, 1)) {
+ $type = 'blob';
+ $size = length($$bref);
+ }
+ }
return warn("W: $oid is $type (!= blob)\n") if $type ne 'blob';
return warn("E: $oid is empty\n") unless $size;
- my ($write_cb, $smsg) = @$arg;
die "BUG: expected=$smsg->{blob} got=$oid" if $smsg->{blob} ne $oid;
$write_cb->($bref, $smsg);
}
@@ -644,6 +650,7 @@ sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
$lei->_lei_atfork_child;
+ $self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse};
$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
@@ -665,6 +672,7 @@ sub poke_dst {
sub write_mail { # via ->wq_io_do
my ($self, $smsg, $eml) = @_;
return $self->{wcb}->(undef, $smsg, $eml) if $eml;
+ $smsg->{-lms_ro} = $self->{-lms_ro};
$self->{lei}->{ale}->git->cat_async($smsg->{blob}, \&git_to_mail,
[$self->{wcb}, $smsg]);
}
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 66dec099..5f96a5b0 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -670,4 +670,22 @@ DELETE FROM eidxq WHERE docid = ?
}
+# returns true if we're vivifying a message for lei/store that was
+# previously external-metadata only
+sub vivify_xvmd {
+ my ($self, $smsg) = @_;
+ my @docids = $self->blob_exists($smsg->{blob});
+ my @vivify_xvmd;
+ for my $id (@docids) {
+ if (my $cur = $self->get_art($id)) {
+ # already indexed if bytes > 0
+ return if $cur->{bytes} > 0;
+ push @vivify_xvmd, $id;
+ } else {
+ warn "W: $smsg->{blob} #$id gone (bug?)\n";
+ }
+ }
+ $smsg->{-vivify_xvmd} = \@vivify_xvmd;
+}
+
1;
diff --git a/t/lei-index.t b/t/lei-index.t
new file mode 100644
index 00000000..3382d42b
--- /dev/null
+++ b/t/lei-index.t
@@ -0,0 +1,58 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use File::Spec;
+require_mods(qw(lei -nntpd));
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my ($tmpdir, $for_destroy) = tmpdir;
+my $env = { PI_CONFIG => $cfg_path };
+
+my $sock = tcp_server;
+my $cmd = [ '-nntpd', '-W0', "--stdout=$tmpdir/n1", "--stderr=$tmpdir/n2" ];
+my $nntpd = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-nntpd $?");
+my $nntp_host_port = tcp_host_port($sock);
+
+$sock = tcp_server;
+$cmd = [ '-imapd', '-W0', "--stdout=$tmpdir/i1", "--stderr=$tmpdir/i2" ];
+my $imapd = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-imapd $?");
+my $imap_host_port = tcp_host_port($sock);
+undef $sock;
+for ('', qw(cur new)) {
+ mkdir "$tmpdir/md/$_" or xbail "mkdir: $!";
+}
+symlink(File::Spec->rel2abs('t/plack-qp.eml'), "$tmpdir/md/cur/x:2,");
+my $expect = do {
+ open my $fh, '<', 't/plack-qp.eml' or xbail $!;
+ local $/;
+ <$fh>;
+};
+test_lei({ tmpdir => $tmpdir }, sub {
+ my $store_path = "$ENV{HOME}/.local/share/lei/store/";
+
+ lei_ok('index', "$tmpdir/md");
+ lei_ok(qw(q mid:qp@example.com));
+ my $res_a = json_utf8->decode($lei_out);
+ my $blob = $res_a->[0]->{'blob'};
+ like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example');
+ lei_ok('blob', $blob);
+ is($lei_out, $expect, 'got expected blob via Maildir');
+ lei_ok(qw(q mid:qp@example.com -f text));
+ like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
+
+ my $all_obj = ['git', "--git-dir=$store_path/ALL.git",
+ qw(cat-file --batch-check --batch-all-objects)];
+ is_deeply([xqx($all_obj)], [], 'no git objects');
+ lei_ok('import', 't/plack-qp.eml');
+ ok(grep(/\A$blob blob /, my @objs = xqx($all_obj)),
+ 'imported blob');
+ lei_ok(qw(q z:0.. --dedupe=none));
+ my $res_b = json_utf8->decode($lei_out);
+ is_deeply($res_b, $res_a, 'no extra DB entries');
+
+ lei_ok('index', "nntp://$nntp_host_port/t.v2");
+ lei_ok('index', "imap://$imap_host_port/t.v2.0");
+ is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
+});
+
+done_testing;
^ permalink raw reply related [relevance 26%]
* s/qlite3/s&/ [was: [PATCH] lei: fix ... folder names for NNTP]
2021-05-04 5:24 66% [PATCH] lei: fix mail_sync.qlite3 folder names for NNTP Eric Wong
@ 2021-05-04 5:26 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-04 5:26 UTC (permalink / raw)
To: meta
ill ix he itle efore ushing :x
^ permalink raw reply [relevance 71%]
* [PATCH] lei: fix mail_sync.qlite3 folder names for NNTP
@ 2021-05-04 5:24 66% Eric Wong
2021-05-04 5:26 71% ` s/qlite3/s&/ [was: [PATCH] lei: fix ... folder names for NNTP] Eric Wong
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-04 5:24 UTC (permalink / raw)
To: meta
We should not have "SCALAR(XXXXXXX)" showing up in SQLite DBs
because we passed a SCALAR ref instead of a non-ref SCALAR.
---
lib/PublicInbox/NetReader.pm | 3 ++-
t/lei-import-nntp.t | 7 ++++---
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 64910fe1..fd0d1682 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -628,6 +628,7 @@ sub _nntp_fetch_all ($$$) {
warn "# $uri fetching ARTICLE $beg..$end\n";
}
my $n = $self->{max_batch};
+ my $url = $$uri;
for ($beg..$end) {
last if $self->{quit};
$art = $_;
@@ -650,7 +651,7 @@ sub _nntp_fetch_all ($$$) {
$raw = join('', @$raw);
$raw =~ s/\r\n/\n/sg;
my ($eml_cb, @args) = @{$self->{eml_each}};
- $eml_cb->($uri, $art, $kw, PublicInbox::Eml->new(\$raw), @args);
+ $eml_cb->($url, $art, $kw, PublicInbox::Eml->new(\$raw), @args);
$last_art = $art;
}
run_commit_cb($self);
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index 12bb002a..662da309 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -16,10 +16,9 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok(qw(q z:1..));
my $out = json_utf8->decode($lei_out);
is_deeply($out, [ undef ], 'nothing imported, yet');
- lei_ok('import', "nntp://$host_port/t.v2");
- diag $lei_err;
+ my $url = "nntp://$host_port/t.v2";
+ lei_ok('import', $url);
lei_ok(qw(q z:1..));
- diag $lei_err;
$out = json_utf8->decode($lei_out);
ok(scalar(@$out) > 1, 'got imported messages');
is(pop @$out, undef, 'trailing JSON null element was null');
@@ -29,5 +28,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $f = "$ENV{HOME}/.local/share/lei/store/mail_sync.sqlite3";
ok(-s $f, 'mail_sync exists tracked for redundant imports');
+ lei_ok 'ls-mail-sync';
+ like($lei_out, qr!\A\Q$url\E\n\z!, 'ls-mail-sync output as-expected');
});
done_testing;
^ permalink raw reply related [relevance 66%]
* Re: [PATCH 4/5] lei ls-mail-sync: fix handling of non-wildcard filters
2021-05-04 4:45 68% ` [PATCH 4/5] lei ls-mail-sync: fix handling of non-wildcard filters Kyle Meyer
@ 2021-05-04 5:14 66% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-04 5:14 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> If lei_ls_mail_sync() is given a filter without any wildcards and
> --globoff is unspecified, glob2re() will return undef, resulting in
> the final regular expression being undefined. Add a fallback value.
> ---
>
> I'm not sure if this is the cleanest approach; repeating
> qr/\Q$filter\E/ seems ugly. I considered something closer to what
> lei_ls_external() does, but decided against it because it leads to
> --globoff with no filter showing no output, which I think is
> surprising (even if passing --globoff with no filter doesn't make
> any sense).
Thanks for bringing this up.
My code there was an insomnia-generated disaster :x
Mixing a ternary statement with a trailing "if" was just gross.
Below is a shorter, less-redundant rewrite based on your fix.
> This probably deserves a test, but I'm out of time for tonight. If
> the change looks okay, I'm happy to look into adding a test
> tomorrow.
Yes, but probably no rush. I still need to get mail-sync
working... (sidetracked into working on "lei index" :x)
----------8<------------------------
Subject: [PATCH] lei ls-mail-sync: fix handling of non-wildcard filters
If lei_ls_mail_sync() is given a filter without any wildcards
and --globoff is unspecified, glob2re() will return undef,
resulting in the final regular expression being undefined.
Always use a fallback value when there's no RE.
Based-on-patch-by: Kyle Meyer <kyle@kyleam.com>
Link: https://public-inbox.org/meta/20210504044559.12941-5-kyle@kyleam.com/
---
lib/PublicInbox/LeiLsMailSync.pm | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiLsMailSync.pm b/lib/PublicInbox/LeiLsMailSync.pm
index 532ea9b5..505c0b3f 100644
--- a/lib/PublicInbox/LeiLsMailSync.pm
+++ b/lib/PublicInbox/LeiLsMailSync.pm
@@ -12,9 +12,8 @@ sub lei_ls_mail_sync {
my $sto = $lei->_lei_store or return;
my $lms = $sto->search->lms or return;
my $opt = $lei->{opt};
- my $re;
- $re = defined($filter) ? qr/\Q$filter\E/ : qr/./ if $opt->{globoff};
- $re //= $lei->glob2re($filter // '*');
+ my $re = $opt->{globoff} ? undef : $lei->glob2re($filter // '*');
+ $re //= qr/\Q$filter\E/;
my @f = $lms->folders;
@f = $opt->{'invert-match'} ? grep(!/$re/, @f) : grep(/$re/, @f);
if ($opt->{'local'} && !$opt->{remote}) {
^ permalink raw reply related [relevance 66%]
* [PATCH 5/5] lei: add help output for --invert match
` (3 preceding siblings ...)
2021-05-04 4:45 68% ` [PATCH 4/5] lei ls-mail-sync: fix handling of non-wildcard filters Kyle Meyer
@ 2021-05-04 4:45 71% ` Kyle Meyer
4 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-04 4:45 UTC (permalink / raw)
To: meta
ls-external and ls-mail-sync accept an --invert-match option. Show it
in the --help output.
---
lib/PublicInbox/LEI.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7be68121..c5fdfeb8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -270,6 +270,7 @@ my %OPTDESC = (
'incremental! import' => 'import already seen IMAP and NNTP articles',
'globoff|g' => "do not match locations using '*?' wildcards ".
"and\xa0'[]'\x{a0}ranges",
+'invert-match|v' => 'select non-matching lines',
'color!' => 'disable color (for --format=text)',
'verbose|v+' => 'be more verbose',
'external!' => 'do not use externals',
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 4/5] lei ls-mail-sync: fix handling of non-wildcard filters
` (2 preceding siblings ...)
2021-05-04 4:45 71% ` [PATCH 3/5] lei ls-mail-sync: accept a filter Kyle Meyer
@ 2021-05-04 4:45 68% ` Kyle Meyer
2021-05-04 5:14 66% ` Eric Wong
2021-05-04 4:45 71% ` [PATCH 5/5] lei: add help output for --invert match Kyle Meyer
4 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-05-04 4:45 UTC (permalink / raw)
To: meta
If lei_ls_mail_sync() is given a filter without any wildcards and
--globoff is unspecified, glob2re() will return undef, resulting in
the final regular expression being undefined. Add a fallback value.
---
I'm not sure if this is the cleanest approach; repeating
qr/\Q$filter\E/ seems ugly. I considered something closer to what
lei_ls_external() does, but decided against it because it leads to
--globoff with no filter showing no output, which I think is
surprising (even if passing --globoff with no filter doesn't make
any sense).
This probably deserves a test, but I'm out of time for tonight. If
the change looks okay, I'm happy to look into adding a test
tomorrow.
lib/PublicInbox/LeiLsMailSync.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiLsMailSync.pm b/lib/PublicInbox/LeiLsMailSync.pm
index 532ea9b5..06e25a63 100644
--- a/lib/PublicInbox/LeiLsMailSync.pm
+++ b/lib/PublicInbox/LeiLsMailSync.pm
@@ -14,7 +14,7 @@ sub lei_ls_mail_sync {
my $opt = $lei->{opt};
my $re;
$re = defined($filter) ? qr/\Q$filter\E/ : qr/./ if $opt->{globoff};
- $re //= $lei->glob2re($filter // '*');
+ $re //= $lei->glob2re($filter // '*') // qr/\Q$filter\E/;;
my @f = $lms->folders;
@f = $opt->{'invert-match'} ? grep(!/$re/, @f) : grep(/$re/, @f);
if ($opt->{'local'} && !$opt->{remote}) {
--
2.31.1
^ permalink raw reply related [relevance 68%]
* [PATCH 3/5] lei ls-mail-sync: accept a filter
2021-05-04 4:45 71% ` [PATCH 1/5] lei ls-mail-sync: update reference to ls-sync Kyle Meyer
2021-05-04 4:45 71% ` [PATCH 2/5] lei ls-mail-sync: drop repeated -z/0 option Kyle Meyer
@ 2021-05-04 4:45 71% ` Kyle Meyer
2021-05-04 4:45 68% ` [PATCH 4/5] lei ls-mail-sync: fix handling of non-wildcard filters Kyle Meyer
2021-05-04 4:45 71% ` [PATCH 5/5] lei: add help output for --invert match Kyle Meyer
4 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-04 4:45 UTC (permalink / raw)
To: meta
lei_ls_mail_sync() is written to accept a filter, and ls-mail-sync has
related command-line options (--globoff, --invert-match), but a
positional argument isn't actually accepted. Add it.
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e9c1675a..7be68121 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -170,7 +170,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-external' => [ '[FILTER]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
-'ls-mail-sync' => [ '', 'list mail sync folders',
+'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 1/5] lei ls-mail-sync: update reference to ls-sync
@ 2021-05-04 4:45 71% ` Kyle Meyer
2021-05-04 4:45 71% ` [PATCH 2/5] lei ls-mail-sync: drop repeated -z/0 option Kyle Meyer
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-04 4:45 UTC (permalink / raw)
To: meta
ls-sync was renamed to ls-mail-sync in cb0e9d42b799c748. Update a
stale reference to the old name.
---
lib/PublicInbox/LeiLsMailSync.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiLsMailSync.pm b/lib/PublicInbox/LeiLsMailSync.pm
index 2b3d326d..532ea9b5 100644
--- a/lib/PublicInbox/LeiLsMailSync.pm
+++ b/lib/PublicInbox/LeiLsMailSync.pm
@@ -1,7 +1,7 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# front-end for the "lei ls-sync" sub-command
+# front-end for the "lei ls-mail-sync" sub-command
package PublicInbox::LeiLsMailSync;
use strict;
use v5.10.1;
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH 2/5] lei ls-mail-sync: drop repeated -z/0 option
2021-05-04 4:45 71% ` [PATCH 1/5] lei ls-mail-sync: update reference to ls-sync Kyle Meyer
@ 2021-05-04 4:45 71% ` Kyle Meyer
2021-05-04 4:45 71% ` [PATCH 3/5] lei ls-mail-sync: accept a filter Kyle Meyer
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Kyle Meyer @ 2021-05-04 4:45 UTC (permalink / raw)
To: meta
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 599cfab2..e9c1675a 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -171,7 +171,7 @@ our %CMD = ( # sorted in order of importance/use:
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
'ls-mail-sync' => [ '', 'list mail sync folders',
- qw(z|0 z|0 globoff|g invert-match|v local remote), @c_opt ],
+ qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
--
2.31.1
^ permalink raw reply related [relevance 71%]
* [PATCH] lei up: fix dedupe with remote externals on Maildir + IMAP
@ 2021-05-03 20:57 57% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-03 20:57 UTC (permalink / raw)
To: meta
LeiToMail Maildir and IMAP write callbacks need to account for
the caller-supplied smsg. We'll also make better use of the
user-supplied smsg object by ensuring blob deduplication happens
ASAP.
Fixes: e76683309ca4f254 ("lei <q|up>: distinguish between mset and l2m counts")
---
lib/PublicInbox/LeiSavedSearch.pm | 15 ++++++++-------
lib/PublicInbox/LeiToMail.pm | 6 ++++--
t/lei-q-remote-import.t | 6 ++++++
3 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 8177c98e..92ced28b 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -170,23 +170,24 @@ sub cfg_set { # called by LeiXSearch
sub is_dup {
my ($self, $eml, $smsg) = @_;
my $oidx = $self->{oidx} // die 'BUG: no {oidx}';
- my $blob = $smsg ? $smsg->{blob} : undef;
- my $lk = $self->lock_for_scope_fast;
- return 1 if $blob && $oidx->blob_exists($blob);
+ my $lk;
if ($self->{-dedupe_mid}) {
+ $lk //= $self->lock_for_scope_fast;
for my $mid (@{mids_for_index($eml)}) {
my ($id, $prv);
return 1 if $oidx->next_by_mid($mid, \$id, \$prv);
}
}
+ my $blob = $smsg ? $smsg->{blob} : git_sha(1, $eml)->hexdigest;
+ $lk //= $self->lock_for_scope_fast;
+ return 1 if $oidx->blob_exists($blob);
if (my $xoids = PublicInbox::LeiSearch::xoids_for($self, $eml, 1)) {
for my $docid (values %$xoids) {
$oidx->add_xref3($docid, -1, $blob, '.');
}
$oidx->commit_lazy;
if ($self->{-dedupe_oid}) {
- $smsg->{blob} //= git_sha(1, $eml)->hexdigest;
- exists $xoids->{$smsg->{blob}} ? 1 : undef;
+ exists $xoids->{$blob} ? 1 : undef;
} else {
1;
}
@@ -197,11 +198,11 @@ sub is_dup {
$smsg->{bytes} = 0;
$smsg->populate($eml);
}
+ $smsg->{blob} //= $blob;
$oidx->begin_lazy;
$smsg->{num} = $oidx->adj_counter('eidx_docid', '+');
- $smsg->{blob} //= git_sha(1, $eml)->hexdigest;
$oidx->add_overview($eml, $smsg);
- $oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
+ $oidx->add_xref3($smsg->{num}, -1, $blob, '.');
$oidx->commit_lazy;
undef;
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 71acf952..64061788 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -283,7 +283,8 @@ sub _maildir_write_cb ($$) {
my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
return if $dedupe && $dedupe->is_dup($eml //
- PublicInbox::Eml->new($$bref));
+ PublicInbox::Eml->new($$bref),
+ $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
my $n = _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
$sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
@@ -305,7 +306,8 @@ sub _imap_write_cb ($$) {
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
return if $dedupe && $dedupe->is_dup($eml //
- PublicInbox::Eml->new($$bref));
+ PublicInbox::Eml->new($$bref),
+ $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
my $uid = eval { $append->($mic, $folder, $bref, $smsg, $eml) };
if (my $err = $@) {
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 32c5172b..80067061 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -91,5 +91,11 @@ EOF
lei_ok(qw(q -o mboxrd:/dev/stdout m:never-before-seen@example.com));
like($lei_out, qr/seen\@example\.com>\nStatus: RO\n\nwhatever/sm,
'--import-before imported totally unseen message');
+
+ lei_ok(qw(q --save z:0.. -o), "$ENV{HOME}/md", '--only', $url);
+ my @f = glob("$ENV{HOME}/md/*/*");
+ lei_ok('up', "$ENV{HOME}/md");
+ is_deeply(\@f, [ glob("$ENV{HOME}/md/*/*") ],
+ 'lei up remote dedupe works on maildir');
});
done_testing;
^ permalink raw reply related [relevance 57%]
* yes [Re: should lei attempt to index mail outside of git?]
@ 2021-05-02 6:12 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-02 6:12 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> Currently, every mail lei indexes has a git blob associated with it.
It still will. But we have PublicInbox::ContentHash::git_sha
> I understand some folks might want to keep using their existing
> storage and not have a redundant, expensive-to-erase copy of the
> mail in git; but just want an indexing-only solution like mairix.
At least for Maildir, IMAP, and NNTP where random access is
reliable and fast, it should be doable.
> So, is this a feature worth implementing?
Since we have ~/.local/share/lei/store/mail_sync.sqlite3
nowadays, I think it's possible to make LeiToMail fallback
to retrieving from Maildir, IMAP, NNTP; at least. mbox
can be a pain since message offsets can change.
^ permalink raw reply [relevance 71%]
* [PATCH 4/6] lei: simplify workers_start API
2021-05-02 6:05 70% [PATCH 0/6] lei: more steps towards kw sync Eric Wong
2021-05-02 6:05 70% ` [PATCH 1/6] lei <q|up>: combine written/results into one line Eric Wong
2021-05-02 6:05 66% ` [PATCH 2/6] lei_input: common net_merge_all_done for lei <import|tag> Eric Wong
@ 2021-05-02 6:05 53% ` Eric Wong
2021-05-02 6:05 49% ` [PATCH 6/6] lei <q|up>: writes to Maildirs and IMAP use mail-sync Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-02 6:05 UTC (permalink / raw)
To: meta
In most cases, we just name the worker process based
on the command. The only change is for LeiMirror
vs "lei add-external --mirror", but I doubt it matters.
---
lib/PublicInbox/LEI.pm | 3 ++-
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 2 +-
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 2 +-
lib/PublicInbox/LeiP2q.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 2 +-
8 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5d701d5e..d5c6bd52 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -542,7 +542,7 @@ sub pkt_op_pair {
}
sub workers_start {
- my ($lei, $wq, $ident, $jobs, $ops) = @_;
+ my ($lei, $wq, $jobs, $ops) = @_;
$ops = {
'!' => [ \&fail_handler, $lei ],
'|' => [ \&sigpipe_handler, $lei ],
@@ -552,6 +552,7 @@ sub workers_start {
};
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
+ my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 0a957358..710430a2 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -151,7 +151,7 @@ EOM
}
require PublicInbox::SolverGit;
my $self = bless { lxs => $lxs, oid_b => $blob }, __PACKAGE__;
- my ($op_c, $ops) = $lei->workers_start($self, 'lei-blob', 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_solve_blob', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 5b27ec2d..395a80f8 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -55,7 +55,7 @@ sub lei_convert { # the main "lei convert" method
my $devfd = $lei->path_to_fd($ovv->{dst}) // return;
$lei->{opt}->{augment} = 1 if $devfd < 0;
$self->prepare_inputs($lei, \@inputs) or return;
- my ($op_c, $ops) = $lei->workers_start($self, 'lei-convert', 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('process_inputs', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 394138b4..6a57df47 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -76,7 +76,7 @@ sub lei_import { # the main "lei import" method
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
$lei->{-eml_noisy} = 1;
- (my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index a00e78fc..6cea6ae8 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -72,7 +72,7 @@ sub do_ls_search_long {
sub bg_worker ($$$) {
my ($lei, $pfx, $json) = @_;
my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
- my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index db97b98c..a37e1d5c 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -282,7 +282,7 @@ sub start {
require PublicInbox::Inbox;
require PublicInbox::Admin;
require PublicInbox::InboxWritable;
- my ($op, $ops) = $lei->workers_start($self, 'lei_mirror', 1);
+ my ($op, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_mirror', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index b4893489..f381a31c 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -188,7 +188,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
} else {
$self->{input} = $input;
}
- my ($op, $ops) = $lei->workers_start($self, 'lei-p2q', 1);
+ my ($op, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_p2q', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 6025c93e..c650e886 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -44,7 +44,7 @@ sub lei_tag { # the "lei tag" method
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{vmd_mod} = $vmd_mod;
my $j = $self->{-wq_nr_workers} = 1; # locked for now
- (my $op_c, $ops) = $lei->workers_start($self, 'lei-tag', $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
^ permalink raw reply related [relevance 53%]
* [PATCH 6/6] lei <q|up>: writes to Maildirs and IMAP use mail-sync
2021-05-02 6:05 70% [PATCH 0/6] lei: more steps towards kw sync Eric Wong
` (2 preceding siblings ...)
2021-05-02 6:05 53% ` [PATCH 4/6] lei: simplify workers_start API Eric Wong
@ 2021-05-02 6:05 49% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-02 6:05 UTC (permalink / raw)
To: meta
This will allow keyword updates from other folders to propagate
to folders where search results may be duplicated.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 7 +++----
lib/PublicInbox/LeiStore.pm | 8 ++++----
lib/PublicInbox/LeiToMail.pm | 24 +++++++++++++++++-------
4 files changed, 25 insertions(+), 16 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d5c6bd52..599cfab2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -145,7 +145,7 @@ our %CMD = ( # sorted in order of importance/use:
qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
- color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
+ color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index efe328cc..1999a534 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -29,10 +29,9 @@ sub _start_query { # used by "lei q" and "lei up"
return $self->fail("`$mj' writer jobs must be >= 1");
}
my $l2m = $self->{l2m};
- if ($l2m && ($opt->{'import-remote'} //= 1) |
- # we use \1 (a ref) to distinguish between
- # user-supplied and default value
- (($opt->{'import-before'} //= \1) ? 1 : 0)) {
+ # we use \1 (a ref) to distinguish between default vs. user-supplied
+ if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
+ import-before))) {
$self->_lei_store(1)->write_prepare($self);
}
$l2m and $l2m->{-wq_nr_workers} = $mj // do {
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 8af740fd..29362b2e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -193,15 +193,15 @@ sub remove_eml_vmd {
\@docids;
}
-sub set_sync_info ($$$) {
- my ($self, $oidhex, $sync_info) = @_;
+sub set_sync_info {
+ my ($self, $oidhex, $folder, $id) = @_;
($self->{lms} //= do {
require PublicInbox::LeiMailSync;
my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
my $lms = PublicInbox::LeiMailSync->new($f);
$lms->lms_begin;
$lms;
- })->set_src($oidhex, @$sync_info);
+ })->set_src($oidhex, $folder, $id);
}
sub add_eml {
@@ -212,7 +212,7 @@ sub add_eml {
my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
my $im_mark = $im->add($eml, undef, $smsg);
if ($vmd && $vmd->{sync_info}) {
- set_sync_info($self, $smsg->{blob}, $vmd->{sync_info});
+ set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
}
$im_mark or return; # duplicate blob returns undef
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ab4de378..71acf952 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -242,7 +242,7 @@ sub _buf2maildir {
my $kw = $smsg->{kw} // [];
my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw));
my $rand = ''; # chosen by die roll :P
- my ($tmp, $fh, $final, $ok);
+ my ($tmp, $fh, $base, $ok);
my $common = $smsg->{blob} // _rand;
if (defined(my $pct = $smsg->{pct})) { $common .= "=$pct" }
do {
@@ -257,11 +257,12 @@ sub _buf2maildir {
$dst .= 'cur/';
$rand = '';
do {
- $final = $dst.$rand.$common.':2,'.$sfx;
- } while (!($ok = link($tmp, $final)) && $!{EEXIST} &&
+ $base = $rand.$common.':2,'.$sfx
+ } while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} &&
($rand = _rand.','));
- die "link($tmp, $final): $!" unless $ok;
+ die "link($tmp, $dst$base): $!" unless $ok;
unlink($tmp) or warn "W: failed to unlink $tmp: $!\n";
+ \$base;
} else {
my $err = "Error writing $smsg->{blob} to $dst: $!\n";
$_[0] = undef; # clobber dst
@@ -276,13 +277,16 @@ sub _maildir_write_cb ($$) {
$dedupe->prepare_dedupe if $dedupe;
my $dst = $lei->{ovv}->{dst};
my $lse = $lei->{lse}; # may be undef
+ my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
+ my $out = $sto ? 'maildir:'.$lei->rel2abs($dst) : undef;
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
return if $dedupe && $dedupe->is_dup($eml //
PublicInbox::Eml->new($$bref));
$lse->xsmsg_vmd($smsg) if $lse;
- _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+ my $n = _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+ $sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
++$lei->{-nr_write};
}
}
@@ -291,21 +295,27 @@ sub _imap_write_cb ($$) {
my ($self, $lei) = @_;
my $dedupe = $lei->{dedupe};
$dedupe->prepare_dedupe if $dedupe;
- my $imap_append = $lei->{net}->can('imap_append');
+ my $append = $lei->{net}->can('imap_append');
my $mic = $lei->{net}->mic_get($self->{uri});
my $folder = $self->{uri}->mailbox;
my $lse = $lei->{lse}; # may be undef
+ my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
+ my $out = $lei->{ovv}->{dst};
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
return if $dedupe && $dedupe->is_dup($eml //
PublicInbox::Eml->new($$bref));
$lse->xsmsg_vmd($smsg) if $lse;
- eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
+ my $uid = eval { $append->($mic, $folder, $bref, $smsg, $eml) };
if (my $err = $@) {
undef $mic;
die $err;
}
+ # imap_append returns UID if IMAP server has UIDPLUS extension
+ ($sto && $uid =~ /\A[0-9]+\z/) and
+ $sto->ipc_do('set_sync_info',
+ $smsg->{blob}, $out, $uid + 0);
++$lei->{-nr_write};
}
}
^ permalink raw reply related [relevance 49%]
* [PATCH 1/6] lei <q|up>: combine written/results into one line
2021-05-02 6:05 70% [PATCH 0/6] lei: more steps towards kw sync Eric Wong
@ 2021-05-02 6:05 70% ` Eric Wong
2021-05-02 6:05 66% ` [PATCH 2/6] lei_input: common net_merge_all_done for lei <import|tag> Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-02 6:05 UTC (permalink / raw)
To: meta
Having multiple lines of output mean they can be interleaved in
daemon mode. Put stats into one line to reduce screen
real-estate size and improve readability.
---
lib/PublicInbox/LeiXSearch.pm | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d212a732..21b15025 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -359,9 +359,8 @@ sub query_done { # EOF callback for main daemon
}
my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
- my (@out, $start_mua);
+ my $start_mua;
if ($l2m) { # close() calls LeiToMail reap_compress
- @out = (" in $lei->{ovv}->{dst}");
if (my $out = delete $lei->{old_1}) {
if (my $mbout = $lei->{1}) {
close($mbout) or return $lei->fail(<<"");
@@ -379,9 +378,11 @@ Error closing $lei->{ovv}->{dst}: $!
}
}
if ($lei->{-progress}) {
- $lei->qerr('# ', $lei->{-mset_total} // 0, " matches", @out);
+ my $tot = $lei->{-mset_total} // 0;
my $nr = $lei->{-nr_write} // 0;
- $lei->qerr("# $nr written to $lei->{ovv}->{dst}") if $l2m;
+ $lei->qerr($l2m ?
+ "# $nr written to $lei->{ovv}->{dst} ($tot matches)" :
+ "# $tot matches");
}
$lei->start_mua if $start_mua;
$lei->dclose;
^ permalink raw reply related [relevance 70%]
* [PATCH 0/6] lei: more steps towards kw sync
@ 2021-05-02 6:05 70% Eric Wong
2021-05-02 6:05 70% ` [PATCH 1/6] lei <q|up>: combine written/results into one line Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-05-02 6:05 UTC (permalink / raw)
To: meta
6/6 is a fairly significant change, but I think it's
worth doing by default to make things less surprising
for eventual users of "lei export-kw" (I'm not sure if
that's even a good name).
Eric Wong (6):
lei <q|up>: combine written/results into one line
lei_input: common net_merge_all_done for lei <import|tag>
lei_input: reject --mail-sync if using HTTP(S) for now
lei: simplify workers_start API
net_writer: use "FLAGS.SILENT" to set keywords
lei <q|up>: writes to Maildirs and IMAP use mail-sync
lib/PublicInbox/LEI.pm | 5 +++--
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 9 ++-------
lib/PublicInbox/LeiInput.pm | 9 +++++++++
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 2 +-
lib/PublicInbox/LeiP2q.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 7 +++----
lib/PublicInbox/LeiStore.pm | 8 ++++----
lib/PublicInbox/LeiTag.pm | 9 ++-------
lib/PublicInbox/LeiToMail.pm | 24 +++++++++++++++++-------
lib/PublicInbox/LeiXSearch.pm | 9 +++++----
lib/PublicInbox/NetWriter.pm | 14 ++++++--------
t/lei-import-http.t | 5 +++++
xt/net_writer-imap.t | 9 +++++++--
16 files changed, 68 insertions(+), 50 deletions(-)
^ permalink raw reply [relevance 70%]
* [PATCH 2/6] lei_input: common net_merge_all_done for lei <import|tag>
2021-05-02 6:05 70% [PATCH 0/6] lei: more steps towards kw sync Eric Wong
2021-05-02 6:05 70% ` [PATCH 1/6] lei <q|up>: combine written/results into one line Eric Wong
@ 2021-05-02 6:05 66% ` Eric Wong
2021-05-02 6:05 53% ` [PATCH 4/6] lei: simplify workers_start API Eric Wong
2021-05-02 6:05 49% ` [PATCH 6/6] lei <q|up>: writes to Maildirs and IMAP use mail-sync Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-02 6:05 UTC (permalink / raw)
To: meta
I suspect there'll be more lei_input-only things in the future.
---
lib/PublicInbox/LeiImport.pm | 7 +------
lib/PublicInbox/LeiInput.pm | 7 +++++++
lib/PublicInbox/LeiTag.pm | 7 +------
3 files changed, 9 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 575cf125..394138b4 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -48,12 +48,6 @@ sub input_net_cb { # imap_each / nntp_each
input_eml_cb($self, $eml, $vmd);
}
-sub net_merge_all_done { # callback used by LeiAuth
- my ($self) = @_;
- $self->wq_io_do('process_inputs');
- $self->wq_close(1);
-}
-
sub lei_import { # the main "lei import" method
my ($lei, @inputs) = @_;
my $sto = $lei->_lei_store(1);
@@ -99,6 +93,7 @@ sub _complete_import {
no warnings 'once';
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
# the following works even when LeiAuth is lazy-loaded
*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 9bcc86e1..917f682b 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -329,6 +329,13 @@ sub input_only_atfork_child {
undef;
}
+# alias this as "net_merge_all_done" to use as an LeiAuth callback
+sub input_only_net_merge_all_done {
+ my ($self) = @_;
+ $self->wq_io_do('process_inputs');
+ $self->wq_close(1);
+}
+
# like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare
# for update_xvmd -> update_vmd
sub vmd_mod_extract {
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 2170e3f2..6025c93e 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -19,12 +19,6 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
-sub net_merge_all_done { # callback used by LeiAuth
- my ($self) = @_;
- $self->wq_io_do('process_inputs');
- $self->wq_close(1);
-}
-
sub input_maildir_cb { # maildir_each_eml cb
my ($f, $kw, $eml, $self) = @_;
input_eml_cb($self, $eml);
@@ -117,5 +111,6 @@ sub _complete_tag {
no warnings 'once'; # the following works even when LeiAuth is lazy-loaded
*net_merge_all = \&PublicInbox::LeiAuth::net_merge_all;
+*net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
1;
^ permalink raw reply related [relevance 66%]
* [PATCH 6/5] lei import: fix --mail-sync handling in LeiInput
2021-05-01 6:21 56% ` [PATCH 4/5] lei: rename ls-sync to ls-mail-sync Eric Wong
@ 2021-05-01 19:29 59% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-01 19:29 UTC (permalink / raw)
To: meta
"lei inspect" also shows "mail-sync" as a field name
---
lib/PublicInbox/LeiInput.pm | 8 ++++----
lib/PublicInbox/LeiInspect.pm | 2 +-
t/lei-import-imap.t | 4 ++--
t/lei-import-maildir.t | 2 +-
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 86f300c3..9bcc86e1 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -204,7 +204,7 @@ sub prepare_http_input ($$$) {
sub prepare_inputs { # returns undef on error
my ($self, $lei, $inputs) = @_;
my $in_fmt = $lei->{opt}->{'in-format'};
- my $sync = $lei->{opt}->{sync} ? {} : undef; # using LeiMailSync
+ my $sync = $lei->{opt}->{'mail-sync'} ? {} : undef; # using LeiMailSync
if ($lei->{opt}->{stdin}) {
@$inputs and return
$lei->fail("--stdin and @$inputs do not mix");
@@ -286,11 +286,11 @@ $input is `eml', not --in-format=$in_fmt
}
if ($sync && $sync->{no}) {
return $lei->fail(<<"") if !$sync->{ok};
---sync specified but no inputs support it
+--mail-sync specified but no inputs support it
# non-fatal if some inputs support support sync
- $lei->err("# --sync will only be used for @{$sync->{ok}}");
- $lei->err("# --sync is not supported for: @{$sync->{no}}");
+ $lei->err("# --mail-sync will only be used for @{$sync->{ok}}");
+ $lei->err("# --mail-sync is not supported for: @{$sync->{no}}");
}
if ($net) {
$net->{-can_die} = 1;
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 6cfc8083..714d2526 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -18,7 +18,7 @@ sub inspect_blob ($$) {
$ent->{'lei/store'} = \@docids if @docids;
my $lms = $lse->lms;
if (my $loc = $lms ? $lms->locations_for($oidhex) : undef) {
- $ent->{sync} = $loc;
+ $ent->{'mail-sync'} = $loc;
}
}
$ent;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 3a1fff4c..fd15ef4f 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -54,8 +54,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('inspect', "blob:$out->[5]->{blob}");
my $x = json_utf8->decode($lei_out);
is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect');
- is(ref($x->{sync}), 'HASH', 'sync in inspect');
- is(ref($x->{sync}->{$k[0]}), 'ARRAY', 'UID arrays in inspect');
+ is(ref($x->{'mail-sync'}), 'HASH', 'sync in inspect');
+ is(ref($x->{'mail-sync'}->{$k[0]}), 'ARRAY', 'UID arrays in inspect');
my $psgi_attach = 'cfa3622cbeffc9bd6b0fc66c4d60d420ba74f60d';
lei_ok('blob', $psgi_attach);
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index 02fe43e1..b85d3026 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -37,7 +37,7 @@ test_lei(sub {
lei_ok('inspect', "blob:$res->[0]->{blob}");
$inspect = json_utf8->decode($lei_out);
is(ref(delete $inspect->{"lei/store"}), 'ARRAY', 'lei/store IDs');
- is_deeply($inspect, { sync => { "maildir:$md" => [ 'x:2,S' ] } },
+ is_deeply($inspect, { 'mail-sync' => { "maildir:$md" => [ 'x:2,S' ] } },
'maildir sync info as expected');
lei_ok qw(ls-mail-sync);
^ permalink raw reply related [relevance 59%]
* [PATCH 5/5] lei edit-search: support relocating lei.q.output
2021-05-01 6:21 70% [PATCH 0/5] lei: more UI/UX tweaks Eric Wong
` (3 preceding siblings ...)
2021-05-01 6:21 56% ` [PATCH 4/5] lei: rename ls-sync to ls-mail-sync Eric Wong
@ 2021-05-01 6:21 75% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-01 6:21 UTC (permalink / raw)
To: meta
The contents of the old lei.q.output will not be removed,
but will be converted into the new one.
---
lib/PublicInbox/LeiConvert.pm | 5 ++-
lib/PublicInbox/LeiEditSearch.pm | 15 +++++++--
lib/PublicInbox/LeiSavedSearch.pm | 51 +++++++++++++++++++++++++++++++
script/lei | 7 +++++
4 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index cefcaf65..5b27ec2d 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -37,8 +37,11 @@ sub process_inputs { # via wq_do
my ($self) = @_;
local $PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid
$self->SUPER::process_inputs;
- delete $self->{lei}->{1};
+ my $lei = $self->{lei};
+ delete $lei->{1};
delete $self->{wcb}; # commit
+ my $nr = delete($lei->{-nr_write}) // 0;
+ $lei->err("# converted $nr messages") if $lei->{opt}->{verbose};
}
sub lei_convert { # the main "lei convert" method
diff --git a/lib/PublicInbox/LeiEditSearch.pm b/lib/PublicInbox/LeiEditSearch.pm
index fb36fdcd..30ac65bd 100644
--- a/lib/PublicInbox/LeiEditSearch.pm
+++ b/lib/PublicInbox/LeiEditSearch.pm
@@ -13,10 +13,19 @@ sub lei_edit_search {
my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
my @cmd = (qw(git config --edit -f), $lss->{'-f'});
$lei->qerr("# spawning @cmd");
+ $lss->edit_begin($lei);
if ($lei->{oneshot}) {
- exec(@cmd) or die "exec @cmd: $!\n";
- } else {
- $lei->send_exec_cmd([], \@cmd, {});
+ require PublicInbox::Spawn;
+ waitpid(PublicInbox::Spawn::spawn(\@cmd), 0);
+ # non-fatal, editor could fail after successful write
+ $lei->child_error($?) if $?;
+ $lss->edit_done($lei);
+ } else { # run in script/lei foreground
+ require PublicInbox::PktOp;
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ # $op_p will EOF when $EDITOR is done
+ $op_c->{ops} = { '' => [$lss->can('edit_done'), $lss, $lei] };
+ $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p ], \@cmd, {});
}
}
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 79125214..8177c98e 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -259,6 +259,57 @@ sub output2lssdir {
undef;
}
+sub edit_begin {
+ my ($self, $lei) = @_;
+ if (ref($self->{-cfg}->{'lei.q.output'})) {
+ delete $self->{-cfg}->{'lei.q.output'}; # invalid
+ $lei->err(<<EOM);
+$self->{-f} has multiple values of lei.q.output
+please remove redundant ones
+EOM
+ }
+ $lei->{-lss_for_edit} = $self;
+}
+
+sub edit_done {
+ my ($self, $lei) = @_;
+ my $cfg = PublicInbox::Config->git_config_dump($self->{'-f'});
+ my $new_out = $cfg->{'lei.q.output'} // '';
+ return $lei->fail(<<EOM) if ref $new_out;
+$self->{-f} has multiple values of lei.q.output
+please edit again
+EOM
+ return $lei->fail(<<EOM) if $new_out eq '';
+$self->{-f} needs lei.q.output
+please edit again
+EOM
+ my $old_out = $self->{-cfg}->{'lei.q.output'} // '';
+ return if $old_out eq $new_out;
+ my $old_path = $old_out;
+ my $new_path = $new_out;
+ s!$LOCAL_PFX!! for ($old_path, $new_path);
+ my $dir_old = lss_dir_for($lei, \$old_path, 1);
+ my $dir_new = lss_dir_for($lei, \$new_path);
+ return if $dir_new eq $dir_old; # no change, likely
+ return $lei->fail(<<EOM) if -e $dir_new;
+lei.q.output changed from `$old_out' to `$new_out'
+However, $dir_new exists
+EOM
+ # start the conversion asynchronously
+ my $old_sq = PublicInbox::Config::squote_maybe($old_out);
+ my $new_sq = PublicInbox::Config::squote_maybe($new_out);
+ $lei->puts("lei.q.output changed from $old_sq to $new_sq");
+ $lei->qerr("# lei convert $old_sq -o $new_sq");
+ my $v = !$lei->{opt}->{quiet};
+ $lei->{opt} = { output => $new_out, verbose => $v };
+ require PublicInbox::LeiConvert;
+ PublicInbox::LeiConvert::lei_convert($lei, $old_out);
+
+ $lei->fail(<<EOM) if -e $dir_old && !rename($dir_old, $dir_new);
+E: rename($dir_old, $dir_new) error: $!
+EOM
+}
+
no warnings 'once';
*nntp_url = \&cloneurl;
*base_url = \&PublicInbox::Inbox::base_url;
diff --git a/script/lei b/script/lei
index 90a93839..bec6b001 100755
--- a/script/lei
+++ b/script/lei
@@ -33,8 +33,15 @@ my $exec_cmd = sub {
push @rdr, shift(@old), $newfh;
}
my $do_exec = sub {
+ my @non_std; # ex. $op_p from lei_edit_search
while (my ($io, $newfh) = splice(@rdr, 0, 2)) {
+ my $old_io = !!$io;
open $io, '+<&', $newfh or die "open +<&=: $!";
+ push @non_std, $io unless $old_io;
+ }
+ if (@non_std) {
+ require Fcntl;
+ fcntl($_, Fcntl::F_SETFD(), 0) for @non_std;
}
my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
@ENV{keys %env} = values %env;
^ permalink raw reply related [relevance 75%]
* [PATCH 3/5] lei_saved_search: fix excess indent for first lei.q entry
2021-05-01 6:21 70% [PATCH 0/5] lei: more UI/UX tweaks Eric Wong
2021-05-01 6:21 61% ` [PATCH 1/5] xt/lei-onion-convert: test for NNTP+IMAP onions Eric Wong
2021-05-01 6:21 55% ` [PATCH 2/5] lei <q|up>: distinguish between mset and l2m counts Eric Wong
@ 2021-05-01 6:21 71% ` Eric Wong
2021-05-01 6:21 56% ` [PATCH 4/5] lei: rename ls-sync to ls-mail-sync Eric Wong
2021-05-01 6:21 75% ` [PATCH 5/5] lei edit-search: support relocating lei.q.output Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-01 6:21 UTC (permalink / raw)
To: meta
This was harmless, but ugly and possibly confusing to
users who run "lei edit-search".
---
lib/PublicInbox/LeiSavedSearch.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 682a43e8..79125214 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -137,7 +137,7 @@ sub new { # new saved search "lei q --save"
print $fh <<EOM;
; to refresh with new results, run: lei up $sq_dst
[lei]
- $q
+$q
[lei "q"]
output = $dst
EOM
^ permalink raw reply related [relevance 71%]
* [PATCH 0/5] lei: more UI/UX tweaks
@ 2021-05-01 6:21 70% Eric Wong
2021-05-01 6:21 61% ` [PATCH 1/5] xt/lei-onion-convert: test for NNTP+IMAP onions Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-05-01 6:21 UTC (permalink / raw)
To: meta
4/5 is an incompatible change of an incomplete feature
2/5 is something I noticed from using "lei up --all=local"
Editing searches is less surprising, as lei.q.output updates
and auto-conversion are now supported (since there's no
"lei mv-search").
Eric Wong (5):
xt/lei-onion-convert: test for NNTP+IMAP onions
lei <q|up>: distinguish between mset and l2m counts
lei_saved_search: fix excess indent for first lei.q entry
lei: rename ls-sync to ls-mail-sync
lei edit-search: support relocating lei.q.output
MANIFEST | 3 +-
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 5 +-
lib/PublicInbox/LeiEditSearch.pm | 15 ++++-
.../{LeiLsSync.pm => LeiLsMailSync.pm} | 4 +-
lib/PublicInbox/LeiSavedSearch.pm | 53 +++++++++++++++-
lib/PublicInbox/LeiToMail.pm | 36 ++++++-----
lib/PublicInbox/LeiXSearch.pm | 18 ++++--
script/lei | 7 +++
t/lei-import-imap.t | 4 +-
t/lei-import-maildir.t | 4 +-
xt/lei-onion-convert.t | 61 +++++++++++++++++++
12 files changed, 180 insertions(+), 32 deletions(-)
rename lib/PublicInbox/{LeiLsSync.pm => LeiLsMailSync.pm} (93%)
create mode 100644 xt/lei-onion-convert.t
^ permalink raw reply [relevance 70%]
* [PATCH 1/5] xt/lei-onion-convert: test for NNTP+IMAP onions
2021-05-01 6:21 70% [PATCH 0/5] lei: more UI/UX tweaks Eric Wong
@ 2021-05-01 6:21 61% ` Eric Wong
2021-05-01 6:21 55% ` [PATCH 2/5] lei <q|up>: distinguish between mset and l2m counts Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-01 6:21 UTC (permalink / raw)
To: meta
These tests require a running Tor instance (defaulting to
127.0.0.1:9050) and Internet connectivity, but otherwise
work pretty well.
---
MANIFEST | 1 +
xt/lei-onion-convert.t | 61 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 62 insertions(+)
create mode 100644 xt/lei-onion-convert.t
diff --git a/MANIFEST b/MANIFEST
index bc2ad671..82f25735 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -516,6 +516,7 @@ xt/httpd-async-stream.t
xt/imapd-mbsync-oimap.t
xt/imapd-validate.t
xt/lei-auth-fail.t
+xt/lei-onion-convert.t
xt/mem-imapd-tls.t
xt/mem-msgview.t
xt/msgtime_cmp.t
diff --git a/xt/lei-onion-convert.t b/xt/lei-onion-convert.t
new file mode 100644
index 00000000..d38b4b16
--- /dev/null
+++ b/xt/lei-onion-convert.t
@@ -0,0 +1,61 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10; use PublicInbox::TestCommon;
+use PublicInbox::MboxReader;
+my $test_tor = $ENV{TEST_TOR};
+plan skip_all => "TEST_TOR unset" unless $test_tor;
+unless ($test_tor =~ m!\Asocks5h://!i) {
+ my $default = 'socks5h://127.0.0.1:9050';
+ diag "using $default (set TEST_TOR=socks5h://ADDR:PORT to override)";
+ $test_tor = $default;
+}
+my $onion = $ENV{TEST_ONION_HOST} //'ou63pmih66umazou.onion';
+my $ng = 'inbox.comp.mail.public-inbox.meta';
+my $nntp_url = $ENV{TEST_NNTP_ONION_URL} // "nntp://$onion/$ng";
+my $imap_url = $ENV{TEST_IMAP_ONION_URL} // "imap://$onion/$ng.0";
+my @cnv = qw(lei convert -o mboxrd:/dev/stdout);
+my @proxy_cli = ("--proxy=$test_tor");
+my $proxy_cfg = "proxy=$test_tor";
+test_lei(sub {
+ my $run = {};
+ for my $args ([$nntp_url, @proxy_cli], [$imap_url, @proxy_cli],
+ [ $nntp_url, '-c', "nntp.$proxy_cfg" ],
+ [ $imap_url, '-c', "imap.$proxy_cfg" ]) {
+ pipe(my ($r, $w)) or xbail "pipe: $!";
+ my $cmd = [@cnv, @$args];
+ my $td = start_script($cmd, undef, { 1 => $w, run_mode => 0 });
+ $args->[0] =~ s!\A(.+?://).*!$1...!;
+ my $key = "@$args";
+ ok($td, "$key running");
+ $run->{$key} = { td => $td, r => $r };
+ }
+ while (my ($key, $x) = each %$run) {
+ my ($td, $r) = delete(@$x{qw(td r)});
+ eval {
+ PublicInbox::MboxReader->mboxrd($r, sub {
+ my ($eml) = @_;
+ if ($key =~ m!\Anntps?://!i) {
+ for (qw(Xref Newsgroups Path)) {
+ $eml->header_set($_);
+ }
+ }
+ push @{$x->{eml}}, $eml;
+ close $r;
+ $td->kill('-INT');
+ die "$key done\n";
+ });
+ };
+ chomp(my $done = $@);
+ like($done, qr/\Q$key\E done/, $done);
+ $td->join;
+ }
+ my @keys = keys %$run;
+ my $first_key = shift @keys;
+ for my $key (@keys) {
+ is_deeply($run->{$key}, $run->{$first_key},
+ "`$key' matches `$first_key'");
+ }
+});
+
+done_testing;
^ permalink raw reply related [relevance 61%]
* [PATCH 4/5] lei: rename ls-sync to ls-mail-sync
2021-05-01 6:21 70% [PATCH 0/5] lei: more UI/UX tweaks Eric Wong
` (2 preceding siblings ...)
2021-05-01 6:21 71% ` [PATCH 3/5] lei_saved_search: fix excess indent for first lei.q entry Eric Wong
@ 2021-05-01 6:21 56% ` Eric Wong
2021-05-01 19:29 59% ` [PATCH 6/5] lei import: fix --mail-sync handling in LeiInput Eric Wong
2021-05-01 6:21 75% ` [PATCH 5/5] lei edit-search: support relocating lei.q.output Eric Wong
4 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-05-01 6:21 UTC (permalink / raw)
To: meta
This allows tab-completion for "ls-search" to work with fewer
characters ("ls-s<TAB>" instead of "ls-se<TAB>"), and I expect
"ls-search" to be used more frequently than "ls-mail-sync".
This also matches the --mail-sync switch of "lei import"
---
MANIFEST | 2 +-
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/{LeiLsSync.pm => LeiLsMailSync.pm} | 4 ++--
t/lei-import-imap.t | 4 ++--
t/lei-import-maildir.t | 4 ++--
5 files changed, 8 insertions(+), 8 deletions(-)
rename lib/PublicInbox/{LeiLsSync.pm => LeiLsMailSync.pm} (93%)
diff --git a/MANIFEST b/MANIFEST
index 82f25735..b7e55793 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -202,8 +202,8 @@ lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
lib/PublicInbox/LeiLcat.pm
lib/PublicInbox/LeiLsLabel.pm
+lib/PublicInbox/LeiLsMailSync.pm
lib/PublicInbox/LeiLsSearch.pm
-lib/PublicInbox/LeiLsSync.pm
lib/PublicInbox/LeiMailSync.pm
lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bb67fc0b..5d701d5e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -170,7 +170,7 @@ our %CMD = ( # sorted in order of importance/use:
'ls-external' => [ '[FILTER]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
-'ls-sync' => [ '', 'list sync folders',
+'ls-mail-sync' => [ '', 'list mail sync folders',
qw(z|0 z|0 globoff|g invert-match|v local remote), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
diff --git a/lib/PublicInbox/LeiLsSync.pm b/lib/PublicInbox/LeiLsMailSync.pm
similarity index 93%
rename from lib/PublicInbox/LeiLsSync.pm
rename to lib/PublicInbox/LeiLsMailSync.pm
index 71f111a9..2b3d326d 100644
--- a/lib/PublicInbox/LeiLsSync.pm
+++ b/lib/PublicInbox/LeiLsMailSync.pm
@@ -2,12 +2,12 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# front-end for the "lei ls-sync" sub-command
-package PublicInbox::LeiLsSync;
+package PublicInbox::LeiLsMailSync;
use strict;
use v5.10.1;
use PublicInbox::LeiMailSync;
-sub lei_ls_sync {
+sub lei_ls_mail_sync {
my ($lei, $filter) = @_;
my $sto = $lei->_lei_store or return;
my $lms = $sto->search->lms or return;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index c977c68e..3a1fff4c 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -22,8 +22,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
is_deeply(json_utf8->decode($lei_out), {}, 'no inspect stats, yet');
lei_ok('import', $url);
- lei_ok 'ls-sync';
- like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-sync');
+ lei_ok 'ls-mail-sync';
+ like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-mail-sync');
chomp(my $u = $lei_out);
lei_ok('import', $u, \'UIDVALIDITY match in URL');
$u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s;
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index 808e1a73..02fe43e1 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -40,8 +40,8 @@ test_lei(sub {
is_deeply($inspect, { sync => { "maildir:$md" => [ 'x:2,S' ] } },
'maildir sync info as expected');
- lei_ok qw(ls-sync);
- is($lei_out, "maildir:$md\n", 'ls-sync as expected');
+ lei_ok qw(ls-mail-sync);
+ is($lei_out, "maildir:$md\n", 'ls-mail-sync as expected');
lei_ok(qw(import), $md, \'import Maildir again');
$imp_err = $lei_err;
^ permalink raw reply related [relevance 56%]
* [PATCH 2/5] lei <q|up>: distinguish between mset and l2m counts
2021-05-01 6:21 70% [PATCH 0/5] lei: more UI/UX tweaks Eric Wong
2021-05-01 6:21 61% ` [PATCH 1/5] xt/lei-onion-convert: test for NNTP+IMAP onions Eric Wong
@ 2021-05-01 6:21 55% ` Eric Wong
2021-05-01 6:21 71% ` [PATCH 3/5] lei_saved_search: fix excess indent for first lei.q entry Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-05-01 6:21 UTC (permalink / raw)
To: meta
The number of messages we write to --output is usually different
than the mset count due to deduplication from combining multiple
sources.
This change makes the stderr output of "lei up --all=local" way
more useful IMHO.
---
lib/PublicInbox/LeiToMail.pm | 36 ++++++++++++++++++++---------------
lib/PublicInbox/LeiXSearch.pm | 18 ++++++++++++++----
2 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index eda4701c..a546ab42 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -196,9 +196,13 @@ sub _mbox_write_cb ($$) {
return if $dedupe->is_dup($eml, $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
$buf = $eml2mbox->($eml, $smsg);
- return atomic_append($lei, $buf) if $atomic_append;
- my $lk = $ovv->lock_for_scope;
- $lei->out($$buf);
+ if ($atomic_append) {
+ atomic_append($lei, $buf);
+ } else {
+ my $lk = $ovv->lock_for_scope;
+ $lei->out($$buf);
+ }
+ ++$lei->{-nr_write};
}
}
@@ -273,15 +277,13 @@ sub _maildir_write_cb ($$) {
my $dst = $lei->{ovv}->{dst};
my $lse = $lei->{lse}; # may be undef
sub { # for git_to_mail
- my ($buf, $smsg, $eml) = @_;
+ my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
- $buf //= \($eml->as_string);
+ return if $dedupe && $dedupe->is_dup($eml //
+ PublicInbox::Eml->new($$bref));
$lse->xsmsg_vmd($smsg) if $lse;
- return _buf2maildir($dst, $buf, $smsg) if !$dedupe;
- $eml //= PublicInbox::Eml->new($$buf); # copy buf
- return if $dedupe->is_dup($eml, $smsg);
- undef $eml;
- _buf2maildir($dst, $buf, $smsg);
+ _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+ ++$lei->{-nr_write};
}
}
@@ -296,16 +298,15 @@ sub _imap_write_cb ($$) {
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
- if ($dedupe) {
- $eml //= PublicInbox::Eml->new($$bref); # copy bref
- return if $dedupe->is_dup($eml, $smsg);
- }
+ return if $dedupe && $dedupe->is_dup($eml //
+ PublicInbox::Eml->new($$bref));
$lse->xsmsg_vmd($smsg) if $lse;
eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
if (my $err = $@) {
undef $mic;
die $err;
}
+ ++$lei->{-nr_write};
}
}
@@ -659,7 +660,12 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
delete $self->{wcb};
- $self->{lei}->{ale}->git->async_wait_all;
+ my $lei = $self->{lei};
+ $lei->{ale}->git->async_wait_all;
+ my $nr = delete($lei->{-nr_write}) or return;
+ return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
+ require PublicInbox::PktOp;
+ PublicInbox::PktOp::pkt_do($lei->{pkt_op_p}, 'l2m_progress', $nr);
}
# called in top-level lei-daemon when LeiAuth is done
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index b3fd79d0..e04af0dc 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -144,6 +144,11 @@ sub mset_progress {
}
}
+sub l2m_progress {
+ my ($lei, $nr) = @_;
+ $lei->{-nr_write} += $nr;
+}
+
sub query_one_mset { # for --threads and l2m w/o sort
my ($self, $ibxish) = @_;
local $0 = "$0 query_one_mset";
@@ -354,7 +359,7 @@ sub query_done { # EOF callback for main daemon
}
my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
- my @out;
+ my (@out, $start_mua);
if ($l2m) { # close() calls LeiToMail reap_compress
@out = (" in $lei->{ovv}->{dst}");
if (my $out = delete $lei->{old_1}) {
@@ -370,11 +375,15 @@ Error closing $lei->{ovv}->{dst}: $!
$lei->poke_mua;
} else { # mbox users
delete $l2m->{mbl}; # drop dotlock
- $lei->start_mua;
+ $start_mua = 1;
}
}
- $lei->{-progress} and
- $lei->err('# ', $lei->{-mset_total} // 0, " matches", @out);
+ if ($lei->{-progress}) {
+ $lei->qerr('# ', $lei->{-mset_total} // 0, " matches", @out);
+ my $nr = $lei->{-nr_write} // 0;
+ $lei->qerr("# $nr written to $lei->{ovv}->{dst}") if $l2m;
+ }
+ $lei->start_mua if $start_mua;
$lei->dclose;
}
@@ -456,6 +465,7 @@ sub do_query {
'+' => [ \&incr_post_augment, $lei ],
'' => [ \&query_done, $lei ],
'mset_progress' => [ \&mset_progress, $lei ],
+ 'l2m_progress' => [ \&l2m_progress, $lei ],
'x_it' => [ $lei->can('x_it'), $lei ],
'child_error' => [ $lei->can('child_error'), $lei ],
'incr_start_query' => [ \&incr_start_query, $self, $l2m ],
^ permalink raw reply related [relevance 55%]
* [PATCH 4/8] lei: ensure autoflush(1) is on STDERR
2021-04-30 9:24 67% [PATCH 0/8] lei NNTP/IMAP .onion support and misc fixes Eric Wong
2021-04-30 9:24 71% ` [PATCH 1/8] lei sucks: preserve utsname.machine, add "x86" where appropriate Eric Wong
2021-04-30 9:24 71% ` [PATCH 3/8] lei: kill old PIDs when dropping Eric Wong
@ 2021-04-30 9:24 70% ` Eric Wong
2021-04-30 9:24 45% ` [PATCH 6/8] lei: IMAP .onion support via --proxy=s switch Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-30 9:24 UTC (permalink / raw)
To: meta
This fixes error reporting for oneshot tests in xt/lei-auth-failure.t
---
lib/PublicInbox/LEI.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3468094f..6a82d497 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -500,6 +500,7 @@ sub _lei_atfork_child {
}
} else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
+ STDERR->autoflush(1);
}
close($_) for (grep(defined, delete @$self{qw(3 old_1 au_done)}));
if (my $op_c = delete $self->{pkt_op_c}) {
@@ -676,6 +677,7 @@ sub lazy_cb ($$$) {
sub dispatch {
my ($self, $cmd, @argv) = @_;
local $current_lei = $self; # for __WARN__
+ $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
dump_and_clear_log("from previous run\n");
return _help($self, 'no command given') unless defined($cmd);
# do not support Getopt bundling for this
@@ -1006,7 +1008,6 @@ sub accept_dispatch { # Listener {post_accept} callback
}
$i == 4 or return send($sock, 'not enough FDs='.($i-1), MSG_EOR)
}
- $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
# $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);
# $buf = "$argc\0".join("\0", @ARGV).$ENV_STR."\0\0";
substr($buf, -2, 2, '') eq "\0\0" or # s/\0\0\z//
^ permalink raw reply related [relevance 70%]
* [PATCH 6/8] lei: IMAP .onion support via --proxy=s switch
2021-04-30 9:24 67% [PATCH 0/8] lei NNTP/IMAP .onion support and misc fixes Eric Wong
` (2 preceding siblings ...)
2021-04-30 9:24 70% ` [PATCH 4/8] lei: ensure autoflush(1) is on STDERR Eric Wong
@ 2021-04-30 9:24 45% ` Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-30 9:24 UTC (permalink / raw)
To: meta
Mail::IMAPClient provides the ability to pass a pre-connected
Socket to it. We can rely on this functionality to use
IO::Socket::Socks in place whatever socket class
Mail::IMAPClient chooses to use.
The --proxy=s is shared with curl(1), though we only support
socks5h:// at the moment. Is there any need for SOCKS4 or SOCKS5
without name resolution? Tor .onions require socks5h:// for
name resolution and to prevent data leakage.
---
lib/PublicInbox/LEI.pm | 12 ++++++++----
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 4 ++--
lib/PublicInbox/NetReader.pm | 31 ++++++++++++++++++++++++++++---
4 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6a82d497..bb67fc0b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -188,7 +188,8 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
'tag' => [ 'KEYWORDS...',
'set/unset keywords and/or labels on message(s)',
- qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@), @c_opt,
+ qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt,
pass_through('-kw:foo for delete') ],
'forget' => [ '[--stdin|--oid=OID|--by-mid=MID]',
"exclude message(s) on stdin from `q' search results",
@@ -211,11 +212,12 @@ our %CMD = ( # sorted in order of importance/use:
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s
- lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!), @c_opt ],
+ lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
- qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s
- lock=s@ kw!), @c_opt ],
+ qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
+ qw(no-torsocks torsocks=s), PublicInbox::LeiQuery::curl_opt(), @c_opt ],
'p2q' => [ 'FILE|COMMIT_OID|--stdin',
"use a patch to generate a query for `lei q --stdin'",
qw(stdin| want|w=s@ uri debug), @c_opt ],
@@ -277,6 +279,8 @@ my %OPTDESC = (
'path-a|a=s' => 'pre-image pathname associated with OID',
'path-b|b=s' => 'post-image pathname associated with OID',
'git-dir=s@' => 'additional git repository to scan',
+'proxy=s' => [ 'PROTO://HOST[:PORT]', # shared with curl(1)
+ "proxy for (e.g. `socks5h://0:9050')" ],
'torsocks=s' => ['VAL|auto|no|yes',
'whether or not to wrap git and curl commands with torsocks'],
'no-torsocks' => 'alias for --torsocks=no',
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 277ad88d..86f300c3 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -294,7 +294,7 @@ $input is `eml', not --in-format=$in_fmt
}
if ($net) {
$net->{-can_die} = 1;
- if (my $err = $net->errors) {
+ if (my $err = $net->errors($lei)) {
return $lei->fail($err);
}
$net->{quiet} = $lei->{opt}->{quiet};
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index fa3af710..eda4701c 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -351,14 +351,14 @@ sub new {
require PublicInbox::MboxReader;
$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
$self->{base_type} = 'mbox';
- } elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support
+ } elsif ($fmt =~ /\Aimaps?\z/) {
require PublicInbox::NetWriter;
require PublicInbox::URIimap;
my $net = PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
$net->add_url($uri);
- my $err = $net->errors;
+ my $err = $net->errors($lei);
return $lei->fail($err) if $err;
$uri->mailbox or return $lei->fail("No mailbox: $dst");
$self->{uri} = $uri;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index b9365c05..ac23e701 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -7,6 +7,7 @@ use strict;
use v5.10.1;
use parent qw(Exporter PublicInbox::IPC);
use PublicInbox::Eml;
+use PublicInbox::Config;
our %IMAPflags2kw = map {; "\\\u$_" => $_ } qw(seen answered flagged draft);
$IMAPflags2kw{'$Forwarded'} = 'forwarded'; # RFC 5550
@@ -51,7 +52,16 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
%$common, # may set Starttls, Compress, Debug ....
};
require PublicInbox::IMAPClient;
- my $mic = PublicInbox::IMAPClient->new(%$mic_arg) or
+ my %socks;
+ if ($lei && $lei->{socks5h}) {
+ my %opt = %{$lei->{socks5h}};
+ $opt{ConnectAddr} = delete $mic_arg->{Server};
+ $opt{ConnectPort} = delete $mic_arg->{Port};
+ $socks{Socket} = IO::Socket::Socks->new(%opt) or die
+ "E: <$url> ".eval('$IO::Socket::Socks::SOCKS_ERROR');
+ $self->{mic_socks5h} = \%opt;
+ }
+ my $mic = PublicInbox::IMAPClient->new(%$mic_arg, %socks) or
die "E: <$url> new: $@\n";
# default to using STARTTLS if it's available, but allow
@@ -331,7 +341,7 @@ sub add_url {
}
sub errors {
- my ($self) = @_;
+ my ($self, $lei) = @_;
if (my $u = $self->{unsupported_url}) {
return "Unsupported URL(s): @$u";
}
@@ -343,6 +353,16 @@ sub errors {
eval { require Net::NNTP } or
die "Net::NNTP is required for NNTP:\n$@\n";
}
+ if ($lei && (($lei->{opt}->{proxy}//'') =~ m!\Asocks5h://
+ (?: \[ ([^\]]+) \] | ([^:/]+) )
+ (?::([0-9]+))?/?(?:,|\z)!ix)) {
+ my ($h, $p) = ($1 // $2, $3 + 0);
+ $h = '127.0.0.1' if $h eq '0';
+ eval { require IO::Socket::Socks } or die <<EOM;
+IO::Socket::Socks missing for socks5h://$h:$p
+EOM
+ $lei->{socks5h} = { ProxyAddr => $h, ProxyPort => $p };
+ }
undef;
}
@@ -507,7 +527,12 @@ sub mic_get {
$mic_arg->{Authcallback} = $self->can($cb_name);
}
}
- my $mic = PublicInbox::IMAPClient->new(%$mic_arg);
+ my %socks;
+ if (my $s5h = $self->{mic_socks5h}) {
+ $socks{Socket} = IO::Socket::Socks->new(%$s5h) or die
+ "E: <$$uri> ".eval('$IO::Socket::Socks::SOCKS_ERROR');
+ }
+ my $mic = PublicInbox::IMAPClient->new(%$mic_arg, %socks);
$cached //= {}; # invalid placeholder if no cache enabled
$mic && $mic->IsConnected ? ($cached->{$sec} = $mic) : undef;
}
^ permalink raw reply related [relevance 45%]
* [PATCH 1/8] lei sucks: preserve utsname.machine, add "x86" where appropriate
2021-04-30 9:24 67% [PATCH 0/8] lei NNTP/IMAP .onion support and misc fixes Eric Wong
@ 2021-04-30 9:24 71% ` Eric Wong
2021-04-30 9:24 71% ` [PATCH 3/8] lei: kill old PIDs when dropping Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-30 9:24 UTC (permalink / raw)
To: meta
It's helpful for us to distinguish x86 kernels from x86_64
kernels when using an x86 userspace. OSes are dropping i386
support and only support i486 and newer, so "x86" is a more
appropriate description for that platform than "i386".
---
lib/PublicInbox/LeiSucks.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index d364a856..2ce64d62 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -18,7 +18,8 @@ sub lei_sucks {
$lei->start_pager if -t $lei->{1};
my ($os, undef, $rel, undef, $mac)= POSIX::uname();
if ($mac eq 'x86_64' && $Config{ptrsize} == 4) {
- $mac = $Config{cppsymbols} =~ /\b__ILP32__=1\b/ ? 'x32' : 'i386'
+ $mac .= $Config{cppsymbols} =~ /\b__ILP32__=1\b/ ?
+ ',u=x32' : ',u=x86';
}
eval { require PublicInbox };
my $pi_ver = eval('$PublicInbox::VERSION') // '(???)';
^ permalink raw reply related [relevance 71%]
* [PATCH 3/8] lei: kill old PIDs when dropping
2021-04-30 9:24 67% [PATCH 0/8] lei NNTP/IMAP .onion support and misc fixes Eric Wong
2021-04-30 9:24 71% ` [PATCH 1/8] lei sucks: preserve utsname.machine, add "x86" where appropriate Eric Wong
@ 2021-04-30 9:24 71% ` Eric Wong
2021-04-30 9:24 70% ` [PATCH 4/8] lei: ensure autoflush(1) is on STDERR Eric Wong
2021-04-30 9:24 45% ` [PATCH 6/8] lei: IMAP .onion support via --proxy=s switch Eric Wong
3 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-30 9:24 UTC (permalink / raw)
To: meta
This ensures hitting Ctrl-C on a long-running "lei convert" or
similar will stop the WQ worker, even after we've closed
the WQ socketpair in the daemon.
---
lib/PublicInbox/LEI.pm | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 52ce8ec2..3468094f 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -387,7 +387,14 @@ my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
sub _drop_wq {
my ($self) = @_;
- for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) { $wq->DESTROY }
+ for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) {
+ if ($wq->wq_kill) {
+ $wq->wq_close(0, undef, $self);
+ } elsif ($wq->wq_kill_old) {
+ $wq->wq_wait_old(undef, $self);
+ }
+ $wq->DESTROY;
+ }
}
# pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
^ permalink raw reply related [relevance 71%]
* [PATCH 0/8] lei NNTP/IMAP .onion support and misc fixes
@ 2021-04-30 9:24 67% Eric Wong
2021-04-30 9:24 71% ` [PATCH 1/8] lei sucks: preserve utsname.machine, add "x86" where appropriate Eric Wong
` (3 more replies)
0 siblings, 4 replies; 200+ results
From: Eric Wong @ 2021-04-30 9:24 UTC (permalink / raw)
To: meta
Attempting to use torsocks(1) for NNTP or IMAP could get tricky.
Fortunately, IO::Socket::Socks is packaged for on CentOS 7,
FreeBSD, and Debian, so it seems to be a reasonable way to
support NNTP and IMAP Tor onions.
--proxy= (shared with curl) is supported for one-off
command-line use, but imap.proxy and nntp.proxy are both
supported along with URL-matching variants with git 1.8.5 (or
git 2.26 for wildcard URL matching).
Only socks5h:// proxies are supported (the default with
IO::Socket::Socks), which is what Tor uses. I doubt its worth
the effort (and potential for DNS request leaks) to support
prior versions of SOCKS in 2021.
Eric Wong (8):
lei sucks: preserve utsname.machine, add "x86" where appropriate
lei_curl: improve correctness of LD_PRELOAD check
lei: kill old PIDs when dropping
lei: ensure autoflush(1) is on STDERR
net_reader: {nn,mic}_for: use prototypes for internal subs
lei: IMAP .onion support via --proxy=s switch
net_reader: Net::NNTP --proxy=socks5h:// support
net_reader: support (imap|nntp).proxy in config file
MANIFEST | 2 +
lib/PublicInbox/Config.pm | 1 +
lib/PublicInbox/LEI.pm | 24 ++++++++---
lib/PublicInbox/LeiCurl.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiSucks.pm | 3 +-
lib/PublicInbox/LeiToMail.pm | 4 +-
lib/PublicInbox/NetNNTPSocks.pm | 33 +++++++++++++++
lib/PublicInbox/NetReader.pm | 72 +++++++++++++++++++++++++++------
xt/net_nntp_socks.t | 22 ++++++++++
10 files changed, 141 insertions(+), 24 deletions(-)
create mode 100644 lib/PublicInbox/NetNNTPSocks.pm
create mode 100644 xt/net_nntp_socks.t
^ permalink raw reply [relevance 67%]
* [PATCH 3/4] lei import: support UIDVALIDITY in IMAP URL
2021-04-29 9:46 69% [PATCH 0/4] some lei synchronization work Eric Wong
2021-04-29 9:46 33% ` [PATCH 2/4] lei import: avoid IMAPTracker, use LeiMailSync more Eric Wong
@ 2021-04-29 9:46 61% ` Eric Wong
2021-04-29 9:46 57% ` [PATCH 4/4] lei import: support shell completion of known folders Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-29 9:46 UTC (permalink / raw)
To: meta
Specifying a UIDVALIDITY value allows the user to enforce
a strict match and force failure. This necessitated changes
to NetReader to allow die() and make error reporting more
suitable for CLI usage rather than daemonized usage of -watch.
---
lib/PublicInbox/LeiInput.pm | 10 +++++++++-
lib/PublicInbox/NetReader.pm | 7 +++++++
t/lei-import-imap.t | 4 ++++
3 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index ce675f40..277ad88d 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -293,6 +293,7 @@ $input is `eml', not --in-format=$in_fmt
$lei->err("# --sync is not supported for: @{$sync->{no}}");
}
if ($net) {
+ $net->{-can_die} = 1;
if (my $err = $net->errors) {
return $lei->fail($err);
}
@@ -306,10 +307,17 @@ $input is `eml', not --in-format=$in_fmt
sub process_inputs {
my ($self) = @_;
+ my $err;
for my $input (@{$self->{inputs}}) {
- $self->input_path_url($input);
+ eval { $self->input_path_url($input) };
+ next unless $@;
+ $err = "$input: $@";
+ last;
}
+ # always commit first, even on error partial work is acceptable for
+ # lei <import|tag|convert>
my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto};
+ $self->{lei}->fail($err) if $err;
}
sub input_only_atfork_child {
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 81d25ead..3fc37b10 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -407,6 +407,11 @@ sub _imap_fetch_all ($$$) {
return "E: $orig_uri cannot get UIDVALIDITY";
$r_uidnext //= $mic->uidnext($mbx) //
return "E: $orig_uri cannot get UIDNEXT";
+ my $expect = $orig_uri->uidvalidity // $r_uidval;
+ return <<EOF if $expect != $r_uidval;
+E: $orig_uri UIDVALIDITY mismatch (got $r_uidval)
+EOF
+
my $uri = $orig_uri->clone;
my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval);
return <<EOF if $l_uidval != $r_uidval;
@@ -520,6 +525,7 @@ sub imap_each {
} else {
$err = "E: <$uri> not connected: $!";
}
+ die $err if $err && $self->{-can_die};
warn $err if $err;
$mic;
}
@@ -620,6 +626,7 @@ sub nntp_each {
} else {
$err = "E: <$uri> not connected: $!";
}
+ die $err if $err && $self->{-can_die};
warn $err if $err;
$nn;
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 611328b4..c977c68e 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -24,6 +24,10 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('import', $url);
lei_ok 'ls-sync';
like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-sync');
+ chomp(my $u = $lei_out);
+ lei_ok('import', $u, \'UIDVALIDITY match in URL');
+ $u =~ s/;UIDVALIDITY=(\d+)\s*/;UIDVALIDITY=9$1/s;
+ ok(!lei('import', $u), 'UIDVALIDITY mismatch in URL rejected');
lei_ok('inspect', $url);
my $inspect = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 61%]
* [PATCH 4/4] lei import: support shell completion of known folders
2021-04-29 9:46 69% [PATCH 0/4] some lei synchronization work Eric Wong
2021-04-29 9:46 33% ` [PATCH 2/4] lei import: avoid IMAPTracker, use LeiMailSync more Eric Wong
2021-04-29 9:46 61% ` [PATCH 3/4] lei import: support UIDVALIDITY in IMAP URL Eric Wong
@ 2021-04-29 9:46 57% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-29 9:46 UTC (permalink / raw)
To: meta
This also fixes completion of "lei up" for IMAP folders.
---
contrib/completion/lei-completion.bash | 1 +
lib/PublicInbox/LeiExternal.pm | 22 ++++++++++++----------
lib/PublicInbox/LeiImport.pm | 8 ++++++++
lib/PublicInbox/LeiUp.pm | 4 ++--
4 files changed, 23 insertions(+), 12 deletions(-)
diff --git a/contrib/completion/lei-completion.bash b/contrib/completion/lei-completion.bash
index 2c28d44a..5c137e68 100644
--- a/contrib/completion/lei-completion.bash
+++ b/contrib/completion/lei-completion.bash
@@ -9,6 +9,7 @@ _lei() {
*':'* | *'='* | '//'*) compopt -o nospace ;;
*) compopt +o nospace ;; # the default
esac
+ wordlist="${wordlist//;/\\\\;}" # escape ';' for ';UIDVALIDITY' and such
COMPREPLY=($(compgen -W "$wordlist" -- "${COMP_WORDS[COMP_CWORD]}"))
return 0
}
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 3858085e..6fd3efef 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -215,7 +215,8 @@ sub lei_forget_external {
}
}
-sub complete_url_common {
+# returns an anonymous sub which returns an array of potential results
+sub complete_url_prepare {
my $argv = $_[-1];
# Workaround bash word-splitting URLs to ['https', ':', '//' ...]
# Maybe there's a better way to go about this in
@@ -239,37 +240,38 @@ sub complete_url_common {
}
$re = quotemeta($re);
}
- ($cur, $re);
+ my $match_cb = sub {
+ # only return the part specified on the CLI
+ # don't duplicate if already 100% completed
+ $_[0] =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ()
+ };
+ wantarray ? ($re, $cur, $match_cb) : $match_cb;
}
# shell completion helper called by lei__complete
sub _complete_forget_external {
my ($self, @argv) = @_;
my $cfg = $self->_lei_cfg;
- my ($cur, $re) = complete_url_common(\@argv);
+ my ($cur, $re, $match_cb) = complete_url_prepare(\@argv);
# FIXME: bash completion off "http:" or "https:" when the last
# character is a colon doesn't work properly even if we're
# returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
# be a bash issue.
map {
- my $x = substr($_, length('external.'));
- # only return the part specified on the CLI
- # don't duplicate if already 100% completed
- $x =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ();
+ $match_cb->(substr($_, length('external.')));
} grep(/\Aexternal\.$re\Q$cur/, @{$cfg->{-section_order}});
}
sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
my ($self, @argv) = @_;
my $cfg = $self->_lei_cfg;
- my ($cur, $re) = complete_url_common(\@argv);
+ my $match_cb = complete_url_prepare(\@argv);
require URI;
map {
my $u = URI->new(substr($_, length('external.')));
my ($base) = ($u->path =~ m!((?:/?.*)?/)[^/]+/?\z!);
$u->path($base);
- $u = $u->as_string;
- $u =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ();
+ $match_cb->($u->as_string);
} grep(m!\Aexternal\.https?://!, @{$cfg->{-section_order}});
}
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 277f4f95..def121ab 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -89,6 +89,14 @@ sub lei_import { # the main "lei import" method
$op_c->op_wait_event($ops);
}
+sub _complete_import {
+ my ($lei, @argv) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ map { $match_cb->($_) } $lms->folders;
+}
+
no warnings 'once';
*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index f4ff070b..4399c4fb 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -99,8 +99,8 @@ sub lei_up {
sub _complete_up {
my ($lei, @argv) = @_;
- my ($cur, $re) = $lei->complete_url_common(\@argv);
- grep(/\A$re\Q$cur/, PublicInbox::LeiSavedSearch::list($lei));
+ my $match_cb = $lei->complete_url_prepare(\@argv);
+ map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
}
1;
^ permalink raw reply related [relevance 57%]
* [PATCH 2/4] lei import: avoid IMAPTracker, use LeiMailSync more
2021-04-29 9:46 69% [PATCH 0/4] some lei synchronization work Eric Wong
@ 2021-04-29 9:46 33% ` Eric Wong
2021-04-29 9:46 61% ` [PATCH 3/4] lei import: support UIDVALIDITY in IMAP URL Eric Wong
2021-04-29 9:46 57% ` [PATCH 4/4] lei import: support shell completion of known folders Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-29 9:46 UTC (permalink / raw)
To: meta
IMAPTracker has a UNIQUE constraint on the `url' column,
which may cause compatibility and/or rollback problems
in attempting to deal with UIDVALIDITY changes.
Having multiple sources of truth leads to confusion and bugs,
so relying on LeiMailSync exclusively ought to simplify things.
Furthermore, since LeiMailSync is only written to by LeiStore,
it is safer in that it won't mark a UID or article as imported
until git-fast-import has seen it, and the SQLite commit always
happens after "done\n" is sent to fast-import.
This mostly reverts recent commits to IMAPTracker to support
lei, those are:
1) commit 7632d8f7590daf70c65d4270e750c36552fa9389
("net_reader: restart on first UID when UIDVALIDITY changes")
2) commit 311a5d37ad275cd75b1e64d87827c4d13fe4bfab
("imap_tracker: prepare for use with lei").
This means public-inbox-watch will not change between 1.6 and
1.7: -watch stops synching a folder when UIDVALIDITY changes.
---
Documentation/lei-store-format.pod | 2 +-
lib/PublicInbox/IMAPTracker.pm | 35 ++++++-----------
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 12 ++----
lib/PublicInbox/LeiInput.pm | 10 ++---
lib/PublicInbox/LeiMailSync.pm | 2 +-
lib/PublicInbox/NetReader.pm | 60 ++++++++++++++++++------------
lib/PublicInbox/URIimap.pm | 2 +
t/lei-import-imap.t | 4 +-
t/lei-import-nntp.t | 4 +-
10 files changed, 64 insertions(+), 69 deletions(-)
diff --git a/Documentation/lei-store-format.pod b/Documentation/lei-store-format.pod
index 3e1ddc65..71aa72cb 100644
--- a/Documentation/lei-store-format.pod
+++ b/Documentation/lei-store-format.pod
@@ -32,7 +32,7 @@ prevent them from being accidentally treated as a v2 inbox.
~/.local/share/lei/store
- ipc.lock # lock file for internal lei IPC
- local/$EPOCH.git # normal bare git repositories
- - net_last.sqlite3 # import state for IMAP & NNTP
+ - mail_sync.sqlite3 # sync state IMAP, Maildir, NNTP
Additionally, the following share the same roles they do in extindex:
diff --git a/lib/PublicInbox/IMAPTracker.pm b/lib/PublicInbox/IMAPTracker.pm
index fe813582..5eb33cf7 100644
--- a/lib/PublicInbox/IMAPTracker.pm
+++ b/lib/PublicInbox/IMAPTracker.pm
@@ -39,20 +39,12 @@ sub dbh_new ($) {
$dbh;
}
-sub get_last ($;$) {
- my ($self, $validity) = @_;
- my $sth;
- if (defined $validity) {
- $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
-SELECT uid_validity, uid FROM imap_last WHERE url = ? AND uid_validity = ?
-
- $sth->execute($self->{url}, $validity);
- } else {
- $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
+sub get_last ($) {
+ my ($self) = @_;
+ my $sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
SELECT uid_validity, uid FROM imap_last WHERE url = ?
- $sth->execute($self->{url});
- }
+ $sth->execute($self->{url});
$sth->fetchrow_array;
}
@@ -70,19 +62,16 @@ VALUES (?, ?, ?)
}
sub new {
- my ($class, $url, $dbname) = @_;
+ my ($class, $url) = @_;
- unless (defined($dbname)) {
- # original name for compatibility with old setups:
- $dbname = PublicInbox::Config->config_dir() . '/imap.sqlite3';
+ # original name for compatibility with old setups:
+ my $dbname = PublicInbox::Config->config_dir() . '/imap.sqlite3';
- # use the new XDG-compliant name for new setups:
- if (!-f $dbname) {
- $dbname = ($ENV{XDG_DATA_HOME} //
- (($ENV{HOME} // '/nonexistent').
- '/.local/share')) .
- '/public-inbox/imap.sqlite3';
- }
+ # use the new XDG-compliant name for new setups:
+ if (!-f $dbname) {
+ $dbname = ($ENV{XDG_DATA_HOME} //
+ (($ENV{HOME} // '/nonexistent').'/.local/share')) .
+ '/public-inbox/imap.sqlite3';
}
if (!-f $dbname) {
require File::Path;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 1ea7c9ca..52ce8ec2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -211,7 +211,7 @@ our %CMD = ( # sorted in order of importance/use:
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s
- lock=s@ in-format|F=s kw! verbose|v+ incremental! sync!), @c_opt ],
+ lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!), @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 26127ece..277f4f95 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -41,18 +41,13 @@ sub input_maildir_cb { # maildir_each_eml cb
input_eml_cb($self, $eml, $vmd);
}
-sub input_imap_cb { # imap_each
+sub input_net_cb { # imap_each / nntp_each
my ($url, $uid, $kw, $eml, $self) = @_;
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
$vmd->{sync_info} = [ $url, $uid ] if $self->{-mail_sync};
input_eml_cb($self, $eml, $vmd);
}
-sub input_nntp_cb { # nntp_each
- my ($url, $num, $kw, $eml, $self) = @_;
- input_eml_cb($self, $eml, $self->{-import_kw} ? { kw => $kw } : undef);
-}
-
sub net_merge_complete { # callback used by LeiAuth
my ($self) = @_;
$self->wq_io_do('process_inputs');
@@ -69,7 +64,7 @@ sub lei_import { # the main "lei import" method
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
$self->{all_vmd} = $vmd_mod if scalar keys %$vmd_mod;
$self->prepare_inputs($lei, \@inputs) or return;
- $self->{-mail_sync} = $lei->{opt}->{sync} // 1;
+ $self->{-mail_sync} = $lei->{opt}->{'mail-sync'} // 1;
$lei->ale; # initialize for workers to read
my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
@@ -77,8 +72,7 @@ sub lei_import { # the main "lei import" method
# $j = $net->net_concurrency($j); TODO
if ($lei->{opt}->{incremental} // 1) {
$net->{incremental} = 1;
- $net->{itrk_fn} = $lei->store_path .
- '/net_last.sqlite3';
+ $net->{-lms_ro} = $lei->_lei_store->search->lms // 0;
}
} else {
my $nproc = $self->detect_nproc;
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 785e607d..ce675f40 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -110,14 +110,12 @@ sub input_path_url {
my $ifmt = lc($lei->{opt}->{'in-format'} // '');
# TODO auto-detect?
if ($input =~ m!\Aimaps?://!i) {
- $lei->{net}->imap_each($input, $self->can('input_imap_cb') //
- $self->can('input_net_cb'),
- $self, @args);
+ $lei->{net}->imap_each($input, $self->can('input_net_cb'),
+ $self, @args);
return;
} elsif ($input =~ m!\A(?:nntps?|s?news)://!i) {
- $lei->{net}->nntp_each($input, $self->can('input_nntp_cb') //
- $self->can('input_net_cb'),
- $self, @args);
+ $lei->{net}->nntp_each($input, $self->can('input_net_cb'),
+ $self, @args);
return;
} elsif ($input =~ m!\Ahttps?://!i) {
handle_http_input($self, $input, @args);
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 52f26d69..2ce189fa 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -143,7 +143,7 @@ sub each_src {
}
sub location_stats {
- my ($self, $folder, $cb, @args) = @_;
+ my ($self, $folder) = @_;
my $dbh = $self->{dbh} //= dbh_new($self);
my $fid;
my $ret = {};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 5978752f..81d25ead 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -235,7 +235,7 @@ sub imap_common_init ($;$) {
$self->{quiet} = 1 if $lei && $lei->{opt}->{quiet};
eval { require PublicInbox::IMAPClient } or
die "Mail::IMAPClient is required for IMAP:\n$@\n";
- eval { require PublicInbox::IMAPTracker } or
+ ($lei || eval { require PublicInbox::IMAPTracker }) or
die "DBD::SQLite is required for IMAP\n:$@\n";
require PublicInbox::URIimap;
my $cfg = $self->{pi_cfg} // $lei->_lei_cfg;
@@ -283,7 +283,7 @@ sub nntp_common_init ($;$) {
$self->{quiet} = 1 if $lei && $lei->{opt}->{quiet};
eval { require Net::NNTP } or
die "Net::NNTP is required for NNTP:\n$@\n";
- eval { require PublicInbox::IMAPTracker } or
+ ($lei || eval { require PublicInbox::IMAPTracker }) or
die "DBD::SQLite is required for NNTP\n:$@\n";
my $cfg = $self->{pi_cfg} // $lei->_lei_cfg;
my $nn_args = {}; # scheme://authority => Net::NNTP->new arg
@@ -373,17 +373,28 @@ sub run_commit_cb ($) {
$cb->(@args);
}
-sub _itrk ($$) {
- my ($self, $uri) = @_;
- return unless $self->{incremental};
- # itrk_fn is set by lei
- PublicInbox::IMAPTracker->new($$uri, $self->{itrk_fn});
+sub _itrk_last ($$;$) {
+ my ($self, $uri, $r_uidval) = @_;
+ return (undef, undef, $r_uidval) unless $self->{incremental};
+ my ($itrk, $l_uid, $l_uidval);
+ if (defined(my $lms = $self->{-lms_ro})) { # LeiMailSync or 0
+ $uri->uidvalidity($r_uidval) if defined $r_uidval;
+ my $x;
+ $l_uid = ($lms && ($x = $lms->location_stats($$uri))) ?
+ $x->{'uid.max'} : undef;
+ # itrk remains undef, lei/store worker writes to
+ # mail_sync.sqlite3
+ } else {
+ $itrk = PublicInbox::IMAPTracker->new($$uri);
+ ($l_uidval, $l_uid) = $itrk->get_last($$uri);
+ }
+ ($itrk, $l_uid, $l_uidval //= $r_uidval);
}
sub _imap_fetch_all ($$$) {
- my ($self, $mic, $uri) = @_;
- my $sec = uri_section($uri);
- my $mbx = $uri->mailbox;
+ my ($self, $mic, $orig_uri) = @_;
+ my $sec = uri_section($orig_uri);
+ my $mbx = $orig_uri->mailbox;
$mic->Clear(1); # trim results history
$mic->examine($mbx) or return "E: EXAMINE $mbx ($sec) failed: $!";
my ($r_uidval, $r_uidnext);
@@ -393,20 +404,22 @@ sub _imap_fetch_all ($$$) {
last if $r_uidval && $r_uidnext;
}
$r_uidval //= $mic->uidvalidity($mbx) //
- return "E: $uri cannot get UIDVALIDITY";
+ return "E: $orig_uri cannot get UIDVALIDITY";
$r_uidnext //= $mic->uidnext($mbx) //
- return "E: $uri cannot get UIDNEXT";
- my $url = ref($uri)->new($$uri);
- $url->uidvalidity($r_uidval);
- $url = $$url;
- my $itrk = _itrk($self, $uri);
- my $l_uid;
- $l_uid = $itrk->get_last($r_uidval) if $itrk;
+ return "E: $orig_uri cannot get UIDNEXT";
+ my $uri = $orig_uri->clone;
+ my ($itrk, $l_uid, $l_uidval) = _itrk_last($self, $uri, $r_uidval);
+ return <<EOF if $l_uidval != $r_uidval;
+E: $uri UIDVALIDITY mismatch
+E: local=$l_uidval != remote=$r_uidval
+EOF
+ $uri->uidvalidity($r_uidval);
$l_uid //= 0;
my $r_uid = $r_uidnext - 1;
- if ($l_uid > $r_uid) {
- return "E: $uri local UID exceeds remote ($l_uid > $r_uid)\n";
- }
+ return <<EOF if $l_uid > $r_uid;
+E: $uri local UID exceeds remote ($l_uid > $r_uid)
+E: $uri strangely, UIDVALIDLITY matches ($l_uidval)
+EOF
return if $l_uid >= $r_uid; # nothing to do
$l_uid ||= 1;
my ($mod, $shard) = @{$self->{shard_info} // []};
@@ -458,7 +471,7 @@ sub _imap_fetch_all ($$$) {
# messages get deleted, so holes appear
my $per_uid = delete $r->{$uid} // next;
my $raw = delete($per_uid->{$key}) // next;
- _imap_do_msg($self, $url, $uid, \$raw,
+ _imap_do_msg($self, $$uri, $uid, \$raw,
$per_uid->{FLAGS});
$last_uid = $uid;
last if $self->{quit};
@@ -547,8 +560,7 @@ sub _nntp_fetch_all ($$$) {
# IMAPTracker is also used for tracking NNTP, UID == article number
# LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
# expensive. So we assume newsgroups don't change:
- my $itrk = _itrk($self, $uri);
- my (undef, $l_art) = $itrk ? $itrk->get_last : ();
+ my ($itrk, $l_art) = _itrk_last($self, $uri);
# allow users to specify articles to refetch
# cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt
diff --git a/lib/PublicInbox/URIimap.pm b/lib/PublicInbox/URIimap.pm
index df9f5fd9..f6244137 100644
--- a/lib/PublicInbox/URIimap.pm
+++ b/lib/PublicInbox/URIimap.pm
@@ -144,4 +144,6 @@ sub scheme {
sub as_string { ${$_[0]} }
+sub clone { ref($_[0])->new(as_string($_[0])) }
+
1;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index cf1fa49d..611328b4 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -45,8 +45,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
is_deeply(\%r, { 'HASH' => scalar(@$out) }, 'all hashes');
lei_ok([qw(tag +kw:seen), $url], undef, undef);
- my $f = "$ENV{HOME}/.local/share/lei/store/net_last.sqlite3";
- ok(-s $f, 'net tracked for redundant imports');
+ my $f = "$ENV{HOME}/.local/share/lei/store/mail_sync.sqlite3";
+ ok(-s $f, 'mail_sync tracked for redundant imports');
lei_ok('inspect', "blob:$out->[5]->{blob}");
my $x = json_utf8->decode($lei_out);
is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect');
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index d795a86a..12bb002a 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -27,7 +27,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
for (@$out) { $r{ref($_)}++ }
is_deeply(\%r, { 'HASH' => scalar(@$out) }, 'all hashes');
- my $f = "$ENV{HOME}/.local/share/lei/store/net_last.sqlite3";
- ok(-s $f, 'net tracked for redundant imports');
+ my $f = "$ENV{HOME}/.local/share/lei/store/mail_sync.sqlite3";
+ ok(-s $f, 'mail_sync exists tracked for redundant imports');
});
done_testing;
^ permalink raw reply related [relevance 33%]
* [PATCH 0/4] some lei synchronization work
@ 2021-04-29 9:46 69% Eric Wong
2021-04-29 9:46 33% ` [PATCH 2/4] lei import: avoid IMAPTracker, use LeiMailSync more Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-04-29 9:46 UTC (permalink / raw)
To: meta
2/4 is a fairly big change to recent-ish behavior; but ought to
help us avoid problems by avoiding mismatched sources of truth.
-watch once again halts on UIDVALIDITY changes (matching <=1.6
behavior).
Tor .onion URLs for NNTP and IMAP also aren't supported unless
the entire lei-daemon process is running under torsocks.
Net::NNTP doesn't seem to have provisions for using
user-supplied classes like IO::Socket::SOCKS, either.
Mail::IMAPClient supports ->Socket and ->RawSocket, at least...
Eric Wong (4):
content_hash: git_sha: allow unblessed SCALAR refs
lei import: avoid IMAPTracker, use LeiMailSync more
lei import: support UIDVALIDITY in IMAP URL
lei import: support shell completion of known folders
Documentation/lei-store-format.pod | 2 +-
contrib/completion/lei-completion.bash | 1 +
lib/PublicInbox/ContentHash.pm | 6 +--
lib/PublicInbox/IMAPTracker.pm | 35 +++++---------
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiExternal.pm | 22 +++++----
lib/PublicInbox/LeiImport.pm | 20 ++++----
lib/PublicInbox/LeiInput.pm | 20 +++++---
lib/PublicInbox/LeiMailSync.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 4 +-
lib/PublicInbox/LeiViewText.pm | 1 -
lib/PublicInbox/NetReader.pm | 67 +++++++++++++++++---------
lib/PublicInbox/URIimap.pm | 2 +
t/lei-import-imap.t | 8 ++-
t/lei-import-nntp.t | 4 +-
t/solver_git.t | 7 +--
16 files changed, 112 insertions(+), 91 deletions(-)
^ permalink raw reply [relevance 69%]
* Re: [PATCH 2/3] doc: lei q: split =item aliases onto separate lines
2021-04-29 1:39 71% ` Kyle Meyer
@ 2021-04-29 1:57 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-29 1:57 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Kyle Meyer <kyle@kyleam.com> wrote:
> Eric Wong writes:
>
> > It makes L</--augment> look nicer without resorting to
> > L<--augment|/-a, --augment> and similarly verbose nastiness.
> >
> > Having each option as a separate =item (with a blank line in
> > between each =item) seems to be the preferred style used within
> > Perl core documentation (I used perlrun.pod as an example),
> > so we'll follow Perl core style, here.
>
> Okay, thanks providing a rationale for using separate lines. I went
> with the single line style based on some combination of a subjective
> visual preference and looking at other manpages (presumably git's).
Yeah, also the existing public-inbox-* manpages were using
commas or slash; probably because I based them on git manpages :x
> > This needs to be done for other manpages, at some point...
>
> I should be able to get to another round of lei doc updates this
> weekend.
Thanks in advance. Also, in response to
https://public-inbox.org/meta/20210227180328.28057-1-kyle@kyleam.com/
w.r.t. lei-convert; I think it's helpful to document
since the WWW interface provides gzipped mboxrd and
IMAP|Maildir are probably the most commonly used.
^ permalink raw reply [relevance 71%]
* Re: [PATCH 2/3] doc: lei q: split =item aliases onto separate lines
2021-04-28 4:51 54% ` [PATCH 2/3] doc: lei q: split =item aliases onto separate lines Eric Wong
@ 2021-04-29 1:39 71% ` Kyle Meyer
2021-04-29 1:57 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Kyle Meyer @ 2021-04-29 1:39 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> It makes L</--augment> look nicer without resorting to
> L<--augment|/-a, --augment> and similarly verbose nastiness.
>
> Having each option as a separate =item (with a blank line in
> between each =item) seems to be the preferred style used within
> Perl core documentation (I used perlrun.pod as an example),
> so we'll follow Perl core style, here.
Okay, thanks providing a rationale for using separate lines. I went
with the single line style based on some combination of a subjective
visual preference and looking at other manpages (presumably git's).
> This needs to be done for other manpages, at some point...
I should be able to get to another round of lei doc updates this
weekend.
^ permalink raw reply [relevance 71%]
* [PATCH 2/2] lei: avoid close(STD{IN,OUT,ERR}) in oneshot mode
@ 2021-04-28 19:37 63% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 19:37 UTC (permalink / raw)
To: meta
This seems to fix the occasional "make check-run" failures I've
been chasing.
Some parts of our code assumes we can close($lei->{1})
and similar, which causes IO::Handle::autoflush to behave
badly when STDOUT is the "select"-ed FH of the Perl process.
Since oneshot mode is (hopefully) the uncommon case, we'll
just accept the cost of extra FDs and minimize differences
between lei in oneshot vs daemon mode.
---
lib/PublicInbox/LEI.pm | 18 ++++--------------
t/lei.t | 1 +
2 files changed, 5 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 7ffcf163..1ea7c9ca 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -487,19 +487,14 @@ sub _lei_atfork_child {
# we need to explicitly close things which are on stack
if ($persist) {
chdir '/' or die "chdir(/): $!";
- my @io = delete @$self{qw(0 1 2 sock)};
- unless ($self->{oneshot}) {
- close($_) for @io;
- }
+ close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
if (my $cfg = $self->{cfg}) {
delete $cfg->{-lei_store};
}
} else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
}
- for (delete @$self{qw(3 old_1 au_done)}) {
- close($_) if defined($_);
- }
+ close($_) for (grep(defined, delete @$self{qw(3 old_1 au_done)}));
if (my $op_c = delete $self->{pkt_op_c}) {
close(delete $op_c->{sock});
}
@@ -1213,13 +1208,8 @@ sub oneshot {
local $quit = $exit if $exit;
local %PATH2CFG;
umask(077) // die("umask(077): $!");
- my $self = bless {
- oneshot => 1,
- 0 => *STDIN{GLOB},
- 1 => *STDOUT{GLOB},
- 2 => *STDERR{GLOB},
- env => \%ENV
- }, __PACKAGE__;
+ my $self = bless { oneshot => 1, env => \%ENV }, __PACKAGE__;
+ for (0..2) { open($self->{$_}, '+<&=', $_) or die "open fd=$_: $!" }
dispatch($self, @ARGV);
x_it($self, $self->{child_error}) if $self->{child_error};
}
diff --git a/t/lei.t b/t/lei.t
index 6d276050..8211c01d 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -154,6 +154,7 @@ my $test_fail = sub {
}
}
lei_ok('sucks', \'yes, but hopefully less every day');
+ like($lei_out, qr/loaded features/, 'loaded features shown');
SKIP: {
skip 'no curl', 3 unless which('curl');
lei(qw(q --only http://127.0.0.1:99999/bogus/ t:m));
^ permalink raw reply related [relevance 63%]
* [PATCH 11/11] lei (lcat|q): support --no-color and --color
2021-04-28 7:51 68% [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (4 preceding siblings ...)
2021-04-28 7:52 44% ` [PATCH 08/11] lei: simple WQ workers use {wq1} field Eric Wong
@ 2021-04-28 7:52 60% ` Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
This should be familiar to git users who wish to force color
when writing to pipes or disable color.
---
lib/PublicInbox/LEI.pm | 9 +++++----
lib/PublicInbox/LeiViewText.pm | 2 +-
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 403f9ed8..7ffcf163 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -144,8 +144,8 @@ our %CMD = ( # sorted in order of importance/use:
@lxs_opt,
qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
- import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
- opt_dash('limit|n=i', '[0-9]+') ],
+ import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+ color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
@@ -155,8 +155,8 @@ our %CMD = ( # sorted in order of importance/use:
# some of these options are ridiculous for lcat
@lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i jobs|j=s globoff|g augment|a
- import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
- opt_dash('limit|n=i', '[0-9]+') ],
+ import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+ color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
'blob' => [ 'OID', 'show a git blob, reconstructing from mail if necessary',
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@@ -268,6 +268,7 @@ my %OPTDESC = (
'incremental! import' => 'import already seen IMAP and NNTP articles',
'globoff|g' => "do not match locations using '*?' wildcards ".
"and\xa0'[]'\x{a0}ranges",
+'color!' => 'disable color (for --format=text)',
'verbose|v+' => 'be more verbose',
'external!' => 'do not use externals',
'mail!' => 'do not look in mail storage for OID',
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index e0d62c0d..d0f8b7f4 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -69,7 +69,7 @@ sub uncolored { ${$_[0]->{obuf}} .= $_[2] }
sub new {
my ($cls, $lei) = @_;
my $self = bless { %{$lei->{opt}}, -colored => \&uncolored }, $cls;
- return $self unless $self->{color} || -t $lei->{1};
+ return $self unless $self->{color} //= -t $lei->{1};
my $cmd = [ qw(git config -z --includes -l) ];
my ($r, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
my $cfg = PublicInbox::Config::config_fh_parse($r, "\0", "\n");
^ permalink raw reply related [relevance 60%]
* [PATCH 08/11] lei: simple WQ workers use {wq1} field
2021-04-28 7:51 68% [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (3 preceding siblings ...)
2021-04-28 7:52 59% ` [PATCH 07/11] lei: quiet down Eml-related warnings consistently Eric Wong
@ 2021-04-28 7:52 44% ` Eric Wong
2021-04-28 7:52 60% ` [PATCH 11/11] lei (lcat|q): support --no-color and --color Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
This lets us share more code and reduces cognitive overhead when
it comes to picking names (because {lsss} was ridiculous).
We'll need to ensure the first error set in lei is the actual
error we exit with, otherwise things can get confusing and
errors may get lost.
---
lib/PublicInbox/LEI.pm | 16 ++++++++++++----
lib/PublicInbox/LeiBlob.pm | 8 +-------
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 9 ++-------
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
lib/PublicInbox/LeiP2q.pm | 8 +-------
lib/PublicInbox/LeiTag.pm | 9 ++-------
script/lei | 4 ++--
9 files changed, 24 insertions(+), 38 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index cfbf12f0..403f9ed8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -382,7 +382,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol lsss); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
sub _drop_wq {
my ($self) = @_;
@@ -542,7 +542,7 @@ sub workers_start {
'child_error' => [ \&child_error, $lei ],
($ops ? %$ops : ()),
};
- $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&dclose, $lei ];
+ $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
delete $lei->{pkt_op_p};
@@ -1237,9 +1237,17 @@ sub DESTROY {
sub wq_done_wait { # dwaitpid callback
my ($arg, $pid) = @_;
- my ($wq, $lei, $e) = @$arg;
- $? and $lei->child_error($?, $e ? "$e errors during $lei->{cmd}" : ());
+ my ($wq, $lei) = @$arg;
+ my $err_type = $lei->{-err_type};
+ $? and $lei->child_error($?,
+ $err_type ? "$err_type errors during $lei->{cmd}" : ());
$lei->dclose;
}
+sub wq_eof { # EOF callback for main daemon
+ my ($lei) = @_;
+ my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
+ $wq1->wq_wait_old(\&wq_done_wait, $lei);
+}
+
1;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index ff079e65..0a957358 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -10,12 +10,6 @@ use parent qw(PublicInbox::IPC);
use PublicInbox::Spawn qw(spawn popen_rd which);
use PublicInbox::DS;
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $sol = delete $lei->{sol} // return $lei->dclose; # already failed
- $sol->wq_wait_old($lei->can('wq_done_wait'), $lei);
-}
-
sub get_git_dir ($$) {
my ($lei, $d) = @_;
return $d if -d "$d/objects" && -d "$d/refs" && -e "$d/HEAD";
@@ -158,7 +152,7 @@ EOM
require PublicInbox::SolverGit;
my $self = bless { lxs => $lxs, oid_b => $blob }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 'lei-blob', 1);
- $lei->{sol} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_solve_blob', []);
$self->wq_close(1);
$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 14bed901..cefcaf65 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -53,7 +53,7 @@ sub lei_convert { # the main "lei convert" method
$lei->{opt}->{augment} = 1 if $devfd < 0;
$self->prepare_inputs($lei, \@inputs) or return;
my ($op_c, $ops) = $lei->workers_start($self, 'lei-convert', 1);
- $lei->{cnv} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('process_inputs', []);
$self->wq_close(1);
$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index f2a0c95a..26127ece 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -53,12 +53,6 @@ sub input_nntp_cb { # nntp_each
input_eml_cb($self, $eml, $self->{-import_kw} ? { kw => $kw } : undef);
}
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $imp = delete $lei->{imp} // return $lei->fail('BUG: {imp} gone');
- $imp->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
-}
-
sub net_merge_complete { # callback used by LeiAuth
my ($self) = @_;
$self->wq_io_do('process_inputs');
@@ -95,7 +89,8 @@ sub lei_import { # the main "lei import" method
$self->{-wq_nr_workers} = $j // 1; # locked
$lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
- $lei->{imp} = $self;
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
net_merge_complete($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 9ac4870f..a00e78fc 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -73,7 +73,7 @@ sub bg_worker ($$$) {
my ($lei, $pfx, $json) = @_;
my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1);
- $lei->{lsss} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
$self->wq_close(1);
$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 50ab4c85..db97b98c 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -26,7 +26,7 @@ sub do_finish_mirror { # dwaitpid callback
sub _lei_wq_eof { # EOF callback for main daemon
my ($lei) = @_;
- my $mrr = delete $lei->{mrr} or return;
+ my $mrr = delete $lei->{wq1} or return $lei->fail;
$mrr->wq_wait_old(\&do_finish_mirror, $lei);
}
@@ -283,7 +283,7 @@ sub start {
require PublicInbox::Admin;
require PublicInbox::InboxWritable;
my ($op, $ops) = $lei->workers_start($self, 'lei_mirror', 1);
- $lei->{mrr} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_mirror', []);
$self->wq_close(1);
$op->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index deb31974..b4893489 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -189,7 +189,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
$self->{input} = $input;
}
my ($op, $ops) = $lei->workers_start($self, 'lei-p2q', 1);
- $lei->{p2q} = $self;
+ $lei->{wq1} = $self;
$self->wq_io_do('do_p2q', []);
$self->wq_close(1);
$op->op_wait_event($ops);
@@ -201,10 +201,4 @@ sub ipc_atfork_child {
$self->SUPER::ipc_atfork_child;
}
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $p2q = delete $lei->{p2q} // return $lei->dclose;
- $p2q->wq_wait_old($lei->can('wq_done_wait'), $lei);
-}
-
1;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 3cda2eca..989a6954 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -19,12 +19,6 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
-sub _lei_wq_eof { # EOF callback for main daemon
- my ($lei) = @_;
- my $tag = delete $lei->{tag} // return $lei->dclose;
- $tag->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
-}
-
sub net_merge_complete { # callback used by LeiAuth
my ($self) = @_;
$self->wq_io_do('process_inputs');
@@ -57,7 +51,8 @@ sub lei_tag { # the "lei tag" method
$self->{vmd_mod} = $vmd_mod;
my $j = $self->{-wq_nr_workers} = 1; # locked for now
(my $op_c, $ops) = $lei->workers_start($self, 'lei-tag', $j, $ops);
- $lei->{tag} = $self;
+ $lei->{wq1} = $self;
+ $lei->{-err_type} = 'non-fatal';
net_merge_complete($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
}
diff --git a/script/lei b/script/lei
index db302422..90a93839 100755
--- a/script/lei
+++ b/script/lei
@@ -116,10 +116,10 @@ Falling back to (slow) one-shot mode
} elsif ($buf eq '-WINCH') {
kill($buf, @parent); # for MUA
} elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
- $x_it_code = $1 + 0;
+ $x_it_code ||= $1 + 0;
last;
} elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
- $x_it_code = $1 + 0;
+ $x_it_code ||= $1 + 0;
} else {
$sigchld->();
die $buf;
^ permalink raw reply related [relevance 44%]
* [PATCH 01/11] t/lei-p2q: add diagnostics
2021-04-28 7:51 68% [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
@ 2021-04-28 7:51 71% ` Eric Wong
2021-04-28 7:51 52% ` [PATCH 02/11] tests: restore CWD with "lei -C" and run_script Eric Wong
` (4 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
Maybe this helps fix occasional problems in daemon mode,
but I haven't seen anything on failure, yet...
---
t/lei-p2q.t | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index be2d437c..f8b073cf 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -9,10 +9,10 @@ test_lei(sub {
ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
'p2q fails on bogus arg');
lei_ok(qw(p2q -w dfpost t/data/0001.patch));
- is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname');
+ is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname') or diag $lei_err;
open my $fh, '+<', 't/data/0001.patch' or xbail "open: $!";
lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
- is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin');
+ is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin') or diag $lei_err;
lei_ok(qw(p2q --uri t/data/0001.patch -w), 'dfpost,dfn');
is($lei_out, "dfpost%3A6e006fd73b1d+".
^ permalink raw reply related [relevance 71%]
* [PATCH 05/11] lei-daemon: note FD count mismatch to client
2021-04-28 7:51 68% [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
2021-04-28 7:51 71% ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
2021-04-28 7:51 52% ` [PATCH 02/11] tests: restore CWD with "lei -C" and run_script Eric Wong
@ 2021-04-28 7:51 71% ` Eric Wong
2021-04-28 7:52 59% ` [PATCH 07/11] lei: quiet down Eml-related warnings consistently Eric Wong
` (2 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
This should help in some error diagnostics
---
lib/PublicInbox/LEI.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ef72758c..a949ae3e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -998,7 +998,7 @@ sub accept_dispatch { # Listener {post_accept} callback
open($self->{$i++}, '+<&=', $fd) and next;
send($sock, "open(+<&=$fd) (FD=$i): $!", MSG_EOR);
}
- return if scalar(@fds) != 4;
+ $i == 4 or return send($sock, 'not enough FDs='.($i-1), MSG_EOR)
}
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
# $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);
^ permalink raw reply related [relevance 71%]
* [PATCH 00/11] lei: misc fixes, more lcat color support
@ 2021-04-28 7:51 68% Eric Wong
2021-04-28 7:51 71% ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
` (5 more replies)
0 siblings, 6 replies; 200+ results
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
I'm seeing some odd test failures in "make check-run"
(but not "make check") that I haven't diagnosed, yet.
So there's some diagnostic changes and some golfing
to make internals more consistent.
Eric Wong (11):
t/lei-p2q: add diagnostics
tests: restore CWD with "lei -C" and run_script
view_diff: minor coding style fixes
lei_p2q: add _lei_wq_eof callback
lei-daemon: note FD count mismatch to client
t/run.perl: add (GNU) tail and strace support
lei: quiet down Eml-related warnings consistently
lei: simple WQ workers use {wq1} field
lei_view_text: improve attachment display
lei_view_text: translate background colors from git
lei (lcat|q): support --no-color and --color
lib/PublicInbox/LEI.pm | 30 ++++++++++++++++++++---------
lib/PublicInbox/LeiBlob.pm | 9 +--------
lib/PublicInbox/LeiConvert.pm | 3 +--
lib/PublicInbox/LeiImport.pm | 10 +++-------
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMirror.pm | 4 ++--
lib/PublicInbox/LeiP2q.pm | 6 ++----
lib/PublicInbox/LeiStore.pm | 1 -
lib/PublicInbox/LeiTag.pm | 9 ++-------
lib/PublicInbox/LeiViewText.pm | 35 +++++++++++++++++++++++-----------
lib/PublicInbox/LeiXSearch.pm | 1 -
lib/PublicInbox/TestCommon.pm | 12 ++++++++----
lib/PublicInbox/ViewDiff.pm | 9 ++++-----
script/lei | 4 ++--
t/lei-externals.t | 5 -----
t/lei-p2q.t | 4 ++--
t/lei-q-save.t | 4 ----
t/run.perl | 7 +++++++
t/solver_git.t | 5 -----
19 files changed, 80 insertions(+), 80 deletions(-)
^ permalink raw reply [relevance 68%]
* [PATCH 07/11] lei: quiet down Eml-related warnings consistently
2021-04-28 7:51 68% [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
` (2 preceding siblings ...)
2021-04-28 7:51 71% ` [PATCH 05/11] lei-daemon: note FD count mismatch to client Eric Wong
@ 2021-04-28 7:52 59% ` Eric Wong
2021-04-28 7:52 44% ` [PATCH 08/11] lei: simple WQ workers use {wq1} field Eric Wong
2021-04-28 7:52 60% ` [PATCH 11/11] lei (lcat|q): support --no-color and --color Eric Wong
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 7:52 UTC (permalink / raw)
To: meta
"lei import" is probably the only place where it users
might care about warnings.
---
lib/PublicInbox/LEI.pm | 3 +++
lib/PublicInbox/LeiBlob.pm | 1 -
lib/PublicInbox/LeiConvert.pm | 1 -
lib/PublicInbox/LeiImport.pm | 1 +
lib/PublicInbox/LeiP2q.pm | 4 +---
lib/PublicInbox/LeiStore.pm | 1 -
lib/PublicInbox/LeiXSearch.pm | 1 -
7 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a949ae3e..cfbf12f0 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -23,6 +23,7 @@ use PublicInbox::Sigfd;
use PublicInbox::DS qw(now dwaitpid);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::Lock;
+use PublicInbox::Eml;
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
@@ -509,6 +510,8 @@ sub _lei_atfork_child {
%PATH2CFG = ();
undef $errors_log;
$quit = \&CORE::exit;
+ $self->{-eml_noisy} or # only "lei import" sets this atm
+ $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$current_lei = $persist ? undef : $self; # for SIG{__WARN__}
}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 0b96bd04..ff079e65 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -167,7 +167,6 @@ EOM
sub ipc_atfork_child {
my ($self) = @_;
$self->{lei}->_lei_atfork_child;
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 0c324169..14bed901 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -68,7 +68,6 @@ sub ipc_atfork_child {
$net->{mics_cached} = $net->imap_common_init($lei);
$net->{nn_cached} = $net->nntp_common_init($lei);
}
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$l2m->pre_augment($lei);
$l2m->do_augment($lei);
$l2m->post_augment($lei);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index e0d899cc..f2a0c95a 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -93,6 +93,7 @@ sub lei_import { # the main "lei import" method
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
+ $lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
$lei->{imp} = $self;
net_merge_complete($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index 07357e32..deb31974 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -197,9 +197,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
sub ipc_atfork_child {
my ($self) = @_;
- my $lei = $self->{lei};
- $lei->_lei_atfork_child;
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
+ $self->{lei}->_lei_atfork_child;
$self->SUPER::ipc_atfork_child;
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 1cf7ffc1..fcc9224d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -411,7 +411,6 @@ sub ipc_atfork_child {
close $err->[0];
$self->{-err_wr} = $err->[1];
}
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 018b60f9..b3fd79d0 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -443,7 +443,6 @@ sub incr_start_query { # called whenever an l2m shard starts do_post_auth
sub ipc_atfork_child {
my ($self) = @_;
$self->{lei}->_lei_atfork_child;
- $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
}
^ permalink raw reply related [relevance 59%]
* [PATCH 02/11] tests: restore CWD with "lei -C" and run_script
2021-04-28 7:51 68% [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
2021-04-28 7:51 71% ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
@ 2021-04-28 7:51 52% ` Eric Wong
2021-04-28 7:51 71% ` [PATCH 05/11] lei-daemon: note FD count mismatch to client Eric Wong
` (3 subsequent siblings)
5 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 7:51 UTC (permalink / raw)
To: meta
This simplifies test_lei users in t/*.t
---
lib/PublicInbox/TestCommon.pm | 12 ++++++++----
t/lei-externals.t | 5 -----
t/lei-q-save.t | 4 ----
t/solver_git.t | 5 -----
4 files changed, 8 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 09256852..460c9da0 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -12,6 +12,7 @@ use IO::Socket::INET;
use File::Spec;
our @EXPORT;
my $lei_loud = $ENV{TEST_LEI_ERR_LOUD};
+our ($lei_opt, $lei_out, $lei_err, $lei_cwdfh);
BEGIN {
@EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
run_script start_script key2sub xsys xsys_e xqx eml_load tick
@@ -306,14 +307,16 @@ sub run_script ($;$$) {
local %SIG = %SIG;
local $0 = join(' ', @$cmd);
my $orig_io = _prepare_redirects($fhref);
- my $cwdfh;
+ my $cwdfh = $lei_cwdfh;
if (my $d = $opt->{'-C'}) {
- opendir $cwdfh, '.' or die "opendir .: $!";
+ unless ($cwdfh) {
+ opendir $cwdfh, '.' or die "opendir .: $!";
+ }
chdir $d or die "chdir $d: $!";
}
_run_sub($sub, $key, \@argv);
eval { PublicInbox::Inbox::cleanup_task() };
- die "chdir(restore): $!" if $cwdfh && !chdir($cwdfh);
+ die "fchdir(restore): $!" if $cwdfh && !chdir($cwdfh);
_undo_redirects($orig_io);
select STDOUT;
}
@@ -469,7 +472,6 @@ sub have_xapian_compact () {
PublicInbox::Spawn::which($ENV{XAPIAN_COMPACT} || 'xapian-compact');
}
-our ($lei_opt, $lei_out, $lei_err);
# favor lei() or lei_ok() over $lei for new code
sub lei (@) {
my ($cmd, $env, $xopt) = @_;
@@ -515,6 +517,8 @@ sub test_lei {
SKIP: {
my ($cb) = pop @_;
my $test_opt = shift // {};
+ local $lei_cwdfh;
+ opendir $lei_cwdfh, '.' or xbail "opendir .: $!";
require_git(2.6, 1) or skip('git 2.6+ required for lei test', 2);
require_mods(qw(json DBD::SQLite Search::Xapian), 2);
require PublicInbox::Config;
diff --git a/t/lei-externals.t b/t/lei-externals.t
index 2291dd99..16241e02 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -4,7 +4,6 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use Fcntl qw(SEEK_SET);
use PublicInbox::Spawn qw(which);
-use PublicInbox::OnDestroy;
require_git 2.6;
require_mods(qw(json DBD::SQLite Search::Xapian));
use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
@@ -266,10 +265,6 @@ test_lei(sub {
{
skip 'TEST_LEI_DAEMON_PERSIST_DIR in use', 1 if
$ENV{TEST_LEI_DAEMON_PERSIST_DIR};
- opendir my $dh, '.' or BAIL_OUT "opendir(.) $!";
- my $od = PublicInbox::OnDestroy->new($$, sub {
- chdir $dh or BAIL_OUT "chdir: $!"
- });
my @q = qw(q -o mboxcl2:rel.mboxcl2 bye);
lei_ok('-C', $home, @q);
is(unlink("$home/rel.mboxcl2"), 1, '-C works before q');
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 170f7ce5..9f65e4a2 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -36,11 +36,9 @@ test_lei(sub {
# ensure "lei up" works, since it compliments "lei q --save"
$in = $doc2->as_string;
lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
- opendir my $dh, '.' or xbail "opendir .: $!";
lei_ok qw(up -q md -C), $home;
lei_ok qw(up -q . -C), "$home/md";
lei_ok qw(up -q), "/$home/md";
- chdir($dh) or xbail "fchdir . $!";
my %after = map { $_ => 1 } glob("$home/md/cur/*");
is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
is(scalar(keys %after), 1, 'one new message added');
@@ -89,7 +87,6 @@ test_lei(sub {
print $mb $pre_existing;
close $mb or xbail "close: $!";
lei_ok(qw(q --save -o mboxrd:mbrd m:qp@example.com -C), $home);
- chdir($dh) or xbail "fchdir . $!";
open $mb, '<', "$home/mbrd" or xbail "open $!";
is_deeply([grep(/pre-existing/, <$mb>)], [],
'pre-existing messsage gone w/o augment');
@@ -103,7 +100,6 @@ test_lei(sub {
print $mb $pre_existing;
close $mb or xbail "close: $!";
lei_ok(qw(q -a --save -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
- chdir($dh) or xbail "fchdir . $!";
open $mb, '<', "$home/mbrd-aug" or xbail "open $!";
$mb = do { local $/; <$mb> };
like($mb, qr/pre-existing/, 'pre-existing message preserved w/ -a');
diff --git a/t/solver_git.t b/t/solver_git.t
index 8acf907f..6875e26b 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -50,11 +50,6 @@ test_lei({tmpdir => $tmpdir}, sub {
'non-existent blob fails');
SKIP: {
skip '/.git exists', 1 if -e '/.git';
- require PublicInbox::OnDestroy;
- opendir my $dh, '.' or xbail "opendir: $!";
- my $end = PublicInbox::OnDestroy->new($$, sub {
- chdir $dh or xbail "chdir: $!";
- });
lei_ok(qw(-C / blob 69df7d5 -I), $ibx->{inboxdir},
"--git-dir=$git_dir");
is($lei_out, $prev, '--git-dir works');
^ permalink raw reply related [relevance 52%]
* [PATCH 0/3] doc: lei updates around lei-q
@ 2021-04-28 4:51 90% Eric Wong
2021-04-28 4:51 64% ` [PATCH 1/3] doc: lei: use /tmp for search results pathnames Eric Wong
` (2 more replies)
0 siblings, 3 replies; 200+ results
From: Eric Wong @ 2021-04-28 4:51 UTC (permalink / raw)
To: meta
1/3 is probably a game changer and will help anybody supporting
this project sleep easier at night :)
Eric Wong (3):
doc: lei: use /tmp for search results pathnames
doc: lei q: split =item aliases onto separate lines
doc: lei q: split --output and --format, note "text"
Documentation/lei-overview.pod | 4 +-
Documentation/lei-q.pod | 88 +++++++++++++++++++++++-----------
2 files changed, 63 insertions(+), 29 deletions(-)
^ permalink raw reply [relevance 90%]
* [PATCH 1/3] doc: lei: use /tmp for search results pathnames
2021-04-28 4:51 90% [PATCH 0/3] doc: lei updates around lei-q Eric Wong
@ 2021-04-28 4:51 64% ` Eric Wong
2021-04-28 4:51 54% ` [PATCH 2/3] doc: lei q: split =item aliases onto separate lines Eric Wong
2021-04-28 4:51 63% ` [PATCH 3/3] doc: lei q: split --output and --format, note "text" Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 4:51 UTC (permalink / raw)
To: meta
This drives the point home about results being volatile
and discardable.
---
Documentation/lei-overview.pod | 4 ++--
Documentation/lei-q.pod | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index 70dbf2b5..6b5fa721 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -82,7 +82,7 @@ Search for messages whose subject includes "lei" and "skeleton".
Do the same, but also report unmatched messages that are in the same
thread as a matched message.
-=item $ lei q -t -o mdir --mua=mutt s:lei s:skeleton
+=item $ lei q -t -o /tmp/mdir --mua=mutt s:lei s:skeleton
Write results to a Maildir at "mdir". Mutt will be invoked
to open mfolder (C<mutt -f %f>) while results are being fetched
@@ -92,7 +92,7 @@ and written.
Search for all flagged messages that also have a "next" label.
-=item $ lei p2q HEAD | lei q --stdin -tt -o mdir
+=item $ lei p2q HEAD | lei q --stdin -tt -o /tmp/mdir
Search for messages that have post-image git blob IDs that match those
of the current repository's HEAD commit, writing them to the Maildir
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index a84fc440..b938746a 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -31,9 +31,9 @@ Read search terms from stdin.
Warning: this clobbers and overwrites the output destination unless
L</-a, --augment> is specified.
-Destination for results (e.g., C<path/to/Maildir>,
+Destination for results (e.g., C</tmp/results-Maildir>,
C<imaps://user@mail.example.com/INBOX.test>, or
-C<mboxcl2:path/to/mbox>). The prefix may be a supported protocol:
+C<mboxcl2:/tmp/results-mboxcl2>). The prefix may be a supported protocol:
C<imap://> or C<imaps://>. URLs requiring
authentication must use L<netrc(5)> and/or L<git-credential(1)> to
fill in the username and password.
^ permalink raw reply related [relevance 64%]
* [PATCH 3/3] doc: lei q: split --output and --format, note "text"
2021-04-28 4:51 90% [PATCH 0/3] doc: lei updates around lei-q Eric Wong
2021-04-28 4:51 64% ` [PATCH 1/3] doc: lei: use /tmp for search results pathnames Eric Wong
2021-04-28 4:51 54% ` [PATCH 2/3] doc: lei q: split =item aliases onto separate lines Eric Wong
@ 2021-04-28 4:51 63% ` Eric Wong
2 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-28 4:51 UTC (permalink / raw)
To: meta
I guess --format makes sense for stdout, after all;
and I'm enjoying "-f text" quite a bit, so far.
---
Documentation/lei-q.pod | 26 +++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 2b9936b8..bf7a5f70 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -42,15 +42,11 @@ C<imap://> or C<imaps://>. URLs requiring
authentication must use L<netrc(5)> and/or L<git-credential(1)> to
fill in the username and password.
-The prefix can instead specify the format of the output: C<maildir>,
-C<mboxrd>, C<mboxcl2>, C<mboxcl>, C<mboxo>, C<json>, C<jsonl>, or
-C<concatjson>. When a format isn't specified, it's chosen based on
-the destination. C<json> is used for the default destination
-(stdout), and C<maildir> is used for an existing directory or
-non-existing path.
+A prefix can specify the format of the output: C<maildir>,
+C<mboxrd>, C<mboxcl2>, C<mboxcl>, C<mboxo>. For a description of
+mail formats, see L<lei-mail-formats(5)>.
-=for comment
-TODO: Provide description of formats?
+C<maildir> is the default for an existing directory or non-existing path.
Default: C<-> (stdout)
@@ -58,9 +54,17 @@ Default: C<-> (stdout)
=item -f FORMAT
-Format of results. This option exists as a convenient way to specify
-the format for the default stdout destination. Using a C<format:>
-prefix with the C<--output> destination is preferred otherwise.
+Format of results to stdout. This option exists as a convenient
+way to specify the format for the default stdout destination.
+C<text>, C<json>, C<jsonl>, or C<concatjson> are all supported,
+as are the various mbox variants described in L</--output>.
+
+When a format isn't specified, it's chosen based on the
+L</--output> destination or prefix. C<json> is used for the
+default destination (stdout).
+
+Using a C<format:> prefix with the C<--output> destination is
+preferred when not writing to stdout.
=item --pretty
^ permalink raw reply related [relevance 63%]
* [PATCH 2/3] doc: lei q: split =item aliases onto separate lines
2021-04-28 4:51 90% [PATCH 0/3] doc: lei updates around lei-q Eric Wong
2021-04-28 4:51 64% ` [PATCH 1/3] doc: lei: use /tmp for search results pathnames Eric Wong
@ 2021-04-28 4:51 54% ` Eric Wong
2021-04-29 1:39 71% ` Kyle Meyer
2021-04-28 4:51 63% ` [PATCH 3/3] doc: lei q: split --output and --format, note "text" Eric Wong
2 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-04-28 4:51 UTC (permalink / raw)
To: meta
It makes L</--augment> look nicer without resorting to
L<--augment|/-a, --augment> and similarly verbose nastiness.
Having each option as a separate =item (with a blank line in
between each =item) seems to be the preferred style used within
Perl core documentation (I used perlrun.pod as an example),
so we'll follow Perl core style, here.
This needs to be done for other manpages, at some point...
---
Documentation/lei-q.pod | 58 +++++++++++++++++++++++++++++++----------
1 file changed, 44 insertions(+), 14 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index b938746a..2b9936b8 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -26,10 +26,14 @@ TODO: mention curl options?
Read search terms from stdin.
-=item -o MFOLDER, --output=MFOLDER, --mfolder=MFOLDER
+=item --output=MFOLDER
+
+=item -o MFOLDER
+
+=item --mfolder=MFOLDER
Warning: this clobbers and overwrites the output destination unless
-L</-a, --augment> is specified.
+L</--augment> is specified.
Destination for results (e.g., C</tmp/results-Maildir>,
C<imaps://user@mail.example.com/INBOX.test>, or
@@ -50,7 +54,9 @@ TODO: Provide description of formats?
Default: C<-> (stdout)
-=item -f FORMAT, --format=FORMAT
+=item --format=FORMAT
+
+=item -f FORMAT
Format of results. This option exists as a convenient way to specify
the format for the default stdout destination. Using a C<format:>
@@ -81,7 +87,9 @@ This option may be given multiple times.
Default: C<:WINCH,:bell> when C<--mua> is specified and C<--output>
doesn't point to stdout, nothing otherwise.
-=item -a, --augment
+=item --augment
+
+=item -a
Augment output destination instead of clobbering it.
@@ -90,7 +98,9 @@ Augment output destination instead of clobbering it.
Do not importing keywords before writing to an existing output
destination.
-=item -t, --threads
+=item --threads
+
+=item -t
Return all messages in the same thread as the actual match(es).
@@ -102,7 +112,9 @@ of the same thread.
TODO: Warning: this flag may become persistent and saved in
lei/store unless an MUA unflags it! (Behavior undecided)
-=item -d STRATEGY, --dedupe=STRATEGY
+=item --dedupe=STRATEGY
+
+=item -d STRATEGY
Strategy for deduplicating messages: C<content>, C<oid>, C<mid>, or
C<none>.
@@ -126,7 +138,9 @@ Limit operations to those requiring network access.
Don't include results from externals.
-=item -I LOCATION, --include=LOCATION
+=item --include=LOCATION
+
+=item -I LOCATION
Include specified external in search. This option may be given
multiple times.
@@ -141,7 +155,9 @@ multiple times.
Use only the specified external for search. This option may be given
multiple times, in which case the search uses only the specified set.
-=item -g, --globoff
+=item --globoff
+
+=item -g
Do not match locations using C<*?> wildcards and C<[]> ranges. This
option applies to C<--include>, C<--exclude>, and C<--only>.
@@ -158,7 +174,11 @@ C<none>.
Default: fcntl,dotlock
-=item -NUMBER, -n NUMBER, --limit=NUMBER
+=item --limit=NUMBER
+
+=item -NUMBER
+
+=item -n NUMBER
Limit the number of matches.
@@ -170,26 +190,36 @@ Shift start of search results.
Default: 0
-=item -r, --reverse
+=item --reverse
+
+=item -r
Reverse the results. Note that this applies before C<--limit>.
-=item -s KEY, --sort=KEY
+=item --sort=KEY
+
+=item -s KEY
Order the results by KEY. Valid keys are C<received>, C<relevance>,
and C<docid>.
Default: C<received>
-=item -v, --verbose
+=item --verbose
+
+=item -v
Provide more feedback on stderr.
-=item -q, --quiet
+=item --quiet
+
+=item -q
Suppress feedback messages.
-=item --torsocks=auto|no|yes, --no-torsocks
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
^ permalink raw reply related [relevance 54%]
* [PATCH 0/5] lei lcat - local cat (not lolcat :P)
@ 2021-04-27 11:07 64% Eric Wong
2021-04-27 11:07 55% ` [PATCH 1/5] lei: add "ls-sync" command for listing sync folders Eric Wong
` (4 more replies)
0 siblings, 5 replies; 200+ results
From: Eric Wong @ 2021-04-27 11:07 UTC (permalink / raw)
To: meta
"lei lcat" is a convenience command to extract Message-IDs
from URLs and <$MSGID> or "id:$MSGID" args (or stdin) and
attempt to display them as text.
--format=text is now the default for lcat, and an option for
"lei q" for stdout users. It decodes base64 and QP just like
the WWW interface. It also supports ANSI terminal colors and
loads the diff ones from the users' existing git config.
It's actually my first time using Term::ANSIColor, even though
it's bundled with Perl since 5.6.
I got sidetracked on the sync stuff, but "ls-sync" exists, now.
I'm not sure how sync would work, especially since I want to
avoid reconnecting for imports...
Eric Wong (5):
lei: add "ls-sync" command for listing sync folders
lei blob: support retrieving attachments via $OID:$IDX
lei: standardize on _lei_wq_eof callback for workers
lei lcat: extract Message-IDs from URLs and show them
lei q + lcat: support --format=text output
MANIFEST | 5 +
lib/PublicInbox/Hval.pm | 2 +-
lib/PublicInbox/LEI.pm | 12 +-
lib/PublicInbox/LeiBlob.pm | 37 ++++-
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiExternal.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 6 +-
lib/PublicInbox/LeiLcat.pm | 125 +++++++++++++++++
lib/PublicInbox/LeiLsSync.pm | 29 ++++
lib/PublicInbox/LeiMirror.pm | 6 +-
lib/PublicInbox/LeiP2q.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 8 +-
lib/PublicInbox/LeiToMail.pm | 63 ++++++++-
lib/PublicInbox/LeiViewText.pm | 237 +++++++++++++++++++++++++++++++++
lib/PublicInbox/ViewDiff.pm | 4 +-
t/lei-import-imap.t | 8 ++
t/lei-import-maildir.t | 3 +
t/lei-lcat.t | 16 +++
t/lei_lcat.t | 44 ++++++
19 files changed, 584 insertions(+), 27 deletions(-)
create mode 100644 lib/PublicInbox/LeiLcat.pm
create mode 100644 lib/PublicInbox/LeiLsSync.pm
create mode 100644 lib/PublicInbox/LeiViewText.pm
create mode 100644 t/lei-lcat.t
create mode 100644 t/lei_lcat.t
^ permalink raw reply [relevance 64%]
* [PATCH 2/5] lei blob: support retrieving attachments via $OID:$IDX
2021-04-27 11:07 64% [PATCH 0/5] lei lcat - local cat (not lolcat :P) Eric Wong
2021-04-27 11:07 55% ` [PATCH 1/5] lei: add "ls-sync" command for listing sync folders Eric Wong
@ 2021-04-27 11:07 61% ` Eric Wong
2021-04-27 11:07 49% ` [PATCH 3/5] lei: standardize on _lei_wq_eof callback for workers Eric Wong
` (2 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-27 11:07 UTC (permalink / raw)
To: meta
We'll be supporting some sort of text view for pager or
piping to an $EDITOR buffer.
---
lib/PublicInbox/LeiBlob.pm | 32 ++++++++++++++++++++++++++++++--
t/lei-import-imap.t | 6 ++++++
2 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index e4cd4cca..4e52c8a5 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -84,12 +84,25 @@ sub do_solve_blob { # via wq_do
$solver->solve($lei->{env}, $log, $self->{oid_b}, $hints);
}
+sub cat_attach_i { # Eml->each_part callback
+ my ($part, $depth, $idx) = @{$_[0]};
+ my $lei = $_[1];
+ my $want = $lei->{-attach_idx} // return;
+ return if $idx ne $want; # [0-9]+(?:\.[0-9]+)+
+ delete $lei->{-attach_idx};
+ $lei->out($part->body);
+}
+
sub lei_blob {
my ($lei, $blob) = @_;
$lei->start_pager if -t $lei->{1};
my $opt = $lei->{opt};
my $has_hints = grep(defined, @$opt{qw(oid-a path-a path-b)});
my $lxs;
+ if ($blob =~ s/:([0-9\.]+)\z//) {
+ $lei->{-attach_idx} = $1;
+ $opt->{mail} = 1;
+ }
# first, see if it's a blob returned by "lei q" JSON output:k
if ($opt->{mail} // ($has_hints ? 0 : 1)) {
@@ -97,7 +110,7 @@ sub lei_blob {
$lxs = $lei->lxs_prepare;
$lei->ale->refresh_externals($lxs);
}
- my $rdr = { 1 => $lei->{1} };
+ my $rdr = {};
if ($opt->{mail}) {
$rdr->{2} = $lei->{2};
} else {
@@ -105,7 +118,22 @@ sub lei_blob {
}
my $cmd = [ 'git', '--git-dir='.$lei->ale->git->{git_dir},
'cat-file', 'blob', $blob ];
- waitpid(spawn($cmd, $lei->{env}, $rdr), 0);
+ if (defined $lei->{-attach_idx}) {
+ my $fh = popen_rd($cmd, $lei->{env}, $rdr);
+ require PublicInbox::Eml;
+ my $str = do { local $/; <$fh> };
+ if (close $fh) {
+ my $eml = PublicInbox::Eml->new(\$str);
+ $eml->each_part(\&cat_attach_i, $lei, 1);
+ my $idx = delete $lei->{-attach_idx};
+ defined($idx) and return $lei->fail(<<EOM);
+E: attachment $idx not found in $blob
+EOM
+ }
+ } else {
+ $rdr->{1} = $lei->{1};
+ waitpid(spawn($cmd, $lei->{env}, $rdr), 0);
+ }
return if $? == 0;
return $lei->child_error($?) if $opt->{mail};
}
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 376a8b48..cf1fa49d 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -52,5 +52,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect');
is(ref($x->{sync}), 'HASH', 'sync in inspect');
is(ref($x->{sync}->{$k[0]}), 'ARRAY', 'UID arrays in inspect');
+
+ my $psgi_attach = 'cfa3622cbeffc9bd6b0fc66c4d60d420ba74f60d';
+ lei_ok('blob', $psgi_attach);
+ like($lei_out, qr!^Content-Type: multipart/mixed;!sm, 'got full blob');
+ lei_ok('blob', "$psgi_attach:2");
+ is($lei_out, "b64\xde\xad\xbe\xef\n", 'got attachment');
});
done_testing;
^ permalink raw reply related [relevance 61%]
* [PATCH 1/5] lei: add "ls-sync" command for listing sync folders
2021-04-27 11:07 64% [PATCH 0/5] lei lcat - local cat (not lolcat :P) Eric Wong
@ 2021-04-27 11:07 55% ` Eric Wong
2021-04-27 11:07 61% ` [PATCH 2/5] lei blob: support retrieving attachments via $OID:$IDX Eric Wong
` (3 subsequent siblings)
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-27 11:07 UTC (permalink / raw)
To: meta
This will be useful, later.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 2 ++
lib/PublicInbox/LeiExternal.pm | 2 +-
lib/PublicInbox/LeiLsSync.pm | 29 +++++++++++++++++++++++++++++
t/lei-import-imap.t | 2 ++
t/lei-import-maildir.t | 3 +++
6 files changed, 38 insertions(+), 1 deletion(-)
create mode 100644 lib/PublicInbox/LeiLsSync.pm
diff --git a/MANIFEST b/MANIFEST
index ce824fcf..d4e7d66f 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -202,6 +202,7 @@ lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
lib/PublicInbox/LeiLsLabel.pm
lib/PublicInbox/LeiLsSearch.pm
+lib/PublicInbox/LeiLsSync.pm
lib/PublicInbox/LeiMailSync.pm
lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 39278de6..c170572b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -161,6 +161,8 @@ our %CMD = ( # sorted in order of importance/use:
'ls-external' => [ '[FILTER]', 'list publicinbox|extindex locations',
qw(format|f=s z|0 globoff|g invert-match|v local remote), @c_opt ],
'ls-label' => [ '', 'list labels', qw(z|0 stats:s), @c_opt ],
+'ls-sync' => [ '', 'list sync folders',
+ qw(z|0 z|0 globoff|g invert-match|v local remote), @c_opt ],
'forget-external' => [ 'LOCATION...|--prune',
'exclude further results from a publicinbox|extindex',
qw(prune), @c_opt ],
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index b0ebe947..3858085e 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -50,7 +50,7 @@ my %re_map = ( '*' => '[^/]*?', '?' => '[^/]',
'[' => '[', ']' => ']', ',' => ',' );
sub glob2re {
- my ($re) = @_;
+ my $re = $_[-1];
my $p = '';
my $in_bracket = 0;
my $qm = 0;
diff --git a/lib/PublicInbox/LeiLsSync.pm b/lib/PublicInbox/LeiLsSync.pm
new file mode 100644
index 00000000..71f111a9
--- /dev/null
+++ b/lib/PublicInbox/LeiLsSync.pm
@@ -0,0 +1,29 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# front-end for the "lei ls-sync" sub-command
+package PublicInbox::LeiLsSync;
+use strict;
+use v5.10.1;
+use PublicInbox::LeiMailSync;
+
+sub lei_ls_sync {
+ my ($lei, $filter) = @_;
+ my $sto = $lei->_lei_store or return;
+ my $lms = $sto->search->lms or return;
+ my $opt = $lei->{opt};
+ my $re;
+ $re = defined($filter) ? qr/\Q$filter\E/ : qr/./ if $opt->{globoff};
+ $re //= $lei->glob2re($filter // '*');
+ my @f = $lms->folders;
+ @f = $opt->{'invert-match'} ? grep(!/$re/, @f) : grep(/$re/, @f);
+ if ($opt->{'local'} && !$opt->{remote}) {
+ @f = grep(!m!\A[a-z\+]+://!i, @f);
+ } elsif ($opt->{remote} && !$opt->{'local'}) {
+ @f = grep(m!\A[a-z\+]+://!i, @f);
+ }
+ my $ORS = $opt->{z} ? "\0" : "\n";
+ $lei->out(join($ORS, @f, ''));
+}
+
+1;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 4a3bd6d8..376a8b48 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -22,6 +22,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
is_deeply(json_utf8->decode($lei_out), {}, 'no inspect stats, yet');
lei_ok('import', $url);
+ lei_ok 'ls-sync';
+ like($lei_out, qr!\A\Q$url\E;UIDVALIDITY=\d+\n\z!, 'ls-sync');
lei_ok('inspect', $url);
my $inspect = json_utf8->decode($lei_out);
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index 3e3d9188..808e1a73 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -40,6 +40,9 @@ test_lei(sub {
is_deeply($inspect, { sync => { "maildir:$md" => [ 'x:2,S' ] } },
'maildir sync info as expected');
+ lei_ok qw(ls-sync);
+ is($lei_out, "maildir:$md\n", 'ls-sync as expected');
+
lei_ok(qw(import), $md, \'import Maildir again');
$imp_err = $lei_err;
lei_ok(qw(q -d none s:boolean), \'lei q w/o dedupe');
^ permalink raw reply related [relevance 55%]
* [PATCH 3/5] lei: standardize on _lei_wq_eof callback for workers
2021-04-27 11:07 64% [PATCH 0/5] lei lcat - local cat (not lolcat :P) Eric Wong
2021-04-27 11:07 55% ` [PATCH 1/5] lei: add "ls-sync" command for listing sync folders Eric Wong
2021-04-27 11:07 61% ` [PATCH 2/5] lei blob: support retrieving attachments via $OID:$IDX Eric Wong
@ 2021-04-27 11:07 49% ` Eric Wong
2021-04-27 11:07 38% ` [PATCH 4/5] lei lcat: extract Message-IDs from URLs and show them Eric Wong
2021-04-27 11:07 32% ` [PATCH 5/5] lei q + lcat: support --format=text output Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-27 11:07 UTC (permalink / raw)
To: meta
Simplify our internals a little bit.
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiBlob.pm | 5 ++---
lib/PublicInbox/LeiConvert.pm | 2 +-
lib/PublicInbox/LeiImport.pm | 6 +++---
lib/PublicInbox/LeiMirror.pm | 6 ++----
lib/PublicInbox/LeiP2q.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 8 ++++----
7 files changed, 14 insertions(+), 17 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index c170572b..effc905a 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -531,7 +531,7 @@ sub workers_start {
'child_error' => [ \&child_error, $lei ],
($ops ? %$ops : ()),
};
- $ops->{''} //= [ \&dclose, $lei ];
+ $ops->{''} //= [ $wq->can('_lei_wq_eof') || \&dclose, $lei ];
my $end = $lei->pkt_op_pair;
$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
delete $lei->{pkt_op_p};
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 4e52c8a5..0b96bd04 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -10,7 +10,7 @@ use parent qw(PublicInbox::IPC);
use PublicInbox::Spawn qw(spawn popen_rd which);
use PublicInbox::DS;
-sub sol_done { # EOF callback for main daemon
+sub _lei_wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $sol = delete $lei->{sol} // return $lei->dclose; # already failed
$sol->wq_wait_old($lei->can('wq_done_wait'), $lei);
@@ -157,8 +157,7 @@ EOM
}
require PublicInbox::SolverGit;
my $self = bless { lxs => $lxs, oid_b => $blob }, __PACKAGE__;
- my ($op_c, $ops) = $lei->workers_start($self, 'lei_solve', 1,
- { '' => [ \&sol_done, $lei ] });
+ my ($op_c, $ops) = $lei->workers_start($self, 'lei-blob', 1);
$lei->{sol} = $self;
$self->wq_io_do('do_solve_blob', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 0ce49ea9..0c324169 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -52,7 +52,7 @@ sub lei_convert { # the main "lei convert" method
my $devfd = $lei->path_to_fd($ovv->{dst}) // return;
$lei->{opt}->{augment} = 1 if $devfd < 0;
$self->prepare_inputs($lei, \@inputs) or return;
- my ($op_c, $ops) = $lei->workers_start($self, 'lei_convert', 1);
+ my ($op_c, $ops) = $lei->workers_start($self, 'lei-convert', 1);
$lei->{cnv} = $self;
$self->wq_io_do('process_inputs', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index daaa6753..e0d899cc 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -53,7 +53,7 @@ sub input_nntp_cb { # nntp_each
input_eml_cb($self, $eml, $self->{-import_kw} ? { kw => $kw } : undef);
}
-sub import_done { # EOF callback for main daemon
+sub _lei_wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $imp = delete $lei->{imp} // return $lei->fail('BUG: {imp} gone');
$imp->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
@@ -90,10 +90,10 @@ sub lei_import { # the main "lei import" method
my $nproc = $self->detect_nproc;
$j = $nproc if $j > $nproc;
}
- my $ops = { '' => [ \&import_done, $lei ] };
+ my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{-wq_nr_workers} = $j // 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, 'lei_import', $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
$lei->{imp} = $self;
net_merge_complete($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 15adb71b..50ab4c85 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -24,7 +24,7 @@ sub do_finish_mirror { # dwaitpid callback
$lei->dclose;
}
-sub mirror_done { # EOF callback for main daemon
+sub _lei_wq_eof { # EOF callback for main daemon
my ($lei) = @_;
my $mrr = delete $lei->{mrr} or return;
$mrr->wq_wait_old(\&do_finish_mirror, $lei);
@@ -282,9 +282,7 @@ sub start {
require PublicInbox::Inbox;
require PublicInbox::Admin;
require PublicInbox::InboxWritable;
- my ($op, $ops) = $lei->workers_start($self, 'lei_mirror', 1, {
- '' => [ \&mirror_done, $lei ]
- });
+ my ($op, $ops) = $lei->workers_start($self, 'lei_mirror', 1);
$lei->{mrr} = $self;
$self->wq_io_do('do_mirror', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index cb2309c7..3248afd7 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -188,7 +188,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
} else {
$self->{input} = $input;
}
- my ($op, $ops) = $lei->workers_start($self, 'lei_p2q', 1);
+ my ($op, $ops) = $lei->workers_start($self, 'lei-p2q', 1);
$lei->{p2q} = $self;
$self->wq_io_do('do_p2q', []);
$self->wq_close(1);
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index f5791947..3cda2eca 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -19,9 +19,9 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
-sub tag_done { # EOF callback for main daemon
+sub _lei_wq_eof { # EOF callback for main daemon
my ($lei) = @_;
- my $tag = delete $lei->{tag} or return;
+ my $tag = delete $lei->{tag} // return $lei->dclose;
$tag->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
}
@@ -52,11 +52,11 @@ sub lei_tag { # the "lei tag" method
$self->prepare_inputs($lei, \@argv) or return;
grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or
return $lei->fail('no keywords or labels specified');
- my $ops = { '' => [ \&tag_done, $lei ] };
+ my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
$self->{vmd_mod} = $vmd_mod;
my $j = $self->{-wq_nr_workers} = 1; # locked for now
- (my $op_c, $ops) = $lei->workers_start($self, 'lei_tag', $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 'lei-tag', $j, $ops);
$lei->{tag} = $self;
net_merge_complete($self) unless $lei->{auth};
$op_c->op_wait_event($ops);
^ permalink raw reply related [relevance 49%]
* [PATCH 4/5] lei lcat: extract Message-IDs from URLs and show them
2021-04-27 11:07 64% [PATCH 0/5] lei lcat - local cat (not lolcat :P) Eric Wong
` (2 preceding siblings ...)
2021-04-27 11:07 49% ` [PATCH 3/5] lei: standardize on _lei_wq_eof callback for workers Eric Wong
@ 2021-04-27 11:07 38% ` Eric Wong
2021-04-27 11:07 32% ` [PATCH 5/5] lei q + lcat: support --format=text output Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-27 11:07 UTC (permalink / raw)
To: meta
It's a wrapper around "lei q" which extracts Message-IDs
from URLs, "<$MSGID>", "id:$MSGID" and attempts to display the
local version of the message.
Its main purpose is to extract Message-IDs out of
commonly-understood URLs to save users bandwidth and time
by displaying the message locally. When reading from stdin,
it will discard things it doesn't understand, so you can just
pipe an entire "Link: $URL" line to it and it'll attempt to
pluck the Message-ID out of the URL.
---
MANIFEST | 3 +
lib/PublicInbox/LEI.pm | 8 +++
lib/PublicInbox/LeiLcat.pm | 125 +++++++++++++++++++++++++++++++++++++
t/lei-lcat.t | 16 +++++
t/lei_lcat.t | 44 +++++++++++++
5 files changed, 196 insertions(+)
create mode 100644 lib/PublicInbox/LeiLcat.pm
create mode 100644 t/lei-lcat.t
create mode 100644 t/lei_lcat.t
diff --git a/MANIFEST b/MANIFEST
index d4e7d66f..d3b46f8b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -200,6 +200,7 @@ lib/PublicInbox/LeiImport.pm
lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
lib/PublicInbox/LeiInspect.pm
+lib/PublicInbox/LeiLcat.pm
lib/PublicInbox/LeiLsLabel.pm
lib/PublicInbox/LeiLsSearch.pm
lib/PublicInbox/LeiLsSync.pm
@@ -400,6 +401,7 @@ t/lei-import-imap.t
t/lei-import-maildir.t
t/lei-import-nntp.t
t/lei-import.t
+t/lei-lcat.t
t/lei-mirror.t
t/lei-p2q.t
t/lei-q-kw.t
@@ -411,6 +413,7 @@ t/lei-tag.t
t/lei.t
t/lei_dedupe.t
t/lei_external.t
+t/lei_lcat.t
t/lei_mail_sync.t
t/lei_overview.t
t/lei_saved_search.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index effc905a..ef72758c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -149,6 +149,14 @@ our %CMD = ( # sorted in order of importance/use:
'up' => [ 'OUTPUT|--all', 'update saved search',
qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
+'lcat' => [ '--stdin|MSGID_OR_URL..', 'display local copy of message(s)',
+ 'stdin|', # /|\z/ must be first for lone dash
+ # some of these options are ridiculous for lcat
+ @lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
+ sort|s=s reverse|r offset=i jobs|j=s globoff|g augment|a
+ import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
+ opt_dash('limit|n=i', '[0-9]+') ],
+
'blob' => [ 'OID', 'show a git blob, reconstructing from mail if necessary',
qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@lxs_opt, @c_opt ],
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
new file mode 100644
index 00000000..f10452be
--- /dev/null
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -0,0 +1,125 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# lcat: local cat, display a local message by Message-ID or blob,
+# extracting from URL necessary
+# "lei lcat <URL|SPEC>"
+package PublicInbox::LeiLcat;
+use strict;
+use v5.10.1;
+use PublicInbox::LeiViewText;
+use URI::Escape qw(uri_unescape);
+use URI;
+use PublicInbox::MID qw($MID_EXTRACT);
+
+sub lcat_redispatch {
+ my ($lei, $out, $op_p) = @_;
+ my $l = bless { %$lei }, ref($lei);
+ delete $l->{sock};
+ $l->{''} = $op_p; # daemon only
+ eval {
+ $l->qerr("# updating $out");
+ up1($l, $out);
+ $l->qerr("# $out done");
+ };
+ $l->err($@) if $@;
+}
+
+sub extract_1 ($$) {
+ my ($lei, $x) = @_;
+ if ($x =~ m!\b([a-z]+?://\S+)!i) {
+ my $u = $1;
+ $u =~ s/[\>\]\)\,\.\;]+\z//;
+ $u = URI->new($u);
+ my $p = $u->path;
+ my $term;
+ if ($p =~ m!([^/]+\@[^/]+)!) { # common msgid pattern
+ $term = 'mid:'.uri_unescape($1);
+
+ # is it a URL which returns the full thread?
+ if ($u->scheme =~ /\Ahttps?/i &&
+ $p =~ m!/(?:T/?|t/?|t\.mbox\.gz|t\.atom)\b!) {
+
+ $lei->{mset_opt}->{threads} = 1;
+ }
+ } elsif ($u->scheme =~ /\Ahttps?/i &&
+ # some msgids don't have '@', see if it looks like
+ # a public-inbox URL:
+ $p =~ m!/([^/]+)/(raw|t/?|T/?|
+ t\.mbox\.gz|t\.atom)\z!x) {
+ $lei->{mset_opt}->{threads} = 1 if $2 && $2 ne 'raw';
+ $term = 'mid:'.uri_unescape($1);
+ }
+ $term;
+ } elsif ($x =~ $MID_EXTRACT) { # <$MSGID>
+ "mid:$1";
+ } elsif ($x =~ /\b((?:m|mid):\S+)/) { # our own prefixes (and mairix)
+ $1;
+ } elsif ($x =~ /\bid:(\S+)/) { # notmuch convention
+ "mid:$1";
+ } else {
+ undef;
+ }
+}
+
+sub extract_all {
+ my ($lei, @argv) = @_;
+ my $strict = !$lei->{opt}->{stdin};
+ my @q;
+ for my $x (@argv) {
+ if (my $term = extract_1($lei,$x)) {
+ push @q, $term;
+ } elsif ($strict) {
+ return $lei->fail(<<"");
+could not extract Message-ID from $x
+
+ }
+ }
+ @q ? join(' OR ', @q) : $lei->fail("no Message-ID in: @argv");
+}
+
+sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
+ my ($lei) = @_; # $_[1] = $rbuf
+ if (defined($_[1])) {
+ $_[1] eq '' and return eval {
+ if (my $dfd = $lei->{3}) {
+ chdir($dfd) or return $lei->fail("fchdir: $!");
+ }
+ my @argv = split(/\s+/, $lei->{mset_opt}->{qstr});
+ $lei->{mset_opt}->{qstr} = extract_all($lei, @argv)
+ or return;
+ $lei->_start_query;
+ };
+ $lei->{mset_opt}->{qstr} .= $_[1];
+ } else {
+ $lei->fail("error reading stdin: $!");
+ }
+}
+
+sub lei_lcat {
+ my ($lei, @argv) = @_;
+ my $lxs = $lei->lxs_prepare or return;
+ $lei->ale->refresh_externals($lxs);
+ my $sto = $lei->_lei_store(1);
+ $lei->{lse} = $sto->search;
+ my $opt = $lei->{opt};
+ my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
+ $mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
+ $mset_opt{limit} //= 10000;
+ $opt->{sort} //= 'relevance';
+ $mset_opt{relevance} = 1;
+ $lei->{mset_opt} = \%mset_opt;
+ $opt->{'format'} //= 'mboxrd' unless defined($opt->{output});
+ if ($lei->{opt}->{stdin}) {
+ return $lei->fail(<<'') if @argv;
+no args allowed on command-line with --stdin
+
+ require PublicInbox::InputPipe;
+ PublicInbox::InputPipe::consume($lei->{0}, \&_stdin, $lei);
+ return;
+ }
+ $lei->{mset_opt}->{qstr} = extract_all($lei, @argv) or return;
+ $lei->_start_query;
+}
+
+1;
diff --git a/t/lei-lcat.t b/t/lei-lcat.t
new file mode 100644
index 00000000..e5f00706
--- /dev/null
+++ b/t/lei-lcat.t
@@ -0,0 +1,16 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+
+test_lei(sub {
+ my $in = "\nMessage-id: <qp\@example.com>\n";
+ lei_ok([qw(lcat --stdin)], undef, { 0 => \$in, %$lei_opt });
+ unlike($lei_out, qr/\S/, 'nothing, yet');
+ lei_ok('import', 't/plack-qp.eml');
+ lei_ok([qw(lcat --stdin)], undef, { 0 => \$in, %$lei_opt });
+ like($lei_out, qr/qp\@example\.com/, 'got a result');
+});
+
+done_testing;
diff --git a/t/lei_lcat.t b/t/lei_lcat.t
new file mode 100644
index 00000000..536abdea
--- /dev/null
+++ b/t/lei_lcat.t
@@ -0,0 +1,44 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# unit test for "lei lcat" internals, see t/lei-lcat.t for functional test
+use strict;
+use v5.10.1;
+use Test::More;
+use_ok 'PublicInbox::LeiLcat';
+my $cb = \&PublicInbox::LeiLcat::extract_1;
+my $ck = sub {
+ my ($txt, $exp, $t) = @_;
+ my $lei = {};
+ is($cb->($lei, $txt), $exp, $txt);
+ ($t ? is_deeply($lei, { mset_opt => { threads => 1 } }, "-t $exp")
+ : is_deeply($lei, {}, "no -t for $exp")) or diag explain($lei);
+};
+
+for my $txt (qw(https://example.com/inbox/foo@bar/
+ https://example.com/inbox/foo@bar
+ https://example.com/inbox/foo@bar/raw
+ id:foo@bar
+ mid:foo@bar
+ <foo@bar>
+ <https://example.com/inbox/foo@bar>
+ <https://example.com/inbox/foo@bar/raw>
+ <https://example.com/inbox/foo@bar/>
+ <nntp://example.com/foo@bar>)) {
+ $ck->($txt, 'mid:foo@bar');
+}
+
+for my $txt (qw(https://example.com/inbox/foo@bar/T/
+ https://example.com/inbox/foo@bar/t/
+ https://example.com/inbox/foo@bar/t.mbox.gz
+ <https://example.com/inbox/foo@bar/t.atom>
+ <https://example.com/inbox/foo@bar/t/>)) {
+ $ck->($txt, 'mid:foo@bar', '-t');
+}
+
+$ck->('https://example.com/x/foobar/T/', 'mid:foobar', '-t');
+$ck->('https://example.com/x/foobar/raw', 'mid:foobar');
+is($cb->(my $lei = {}, 'asdf'), undef, 'no Message-ID');
+is($cb->($lei = {}, 'm:x'), 'm:x', 'bare m: accepted');
+
+done_testing;
^ permalink raw reply related [relevance 38%]
* [PATCH 5/5] lei q + lcat: support --format=text output
2021-04-27 11:07 64% [PATCH 0/5] lei lcat - local cat (not lolcat :P) Eric Wong
` (3 preceding siblings ...)
2021-04-27 11:07 38% ` [PATCH 4/5] lei lcat: extract Message-IDs from URLs and show them Eric Wong
@ 2021-04-27 11:07 32% ` Eric Wong
4 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-27 11:07 UTC (permalink / raw)
To: meta
This is mainly for "lei lcat" where it's the default,
but I find it useful anyways compared to the JSON view.
Colors are loaded from ~/.config/lei/config, and fall back
to using diff colors from a normal git config
(e.g. ~/.gitconfig).
---
MANIFEST | 1 +
lib/PublicInbox/Hval.pm | 2 +-
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 63 ++++++++-
lib/PublicInbox/LeiViewText.pm | 237 +++++++++++++++++++++++++++++++++
lib/PublicInbox/ViewDiff.pm | 4 +-
6 files changed, 301 insertions(+), 8 deletions(-)
create mode 100644 lib/PublicInbox/LeiViewText.pm
diff --git a/MANIFEST b/MANIFEST
index d3b46f8b..5933ddf4 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -218,6 +218,7 @@ lib/PublicInbox/LeiSucks.pm
lib/PublicInbox/LeiTag.pm
lib/PublicInbox/LeiToMail.pm
lib/PublicInbox/LeiUp.pm
+lib/PublicInbox/LeiViewText.pm
lib/PublicInbox/LeiXSearch.pm
lib/PublicInbox/Linkify.pm
lib/PublicInbox/Listener.pm
diff --git a/lib/PublicInbox/Hval.pm b/lib/PublicInbox/Hval.pm
index eab4738e..00b3c8b4 100644
--- a/lib/PublicInbox/Hval.pm
+++ b/lib/PublicInbox/Hval.pm
@@ -34,7 +34,7 @@ my %escape_sequence = (
"\x7f" => '\\x7f', # DEL
);
-my %xhtml_map = (
+our %xhtml_map = (
'"' => '"',
'&' => '&',
"'" => ''',
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index f10452be..87729acf 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -109,7 +109,7 @@ sub lei_lcat {
$opt->{sort} //= 'relevance';
$mset_opt{relevance} = 1;
$lei->{mset_opt} = \%mset_opt;
- $opt->{'format'} //= 'mboxrd' unless defined($opt->{output});
+ $opt->{'format'} //= 'text' unless defined($opt->{output});
if ($lei->{opt}->{stdin}) {
return $lei->fail(<<'') if @argv;
no args allowed on command-line with --stdin
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 8b2f82dc..fa3af710 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -309,6 +309,26 @@ sub _imap_write_cb ($$) {
}
}
+sub _text_write_cb ($$) {
+ my ($self, $lei) = @_;
+ my $dedupe = $lei->{dedupe};
+ $dedupe->prepare_dedupe if $dedupe;
+ my $lvt = $lei->{lvt};
+ my $ovv = $lei->{ovv};
+ $lei->{1} // die "no stdout ($ovv->{dst})"; # redirected earlier
+ $lei->{1}->autoflush(1);
+ binmode $lei->{1}, ':utf8';
+ my $lse = $lei->{lse}; # may be undef
+ sub { # for git_to_mail
+ my ($bref, $smsg, $eml) = @_;
+ $lse->xsmsg_vmd($smsg) if $lse;
+ $eml //= PublicInbox::Eml->new($bref); # copy bref
+ return if $dedupe && $dedupe->is_dup($eml, $smsg);
+ my $lk = $ovv->lock_for_scope;
+ $lei->out(${$lvt->eml_to_text($smsg, $eml)}, "\n");
+ }
+}
+
sub write_cb { # returns a callback for git_to_mail
my ($self, $lei) = @_;
# _mbox_write_cb, _maildir_write_cb or _imap_write_cb
@@ -329,8 +349,6 @@ sub new {
$lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
} elsif (substr($fmt, 0, 4) eq 'mbox') {
require PublicInbox::MboxReader;
- (-d $dst || (-e _ && !-w _)) and die
- "$dst exists and is not a writable file\n";
$self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
$self->{base_type} = 'mbox';
} elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support
@@ -347,9 +365,23 @@ sub new {
$dst = $lei->{ovv}->{dst} = $$uri; # canonicalized
$lei->{net} = $net;
$self->{base_type} = 'imap';
+ } elsif ($fmt eq 'text') {
+ require PublicInbox::LeiViewText;
+ $lei->{lvt} = PublicInbox::LeiViewText->new($lei);
+ $self->{base_type} = 'text';
} else {
die "bad mail --format=$fmt\n";
}
+ if ($self->{base_type} =~ /\A(?:text|mbox)\z/) {
+ (-d $dst || (-e _ && !-w _)) and die
+ "$dst exists and is not a writable file\n";
+ }
+ if ($self->{base_type} eq 'text') {
+ my @err = map {
+ defined($lei->{opt}->{$_}) ? "--$_" : ();
+ } (qw(mua save));
+ die "@err incompatible with $fmt\n" if @err;
+ }
$self->{dst} = $dst;
$lei->{dedupe} = $lei->{lss} // do {
my $dd_cls = 'PublicInbox::'.
@@ -429,6 +461,29 @@ sub _do_augment_imap {
}
}
+sub _pre_augment_text {
+ my ($self, $lei) = @_;
+ my $dst = $lei->{ovv}->{dst};
+ my $out;
+ my $devfd = $lei->path_to_fd($dst) // die "bad $dst";
+ if ($devfd >= 0) {
+ $out = $lei->{$devfd};
+ } else { # normal-looking path
+ if (-p $dst) {
+ open $out, '>', $dst or die "open($dst): $!";
+ } elsif (-f _ || !-e _) {
+ # text allows augment, HTML/Atom won't
+ my $mode = $lei->{opt}->{augment} ? '>>' : '>';
+ open $out, $mode, $dst or die "open($mode, $dst): $!";
+ } else {
+ die "$dst is not a file or FIFO\n";
+ }
+ }
+ $lei->{ovv}->ovv_out_lk_init if !$lei->{ovv}->{lock_path};
+ $lei->{1} = $out;
+ undef;
+}
+
sub _pre_augment_mbox {
my ($self, $lei) = @_;
my $dst = $lei->{ovv}->{dst};
@@ -523,8 +578,8 @@ sub pre_augment { # fast (1 disk seek), runs in same process as post_augment
sub do_augment { # slow, runs in wq worker
my ($self, $lei) = @_;
# _do_augment_maildir, _do_augment_mbox, or _do_augment_imap
- my $m = "_do_augment_$self->{base_type}";
- $self->$m($lei);
+ my $m = $self->can("_do_augment_$self->{base_type}") or return;
+ $m->($self, $lei);
}
# fast (spawn compressor or mkdir), runs in same process as pre_augment
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
new file mode 100644
index 00000000..6f5fca49
--- /dev/null
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -0,0 +1,237 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# PublicInbox::Eml to (optionally colorized) text coverter for terminals
+# the non-HTML counterpart to PublicInbox::View
+package PublicInbox::LeiViewText;
+use strict;
+use v5.10.1;
+use PublicInbox::MsgIter qw(msg_part_text);
+use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::MID qw(references);
+use PublicInbox::View;
+use PublicInbox::Hval;
+use PublicInbox::ViewDiff;
+use PublicInbox::Spawn qw(popen_rd);
+use Term::ANSIColor;
+
+sub _xs {
+ # xhtml_map works since we don't search for HTML ([&<>'"])
+ $_[0] =~ s/([\x7f\x00-\x1f])/$PublicInbox::Hval::xhtml_map{$1}/sge;
+}
+
+my %DEFAULT_COLOR = (
+ # mutt names, loaded from ~/.config/lei/config
+ quoted => 'blue',
+ hdrdefault => 'cyan',
+ status => 'bright_cyan', # smsg stuff
+
+ # git names and defaults, falls back to ~/.gitconfig
+ new => 'green',
+ old => 'red',
+ meta => 'bold',
+ frag => 'cyan',
+ func => undef,
+ context => undef,
+);
+
+sub my_colored {
+ my ($self, $slot) = @_; # $_[2] = buffer
+ my $val = $self->{"color.$slot"} //=
+ $self->{-leicfg}->{"color.$slot"} //
+ $self->{-gitcfg}->{"color.diff.$slot"} //
+ $self->{-gitcfg}->{"diff.color.$slot"} //
+ $DEFAULT_COLOR{$slot};
+ $val = $val->[-1] if ref($val) eq 'ARRAY';
+ if (defined $val) {
+ # git doesn't use "_", Term::ANSIColor does
+ $val =~ s/\Abright([^_])/bright_$1/i;
+ ${$self->{obuf}} .= Term::ANSIColor::colored($_[2], lc $val);
+ } else {
+ ${$self->{obuf}} .= $_[2];
+ }
+}
+
+sub uncolored { ${$_[0]->{obuf}} .= $_[2] }
+
+sub new {
+ my ($cls, $lei) = @_;
+ my $self = bless { %{$lei->{opt}}, -colored => \&uncolored }, $cls;
+ return $self unless $self->{color} || -t $lei->{1};
+ my $cmd = [ qw(git config -z --includes -l) ];
+ my ($r, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
+ my $cfg = PublicInbox::Config::config_fh_parse($r, "\0", "\n");
+ waitpid($pid, 0);
+ if ($?) {
+ $lei->err("# git-config failed, no color (non-fatal)");
+ return $self;
+ }
+ $self->{-colored} = \&my_colored;
+ $self->{-gitcfg} = $cfg;
+ $self->{-leicfg} = $lei->{cfg};
+ $self;
+}
+
+sub hdr_buf ($$) {
+ my ($self, $eml) = @_;
+ my $hbuf = '';
+ for my $f (qw(From To Cc)) {
+ for my $v ($eml->header($f)) {
+ next if $v !~ /\S/;
+ PublicInbox::View::fold_addresses($v);
+ _xs($v);
+ $hbuf .= "$f: $v\n";
+ }
+ }
+ for my $f (qw(Subject Date Newsgroups Message-ID X-Message-ID)) {
+ for my $v ($eml->header($f)) {
+ _xs($v);
+ $hbuf .= "$f: $v\n";
+ }
+ }
+ if (my @irt = $eml->header_raw('In-Reply-To')) {
+ for my $v (@irt) {
+ _xs($v);
+ $hbuf .= "In-Reply-To: $v\n";
+ }
+ } else {
+ my $refs = references($eml);
+ if (defined(my $irt = pop @$refs)) {
+ _xs($irt);
+ $hbuf .= "In-Reply-To: <$irt>\n";
+ }
+ if (@$refs) {
+ my $max = $self->{-max_cols};
+ $hbuf .= 'References: ' .
+ join("\n\t", map { '<'._xs($_).'>' } @$refs) .
+ ">\n";
+ }
+ }
+ $self->{-colored}->($self, 'hdrdefault', $hbuf .= "\n");
+}
+
+sub attach_note ($$$$;$) {
+ my ($self, $ct, $p, $fn, $err) = @_;
+ my ($part, $depth, $idx) = @$p;
+ my $obuf = $self->{obuf};
+ my $nl = $idx eq '1' ? '' : "\n"; # like join("\n", ...)
+ $$obuf .= <<EOF if $err;
+[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
+EOF
+ my $blob = $self->{-smsg}->{blob} // '';
+ $blob .= ':' if $blob ne '';
+ $$obuf .= "[-- Attachment $blob$idx ";
+ _xs($ct);
+ my $size = length($part->body);
+ my $ts = "Type: $ct, Size: $size bytes";
+ my $d = $part->header('Content-Description') // $fn // '';
+ _xs($d);
+ $$obuf .= $d eq '' ? "$ts --]\n" : "$d --]\n[-- $ts --]\n";
+ hdr_buf($self, $part) if $part->{is_submsg};
+}
+
+sub flush_text_diff ($$) {
+ my ($self, $cur) = @_;
+ my @top = split($PublicInbox::ViewDiff::EXTRACT_DIFFS, $$cur);
+ undef $$cur; # free memory
+ my $dctx;
+ my $obuf = $self->{obuf};
+ my $colored = $self->{-colored};
+ while (defined(my $x = shift @top)) {
+ if (scalar(@top) >= 4 &&
+ $top[1] =~ $PublicInbox::ViewDiff::IS_OID &&
+ $top[0] =~ $PublicInbox::ViewDiff::IS_OID) {
+ splice(@top, 0, 4);
+ $dctx = 1;
+ $colored->($self, 'meta', $x);
+ } elsif ($dctx) {
+ # Quiet "Complex regular subexpression recursion limit"
+ # warning. Perl will truncate matches upon hitting
+ # that limit, giving us more (and shorter) scalars than
+ # would be ideal, but otherwise it's harmless.
+ #
+ # We could replace the `+' metacharacter with `{1,100}'
+ # to limit the matches ourselves to 100, but we can
+ # let Perl do it for us, quietly.
+ no warnings 'regexp';
+
+ for my $s (split(/((?:(?:^\+[^\n]*\n)+)|
+ (?:(?:^-[^\n]*\n)+)|
+ (?:^@@ [^\n]+\n))/xsm, $x)) {
+ if (!defined($dctx)) {
+ ${$self->{obuf}} .= $s;
+ } elsif ($s =~ s/\A(@@ \S+ \S+ @@\s*)//) {
+ $colored->($self, 'frag', $1);
+ $colored->($self, 'func', $s);
+ } elsif ($s =~ /\A\+/) {
+ $colored->($self, 'new', $s);
+ } elsif ($s =~ /\A-- $/sm) { # email sig starts
+ $dctx = undef;
+ ${$self->{obuf}} .= $s;
+ } elsif ($s =~ /\A-/) {
+ $colored->($self, 'old', $s);
+ } else {
+ $colored->($self, 'context', $s);
+ }
+ }
+ } else {
+ ${$self->{obuf}} .= $x;
+ }
+ }
+}
+
+sub add_text_buf { # callback for Eml->each_part
+ my ($p, $self) = @_;
+ my ($part, $depth, $idx) = @$p;
+ my $ct = $part->content_type || 'text/plain';
+ my $fn = $part->filename;
+ my ($s, $err) = msg_part_text($part, $ct);
+ return attach_note($self, $ct, $p, $fn) unless defined $s;
+ hdr_buf($self, $part) if $part->{is_submsg};
+ $s =~ s/\r\n/\n/sg;
+ _xs($s);
+ $s .= "\n" unless substr($s, -1, 1) eq "\n";
+ my $diff = ($s =~ /^--- [^\n]+\n\+{3} [^\n]+\n@@ /ms);
+ my @sections = PublicInbox::MsgIter::split_quotes($s);
+ undef $s; # free memory
+ if (defined($fn) || ($depth > 0 && !$part->{is_submsg}) || $err) {
+ # badly-encoded message with $err? tell the world about it!
+ attach_note($self, $ct, $p, $fn, $err);
+ ${$self->{obuf}} .= "\n";
+ }
+ my $colored = $self->{-colored};
+ for my $cur (@sections) {
+ if ($cur =~ /\A>/) {
+ $colored->($self, 'quoted', $cur);
+ } elsif ($diff) {
+ flush_text_diff($self, \$cur);
+ } else {
+ ${$self->{obuf}} .= $cur;
+ }
+ undef $cur; # free memory
+ }
+}
+
+# returns an arrayref suitable for $lei->out or print
+sub eml_to_text {
+ my ($self, $smsg, $eml) = @_;
+ local $Term::ANSIColor::EACHLINE = "\n";
+ $self->{obuf} = \(my $obuf = '');
+ $self->{-smsg} = $smsg;
+ $self->{-max_cols} = ($self->{columns} //= 80) - 8; # for header wrap
+ my @h = ();
+ for my $f (qw(blob pct)) {
+ push @h, "$f:$smsg->{$f}" if defined $smsg->{$f};
+ }
+ @h = ("# @h\n") if @h;
+ for my $f (qw(kw L)) {
+ my $v = $smsg->{$f} or next;
+ push @h, "# $f:".join(',', @$v)."\n" if @$v;
+ }
+ $self->{-colored}->($self, 'status', join('', @h));
+ hdr_buf($self, $eml);
+ $eml->each_part(\&add_text_buf, $self, 1);
+ delete $self->{obuf};
+}
+
+1;
diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index 8fe7261f..e9a7bf69 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -30,7 +30,7 @@ my $DIFFSTAT_COMMENT =
my $NULL_TO_BLOB = qr/^(index $OID_NULL\.\.)($OID_BLOB)\b/ms;
my $BLOB_TO_NULL = qr/^index ($OID_BLOB)(\.\.$OID_NULL)\b/ms;
my $BLOB_TO_BLOB = qr/^index ($OID_BLOB)\.\.($OID_BLOB)/ms;
-my $EXTRACT_DIFFS = qr/(
+our $EXTRACT_DIFFS = qr/(
(?: # begin header stuff, don't capture filenames, here,
# but instead wait for the --- and +++ lines.
(?:^diff\x20--git\x20$FN\x20$FN$LF)
@@ -41,7 +41,7 @@ my $EXTRACT_DIFFS = qr/(
^index\x20($OID_BLOB)\.\.($OID_BLOB)$ANY*$LF
^---\x20($FN)$LF
^\+{3}\x20($FN)$LF)/msx;
-my $IS_OID = qr/\A$OID_BLOB\z/s;
+our $IS_OID = qr/\A$OID_BLOB\z/s;
# link to line numbers in blobs
sub diff_hunk ($$$$) {
^ permalink raw reply related [relevance 32%]
* Re: lei-managed pseudo mailing lists
2021-04-26 19:46 68% ` Konstantin Ryabitsev
@ 2021-04-26 20:34 71% ` Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-26 20:34 UTC (permalink / raw)
To: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> I'm just worried that if we overuse the alternates, then we may find ourselves
> in a situation where when we repack the "every blob" shared repository, we'll
> end up with a pack that isn't really optimized to be used by any of the
> member repos. So, in a situation where a clone is performed, git-upload-pack
> will have to spend a lot of cycles navigating through the monstrous parent
> pack just to build and re-compress the small subset of objects it needs to
> send.
>
> Git has ways of dealing with this by allowing to set things like pack islands,
> but it's finicky and requires that each child repo is defined as refs in the
> parent repo. We deal with this in grokmirror, but it's messy and requires
> properly tracking child repo additions/removals/etc.
At least for personal use, I've been meaning to look into
automatically managing islands.
> I think it may be one of those cases where wasting disk space on duplicate
> objects is worth the CPU cycle savings.
Agreed for serving public inboxes.
> On Mon, Apr 26, 2021 at 06:47:17PM +0000, Eric Wong wrote:
> > The aforementioned maxuid prevents stuff that's too old from
> > being seen. Otherwise, there's always "public-inbox-learn rm".
>
> How would it handle the situation where we import a new list into lore with a
> 10-year-long archive of messages?
maxuid is either per-inbox or per-extindex.
If the search is going off of inboxes via --only, then it would
not see the new inbox at all. If it's on an extindex like
"all", then yes, the newly-imported historical messages would
show up.
So using "rt:" (Received time) is helpful in the [extindex "all"] case
Also, the approxidate parsing is done every time with "lei up",
so you can have a rolling window with "rt:last.week.." as a
search parameter.
^ permalink raw reply [relevance 71%]
* Re: lei-managed pseudo mailing lists
2021-04-26 18:47 71% ` Eric Wong
@ 2021-04-26 19:46 68% ` Konstantin Ryabitsev
2021-04-26 20:34 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-04-26 19:46 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Mon, Apr 26, 2021 at 06:47:17PM +0000, Eric Wong wrote:
> > I'm thinking we need the ability to make it a real clonable repository --
> > perhaps without its own xapian index? Actual git repositories aren't large,
> > especially if they are only used for direct git operations. Disk space is
> > cheap, it's the IO that's expensive. :)
>
> True, though cache overheads hurt a bit. I also wonder if lei
> can increase traffic to public-inbox-<imapd|nntpd> to reduce
> the need/use of "git clone".
>
> > If these are real clonable repositories, then it would be easy for people to
> > set up replication for just the curated content people want.
>
> Understood. Using --output v2publicinbox:... w/o --shared is
> totally doable.
I'm just worried that if we overuse the alternates, then we may find ourselves
in a situation where when we repack the "every blob" shared repository, we'll
end up with a pack that isn't really optimized to be used by any of the
member repos. So, in a situation where a clone is performed, git-upload-pack
will have to spend a lot of cycles navigating through the monstrous parent
pack just to build and re-compress the small subset of objects it needs to
send.
Git has ways of dealing with this by allowing to set things like pack islands,
but it's finicky and requires that each child repo is defined as refs in the
parent repo. We deal with this in grokmirror, but it's messy and requires
properly tracking child repo additions/removals/etc.
I think it may be one of those cases where wasting disk space on duplicate
objects is worth the CPU cycle savings.
> > Not really worried about deduping blobs, but I'm wondering how to make it work
> > well when search parameters change (see above). E.g.:
> >
> > 1. we create the repo with one set of parameters
> > 2. maintainer then broadens it up to include something else
> > 3. maintainer then decides that it's now *way* too much and narrows it down again
> >
> > We don't really want step 2 to lead to a permanent ballooning of the
> > repository, so perhaps all query changes should force-append a dt: with the
> > open-ended datetime of the change? Or do you already have a way to deal with
> > this situation?
>
> The aforementioned maxuid prevents stuff that's too old from
> being seen. Otherwise, there's always "public-inbox-learn rm".
How would it handle the situation where we import a new list into lore with a
10-year-long archive of messages?
-K
^ permalink raw reply [relevance 68%]
* Re: lei-managed pseudo mailing lists
2021-04-26 18:20 65% ` Konstantin Ryabitsev
@ 2021-04-26 18:47 71% ` Eric Wong
2021-04-26 19:46 68% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-04-26 18:47 UTC (permalink / raw)
To: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> On Mon, Apr 26, 2021 at 05:37:26PM +0000, Eric Wong wrote:
> > > The latter is specifically something I think would be of interest to kernel
> > > folks, so I envision that we'd have something like the following:
> > >
> > > - a maintainer publishes a configuration file we can pass to lei
> >
> > The command-line might be enough, the pathname of the current
> > state/config file is a bit tricky and tied to its output.
> > I suppose "lei import-search" can be a command, though...
>
> Excellent, excellent. How well does it deal with the situation when the search
> parameters change?
"lei edit-search" can be used to zero the maxuid parameters;
and normal v2 deduplication will prevent duplicates from showing up.
It's not automatic, though; though that probably seems like a good
idea to keep manual, anyways, given the step 2. below.
> > > - our backend lei process uses all of lore.kernel.org sources to create and
> > > continuously update a new public-inbox repository with matching search
> > > results
> >
> > There's already some accomodations for that in LeiSavedSearch
> > which can present itself as a PublicInbox::Inbox-ish object to
> > PublicInbox::WWW (untested).
> >
> > Searching an within LSS isn't implemented, yet, but I think it's
> > doable w/o extra Xapian storage.
> >
> > However, git object storage isn't duplicated, which is nice for
> > local use (instaweb-like), but supporting clone/fetch isn't as
> > natural...
>
> I'm thinking we need the ability to make it a real clonable repository --
> perhaps without its own xapian index? Actual git repositories aren't large,
> especially if they are only used for direct git operations. Disk space is
> cheap, it's the IO that's expensive. :)
True, though cache overheads hurt a bit. I also wonder if lei
can increase traffic to public-inbox-<imapd|nntpd> to reduce
the need/use of "git clone".
> If these are real clonable repositories, then it would be easy for people to
> set up replication for just the curated content people want.
Understood. Using --output v2publicinbox:... w/o --shared is
totally doable.
> > Perhaps supporting a v2 inbox as an lei q output destination
> > is in order:
> >
> > lei q --output v2publicinbox:/path/to/v2 --shared SEARCH_TERMS
> >
> > --shared would be "git clone --shared", the new v2 inbox can
> > use ~/.cache/lei/all_locals_ever.git/ as an alternate and not
> > duplicate space for blobs.
>
> Not really worried about deduping blobs, but I'm wondering how to make it work
> well when search parameters change (see above). E.g.:
>
> 1. we create the repo with one set of parameters
> 2. maintainer then broadens it up to include something else
> 3. maintainer then decides that it's now *way* too much and narrows it down again
>
> We don't really want step 2 to lead to a permanent ballooning of the
> repository, so perhaps all query changes should force-append a dt: with the
> open-ended datetime of the change? Or do you already have a way to deal with
> this situation?
The aforementioned maxuid prevents stuff that's too old from
being seen. Otherwise, there's always "public-inbox-learn rm".
> > > - we set up a mlmmj list that doesn't receive any direct mail but is only fed
> > > from saved search results; people can subscribe/unsubscribe as they would
> > > with any other mlmmj list
> > >
> > > Any particular reason this wouldn't work?
> >
> > Nope :) As long as all the data formats can interoperate
> > (mostly RFC5322/2822). "lei convert" is nice, too :)
>
> Great! I believe this will help untangle the current situation with "where
> should I send this kernel patch".
>
> I want "just send it to linux-kernel@vger.kernel.org" to be a valid option
> again. Participating subsystems can then define what patches they want to see
> by setting up pseudo-lists and letting participating reviewers/maintainers
> subscribe to them via their preferred mail delivery mechanism.
Yup, that seems easiest for new contributors.
^ permalink raw reply [relevance 71%]
* Re: lei-managed pseudo mailing lists
2021-04-26 17:37 70% ` Eric Wong
@ 2021-04-26 18:20 65% ` Konstantin Ryabitsev
2021-04-26 18:47 71% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-04-26 18:20 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Mon, Apr 26, 2021 at 05:37:26PM +0000, Eric Wong wrote:
> > The latter is specifically something I think would be of interest to kernel
> > folks, so I envision that we'd have something like the following:
> >
> > - a maintainer publishes a configuration file we can pass to lei
>
> The command-line might be enough, the pathname of the current
> state/config file is a bit tricky and tied to its output.
> I suppose "lei import-search" can be a command, though...
Excellent, excellent. How well does it deal with the situation when the search
parameters change?
> > - our backend lei process uses all of lore.kernel.org sources to create and
> > continuously update a new public-inbox repository with matching search
> > results
>
> There's already some accomodations for that in LeiSavedSearch
> which can present itself as a PublicInbox::Inbox-ish object to
> PublicInbox::WWW (untested).
>
> Searching an within LSS isn't implemented, yet, but I think it's
> doable w/o extra Xapian storage.
>
> However, git object storage isn't duplicated, which is nice for
> local use (instaweb-like), but supporting clone/fetch isn't as
> natural...
I'm thinking we need the ability to make it a real clonable repository --
perhaps without its own xapian index? Actual git repositories aren't large,
especially if they are only used for direct git operations. Disk space is
cheap, it's the IO that's expensive. :)
If these are real clonable repositories, then it would be easy for people to
set up replication for just the curated content people want.
> Perhaps supporting a v2 inbox as an lei q output destination
> is in order:
>
> lei q --output v2publicinbox:/path/to/v2 --shared SEARCH_TERMS
>
> --shared would be "git clone --shared", the new v2 inbox can
> use ~/.cache/lei/all_locals_ever.git/ as an alternate and not
> duplicate space for blobs.
Not really worried about deduping blobs, but I'm wondering how to make it work
well when search parameters change (see above). E.g.:
1. we create the repo with one set of parameters
2. maintainer then broadens it up to include something else
3. maintainer then decides that it's now *way* too much and narrows it down again
We don't really want step 2 to lead to a permanent ballooning of the
repository, so perhaps all query changes should force-append a dt: with the
open-ended datetime of the change? Or do you already have a way to deal with
this situation?
> > - we set up a mlmmj list that doesn't receive any direct mail but is only fed
> > from saved search results; people can subscribe/unsubscribe as they would
> > with any other mlmmj list
> >
> > Any particular reason this wouldn't work?
>
> Nope :) As long as all the data formats can interoperate
> (mostly RFC5322/2822). "lei convert" is nice, too :)
Great! I believe this will help untangle the current situation with "where
should I send this kernel patch".
I want "just send it to linux-kernel@vger.kernel.org" to be a valid option
again. Participating subsystems can then define what patches they want to see
by setting up pseudo-lists and letting participating reviewers/maintainers
subscribe to them via their preferred mail delivery mechanism.
-K
^ permalink raw reply [relevance 65%]
* Re: lei-managed pseudo mailing lists
2021-04-26 16:44 70% lei-managed pseudo mailing lists Konstantin Ryabitsev
@ 2021-04-26 17:37 70% ` Eric Wong
2021-04-26 18:20 65% ` Konstantin Ryabitsev
0 siblings, 1 reply; 200+ results
From: Eric Wong @ 2021-04-26 17:37 UTC (permalink / raw)
To: meta
Konstantin Ryabitsev <konstantin@linuxfoundation.org> wrote:
> Hello:
>
> One of the services I think would be interesting to provide is ability for
> people to subscribe to "curated saved searches". For example, a kernel
> subsystem maintainer can define a set of query parameters (a thread mentions
> these files/functions/terms, etc), and allow others to follow this saved
> search either by defining it as a remote source for their own lei command, or
> by subscribing to it as they would to any regular mailing list.
>
> The latter is specifically something I think would be of interest to kernel
> folks, so I envision that we'd have something like the following:
>
> - a maintainer publishes a configuration file we can pass to lei
The command-line might be enough, the pathname of the current
state/config file is a bit tricky and tied to its output.
I suppose "lei import-search" can be a command, though...
> - our backend lei process uses all of lore.kernel.org sources to create and
> continuously update a new public-inbox repository with matching search
> results
There's already some accomodations for that in LeiSavedSearch
which can present itself as a PublicInbox::Inbox-ish object to
PublicInbox::WWW (untested).
Searching an within LSS isn't implemented, yet, but I think it's
doable w/o extra Xapian storage.
However, git object storage isn't duplicated, which is nice for
local use (instaweb-like), but supporting clone/fetch isn't as
natural...
Perhaps supporting a v2 inbox as an lei q output destination
is in order:
lei q --output v2publicinbox:/path/to/v2 --shared SEARCH_TERMS
--shared would be "git clone --shared", the new v2 inbox can
use ~/.cache/lei/all_locals_ever.git/ as an alternate and not
duplicate space for blobs.
> - we set up a mlmmj list that doesn't receive any direct mail but is only fed
> from saved search results; people can subscribe/unsubscribe as they would
> with any other mlmmj list
>
> Any particular reason this wouldn't work?
Nope :) As long as all the data formats can interoperate
(mostly RFC5322/2822). "lei convert" is nice, too :)
^ permalink raw reply [relevance 70%]
* lei-managed pseudo mailing lists
@ 2021-04-26 16:44 70% Konstantin Ryabitsev
2021-04-26 17:37 70% ` Eric Wong
0 siblings, 1 reply; 200+ results
From: Konstantin Ryabitsev @ 2021-04-26 16:44 UTC (permalink / raw)
To: meta
Hello:
One of the services I think would be interesting to provide is ability for
people to subscribe to "curated saved searches". For example, a kernel
subsystem maintainer can define a set of query parameters (a thread mentions
these files/functions/terms, etc), and allow others to follow this saved
search either by defining it as a remote source for their own lei command, or
by subscribing to it as they would to any regular mailing list.
The latter is specifically something I think would be of interest to kernel
folks, so I envision that we'd have something like the following:
- a maintainer publishes a configuration file we can pass to lei
- our backend lei process uses all of lore.kernel.org sources to create and
continuously update a new public-inbox repository with matching search
results
- we set up a mlmmj list that doesn't receive any direct mail but is only fed
from saved search results; people can subscribe/unsubscribe as they would
with any other mlmmj list
Any particular reason this wouldn't work?
-K
^ permalink raw reply [relevance 70%]
* [PATCH] lei p2q: exit with failure if format-patch fails
@ 2021-04-26 8:43 66% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-26 8:43 UTC (permalink / raw)
To: meta
Merely redirecting the failure message from git to our stderr is
insufficient.
---
lib/PublicInbox/LeiP2q.pm | 7 +++++--
t/lei-p2q.t | 4 +++-
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index a8a3dd2c..cb2309c7 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -105,6 +105,7 @@ sub do_p2q { # via wq_do
}
my $smsg = bless {}, 'PublicInbox::Smsg';
my $in = $self->{0};
+ my @cmd;
unless ($in) {
my $input = $self->{input};
my $devfd = $lei->path_to_fd($input) // return;
@@ -114,11 +115,13 @@ sub do_p2q { # via wq_do
open($in, '<', $input) or
return $lei->fail("open < $input: $!");
} else {
- my @cmd = (qw(git format-patch --stdout -1), $input);
+ @cmd = (qw(git format-patch --stdout -1), $input);
$in = popen_rd(\@cmd, undef, { 2 => $lei->{2} });
}
};
- my $eml = PublicInbox::Eml->new(\(do { local $/; <$in> }));
+ my $str = do { local $/; <$in> };
+ @cmd && !close($in) and return $lei->fail("E: @cmd failed: $?");
+ my $eml = PublicInbox::Eml->new(\$str);
$lei->{diff_want} = +{ map { $_ => 1 } @want };
$smsg->populate($eml);
while (my ($pfx, $fields) = each %pfx2smsg) {
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index 87cf9fa7..be2d437c 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -6,9 +6,11 @@ require_git 2.6;
require_mods(qw(json DBD::SQLite Search::Xapian));
test_lei(sub {
+ ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
+ 'p2q fails on bogus arg');
lei_ok(qw(p2q -w dfpost t/data/0001.patch));
is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname');
- open my $fh, '+<', 't/data/0001.patch';
+ open my $fh, '+<', 't/data/0001.patch' or xbail "open: $!";
lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin');
^ permalink raw reply related [relevance 66%]
* [PATCH 7/7] lei import: keep sync info for Maildir and IMAP folders
2021-04-24 9:28 89% [PATCH 0/7] lei sync preparations, "lei inspect" Eric Wong
2021-04-24 9:28 66% ` [PATCH 2/7] t/lei_to_mail: split "lei import" test $HOME directory Eric Wong
@ 2021-04-24 9:28 27% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-04-24 9:28 UTC (permalink / raw)
To: meta
We aren't using it, yet, but the plan is to be able to use
this information to propagate keyword changes back to IMAP
and Maildir folders using some to-be-implemented command.
"lei inspect" is a half-baked new command to make testing this
change easier. It will be updated to support more SQLite+Xapian
introspection duties in the future, including public-inbox
things independent of lei.
---
MANIFEST | 1 +
lib/PublicInbox/LEI.pm | 16 ++++--
lib/PublicInbox/LeiImport.pm | 22 ++++++--
lib/PublicInbox/LeiInput.pm | 41 +++++++++++++--
lib/PublicInbox/LeiInspect.pm | 96 +++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiSearch.pm | 7 +++
lib/PublicInbox/LeiStore.pm | 20 +++++++-
t/lei-import-imap.t | 27 +++++++++-
t/lei-import-maildir.t | 21 ++++++++
9 files changed, 238 insertions(+), 13 deletions(-)
create mode 100644 lib/PublicInbox/LeiInspect.pm
diff --git a/MANIFEST b/MANIFEST
index abaf54b0..79d393c5 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -199,6 +199,7 @@ lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
lib/PublicInbox/LeiInit.pm
lib/PublicInbox/LeiInput.pm
+lib/PublicInbox/LeiInspect.pm
lib/PublicInbox/LeiLsLabel.pm
lib/PublicInbox/LeiLsSearch.pm
lib/PublicInbox/LeiMailSync.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9f49fc03..39278de6 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -64,9 +64,13 @@ sub opt_dash ($$) {
($spec, '<>' => $cb, $GLP_PASS) # for Getopt::Long
}
-sub rel2abs ($$) {
+# rel2abs preserves symlinks in parent, unlike abs_path
+sub rel2abs {
my ($self, $p) = @_;
- return $p if index($p, '/') == 0; # already absolute
+ if (index($p, '/') == 0) { # already absolute
+ $p =~ tr!/!/!s; # squeeze redundant slashes
+ return $p;
+ }
my $pwd = $self->{env}->{PWD};
my $cwd;
if (defined $pwd) {
@@ -84,6 +88,9 @@ sub rel2abs ($$) {
File::Spec->rel2abs($p, $pwd);
}
+# abs_path resolves symlinks in parent iff all parents exist
+sub abs_path { Cwd::abs_path($_[1]) // rel2abs(@_) }
+
sub share_path ($) { # $HOME/.local/share/lei/$FOO
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_DATA_HOME} //
@@ -193,7 +200,7 @@ our %CMD = ( # sorted in order of importance/use:
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s
- lock=s@ in-format|F=s kw! verbose|v+ incremental!), @c_opt ],
+ lock=s@ in-format|F=s kw! verbose|v+ incremental! sync!), @c_opt ],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s
@@ -205,6 +212,9 @@ our %CMD = ( # sorted in order of importance/use:
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
qw(c=s@ C=s@), pass_through('git config') ],
+'inspect' => [ 'ITEMS...', 'inspect lei/store and/or local external',
+ qw(pretty ascii dir=s), @c_opt ],
+
'init' => [ '[DIRNAME]', sub {
"initialize storage, default: ".store_path($_[0]);
}, @c_opt ],
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index e3c756e8..daaa6753 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -13,7 +13,6 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml, $vmd) = @_;
my $xoids = $self->{lei}->{ale}->xoids_for($eml);
if (my $all_vmd = $self->{all_vmd}) {
- $vmd //= {};
@$vmd{keys %$all_vmd} = values %$all_vmd;
}
$self->{lei}->{sto}->ipc_do('set_eml', $eml, $vmd, $xoids);
@@ -31,11 +30,26 @@ sub input_mbox_cb { # MboxReader callback
sub input_maildir_cb { # maildir_each_eml cb
my ($f, $kw, $eml, $self) = @_;
- input_eml_cb($self, $eml, $self->{-import_kw} ? { kw => $kw } : undef);
+ my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
+ if ($self->{-mail_sync}) {
+ if ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) { # ugh...
+ $vmd->{sync_info} = [ "maildir:$1", \(my $n = $2) ];
+ } else {
+ warn "E: $f was not from a Maildir?\n";
+ }
+ }
+ input_eml_cb($self, $eml, $vmd);
}
-sub input_net_cb { # imap_each, nntp_each cb
+sub input_imap_cb { # imap_each
my ($url, $uid, $kw, $eml, $self) = @_;
+ my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
+ $vmd->{sync_info} = [ $url, $uid ] if $self->{-mail_sync};
+ input_eml_cb($self, $eml, $vmd);
+}
+
+sub input_nntp_cb { # nntp_each
+ my ($url, $num, $kw, $eml, $self) = @_;
input_eml_cb($self, $eml, $self->{-import_kw} ? { kw => $kw } : undef);
}
@@ -61,6 +75,8 @@ sub lei_import { # the main "lei import" method
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
$self->{all_vmd} = $vmd_mod if scalar keys %$vmd_mod;
$self->prepare_inputs($lei, \@inputs) or return;
+ $self->{-mail_sync} = $lei->{opt}->{sync} // 1;
+
$lei->ale; # initialize for workers to read
my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
if (my $net = $lei->{net}) {
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 0114f5ee..d11d23d4 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -83,11 +83,13 @@ sub input_path_url {
my $ifmt = lc($lei->{opt}->{'in-format'} // '');
# TODO auto-detect?
if ($input =~ m!\Aimaps?://!i) {
- $lei->{net}->imap_each($input, $self->can('input_net_cb'),
+ $lei->{net}->imap_each($input, $self->can('input_imap_cb') //
+ $self->can('input_net_cb'),
$self, @args);
return;
} elsif ($input =~ m!\A(?:nntps?|s?news)://!i) {
- $lei->{net}->nntp_each($input, $self->can('input_net_cb'),
+ $lei->{net}->nntp_each($input, $self->can('input_nntp_cb') //
+ $self->can('input_net_cb'),
$self, @args);
return;
}
@@ -130,11 +132,13 @@ EOM
sub prepare_inputs { # returns undef on error
my ($self, $lei, $inputs) = @_;
my $in_fmt = $lei->{opt}->{'in-format'};
+ my $sync = $lei->{opt}->{sync} ? {} : undef; # using LeiMailSync
if ($lei->{opt}->{stdin}) {
@$inputs and return
$lei->fail("--stdin and @$inputs do not mix");
check_input_format($lei) or return;
push @$inputs, '/dev/stdin';
+ push @{$sync->{no}}, '/dev/stdin' if $sync;
}
my $net = $lei->{net}; # NetWriter may be created by l2m
my (@f, @d);
@@ -145,6 +149,13 @@ sub prepare_inputs { # returns undef on error
require PublicInbox::NetReader;
$net //= PublicInbox::NetReader->new;
$net->add_url($input);
+ if ($sync) {
+ if ($input =~ m!\Aimaps?://!) {
+ push @{$sync->{ok}}, $input;
+ } else {
+ push @{$sync->{no}}, $input;
+ }
+ }
} elsif ($input_path =~ s/\A([a-z0-9]+)://is) {
my $ifmt = lc $1;
if (($in_fmt // $ifmt) ne $ifmt) {
@@ -152,6 +163,13 @@ sub prepare_inputs { # returns undef on error
--in-format=$in_fmt and `$ifmt:' conflict
}
+ if ($sync) {
+ if ($ifmt =~ /\A(?:maildir|mh)\z/i) {
+ push @{$sync->{ok}}, $input;
+ } else {
+ push @{$sync->{no}}, $input;
+ }
+ }
my $devfd = $lei->path_to_fd($input_path) // return;
if ($devfd >= 0 || (-f $input_path || -p _)) {
require PublicInbox::MboxLock;
@@ -162,6 +180,7 @@ sub prepare_inputs { # returns undef on error
require PublicInbox::MdirReader;
$ifmt eq 'maildir' or return
$lei->fail("$ifmt not supported");
+ $input = $lei->abs_path($input) if $sync;
} else {
return $lei->fail("Unable to handle $input");
}
@@ -170,12 +189,18 @@ sub prepare_inputs { # returns undef on error
$input is `eml', not --in-format=$in_fmt
require PublicInbox::Eml;
+ push @{$sync->{no}}, $input if $sync;
} else {
my $devfd = $lei->path_to_fd($input) // return;
if ($devfd >= 0 || -f $input || -p _) {
- push @f, $input
+ push @{$sync->{no}}, $input if $sync;
+ push @f, $input;
} elsif (-d $input) {
- push @d, $input
+ if ($sync) {
+ $input = $lei->abs_path($input);
+ push @{$sync->{ok}}, $input;
+ }
+ push @d, $input;
} else {
return $lei->fail("Unable to handle $input")
}
@@ -185,6 +210,14 @@ $input is `eml', not --in-format=$in_fmt
if (@d) { # TODO: check for MH vs Maildir, here
require PublicInbox::MdirReader;
}
+ if ($sync && $sync->{no}) {
+ return $lei->fail(<<"") if !$sync->{ok};
+--sync specified but no inputs support it
+
+ # non-fatal if some inputs support support sync
+ $lei->err("# --sync will only be used for @{$sync->{ok}}");
+ $lei->err("# --sync is not supported for: @{$sync->{no}}");
+ }
if ($net) {
if (my $err = $net->errors) {
return $lei->fail($err);
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
new file mode 100644
index 00000000..6cfc8083
--- /dev/null
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -0,0 +1,96 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei inspect" general purpose inspector for stuff in SQLite and
+# Xapian. Will eventually be useful with plain public-inboxes,
+# not just lei/store. This is totally half-baked at the moment
+# but useful for testing.
+package PublicInbox::LeiInspect;
+use strict;
+use v5.10.1;
+use PublicInbox::Config;
+
+sub inspect_blob ($$) {
+ my ($lei, $oidhex) = @_;
+ my $ent = {};
+ if (my $lse = $lei->{lse}) {
+ my @docids = $lse ? $lse->over->blob_exists($oidhex) : ();
+ $ent->{'lei/store'} = \@docids if @docids;
+ my $lms = $lse->lms;
+ if (my $loc = $lms ? $lms->locations_for($oidhex) : undef) {
+ $ent->{sync} = $loc;
+ }
+ }
+ $ent;
+}
+
+sub inspect_sync_folder ($$) {
+ my ($lei, $folder) = @_;
+ my $ent = {};
+ my $lse = $lei->{lse} or return $ent;
+ my $lms = $lse->lms or return $ent;
+ my @folders;
+ if ($folder =~ m!\Aimaps?://!i) {
+ require PublicInbox::URIimap;
+ my $uri = PublicInbox::URIimap->new($folder)->canonical;
+ if (defined($uri->uidvalidity)) {
+ $folders[0] = $$uri;
+ } else {
+ my @maybe = $lms->folders($$uri);
+ @folders = grep {
+ my $u = PublicInbox::URIimap->new($_);
+ $uri->uidvalidity($u->uidvalidity);
+ $$uri eq $$u;
+ } @maybe;
+ }
+ } elsif ($folder =~ m!\A(maildir|mh):(.+)!i) {
+ my $type = $1;
+ $folders[0] = "$type:".$lei->abs_path($2);
+ } elsif (-d $folder) {
+ $folders[0] = 'maildir:'.$lei->abs_path($folder);
+ } else {
+ $lei->fail("$folder not understood");
+ }
+ $lei->qerr("# no folders match $folder (non-fatal)") if !@folders;
+ for my $f (@folders) {
+ $ent->{$f} = $lms->location_stats($f); # may be undef
+ }
+ $ent
+}
+
+sub inspect1 ($$$) {
+ my ($lei, $item, $more) = @_;
+ my $ent;
+ if ($item =~ /\Ablob:(.+)/) {
+ $ent = inspect_blob($lei, $1);
+ } elsif ($item =~ m!\Aimaps?://!i ||
+ $item =~ m!\A(?:maildir|mh):!i || -d $item) {
+ $ent = inspect_sync_folder($lei, $item);
+ } else { # TODO: more things
+ return $lei->fail("$item not understood");
+ }
+ $lei->out($lei->{json}->encode($ent));
+ $lei->out(',') if $more;
+ 1;
+}
+
+sub lei_inspect {
+ my ($lei, @argv) = @_;
+ $lei->{1}->autoflush(0);
+ my $multi = scalar(@argv) > 1;
+ $lei->out('[') if $multi;
+ $lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
+ $lei->{lse} = ($lei->{opt}->{external} // 1) ? do {
+ my $sto = $lei->_lei_store;
+ $sto ? $sto->search : undef;
+ } : undef;
+ if ($lei->{opt}->{pretty} || -t $lei->{1}) {
+ $lei->{json}->pretty(1)->indent(2);
+ }
+ while (defined(my $x = shift @argv)) {
+ inspect1($lei, $x, scalar(@argv)) or return;
+ }
+ $lei->out(']') if $multi;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index ff615d89..cd28a700 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -137,4 +137,11 @@ sub qparse_new {
$qp
}
+sub lms {
+ my ($self) = @_;
+ require PublicInbox::LeiMailSync;
+ my $f = "$self->{topdir}/mail_sync.sqlite3";
+ -f $f ? PublicInbox::LeiMailSync->new($f) : undef;
+}
+
1;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index f8371abf..1cf7ffc1 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -190,13 +190,28 @@ sub remove_eml_vmd {
\@docids;
}
+sub set_sync_info ($$$) {
+ my ($self, $oidhex, $sync_info) = @_;
+ ($self->{lms} //= do {
+ require PublicInbox::LeiMailSync;
+ my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
+ my $lms = PublicInbox::LeiMailSync->new($f);
+ $lms->lms_begin;
+ $lms;
+ })->set_src($oidhex, @$sync_info);
+}
+
sub add_eml {
my ($self, $eml, $vmd, $xoids) = @_;
my $im = $self->importer; # may create new epoch
my ($eidx, $tl) = eidx_init($self); # updates/writes alternates file
my $oidx = $eidx->{oidx}; # PublicInbox::Import::add checks this
my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
- $im->add($eml, undef, $smsg) or return; # duplicate returns undef
+ my $im_mark = $im->add($eml, undef, $smsg);
+ if ($vmd && $vmd->{sync_info}) {
+ set_sync_info($self, $smsg->{blob}, $vmd->{sync_info});
+ }
+ $im_mark or return; # duplicate blob returns undef
local $self->{current_info} = $smsg->{blob};
my $vivify_xvmd = delete($smsg->{-vivify_xvmd}) // []; # exact matches
@@ -379,6 +394,9 @@ sub done {
warn $err;
}
}
+ if (my $lms = delete $self->{lms}) {
+ $lms->lms_commit;
+ }
$self->{priv_eidx}->done; # V2Writable::done
xchg_stderr($self);
die $err if $err;
diff --git a/t/lei-import-imap.t b/t/lei-import-imap.t
index 490ea9be..4a3bd6d8 100644
--- a/t/lei-import-imap.t
+++ b/t/lei-import-imap.t
@@ -12,10 +12,28 @@ my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-imapd: $?");
my $host_port = tcp_host_port($sock);
undef $sock;
test_lei({ tmpdir => $tmpdir }, sub {
+ my $url = "imap://$host_port/t.v2.0";
+
lei_ok(qw(q z:1..));
my $out = json_utf8->decode($lei_out);
is_deeply($out, [ undef ], 'nothing imported, yet');
- lei_ok('import', "imap://$host_port/t.v2.0");
+
+ lei_ok('inspect', $url);
+ is_deeply(json_utf8->decode($lei_out), {}, 'no inspect stats, yet');
+
+ lei_ok('import', $url);
+
+ lei_ok('inspect', $url);
+ my $inspect = json_utf8->decode($lei_out);
+ my @k = keys %$inspect;
+ is(scalar(@k), 1, 'one URL resolved');
+ like($k[0], qr!\A\Q$url\E;UIDVALIDITY=\d+\z!, 'inspect URL matches');
+ my $stats = $inspect->{$k[0]};
+ is_deeply([ sort keys %$stats ],
+ [ qw(uid.count uid.max uid.min) ], 'keys match');
+ ok($stats->{'uid.min'} < $stats->{'uid.max'}, 'min < max');
+ ok($stats->{'uid.count'} > 0, 'count > 0');
+
lei_ok(qw(q z:1..));
$out = json_utf8->decode($lei_out);
ok(scalar(@$out) > 1, 'got imported messages');
@@ -23,9 +41,14 @@ test_lei({ tmpdir => $tmpdir }, sub {
my %r;
for (@$out) { $r{ref($_)}++ }
is_deeply(\%r, { 'HASH' => scalar(@$out) }, 'all hashes');
- lei_ok([qw(tag +kw:seen), "imap://$host_port/t.v2.0"], undef, undef);
+ lei_ok([qw(tag +kw:seen), $url], undef, undef);
my $f = "$ENV{HOME}/.local/share/lei/store/net_last.sqlite3";
ok(-s $f, 'net tracked for redundant imports');
+ lei_ok('inspect', "blob:$out->[5]->{blob}");
+ my $x = json_utf8->decode($lei_out);
+ is(ref($x->{'lei/store'}), 'ARRAY', 'lei/store in inspect');
+ is(ref($x->{sync}), 'HASH', 'sync in inspect');
+ is(ref($x->{sync}->{$k[0]}), 'ARRAY', 'UID arrays in inspect');
});
done_testing;
diff --git a/t/lei-import-maildir.t b/t/lei-import-maildir.t
index 6706b014..3e3d9188 100644
--- a/t/lei-import-maildir.t
+++ b/t/lei-import-maildir.t
@@ -12,6 +12,21 @@ test_lei(sub {
BAIL_OUT "symlink $md $!";
lei_ok(qw(import), $md, \'import Maildir');
my $imp_err = $lei_err;
+
+ my %i;
+ lei_ok('inspect', $md); $i{no_type} = $lei_out;
+ lei_ok('inspect', "maildir:$md"), $i{with_type} = $lei_out;
+ lei_ok(['inspect', $md], undef, { -C => $ENV{HOME}, %$lei_opt });
+ $i{rel_no_type} = $lei_out;
+ lei_ok(['inspect', "maildir:$md"], undef,
+ { -C => $ENV{HOME}, %$lei_opt });
+ $i{rel_with_type} = $lei_out;
+ my %v = map { $_ => 1 } values %i;
+ is(scalar(keys %v), 1, 'inspect handles relative and absolute paths');
+ my $inspect = json_utf8->decode([ keys %v ]->[0]);
+ is_deeply($inspect, {"maildir:$md" => { 'name.count' => 1 }},
+ 'inspect maildir: path had expected output');
+
lei_ok(qw(q s:boolean));
my $res = json_utf8->decode($lei_out);
like($res->[0]->{'s'}, qr/use boolean/, 'got expected result')
@@ -19,6 +34,12 @@ test_lei(sub {
is_deeply($res->[0]->{kw}, ['seen'], 'keyword set');
is($res->[1], undef, 'only got one result');
+ lei_ok('inspect', "blob:$res->[0]->{blob}");
+ $inspect = json_utf8->decode($lei_out);
+ is(ref(delete $inspect->{"lei/store"}), 'ARRAY', 'lei/store IDs');
+ is_deeply($inspect, { sync => { "maildir:$md" => [ 'x:2,S' ] } },
+ 'maildir sync info as expected');
+
lei_ok(qw(import), $md, \'import Maildir again');
$imp_err = $lei_err;
lei_ok(qw(q -d none s:boolean), \'lei q w/o dedupe');
^ permalink raw reply related [relevance 27%]
* [PATCH 0/7] lei sync preparations, "lei inspect"
@ 2021-04-24 9:28 89% Eric Wong
2021-04-24 9:28 66% ` [PATCH 2/7] t/lei_to_mail: split "lei import" test $HOME directory Eric Wong
2021-04-24 9:28 27% ` [PATCH 7/7] lei import: keep sync info for Maildir and IMAP folders Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-04-24 9:28 UTC (permalink / raw)
To: meta
"lei import" now tracks IMAP and Maildir source information in
preparation for propagating keywords back to IMAP|Maildir.
"lei inspect" is a long-overdue debug/diagnostic thing.
Eric Wong (7):
lei_input: drop outdated comment w.r.t. compression
t/lei_to_mail: split "lei import" test $HOME directory
URIimap: support ->uidvalidity and ->iuid
net_reader: imap_each: add UIDVALIDITY to URL arg
doc: lei_design_notes: add a bit on WAL usage
lei_mail_sync: for bidirectional keyword sync
lei import: keep sync info for Maildir and IMAP folders
Documentation/lei_design_notes.txt | 12 ++
MANIFEST | 3 +
lib/PublicInbox/LEI.pm | 16 ++-
lib/PublicInbox/LeiImport.pm | 22 ++-
lib/PublicInbox/LeiInput.pm | 42 +++++-
lib/PublicInbox/LeiInspect.pm | 96 +++++++++++++
lib/PublicInbox/LeiMailSync.pm | 211 +++++++++++++++++++++++++++++
lib/PublicInbox/LeiSearch.pm | 7 +
lib/PublicInbox/LeiStore.pm | 20 ++-
lib/PublicInbox/NetReader.pm | 11 +-
lib/PublicInbox/TestCommon.pm | 2 +
lib/PublicInbox/URIimap.pm | 38 +++++-
t/lei-import-imap.t | 27 +++-
t/lei-import-maildir.t | 21 +++
t/lei_mail_sync.t | 68 ++++++++++
t/lei_to_mail.t | 8 +-
t/net_reader-imap.t | 4 +-
t/uri_imap.t | 32 ++++-
18 files changed, 612 insertions(+), 28 deletions(-)
create mode 100644 lib/PublicInbox/LeiInspect.pm
create mode 100644 lib/PublicInbox/LeiMailSync.pm
create mode 100644 t/lei_mail_sync.t
^ permalink raw reply [relevance 89%]
* [PATCH 2/7] t/lei_to_mail: split "lei import" test $HOME directory
2021-04-24 9:28 89% [PATCH 0/7] lei sync preparations, "lei inspect" Eric Wong
@ 2021-04-24 9:28 66% ` Eric Wong
2021-04-24 9:28 27% ` [PATCH 7/7] lei import: keep sync info for Maildir and IMAP folders Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-04-24 9:28 UTC (permalink / raw)
To: meta
"lei import" behavior will may change w.r.t. keyword
handling. Use separate $HOME between different test_lei
to ensure isolation between the tests.
---
lib/PublicInbox/TestCommon.pm | 2 ++
t/lei_to_mail.t | 8 ++++----
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index b5d0b9f8..49cecacd 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -517,6 +517,7 @@ SKIP: {
require_git(2.6, 1) or skip('git 2.6+ required for lei test', 2);
require_mods(qw(json DBD::SQLite Search::Xapian), 2);
require PublicInbox::Config;
+ require File::Path;
local %ENV = %ENV;
delete $ENV{XDG_DATA_HOME};
delete $ENV{XDG_CONFIG_HOME};
@@ -534,6 +535,7 @@ EOM
$lei_opt = { 1 => \$lei_out, 2 => \$lei_err };
my ($daemon_pid, $for_destroy, $daemon_xrd);
my $tmpdir = $test_opt->{tmpdir};
+ File::Path::mkpath($tmpdir) if (defined $tmpdir && !-d $tmpdir);
($tmpdir, $for_destroy) = tmpdir unless $tmpdir;
state $persist_xrd = $ENV{TEST_LEI_DAEMON_PERSIST_DIR};
SKIP: {
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 51357257..32532a98 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -129,9 +129,9 @@ my $orig = do {
$raw;
};
-test_lei(sub {
- ok(lei(qw(import -F), $mbox, $fn), 'imported mbox');
- ok(lei(qw(q s:x)), 'lei q works') or diag $lei_err;
+test_lei({tmpdir => "$tmpdir/using -F"}, sub {
+ lei_ok(qw(import -F), $mbox, $fn, \'imported mbox');
+ lei_ok(qw(q s:x), \'lei q works') or diag $lei_err;
my $res = json_utf8->decode($lei_out);
my $x = $res->[0];
is($x->{'s'}, 'x', 'subject imported') or diag $lei_out;
@@ -139,7 +139,7 @@ test_lei(sub {
is($res->[1], undef, 'only one result');
});
-test_lei(sub {
+test_lei({tmpdir => "$tmpdir/using TYPE: prefix"}, sub {
lei_ok('import', "$mbox:$fn", \'imported mbox:/path') or diag $lei_err;
lei_ok(qw(q s:x), \'lei q works') or diag $lei_err;
my $res = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 66%]
* [PATCH] lei import: support adding keywords and labels on import
@ 2021-04-23 11:22 40% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-23 11:22 UTC (permalink / raw)
To: meta
This saves some work and makes it easier to set volatile
metadata on a message at import time.
---
lib/PublicInbox/LeiImport.pm | 7 +++++
lib/PublicInbox/LeiInput.pm | 54 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiTag.pm | 56 ++----------------------------------
lib/PublicInbox/SearchIdx.pm | 41 +++++++++++++++-----------
t/lei-import.t | 10 +++++++
5 files changed, 98 insertions(+), 70 deletions(-)
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index accf08f5..e3c756e8 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -12,6 +12,10 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml, $vmd) = @_;
my $xoids = $self->{lei}->{ale}->xoids_for($eml);
+ if (my $all_vmd = $self->{all_vmd}) {
+ $vmd //= {};
+ @$vmd{keys %$all_vmd} = values %$all_vmd;
+ }
$self->{lei}->{sto}->ipc_do('set_eml', $eml, $vmd, $xoids);
}
@@ -53,6 +57,9 @@ sub lei_import { # the main "lei import" method
$sto->write_prepare($lei);
my $self = bless {}, __PACKAGE__;
$self->{-import_kw} = $lei->{opt}->{kw} // 1;
+ my $vmd_mod = $self->vmd_mod_extract(\@inputs);
+ return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
+ $self->{all_vmd} = $vmd_mod if scalar keys %$vmd_mod;
$self->prepare_inputs($lei, \@inputs) or return;
$lei->ale; # initialize for workers to read
my $j = $lei->{opt}->{jobs} // scalar(@{$self->{inputs}}) || 1;
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index e416d3ed..de60a076 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -7,6 +7,38 @@ use strict;
use v5.10.1;
use PublicInbox::DS;
+# JMAP RFC 8621 4.1.1
+# https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
+our @KW = (qw(seen answered flagged draft), # widely-compatible
+ qw(forwarded), # IMAP + Maildir
+ qw(phishing junk notjunk)); # rarely supported
+
+# note: RFC 8621 states "Users may add arbitrary keywords to an Email",
+# but is it good idea? Stick to the system and reserved ones, for now.
+# The widely-compatible ones map to IMAP system flags, Maildir flags
+# and mbox Status/X-Status headers.
+my %KW = map { $_ => 1 } @KW;
+my $L_MAX = 244; # Xapian term limit - length('L')
+
+# RFC 8621, sec 2 (Mailboxes) a "label" for us is a JMAP Mailbox "name"
+# "Servers MAY reject names that violate server policy"
+my %ERR = (
+ L => sub {
+ my ($label) = @_;
+ length($label) >= $L_MAX and
+ return "`$label' too long (must be <= $L_MAX)";
+ $label =~ m{\A[a-z0-9_](?:[a-z0-9_\-\./\@,]*[a-z0-9])?\z}i ?
+ undef : "`$label' is invalid";
+ },
+ kw => sub {
+ my ($kw) = @_;
+ $KW{$kw} ? undef : <<EOM;
+`$kw' is not one of: `seen', `flagged', `answered', `draft'
+`junk', `notjunk', `phishing' or `forwarded'
+EOM
+ }
+);
+
sub check_input_format ($;$) {
my ($lei, $files) = @_;
my $opt_key = 'in-format';
@@ -183,4 +215,26 @@ sub input_only_atfork_child {
undef;
}
+# like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare
+# for update_xvmd -> update_vmd
+sub vmd_mod_extract {
+ my $argv = $_[-1];
+ my $vmd_mod = {};
+ my @new_argv;
+ for my $x (@$argv) {
+ if ($x =~ /\A(\+|\-)(kw|L):(.+)\z/) {
+ my ($op, $pfx, $val) = ($1, $2, $3);
+ if (my $err = $ERR{$pfx}->($val)) {
+ push @{$vmd_mod->{err}}, $err;
+ } else { # set "+kw", "+L", "-L", "-kw"
+ push @{$vmd_mod->{$op.$pfx}}, $val;
+ }
+ } else {
+ push @new_argv, $x;
+ }
+ }
+ @$argv = @new_argv;
+ $vmd_mod;
+}
+
1;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index f019202f..f5791947 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -7,58 +7,6 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
-# JMAP RFC 8621 4.1.1
-# https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
-my @KW = (qw(seen answered flagged draft), # widely-compatible
- qw(forwarded phishing junk notjunk)); # rarely supported
-# note: RFC 8621 states "Users may add arbitrary keywords to an Email",
-# but is it good idea? Stick to the system and reserved ones, for now.
-# The widely-compatible ones map to IMAP system flags, Maildir flags
-# and mbox Status/X-Status headers.
-my %KW = map { $_ => 1 } @KW;
-my $L_MAX = 244; # Xapian term limit - length('L')
-
-# RFC 8621, sec 2 (Mailboxes) a "label" for us is a JMAP Mailbox "name"
-# "Servers MAY reject names that violate server policy"
-my %ERR = (
- L => sub {
- my ($label) = @_;
- length($label) >= $L_MAX and
- return "`$label' too long (must be <= $L_MAX)";
- $label =~ m{\A[a-z0-9_](?:[a-z0-9_\-\./\@,]*[a-z0-9])?\z}i ?
- undef : "`$label' is invalid";
- },
- kw => sub {
- my ($kw) = @_;
- $KW{$kw} ? undef : <<EOM;
-`$kw' is not one of: `seen', `flagged', `answered', `draft'
-`junk', `notjunk', `phishing' or `forwarded'
-EOM
- }
-);
-
-# like Getopt::Long, but for +kw:FOO and -kw:FOO to prepare
-# for update_xvmd -> update_vmd
-sub vmd_mod_extract {
- my $argv = $_[-1];
- my $vmd_mod = {};
- my @new_argv;
- for my $x (@$argv) {
- if ($x =~ /\A(\+|\-)(kw|L):(.+)\z/) {
- my ($op, $pfx, $val) = ($1, $2, $3);
- if (my $err = $ERR{$pfx}->($val)) {
- push @{$vmd_mod->{err}}, $err;
- } else { # set "+kw", "+L", "-L", "-kw"
- push @{$vmd_mod->{$op.$pfx}}, $val;
- }
- } else {
- push @new_argv, $x;
- }
- }
- @$argv = @new_argv;
- $vmd_mod;
-}
-
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) {
@@ -99,7 +47,7 @@ sub lei_tag { # the "lei tag" method
$sto->write_prepare($lei);
my $self = bless { missing => 0 }, __PACKAGE__;
$lei->ale; # refresh and prepare
- my $vmd_mod = vmd_mod_extract(\@argv);
+ my $vmd_mod = $self->vmd_mod_extract(\@argv);
return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
$self->prepare_inputs($lei, \@argv) or return;
grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or
@@ -161,7 +109,7 @@ sub _complete_mark_common ($) {
sub _complete_tag {
my ($self, @argv) = @_;
my @L = eval { $self->_lei_store->search->all_terms('L') };
- my @all = ((map { ("+kw:$_", "-kw:$_") } @KW),
+ my @all = ((map { ("+kw:$_", "-kw:$_") } @PublicInbox::LeiInput::KW),
(map { ("+L:$_", "-L:$_") } @L));
return @all if !@argv;
my ($cur, $re) = _complete_mark_common(\@argv);
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index ca1f3588..f066cc92 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -567,16 +567,39 @@ sub set_vmd {
$self->{xdb}->replace_document($docid, $doc);
}
+sub apply_vmd_mod ($$) {
+ my ($doc, $vmd_mod) = @_;
+ my $updated = 0;
+ my @x = @VMD_MAP;
+ while (my ($field, $pfx) = splice(@x, 0, 2)) {
+ # field: "label" or "kw"
+ for my $val (@{$vmd_mod->{"-$field"} // []}) {
+ eval {
+ $doc->remove_term($pfx . $val);
+ ++$updated;
+ };
+ }
+ for my $val (@{$vmd_mod->{"+$field"} // []}) {
+ $doc->add_boolean_term($pfx . $val);
+ ++$updated;
+ }
+ }
+ $updated;
+}
+
sub add_vmd {
my ($self, $docid, $vmd) = @_;
begin_txn_lazy($self);
my $doc = _get_doc($self, $docid) or return;
my @x = @VMD_MAP;
+ my $updated = 0;
while (my ($field, $pfx) = splice(@x, 0, 2)) {
my $add = $vmd->{$field} // next;
$doc->add_boolean_term($pfx . $_) for @$add;
+ $updated += scalar(@$add);
}
- $self->{xdb}->replace_document($docid, $doc);
+ $updated += apply_vmd_mod($doc, $vmd);
+ $self->{xdb}->replace_document($docid, $doc) if $updated;
}
sub remove_vmd {
@@ -601,21 +624,7 @@ sub update_vmd {
my ($self, $docid, $vmd_mod) = @_;
begin_txn_lazy($self);
my $doc = _get_doc($self, $docid) or return;
- my $updated = 0;
- my @x = @VMD_MAP;
- while (my ($field, $pfx) = splice(@x, 0, 2)) {
- # field: "label" or "kw"
- for my $val (@{$vmd_mod->{"-$field"} // []}) {
- eval {
- $doc->remove_term($pfx . $val);
- ++$updated;
- };
- }
- for my $val (@{$vmd_mod->{"+$field"} // []}) {
- $doc->add_boolean_term($pfx . $val);
- ++$updated;
- }
- }
+ my $updated = apply_vmd_mod($doc, $vmd_mod);
$self->{xdb}->replace_document($docid, $doc) if $updated;
$updated;
}
diff --git a/t/lei-import.t b/t/lei-import.t
index 8635df5a..6e9a853c 100644
--- a/t/lei-import.t
+++ b/t/lei-import.t
@@ -101,6 +101,16 @@ is_deeply($draft_a, $draft_b, 'fake Message-ID lookup') or
lei_ok('blob', '--mail', $draft_b->[0]->{blob});
is($lei_out, $eml_str, 'draft retrieved by blob');
+
+$eml_str = "Message-ID: <inbox\@example.com>\nSubject: label-this\n\n";
+lei_ok([qw(import -F eml - +kw:seen +L:inbox)],
+ undef, { %$lei_opt, 0 => \$eml_str });
+lei_ok(qw(q m:inbox@example.com));
+$res = json_utf8->decode($lei_out);
+is_deeply($res->[0]->{kw}, ['seen'], 'keyword set');
+is_deeply($res->[0]->{L}, ['inbox'], 'label set');
+
+
# see t/lei_to_mail.t for "import -F mbox*"
});
done_testing;
^ permalink raw reply related [relevance 40%]
* [PATCH 2/2] lei up: support symlinked pathnames
2021-04-23 1:45 71% [PATCH 0/2] "lei up" surprise reduction fixes Eric Wong
2021-04-23 1:45 50% ` [PATCH 1/2] lei: saved searches support --dedupe=<mid|oid> Eric Wong
@ 2021-04-23 1:45 55% ` Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-04-23 1:45 UTC (permalink / raw)
To: meta
On my default FreeBSD 11.x system, "/home" is a symlink to
"/usr/home", which causes "lei up" path resolution to fail when
I use outputs in $HOME. Fall back to a slow path of globbing
and matching pathnames based on st_ino+st_dev.
---
lib/PublicInbox/LeiSavedSearch.pm | 40 ++++++++++++++++++++++++-------
t/lei-q-save.t | 6 +++++
2 files changed, 38 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index ed217cf2..af864a50 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -13,6 +13,7 @@ use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::MID qw(mids_for_index);
use Digest::SHA qw(sha256_hex);
+my $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf):!i; # TODO: put in LeiToMail?
# move this to PublicInbox::Config if other things use it:
my %cquote = ("\n" => '\\n', "\t" => '\\t', "\b" => '\\b');
@@ -27,27 +28,50 @@ sub BOOL_FIELDS () {
qw(external local remote import-remote import-before threads)
}
-sub lss_dir_for ($$) {
- my ($lei, $dstref) = @_;
+sub lss_dir_for ($$;$) {
+ my ($lei, $dstref, $on_fs) = @_;
my @n;
if ($$dstref =~ m,\Aimaps?://,i) { # already canonicalized
require PublicInbox::URIimap;
my $uri = PublicInbox::URIimap->new($$dstref)->canonical;
$$dstref = $$uri;
@n = ($uri->mailbox);
- } else { # basename
+ } else {
+ # can't use Cwd::abs_path since dirname($$dstref) may not exist
$$dstref = $lei->rel2abs($$dstref);
+ # Maildirs have trailing '/' internally
$$dstref .= '/' if -d $$dstref;
$$dstref =~ tr!/!/!s;
- @n = ($$dstref =~ m{([^/]+)/*\z});
+ @n = ($$dstref =~ m{([^/]+)/*\z}); # basename
}
push @n, sha256_hex($$dstref);
- $lei->share_path . '/saved-searches/' . join('-', @n);
+ my $lss_dir = $lei->share_path . '/saved-searches/';
+ my $d = $lss_dir . join('-', @n);
+
+ # fall-back to looking up by st_ino + st_dev in case we're in
+ # a symlinked or bind-mounted path
+ if ($on_fs && !-d $d && -e $$dstref) {
+ my @cur = stat(_);
+ my $want = pack('dd', @cur[1,0]); # st_ino + st_dev
+ my ($c, $o, @st);
+ for my $g ("$n[0]-*", '*') {
+ my @maybe = glob("$lss_dir$g/lei.saved-search");
+ for my $f (@maybe) {
+ $c = PublicInbox::Config->git_config_dump($f);
+ $o = $c->{'lei.q.output'} // next;
+ $o =~ s!$LOCAL_PFX!! or next;
+ @st = stat($o) or next;
+ next if pack('dd', @st[1,0]) ne $want;
+ $f =~ m!\A(.+?)/[^/]+\z! and return $1;
+ }
+ }
+ }
+ $d;
}
sub list {
my ($lei, $pfx) = @_;
- my $lss_dir = $lei->share_path.'/saved-searches/';
+ my $lss_dir = $lei->share_path.'/saved-searches';
return () unless -d $lss_dir;
# TODO: persist the cache? Use another format?
my $f = $lei->cache_dir."/saved-tmp.$$.".time.'.config';
@@ -61,7 +85,7 @@ sub list {
unlink($f);
my $out = $cfg->get_all('lei.q.output') or return ();
map {;
- s!\A(?:maildir|mh|mbox.+|mmdf):!!i;
+ s!$LOCAL_PFX!!;
$_;
} @$out
}
@@ -221,7 +245,7 @@ sub cloneurl { [] }
sub output2lssdir {
my ($self, $lei, $dir_ref, $fn_ref) = @_;
my $dst = $$dir_ref; # imap://$MAILBOX, /path/to/maildir, /path/to/mbox
- my $dir = lss_dir_for($lei, \$dst);
+ my $dir = lss_dir_for($lei, \$dst, 1);
my $f = "$dir/lei.saved-search";
if (-f $f && -r _) {
$self->{-cfg} = PublicInbox::Config->git_config_dump($f);
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 26ea5cb8..170f7ce5 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -158,5 +158,11 @@ test_lei(sub {
lei_ok('up', $o);
@m = glob("$o/cur/*");
is(scalar(@m), 2, 'got 2nd result due to different OID');
+
+ SKIP: {
+ symlink($o, "$home/ln -s") or
+ skip "symlinks not supported in $home?: $!", 1;
+ lei_ok('up', "$home/ln -s");
+ };
});
done_testing;
^ permalink raw reply related [relevance 55%]
* [PATCH 0/2] "lei up" surprise reduction fixes
@ 2021-04-23 1:45 71% Eric Wong
2021-04-23 1:45 50% ` [PATCH 1/2] lei: saved searches support --dedupe=<mid|oid> Eric Wong
2021-04-23 1:45 55% ` [PATCH 2/2] lei up: support symlinked pathnames Eric Wong
0 siblings, 2 replies; 200+ results
From: Eric Wong @ 2021-04-23 1:45 UTC (permalink / raw)
To: meta
Eric Wong (2):
lei: saved searches support --dedupe=<mid|oid>
lei up: support symlinked pathnames
lib/PublicInbox/LeiSavedSearch.pm | 67 ++++++++++++++++++++++++++-----
lib/PublicInbox/LeiUp.pm | 4 ++
t/lei-q-save.t | 43 ++++++++++++++++++++
3 files changed, 104 insertions(+), 10 deletions(-)
^ permalink raw reply [relevance 71%]
* [PATCH 1/2] lei: saved searches support --dedupe=<mid|oid>
2021-04-23 1:45 71% [PATCH 0/2] "lei up" surprise reduction fixes Eric Wong
@ 2021-04-23 1:45 50% ` Eric Wong
2021-04-23 1:45 55% ` [PATCH 2/2] lei up: support symlinked pathnames Eric Wong
1 sibling, 0 replies; 200+ results
From: Eric Wong @ 2021-04-23 1:45 UTC (permalink / raw)
To: meta
This is less surprising in case users are used to using --dedupe=
without --save.
---
lib/PublicInbox/LeiSavedSearch.pm | 27 ++++++++++++++++++++--
lib/PublicInbox/LeiUp.pm | 4 ++++
t/lei-q-save.t | 37 +++++++++++++++++++++++++++++++
3 files changed, 66 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index cd9effce..ed217cf2 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -11,6 +11,7 @@ use PublicInbox::LeiSearch;
use PublicInbox::Config;
use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::MID qw(mids_for_index);
use Digest::SHA qw(sha256_hex);
# move this to PublicInbox::Config if other things use it:
@@ -65,6 +66,14 @@ sub list {
} @$out
}
+sub translate_dedupe ($$$) {
+ my ($self, $lei, $dd) = @_;
+ $dd //= 'content';
+ return 1 if $dd eq 'content'; # the default
+ return $self->{"-dedupe_$dd"} = 1 if ($dd eq 'oid' || $dd eq 'mid');
+ $lei->fail("--dedupe=$dd unsupported with --save");
+}
+
sub up { # updating existing saved search via "lei up"
my ($cls, $lei, $dst) = @_;
my $f;
@@ -89,6 +98,8 @@ sub new { # new saved search "lei q --save"
File::Path::make_path($dir); # raises on error
$self->{-cfg} = {};
my $f = $self->{'-f'} = "$dir/lei.saved-search";
+ my $dd = $lei->{opt}->{dedupe};
+ translate_dedupe($self, $lei, $dd) or return;
open my $fh, '>', $f or return $lei->fail("open $f: $!");
my $sq_dst = PublicInbox::Config::squote_maybe($dst);
my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
@@ -105,6 +116,7 @@ sub new { # new saved search "lei q --save"
[lei "q"]
output = $dst
EOM
+ print $fh "\tdedupe = $dd\n" if $dd;
for my $k (ARRAY_FIELDS) {
my $ary = $lei->{opt}->{$k} // next;
for my $x (@$ary) {
@@ -134,14 +146,25 @@ sub is_dup {
my ($self, $eml, $smsg) = @_;
my $oidx = $self->{oidx} // die 'BUG: no {oidx}';
my $blob = $smsg ? $smsg->{blob} : undef;
- return 1 if $blob && $oidx->blob_exists($blob);
my $lk = $self->lock_for_scope_fast;
+ return 1 if $blob && $oidx->blob_exists($blob);
+ if ($self->{-dedupe_mid}) {
+ for my $mid (@{mids_for_index($eml)}) {
+ my ($id, $prv);
+ return 1 if $oidx->next_by_mid($mid, \$id, \$prv);
+ }
+ }
if (my $xoids = PublicInbox::LeiSearch::xoids_for($self, $eml, 1)) {
for my $docid (values %$xoids) {
$oidx->add_xref3($docid, -1, $blob, '.');
}
$oidx->commit_lazy;
- 1;
+ if ($self->{-dedupe_oid}) {
+ $smsg->{blob} //= git_sha(1, $eml)->hexdigest;
+ exists $xoids->{$smsg->{blob}} ? 1 : undef;
+ } else {
+ 1;
+ }
} else {
# n.b. above xoids_for fills out eml->{-lei_fake_mid} if needed
unless ($smsg) {
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 0fb9698b..f4ff070b 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -25,6 +25,10 @@ sub up1 ($$) {
my $o = $lei->{opt}->{output} = $lss->{-cfg}->{'lei.q.output'} //
return $lei->fail("lei.q.output unset in $f");
ref($o) and return $lei->fail("multiple values of lei.q.output in $f");
+ if (defined(my $dd = $lss->{-cfg}->{'lei.q.dedupe'})) {
+ $lss->translate_dedupe($lei, $dd) or return;
+ $lei->{opt}->{dedupe} = $dd;
+ }
for my $k (qw(only include exclude)) {
my $v = $lss->{-cfg}->get_all("lei.q.$k") // next;
$lei->{opt}->{$k} = $v;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 5a2f7fff..26ea5cb8 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -121,5 +121,42 @@ test_lei(sub {
unlike($lei_out, qr/mbrd-aug/,
'forget-search completion cleared after forget');
ok(!lei('up', "$home/mbrd-aug"), 'lei up fails after forget');
+
+ # dedupe=mid
+ my $o = "$home/dd-mid";
+ $in = $doc2->as_string . "\n-------\nappended list sig\n";
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
+ lei_ok(qw(q --dedupe=mid --save m:testmessage@example.com -o), $o);
+ my @m = glob("$o/cur/*");
+ is(scalar(@m), 1, '--dedupe=mid w/ --save');
+ $in = $doc2->as_string . "\n-------\nanother list sig\n";
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
+ lei_ok 'up', $o;
+ is_deeply([glob("$o/cur/*")], \@m, 'lei up dedupe=mid works');
+
+ for my $dd (qw(content)) {
+ $o = "$home/dd-$dd";
+ lei_ok(qw(q --save m:testmessage@example.com -o), $o,
+ "--dedupe=$dd");
+ @m = glob("$o/cur/*");
+ is(scalar(@m), 3, 'all 3 matches with dedupe='.$dd);
+ }
+
+ # dedupe=oid
+ $o = "$home/dd-oid";
+ my $ibx = create_inbox 'ibx', indexlevel => 'medium',
+ tmpdir => "$home/v1", sub {};
+ lei_ok(qw(q --save --dedupe=oid m:qp@example.com -o), $o,
+ '-I', $ibx->{inboxdir});
+ @m = glob("$o/cur/*");
+ is(scalar(@m), 1, 'got first result');
+
+ my $im = $ibx->importer(0);
+ my $diff = "X-Insignificant-Header: x\n".$doc1->as_string;
+ $im->add(PublicInbox::Eml->new($diff));
+ $im->done;
+ lei_ok('up', $o);
+ @m = glob("$o/cur/*");
+ is(scalar(@m), 2, 'got 2nd result due to different OID');
});
done_testing;
^ permalink raw reply related [relevance 50%]
* [PATCH] lei: XDG_RUNTIME_DIR=/dev/null disables daemon mode
@ 2021-04-22 9:44 63% Eric Wong
0 siblings, 0 replies; 200+ results
From: Eric Wong @ 2021-04-22 9:44 UTC (permalink / raw)
To: meta
We'll support this mode of operation for now to quiet down
testing of oneshot mode where the daemon doesn't persist.
---
lib/PublicInbox/TestCommon.pm | 9 ++-------
script/lei | 3 ++-
t/lei-externals.t | 2 +-
3 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 67fe6336..b5d0b9f8 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -468,7 +468,7 @@ sub have_xapian_compact () {
PublicInbox::Spawn::which($ENV{XAPIAN_COMPACT} || 'xapian-compact');
}
-our ($err_skip, $lei_opt, $lei_out, $lei_err);
+our ($lei_opt, $lei_out, $lei_err);
# favor lei() or lei_ok() over $lei for new code
sub lei (@) {
my ($cmd, $env, $xopt) = @_;
@@ -478,8 +478,6 @@ sub lei (@) {
$cmd = [ grep { defined && !ref } @_ ];
}
my $res = run_script(['lei', @$cmd], $env, $xopt // $lei_opt);
- $err_skip and
- $lei_err = join('', grep(!/$err_skip/, split(/^/m, $lei_err)));
if ($lei_err ne '') {
if ($lei_err =~ /Use of uninitialized/ ||
$lei_err =~ m!\bArgument .*? isn't numeric in !) {
@@ -570,10 +568,7 @@ EOM
my $home = "$tmpdir/lei-oneshot";
mkdir($home, 0700) or BAIL_OUT "mkdir: $!";
local $ENV{HOME} = $home;
- # force sun_path[108] overflow:
- my $xrd = "$home/1shot-test".('.sun_path' x 108);
- local $err_skip = qr!\Q$xrd!; # for lei() filtering
- local $ENV{XDG_RUNTIME_DIR} = $xrd;
+ local $ENV{XDG_RUNTIME_DIR} = '/dev/null';
$cb->();
}
if ($daemon_pid) {
diff --git a/script/lei b/script/lei
index 56e9d299..db302422 100755
--- a/script/lei
+++ b/script/lei
@@ -62,6 +62,7 @@ my $exec_cmd = sub {
if ($send_cmd && eval {
my $path = do {
my $runtime_dir = ($ENV{XDG_RUNTIME_DIR} // '') . '/lei';
+ die \0 if $runtime_dir eq '/dev/null/lei'; # oneshot forced
if ($runtime_dir eq '/lei') {
require File::Spec;
$runtime_dir = File::Spec->tmpdir."/lei-$<";
@@ -131,7 +132,7 @@ Falling back to (slow) one-shot mode
}
exit($x_it_code >> 8);
} else { # for systems lacking Socket::MsgHdr or Inline::C
- warn $@ if $@;
+ warn $@ if $@ && !ref($@);
require PublicInbox::LEI;
PublicInbox::LEI::oneshot(__PACKAGE__);
}
diff --git a/t/lei-externals.t b/t/lei-externals.t
index 488bf5ad..2291dd99 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -49,7 +49,7 @@ SKIP: {
ok(WIFSIGNALED($?), "signaled @$out");
is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out");
seek($err, 0, 0);
- my @err = grep(!m{mkdir .*sun_path\b}, <$err>);
+ my @err = <$err>;
is_deeply(\@err, [], "no errors @$out");
}
if (-d $ENV{XDG_RUNTIME_DIR} && -w _) {
^ permalink raw reply related [relevance 63%]
Results 601-800 of ~1329 next (older) | prev (newer) | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-03-03 3:53 should lei attempt to index mail outside of git? Eric Wong
2021-05-02 6:12 71% ` yes [Re: should lei attempt to index mail outside of git?] Eric Wong
2021-04-22 9:44 63% [PATCH] lei: XDG_RUNTIME_DIR=/dev/null disables daemon mode Eric Wong
2021-04-23 1:45 71% [PATCH 0/2] "lei up" surprise reduction fixes Eric Wong
2021-04-23 1:45 50% ` [PATCH 1/2] lei: saved searches support --dedupe=<mid|oid> Eric Wong
2021-04-23 1:45 55% ` [PATCH 2/2] lei up: support symlinked pathnames Eric Wong
2021-04-23 11:22 40% [PATCH] lei import: support adding keywords and labels on import Eric Wong
2021-04-24 9:28 89% [PATCH 0/7] lei sync preparations, "lei inspect" Eric Wong
2021-04-24 9:28 66% ` [PATCH 2/7] t/lei_to_mail: split "lei import" test $HOME directory Eric Wong
2021-04-24 9:28 27% ` [PATCH 7/7] lei import: keep sync info for Maildir and IMAP folders Eric Wong
2021-04-26 8:43 66% [PATCH] lei p2q: exit with failure if format-patch fails Eric Wong
2021-04-26 16:44 70% lei-managed pseudo mailing lists Konstantin Ryabitsev
2021-04-26 17:37 70% ` Eric Wong
2021-04-26 18:20 65% ` Konstantin Ryabitsev
2021-04-26 18:47 71% ` Eric Wong
2021-04-26 19:46 68% ` Konstantin Ryabitsev
2021-04-26 20:34 71% ` Eric Wong
2021-04-27 11:07 64% [PATCH 0/5] lei lcat - local cat (not lolcat :P) Eric Wong
2021-04-27 11:07 55% ` [PATCH 1/5] lei: add "ls-sync" command for listing sync folders Eric Wong
2021-04-27 11:07 61% ` [PATCH 2/5] lei blob: support retrieving attachments via $OID:$IDX Eric Wong
2021-04-27 11:07 49% ` [PATCH 3/5] lei: standardize on _lei_wq_eof callback for workers Eric Wong
2021-04-27 11:07 38% ` [PATCH 4/5] lei lcat: extract Message-IDs from URLs and show them Eric Wong
2021-04-27 11:07 32% ` [PATCH 5/5] lei q + lcat: support --format=text output Eric Wong
2021-04-28 4:51 90% [PATCH 0/3] doc: lei updates around lei-q Eric Wong
2021-04-28 4:51 64% ` [PATCH 1/3] doc: lei: use /tmp for search results pathnames Eric Wong
2021-04-28 4:51 54% ` [PATCH 2/3] doc: lei q: split =item aliases onto separate lines Eric Wong
2021-04-29 1:39 71% ` Kyle Meyer
2021-04-29 1:57 71% ` Eric Wong
2021-04-28 4:51 63% ` [PATCH 3/3] doc: lei q: split --output and --format, note "text" Eric Wong
2021-04-28 7:51 68% [PATCH 00/11] lei: misc fixes, more lcat color support Eric Wong
2021-04-28 7:51 71% ` [PATCH 01/11] t/lei-p2q: add diagnostics Eric Wong
2021-04-28 7:51 52% ` [PATCH 02/11] tests: restore CWD with "lei -C" and run_script Eric Wong
2021-04-28 7:51 71% ` [PATCH 05/11] lei-daemon: note FD count mismatch to client Eric Wong
2021-04-28 7:52 59% ` [PATCH 07/11] lei: quiet down Eml-related warnings consistently Eric Wong
2021-04-28 7:52 44% ` [PATCH 08/11] lei: simple WQ workers use {wq1} field Eric Wong
2021-04-28 7:52 60% ` [PATCH 11/11] lei (lcat|q): support --no-color and --color Eric Wong
2021-04-28 19:37 [PATCH 0/2] "make check-run" fixed Eric Wong
2021-04-28 19:37 63% ` [PATCH 2/2] lei: avoid close(STD{IN,OUT,ERR}) in oneshot mode Eric Wong
2021-04-29 9:46 69% [PATCH 0/4] some lei synchronization work Eric Wong
2021-04-29 9:46 33% ` [PATCH 2/4] lei import: avoid IMAPTracker, use LeiMailSync more Eric Wong
2021-04-29 9:46 61% ` [PATCH 3/4] lei import: support UIDVALIDITY in IMAP URL Eric Wong
2021-04-29 9:46 57% ` [PATCH 4/4] lei import: support shell completion of known folders Eric Wong
2021-04-30 9:24 67% [PATCH 0/8] lei NNTP/IMAP .onion support and misc fixes Eric Wong
2021-04-30 9:24 71% ` [PATCH 1/8] lei sucks: preserve utsname.machine, add "x86" where appropriate Eric Wong
2021-04-30 9:24 71% ` [PATCH 3/8] lei: kill old PIDs when dropping Eric Wong
2021-04-30 9:24 70% ` [PATCH 4/8] lei: ensure autoflush(1) is on STDERR Eric Wong
2021-04-30 9:24 45% ` [PATCH 6/8] lei: IMAP .onion support via --proxy=s switch Eric Wong
2021-05-01 6:21 70% [PATCH 0/5] lei: more UI/UX tweaks Eric Wong
2021-05-01 6:21 61% ` [PATCH 1/5] xt/lei-onion-convert: test for NNTP+IMAP onions Eric Wong
2021-05-01 6:21 55% ` [PATCH 2/5] lei <q|up>: distinguish between mset and l2m counts Eric Wong
2021-05-01 6:21 71% ` [PATCH 3/5] lei_saved_search: fix excess indent for first lei.q entry Eric Wong
2021-05-01 6:21 56% ` [PATCH 4/5] lei: rename ls-sync to ls-mail-sync Eric Wong
2021-05-01 19:29 59% ` [PATCH 6/5] lei import: fix --mail-sync handling in LeiInput Eric Wong
2021-05-01 6:21 75% ` [PATCH 5/5] lei edit-search: support relocating lei.q.output Eric Wong
2021-05-02 6:05 70% [PATCH 0/6] lei: more steps towards kw sync Eric Wong
2021-05-02 6:05 70% ` [PATCH 1/6] lei <q|up>: combine written/results into one line Eric Wong
2021-05-02 6:05 66% ` [PATCH 2/6] lei_input: common net_merge_all_done for lei <import|tag> Eric Wong
2021-05-02 6:05 53% ` [PATCH 4/6] lei: simplify workers_start API Eric Wong
2021-05-02 6:05 49% ` [PATCH 6/6] lei <q|up>: writes to Maildirs and IMAP use mail-sync Eric Wong
2021-05-03 20:57 57% [PATCH] lei up: fix dedupe with remote externals on Maildir + IMAP Eric Wong
2021-05-04 4:45 [PATCH 0/5] Minor ls-mail-sync fixes Kyle Meyer
2021-05-04 4:45 71% ` [PATCH 1/5] lei ls-mail-sync: update reference to ls-sync Kyle Meyer
2021-05-04 4:45 71% ` [PATCH 2/5] lei ls-mail-sync: drop repeated -z/0 option Kyle Meyer
2021-05-04 4:45 71% ` [PATCH 3/5] lei ls-mail-sync: accept a filter Kyle Meyer
2021-05-04 4:45 68% ` [PATCH 4/5] lei ls-mail-sync: fix handling of non-wildcard filters Kyle Meyer
2021-05-04 5:14 66% ` Eric Wong
2021-05-04 4:45 71% ` [PATCH 5/5] lei: add help output for --invert match Kyle Meyer
2021-05-04 5:24 66% [PATCH] lei: fix mail_sync.qlite3 folder names for NNTP Eric Wong
2021-05-04 5:26 71% ` s/qlite3/s&/ [was: [PATCH] lei: fix ... folder names for NNTP] Eric Wong
2021-05-04 9:49 26% [PATCH] lei index: new command to index mail w/o git storage Eric Wong
2021-05-05 10:46 71% [PATCH 0/2] lei rediff + solver-related fix Eric Wong
2021-05-05 10:46 30% ` [PATCH 1/2] lei rediff: regenerate diffs from stdin Eric Wong
2021-05-05 10:46 87% ` [PATCH 2/2] lei blob: support "lei index"-ed mail Eric Wong
2021-05-05 17:49 71% [PATCH 0/3] lei rediff fixes Eric Wong
2021-05-05 17:49 55% ` [PATCH 2/3] lei rediff: capture and regenerate file modes Eric Wong
2021-05-05 17:49 65% ` [PATCH 3/3] lei rediff: do not automatically store patches/mails Eric Wong
2021-05-07 0:13 71% lei timestamp resolution for mail synchronization Eric Wong
2021-05-16 2:41 71% [PATCH] lei rediff: mark --color-moved value as optional Kyle Meyer
2021-05-16 2:42 71% [PATCH] lei rediff: handle stdin like other commands Kyle Meyer
2021-05-16 15:23 71% ` Eric Wong
2021-05-17 3:35 61% [PATCH 0/9] doc: lei manpages, round 5 Kyle Meyer
2021-05-17 3:35 68% ` [PATCH 1/9] doc lei blob: avoid combined description of separate options Kyle Meyer
2021-05-17 3:35 82% ` [PATCH 3/9] doc lei blob: point to lei-q for shared options Kyle Meyer
2021-05-17 3:35 90% ` [PATCH 4/9] doc lei: resort lei-tag entries Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 5/9] doc lei q: fix a typo Kyle Meyer
2021-05-17 3:35 71% ` [PATCH 6/9] doc lei q: add missing value for --lock Kyle Meyer
2021-05-17 3:35 54% ` [PATCH 7/9] doc lei: add manpage for convert Kyle Meyer
2021-05-17 3:35 27% ` [PATCH 8/9] doc lei: add manpages for new commands Kyle Meyer
2021-05-17 8:09 71% ` Eric Wong
2021-05-17 23:24 71% ` Kyle Meyer
2021-05-17 3:35 66% ` [PATCH 9/9] doc lei: update manpages with new options Kyle Meyer
2021-05-17 3:37 71% [PATCH] lei lcat: fix handling of multiple MSGID_OR_URL arguments Kyle Meyer
2021-05-19 8:54 57% [PATCH] lei: relax rules for "new" in Maildir Eric Wong
2021-05-21 4:38 70% [PATCH] lei rediff: fix construction of git-diff options Kyle Meyer
2021-05-21 9:52 71% ` Eric Wong
2021-05-21 10:28 64% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 66% ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
2021-05-21 10:28 53% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
2021-05-21 10:28 71% ` [PATCH 5/8] lei index: support command-line options Eric Wong
2021-05-21 10:28 32% ` [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs Eric Wong
2021-05-21 10:28 46% ` [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Eric Wong
2021-05-23 1:38 71% [PATCH 0/3] lei export-kw: IMAP support Eric Wong
2021-05-23 1:38 49% ` [PATCH 2/3] lei export-kw: support exporting keywords to IMAP Eric Wong
2021-05-23 1:38 47% ` [PATCH 3/3] lei export-kw: relax IMAP URL matching Eric Wong
2021-05-23 8:01 35% [PATCH] lei <q|up>: set \Recent on non-empty mbox and Maildir Eric Wong
2021-05-23 21:36 71% [PATCH 0/2] lei IMAP usability stuff Eric Wong
2021-05-23 21:36 46% ` [PATCH 1/2] lei inspect: use LeiMailSync->match_imap_url Eric Wong
2021-05-24 22:48 71% [PATCH] lei export-kw: fix case-insensitivity typo for IMAP URLs Eric Wong
2021-05-25 11:01 71% [PATCH 0/3] lei forget-mail-sync: drop sync information Eric Wong
2021-05-25 11:01 90% ` [PATCH 1/3] lei export-kw: use lei->abs_path instead of rel2abs Eric Wong
2021-05-25 11:01 55% ` [PATCH 3/3] lei forget-mail-sync: new command to drop sync information Eric Wong
2021-05-25 22:19 71% [PATCH 0/2] ipc: fix "lei q" w/ HTTP(S) externals Eric Wong
2021-05-26 18:08 28% [PATCH] lei: require Socket::MsgHdr or Inline::C, drop oneshot Eric Wong
2021-05-26 23:50 50% [PATCH] lei rm: new command to remove messages from index Eric Wong
2021-05-27 10:49 44% [PATCH] lei: handle a single IMAP message in most places Eric Wong
2021-05-28 0:07 70% ` [PATCH 0/6] lei: odds and ends and a resend Eric Wong
2021-05-28 0:07 71% ` [PATCH 4/6] lei: mark reorder-and-rewrite-local-history as a TODO item Eric Wong
2021-05-28 0:07 46% ` [PATCH 5/6] lei: handle a single IMAP message in most places Eric Wong
2021-05-28 9:37 71% ` Eric Wong
2021-05-28 0:07 71% ` [PATCH 6/6] lei: add TODO item for FUSE mount Eric Wong
2021-05-28 9:45 71% [PATCH 0/3] lei: diagnostic and reliability fix Eric Wong
2021-05-28 9:45 61% ` [PATCH 1/3] t/lei-*: better diagnostics for occasional failures Eric Wong
2021-05-28 9:45 62% ` [PATCH 2/3] lei: restore working directory in more places Eric Wong
2021-05-28 9:45 71% ` [PATCH 3/3] script/lei: drop leftover message about fallback Eric Wong
2021-05-28 19:47 49% [PATCH] lei: retry_reopen on read-only Xapian access Eric Wong
2021-05-28 22:39 39% [PATCH] lei q|up: support v2:/path/to/inboxdir destination Eric Wong
2021-05-29 20:20 71% [PATCH 0/3] lei: v2 and lcat/import/IMAP things Eric Wong
2021-05-29 20:20 36% ` [PATCH 2/3] lei import|lcat: improve+fix single message IMAP support Eric Wong
2021-05-29 20:20 61% ` [PATCH 3/3] lei q: --sort and --save|v2 are incompatible Eric Wong
2021-05-30 6:33 71% [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 6:33 71% ` [PATCH 1/4] lei lcat+inspect: start wiring up completion Eric Wong
2021-05-30 6:33 68% ` [PATCH 2/4] lei lcat: allow IMAP folder URLs w/o UIDVALIDITY Eric Wong
2021-05-30 6:33 63% ` [PATCH 3/4] lei lcat: support maildir: paths, too Eric Wong
2021-05-30 6:33 61% ` [PATCH 4/4] lei: support implicit stdin by default Eric Wong
2021-05-30 11:52 71% ` [PATCH 0/4] lei lcat usability things Eric Wong
2021-05-30 11:45 42% [PATCH] lei import: import IMAP flag changes from old messages Eric Wong
2021-05-31 10:20 86% [PATCH] lei import: reduce writes to lei/store on IMAP sync Eric Wong
2021-06-01 19:20 71% [PATCH] lei: remove "forget" (old name for "rm") Eric Wong
2021-06-02 10:03 58% [PATCH] lei export-kw: do not write directly to mail_sync.sqlite3 Eric Wong
2021-06-03 0:17 40% [PATCH] lei import: speed up kw updates for old IMAP messages Eric Wong
2021-06-03 1:05 38% ` [PATCH v2] " Eric Wong
2021-06-03 5:00 67% [RFC] lei import: support --new-only Eric Wong
2021-06-09 22:39 62% ` [PATCH v2] lei import: support --new-only for IMAP Eric Wong
2021-06-05 21:04 71% [PATCH] INSTALL: note about lei metadata storage Eric Wong
2021-06-05 21:06 71% [PATCH] lei: don't drop WQ workers on normal exit Eric Wong
2021-06-07 19:06 71% [PATCH] lei/store: checkpoint commits mail_sync.sqlite3 Eric Wong
2021-06-08 9:50 69% [PATCH 0/3] lei import: speedup repeated Maildir import Eric Wong
2021-06-08 9:50 71% ` [PATCH 1/3] lei: safety fix for multiple WQ classes Eric Wong
2021-06-08 9:50 48% ` [PATCH 2/3] lei: generalize auxiliary WQ handling Eric Wong
2021-06-08 9:50 32% ` [PATCH 3/3] lei import: speed up repeated Maildir imports Eric Wong
2021-06-08 23:56 70% [PATCH] lei pmdir: fix nproc for <= 4 CPUs Eric Wong
2021-06-09 0:11 70% [PATCH] lei edit-search: fix and add a (weak) test Eric Wong
2021-06-09 7:47 71% [PATCH 0/5] lei Maildir stuff Eric Wong
2021-06-09 7:47 58% ` [PATCH 3/5] lei tag: parallelize Maildir access Eric Wong
2021-06-09 7:47 45% ` [PATCH 5/5] lei prune-mail-sync: new command to prune invalid sync data Eric Wong
2021-06-09 10:03 69% [PATCH] lei/store: do eidx_init before creating R/W lms dbh Eric Wong
2021-06-09 23:27 58% [PATCH] lei tag: less confusing warning about unimported messages Eric Wong
2021-06-11 9:42 41% [PATCH] lei ls-mail-source: list IMAP folders and NNTP groups Eric Wong
2021-06-12 0:10 71% [PATCH 0/5] lei ls-mail-source-related things Eric Wong
2021-06-12 0:10 71% ` [PATCH 1/5] lei: stop pager early on exit Eric Wong
2021-06-12 0:10 48% ` [PATCH 2/5] lei ls-mail-source: write through to URL folder cache Eric Wong
2021-06-12 0:10 71% ` [PATCH 3/5] t/lei-import-http: quiet unnecessary diag message Eric Wong
2021-06-12 0:10 71% ` [PATCH 4/5] lei import: use url_folder_cache for completion Eric Wong
2021-06-13 18:12 71% [PATCH 0/2] lei: support keywords off a single Maildir file Eric Wong
2021-06-13 18:12 59% ` [PATCH 2/2] lei index+import: reject keywords from R/O IMAP Eric Wong
2021-06-17 22:00 71% [PATCH 0/3] lei: internal bug fixups Eric Wong
2021-06-17 22:00 60% ` [PATCH 1/3] lei inspect: learn "num:" and "docid:" prefixes Eric Wong
2021-06-17 22:00 55% ` [PATCH 3/3] lei/store: cull redundant docids based on blob OID Eric Wong
2021-06-18 19:20 71% [PATCH] lei/store: do not put NULL into over.num column Eric Wong
2021-06-20 4:33 71% [PATCH] lei sucks: don't warn or error out on missing dependencies Eric Wong
2021-06-20 8:39 71% [PATCH] lei import: help + completion for --new-only Eric Wong
2021-06-22 10:04 71% [PATCH] lei: use open() perlop for -C (chdir) Eric Wong
2021-07-01 11:31 71% [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
2021-07-01 11:31 71% ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
2021-07-01 11:31 62% ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
2021-07-02 20:42 71% [PATCH] lei inspect: help+completion for --dir option Eric Wong
2021-07-02 21:02 71% [PATCH] lei import: increase flags search batch size, display progress Eric Wong
2021-07-04 19:35 71% [PATCH] lei: drop workers on EOF from clients Eric Wong
2021-07-08 8:42 71% deduplication doesn't always work with "lei up" Eric Wong
2021-07-19 8:59 63% [PATCH 0/2] lei Maildir inotify/kevent support Eric Wong
2021-07-19 8:59 22% ` [PATCH 2/2] lei: start implementing inotify Maildir support Eric Wong
2021-07-21 14:07 47% ` [PATCH 3/2] lei: auto-refresh watches in config, cancel missing Eric Wong
2021-07-23 10:56 71% [PATCH 0/3] lei rm-watch, some error handling stuff Eric Wong
2021-07-23 10:56 36% ` [PATCH 1/3] t/lei*: check error messages on failures Eric Wong
2021-07-23 10:56 58% ` [PATCH 2/3] lei: avoid SQLite COUNT() for dedupe Eric Wong
2021-07-23 10:56 53% ` [PATCH 3/3] lei rm-watch: new command to support removing watches Eric Wong
2021-07-25 11:15 71% [PATCH] t/lei-watch.t: improve test reliability Eric Wong
2021-07-25 12:03 71% [PATCH] doc: lei-{p2q,rediff}: note implicit --stdin Eric Wong
2021-07-25 12:29 71% lei rediff: add --dequote and --requote? Eric Wong
2021-07-28 0:37 [PATCH 0/2] fix "make check-run" reliability Eric Wong
2021-07-28 0:37 71% ` [PATCH 1/2] lei: die on ECONNRESET Eric Wong
2021-07-28 12:34 58% [RFC] lei: address lifetime problems from Linux::Inotify2 Eric Wong
2021-07-29 10:01 62% ` [RFC v2] lei: close inotify FD in forked child Eric Wong
2021-08-04 10:40 71% ` Eric Wong
2021-08-05 2:33 71% [PATCH] lei export-kw: workaround race in updating Maildir locations Eric Wong
2021-08-11 11:26 71% [PATCH 0/3] lei pathname canonicalization fixes Eric Wong
2021-08-11 11:26 42% ` [PATCH 3/3] lei: attempt to canonicalize away "/../" pathnames Eric Wong
2021-08-12 23:40 71% [PATCH 0/2] "lei up" improvements Eric Wong
2021-08-12 23:40 63% ` [PATCH 1/2] lei up: support multiple output folders w/o --all=local Eric Wong
2021-08-12 23:40 71% ` [PATCH 2/2] lei up: note errors if one output destination fails Eric Wong
2021-08-14 0:29 71% [PATCH 0/3] lei: hopefully kill /Document \d+ not found/ errors Eric Wong
2021-08-14 0:29 68% ` [PATCH 1/3] lei: diagnostics for " Eric Wong
2021-08-14 0:29 64% ` [PATCH 2/3] lei <q|up>: wait on remote mboxrd imports synchronously Eric Wong
2021-08-14 0:29 58% ` [PATCH 3/3] lei: hexdigest mocks account for unwanted headers Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).