* [PATCH 1/9] lei q: --save preserves relative time queries
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-16 23:10 ` [PATCH 2/9] lei: expose share_path as a method Eric Wong
` (7 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
Somebody may want a saved search which consistently asks for
messages within a rolling time period window. In other words,
we want to support using "lei q --save dt:last.week.." and keeps
the "dt:last.week.." relative to whenever "lei up" is run. This
ensures relative date-time specifications get used in the future
rather than converting into an absolute date-time from the
initial "lei q" invocation.
---
lib/PublicInbox/LeiQuery.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 5 +++--
t/lei-q-save.t | 25 +++++++++++++++++++++----
3 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 7456f7f9..7ddba4cf 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -143,7 +143,7 @@ no query allowed on command-line with --stdin
PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
return;
}
- $mset_opt{q_raw} = \@argv;
+ $mset_opt{q_raw} = [ @argv ]; # copy
$mset_opt{qstr} =
$self->{lse}->query_argv_to_string($self->{lse}->git, \@argv);
_start_query($self);
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 815008fd..e79cf76a 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -25,12 +25,13 @@ sub new {
} else { # new saved search "lei q --save"
my $saved_dir = $lei->store_path . '/../saved-searches/';
my (@name) = ($lei->{ovv}->{dst} =~ m{([\w\-\.]+)/*\z});
- push @name, to_filename($lei->{mset_opt}->{qstr});
+ my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
+ my $q_raw_str = ref($q) ? "@$q" : $q;
+ push @name, to_filename($q_raw_str);
$dir = $saved_dir . join('-', @name);
require File::Path;
File::Path::make_path($dir); # raises on error
$self->{'-f'} = "$dir/lei.saved-search";
- my $q = $lei->{mset_opt}->{q_raw};
if (ref $q) {
cfg_set($self, '--add', 'lei.q', $_) for @$q;
} else {
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index a6d579cf..6cfac20b 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -2,24 +2,41 @@
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
+use PublicInbox::Smsg;
my $doc1 = eml_load('t/plack-qp.eml');
+$doc1->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 5)}));
my $doc2 = eml_load('t/utf8.eml');
+$doc2->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 4)}));
+
test_lei(sub {
my $home = $ENV{HOME};
- lei_ok qw(import -q t/plack-qp.eml);
- lei_ok qw(q -q --save z:0..), '-o', "$home/md/";
+ my $in = $doc1->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
+ lei_ok qw(q -q --save z:0.. d:last.week..), '-o', "$home/md/";
my %before = map { $_ => 1 } glob("$home/md/cur/*");
is_deeply(eml_load((keys %before)[0]), $doc1, 'doc1 matches');
my @s = glob("$home/.local/share/lei/saved-searches/md-*");
is(scalar(@s), 1, 'got one saved search');
+ my $cfg = PublicInbox::Config->new("$s[0]/lei.saved-search");
+ is_deeply($cfg->{'lei.q'}, ['z:0..', 'd:last.week..'],
+ 'store relative time, not parsed (absolute) timestamp');
# ensure "lei up" works, since it compliments "lei q --save"
- lei_ok qw(import t/utf8.eml);
- lei_ok qw(up), $s[0];
+ $in = $doc2->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
+ lei_ok qw(up -q), $s[0];
my %after = map { $_ => 1 } glob("$home/md/cur/*");
is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
is(scalar(keys %after), 1, 'one new message added');
is_deeply(eml_load((keys %after)[0]), $doc2, 'doc2 matches');
+
+ # check stdin
+ lei_ok [qw(q --save - -o), "mboxcl2:mbcl2" ],
+ undef, { -C => $home, %$lei_opt, 0 => \'d:last.week..'};
+ @s = glob("$home/.local/share/lei/saved-searches/mbcl2-*");
+ $cfg = PublicInbox::Config->new("$s[0]/lei.saved-search");
+ is_deeply $cfg->{'lei.q'}, 'd:last.week..',
+ 'q --stdin stores relative time';
});
done_testing;
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 2/9] lei: expose share_path as a method
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
2021-04-16 23:10 ` [PATCH 1/9] lei q: --save preserves relative time queries Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-16 23:10 ` [PATCH 3/9] lei: saved searches keyed only by path/URL and format Eric Wong
` (6 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
Since saved-searches aren't a part of lei/store, nor
could it be considered cache data... (or can it? it
is discardable, after all).
---
lib/PublicInbox/LEI.pm | 6 ++++--
lib/PublicInbox/LeiSavedSearch.pm | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 4b87c104..52b588a2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -83,13 +83,15 @@ sub rel2abs ($$) {
File::Spec->rel2abs($p, $pwd);
}
-sub store_path ($) {
+sub share_path ($) { # $HOME/.local/share/lei/$FOO
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_DATA_HOME} //
($self->{env}->{HOME} // '/nonexistent').'/.local/share')
- .'/lei/store');
+ .'/lei');
}
+sub store_path ($) { share_path($_[0]) . '/store' }
+
sub _config_path ($) {
my ($self) = @_;
rel2abs($self, ($self->{env}->{XDG_CONFIG_HOME} //
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index e79cf76a..fe8301d6 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -23,7 +23,7 @@ sub new {
return $lei->fail("$f non-existent or unreadable");
$self->{-cfg} = PublicInbox::Config::git_config_dump($f);
} else { # new saved search "lei q --save"
- my $saved_dir = $lei->store_path . '/../saved-searches/';
+ my $saved_dir = $lei->share_path . '/saved-searches/';
my (@name) = ($lei->{ovv}->{dst} =~ m{([\w\-\.]+)/*\z});
my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
my $q_raw_str = ref($q) ? "@$q" : $q;
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 3/9] lei: saved searches keyed only by path/URL and format
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
2021-04-16 23:10 ` [PATCH 1/9] lei q: --save preserves relative time queries Eric Wong
2021-04-16 23:10 ` [PATCH 2/9] lei: expose share_path as a method Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-16 23:10 ` [PATCH 4/9] lei_to_mail: cast to URIimap object early Eric Wong
` (5 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
We want users to be able to edit and refine the query over
time while using the same output destination.
---
lib/PublicInbox/LeiSavedSearch.pm | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index fe8301d6..ebc63091 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -10,9 +10,8 @@ use PublicInbox::OverIdx;
use PublicInbox::LeiSearch;
use PublicInbox::Config;
use PublicInbox::Spawn qw(run_die);
-use PublicInbox::ContentHash qw(content_hash git_sha);
-use PublicInbox::Eml;
-use PublicInbox::Hval qw(to_filename);
+use PublicInbox::ContentHash qw(git_sha);
+use Digest::SHA qw(sha256_hex);
sub new {
my ($cls, $lei, $dir) = @_;
@@ -24,11 +23,11 @@ sub new {
$self->{-cfg} = PublicInbox::Config::git_config_dump($f);
} else { # new saved search "lei q --save"
my $saved_dir = $lei->share_path . '/saved-searches/';
- my (@name) = ($lei->{ovv}->{dst} =~ m{([\w\-\.]+)/*\z});
+ my (@n) = ($lei->{ovv}->{dst} =~ m{([\w\-\.]+)/*\z});
my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
- my $q_raw_str = ref($q) ? "@$q" : $q;
- push @name, to_filename($q_raw_str);
- $dir = $saved_dir . join('-', @name);
+ push @n, sha256_hex("$lei->{ovv}->{fmt}\0$lei->{ovv}->{dst}");
+
+ $dir = $saved_dir . join('-', @n);
require File::Path;
File::Path::make_path($dir); # raises on error
$self->{'-f'} = "$dir/lei.saved-search";
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 4/9] lei_to_mail: cast to URIimap object early
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
` (2 preceding siblings ...)
2021-04-16 23:10 ` [PATCH 3/9] lei: saved searches keyed only by path/URL and format Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-16 23:10 ` [PATCH 5/9] test_common: handle '-C' (chdir) spawn option properly Eric Wong
` (4 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
NetReader->add_url supports URI-like objects, now. We'll be
relying on the canonicalization for LeiSavedSearch.
---
lib/PublicInbox/LeiToMail.pm | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 4ebaf8f3..daa8084b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -335,14 +335,16 @@ sub new {
$self->{base_type} = 'mbox';
} elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support
require PublicInbox::NetWriter;
+ require PublicInbox::URIimap;
my $net = PublicInbox::NetWriter->new;
- $net->add_url($dst);
$net->{quiet} = $lei->{opt}->{quiet};
- my $err = $net->errors($dst);
+ my $uri = PublicInbox::URIimap->new($dst)->canonical;
+ $net->add_url($uri);
+ my $err = $net->errors;
return $lei->fail($err) if $err;
- require PublicInbox::URIimap; # TODO: URI cast early
- $self->{uri} = PublicInbox::URIimap->new($dst);
- $self->{uri}->mailbox or die "No mailbox: $dst";
+ $uri->mailbox or return $lei->fail("No mailbox: $dst");
+ $self->{uri} = $uri;
+ $dst = $lei->{ovv}->{dst} = $$uri; # canonicalized
$lei->{net} = $net;
$self->{base_type} = 'imap';
} else {
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 5/9] test_common: handle '-C' (chdir) spawn option properly
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
` (3 preceding siblings ...)
2021-04-16 23:10 ` [PATCH 4/9] lei_to_mail: cast to URIimap object early Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-17 2:24 ` [SQUASH] test_common: fix -C $DIR with TEST_RUN_MODE=0 Eric Wong
2021-04-16 23:10 ` [PATCH 6/9] lei: fix rel2abs Eric Wong
` (3 subsequent siblings)
8 siblings, 1 reply; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
We use it in t/lei-q-save.t, and were inadvertently writing
to the worktree.
---
lib/PublicInbox/TestCommon.pm | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index d506e4b5..465d4b9d 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -288,6 +288,7 @@ sub run_script ($;$$) {
}
if ($run_mode == 0) {
# spawn an independent new process, like real-world use cases:
+ $spawn_opt->{$_} = $opt->{$_} for (qw(-C));
require PublicInbox::Spawn;
my $cmd = [ key2script($key), @argv ];
my $pid = PublicInbox::Spawn::spawn($cmd, $env, $spawn_opt);
@@ -302,8 +303,14 @@ sub run_script ($;$$) {
local %SIG = %SIG;
local $0 = join(' ', @$cmd);
my $orig_io = _prepare_redirects($fhref);
+ my $cwdfh;
+ if (my $d = $opt->{'-C'}) {
+ opendir $cwdfh, '.' or die "opendir .: $!";
+ chdir $d or die "chdir $d: $!";
+ }
_run_sub($sub, $key, \@argv);
eval { PublicInbox::Inbox::cleanup_task() };
+ die "chdir(restore): $!" if $cwdfh && !chdir($cwdfh);
_undo_redirects($orig_io);
select STDOUT;
}
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [SQUASH] test_common: fix -C $DIR with TEST_RUN_MODE=0
2021-04-16 23:10 ` [PATCH 5/9] test_common: handle '-C' (chdir) spawn option properly Eric Wong
@ 2021-04-17 2:24 ` Eric Wong
2021-04-17 10:00 ` [PATCH 10/9] lei_query: fix relative path handling on --stdin Eric Wong
0 siblings, 1 reply; 12+ messages in thread
From: Eric Wong @ 2021-04-17 2:24 UTC (permalink / raw)
To: meta
---
lib/PublicInbox/TestCommon.pm | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 465d4b9d..2627871a 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -288,9 +288,12 @@ sub run_script ($;$$) {
}
if ($run_mode == 0) {
# spawn an independent new process, like real-world use cases:
- $spawn_opt->{$_} = $opt->{$_} for (qw(-C));
require PublicInbox::Spawn;
my $cmd = [ key2script($key), @argv ];
+ if (my $d = $opt->{'-C'}) {
+ $cmd->[0] = File::Spec->rel2abs($cmd->[0]);
+ $spawn_opt->{'-C'} = $d;
+ }
my $pid = PublicInbox::Spawn::spawn($cmd, $env, $spawn_opt);
if (defined $pid) {
my $r = waitpid($pid, 0) // die "waitpid: $!";
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 10/9] lei_query: fix relative path handling on --stdin
2021-04-17 2:24 ` [SQUASH] test_common: fix -C $DIR with TEST_RUN_MODE=0 Eric Wong
@ 2021-04-17 10:00 ` Eric Wong
0 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-17 10:00 UTC (permalink / raw)
To: meta
Since --stdin could be waiting on user keyboard input or
something else slow, we handle it in the event loop. That
means other commands can change the working directory of
lei-daemon while a query is being trickled to us via stdin.
Rearranging query handling internals to delay opening the
--output destination in commit 26e0fe73de93f451 meant
another command could throw off our --output pathname if
it is relative.
Fixes: 26e0fe73de93f451 ("lei_query: rearrange internals to capture query early")
---
lib/PublicInbox/LeiQuery.pm | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 7ddba4cf..385ba0a9 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -53,6 +53,9 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
my ($self) = @_; # $_[1] = $rbuf
if (defined($_[1])) {
$_[1] eq '' and return eval {
+ if (my $dfd = $self->{3}) {
+ chdir($dfd) or return $self->fail("fchdir: $!");
+ }
$self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr};
$self->{lse}->query_approxidate($self->{lse}->git,
$self->{mset_opt}->{qstr});
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 6/9] lei: fix rel2abs
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
` (4 preceding siblings ...)
2021-04-16 23:10 ` [PATCH 5/9] test_common: handle '-C' (chdir) spawn option properly Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-16 23:10 ` [PATCH 7/9] lei up: support output destination as arg Eric Wong
` (2 subsequent siblings)
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
We don't want pathnames with "GLOB(0xADD12355)" in them.
---
lib/PublicInbox/LEI.pm | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 52b588a2..ebd0f154 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -68,18 +68,19 @@ sub rel2abs ($$) {
my ($self, $p) = @_;
return $p if index($p, '/') == 0; # already absolute
my $pwd = $self->{env}->{PWD};
+ my $cwd;
if (defined $pwd) {
- my $cwd = $self->{3} // getcwd() // die "getcwd(PWD=$pwd): $!";
+ my $xcwd = $self->{3} //
+ ($cwd = getcwd() // die "getcwd(PWD=$pwd): $!");
if (my @st_pwd = stat($pwd)) {
- my @st_cwd = stat($cwd) or die "stat($cwd): $!";
+ my @st_cwd = stat($xcwd) or die "stat($xcwd): $!";
"@st_pwd[1,0]" eq "@st_cwd[1,0]" or
- $self->{env}->{PWD} = $pwd = $cwd;
+ $self->{env}->{PWD} = $pwd = undef;
} else { # PWD was invalid
- delete $self->{env}->{PWD};
- undef $pwd;
+ $self->{env}->{PWD} = $pwd = undef;
}
}
- $pwd //= $self->{env}->{PWD} = getcwd() // die "getcwd(PWD=$pwd): $!";
+ $pwd //= $self->{env}->{PWD} = $cwd // getcwd() // die "getcwd: $!";
File::Spec->rel2abs($p, $pwd);
}
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 7/9] lei up: support output destination as arg
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
` (5 preceding siblings ...)
2021-04-16 23:10 ` [PATCH 6/9] lei: fix rel2abs Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-16 23:10 ` [PATCH 8/9] lei q --save: avoid lei.q.format Eric Wong
2021-04-16 23:10 ` [PATCH 9/9] lei q --save: clobber config file on repeats Eric Wong
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
Specifying a directory in ~/.local/share/lei/saved-searches/
is painful, so support (and start encouraging) the use of
the output.
---
lib/PublicInbox/LeiSavedSearch.pm | 55 ++++++++++++++++++++++++-------
lib/PublicInbox/LeiUp.pm | 4 +--
t/lei-q-save.t | 11 +++++++
3 files changed, 57 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index ebc63091..93b1b23a 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -13,24 +13,39 @@ use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use Digest::SHA qw(sha256_hex);
+sub lss_dir_for ($$) {
+ my ($lei, $dstref) = @_;
+ my @n;
+ if ($$dstref =~ m,\Aimaps?://,i) { # already canonicalized
+ require PublicInbox::URIimap;
+ my $uri = PublicInbox::URIimap->new($$dstref)->canonical;
+ $$dstref = $$uri;
+ @n = ($uri->mailbox);
+ } else { # basename
+ @n = ($$dstref =~ m{([\w\-\.]+)/*\z});
+ $$dstref = $lei->rel2abs($$dstref);
+ }
+ push @n, sha256_hex($$dstref);
+ $lei->share_path . '/saved-searches/' . join('-', @n);
+}
+
sub new {
my ($cls, $lei, $dir) = @_;
- my $self = bless { ale => $lei->ale, -cfg => {} }, $cls;
+ my $self = bless { ale => $lei->ale }, $cls;
if (defined $dir) { # updating existing saved search via "lei up"
- my $f = $self->{'-f'} = "$dir/lei.saved-search";
- -f $f && -r _ or
+ my $f = "$dir/lei.saved-search";
+ ((-f $f && -r _) || output2lssdir($self, $lei, \$dir, \$f)) or
return $lei->fail("$f non-existent or unreadable");
- $self->{-cfg} = PublicInbox::Config::git_config_dump($f);
+ $self->{-cfg} //= PublicInbox::Config::git_config_dump($f);
+ $self->{'-f'} = $f;
} else { # new saved search "lei q --save"
- my $saved_dir = $lei->share_path . '/saved-searches/';
- my (@n) = ($lei->{ovv}->{dst} =~ m{([\w\-\.]+)/*\z});
- my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
- push @n, sha256_hex("$lei->{ovv}->{fmt}\0$lei->{ovv}->{dst}");
-
- $dir = $saved_dir . join('-', @n);
+ my $dst = $lei->{ovv}->{dst};
+ $dir = lss_dir_for($lei, \$dst);
require File::Path;
File::Path::make_path($dir); # raises on error
+ $self->{-cfg} = {};
$self->{'-f'} = "$dir/lei.saved-search";
+ my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
if (ref $q) {
cfg_set($self, '--add', 'lei.q', $_) for @$q;
} else {
@@ -38,7 +53,8 @@ sub new {
}
my $fmt = $lei->{opt}->{'format'};
cfg_set($self, 'lei.q.format', $fmt) if defined $fmt;
- cfg_set($self, 'lei.q.output', $lei->{opt}->{output});
+ $dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i;
+ cfg_set($self, 'lei.q.output', $dst);
for my $k (qw(only include exclude)) {
my $ary = $lei->{opt}->{$k} // next;
for my $x (@$ary) {
@@ -127,6 +143,23 @@ sub mm { undef }
sub altid_map { {} }
sub cloneurl { [] }
+
+# find existing directory containing a `lei.saved-search' file based on
+# $dir_ref which is an output
+sub output2lssdir {
+ my ($self, $lei, $dir_ref, $fn_ref) = @_;
+ my $dst = $$dir_ref; # imap://$MAILBOX, /path/to/maildir, /path/to/mbox
+ my $dir = lss_dir_for($lei, \$dst);
+ my $f = "$dir/lei.saved-search";
+ if (-f $f && -r _) {
+ $self->{-cfg} = PublicInbox::Config::git_config_dump($f);
+ $$dir_ref = $dir;
+ $$fn_ref = $f;
+ return 1;
+ }
+ undef;
+}
+
no warnings 'once';
*nntp_url = \&cloneurl;
*base_url = \&PublicInbox::Inbox::base_url;
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 386a7566..7ddb1dd0 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -9,9 +9,9 @@ use PublicInbox::LeiSavedSearch;
use PublicInbox::LeiOverview;
sub lei_up {
- my ($lei, $dir) = @_;
+ my ($lei, $out) = @_;
$lei->{lse} = $lei->_lei_store(1)->search;
- my $lss = PublicInbox::LeiSavedSearch->new($lei, $dir) or return;
+ my $lss = PublicInbox::LeiSavedSearch->new($lei, $out) or return;
my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
$mset_opt->{limit} = $lei->{opt}->{limit} // 10000;
my $q = $mset_opt->{q_raw} = $lss->{-cfg}->{'lei.q'} //
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 6cfac20b..d43f508b 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -7,6 +7,8 @@ my $doc1 = eml_load('t/plack-qp.eml');
$doc1->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 5)}));
my $doc2 = eml_load('t/utf8.eml');
$doc2->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 4)}));
+my $doc3 = eml_load('t/msg_iter-order.eml');
+$doc3->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 4)}));
test_lei(sub {
my $home = $ENV{HOME};
@@ -38,5 +40,14 @@ test_lei(sub {
$cfg = PublicInbox::Config->new("$s[0]/lei.saved-search");
is_deeply $cfg->{'lei.q'}, 'd:last.week..',
'q --stdin stores relative time';
+ my $size = -s "$home/mbcl2";
+ ok(defined($size) && $size > 0, 'results written');
+ lei_ok([qw(up mbcl2)], undef, { -C => $home, %$lei_opt });
+ is(-s "$home/mbcl2", $size, 'size unchanged on noop up');
+
+ $in = $doc3->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
+ lei_ok([qw(up mbcl2)], undef, { -C => $home, %$lei_opt });
+ ok(-s "$home/mbcl2" > $size, 'size increased after up');
});
done_testing;
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 8/9] lei q --save: avoid lei.q.format
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
` (6 preceding siblings ...)
2021-04-16 23:10 ` [PATCH 7/9] lei up: support output destination as arg Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
2021-04-16 23:10 ` [PATCH 9/9] lei q --save: clobber config file on repeats Eric Wong
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
It is redundant since we stuff everything into the lei.q.output
config key.
---
lib/PublicInbox/LeiSavedSearch.pm | 2 --
lib/PublicInbox/LeiUp.pm | 1 -
2 files changed, 3 deletions(-)
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 93b1b23a..a8bf470b 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -51,8 +51,6 @@ sub new {
} else {
cfg_set($self, 'lei.q', $q);
}
- my $fmt = $lei->{opt}->{'format'};
- cfg_set($self, 'lei.q.format', $fmt) if defined $fmt;
$dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i;
cfg_set($self, 'lei.q.output', $dst);
for my $k (qw(only include exclude)) {
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 7ddb1dd0..9fe4901b 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -24,7 +24,6 @@ sub lei_up {
}
$lei->{opt}->{output} = $lss->{-cfg}->{'lei.q.output'} //
return $lei->fail("lei.q.output unset in $lss->{-f}");
- $lei->{opt}->{'format'} //= $lss->{-cfg}->{'lei.q.format'}; # optional
my $to_avref = $lss->{-cfg}->can('_array');
for my $k (qw(only include exclude)) {
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH 9/9] lei q --save: clobber config file on repeats
2021-04-16 23:10 [PATCH 0/9] lei saved search usability improvements Eric Wong
` (7 preceding siblings ...)
2021-04-16 23:10 ` [PATCH 8/9] lei q --save: avoid lei.q.format Eric Wong
@ 2021-04-16 23:10 ` Eric Wong
8 siblings, 0 replies; 12+ messages in thread
From: Eric Wong @ 2021-04-16 23:10 UTC (permalink / raw)
To: meta
A user may wish to clobber/refine existing search parameters
by issuing "lei q --save" again. Support that by overwriting
the lei.saved-search state file entirely.
We continue to preserve over.sqlite3 for deduplication purposes.
This way, we don't get something redundant like:
[lei]
q = term1
q = term2
q = term1
q = term2
q = term3
...whenever a user wants to refine their search. Instead,
we'll just have:
[lei]
q = term1
q = term2
q = term3
On the second go.
---
lib/PublicInbox/Config.pm | 9 +++++++++
lib/PublicInbox/LeiSavedSearch.pm | 10 +++++++++-
lib/PublicInbox/Reply.pm | 10 ++--------
3 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 26ac298e..603dad98 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -559,4 +559,13 @@ sub json {
};
}
+sub squote_maybe ($) {
+ my ($val) = @_;
+ if ($val =~ m{([^\w@\./,\%\+\-])}) {
+ $val =~ s/(['!])/'\\$1'/g; # '!' for csh
+ return "'$val'";
+ }
+ $val;
+}
+
1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index a8bf470b..932b2aa4 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -13,6 +13,8 @@ use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use Digest::SHA qw(sha256_hex);
+*squote_maybe = \&PublicInbox::Config::squote_maybe;
+
sub lss_dir_for ($$) {
my ($lei, $dstref) = @_;
my @n;
@@ -44,7 +46,13 @@ sub new {
require File::Path;
File::Path::make_path($dir); # raises on error
$self->{-cfg} = {};
- $self->{'-f'} = "$dir/lei.saved-search";
+ my $f = $self->{'-f'} = "$dir/lei.saved-search";
+ open my $fh, '>', $f or return $lei->fail("open $f: $!");
+ my $sq_dst = squote_maybe($dst);
+ print $fh <<EOM or return $lei->fail("print $f: $!");
+; to refresh with new results, run: lei up $sq_dst
+EOM
+ close $fh or return $lei->fail("close $f: $!");
my $q = $lei->{mset_opt}->{q_raw} // die 'BUG: {q_raw} missing';
if (ref $q) {
cfg_set($self, '--add', 'lei.q', $_) for @$q;
diff --git a/lib/PublicInbox/Reply.pm b/lib/PublicInbox/Reply.pm
index 2a1066d2..79dd46a7 100644
--- a/lib/PublicInbox/Reply.pm
+++ b/lib/PublicInbox/Reply.pm
@@ -9,15 +9,9 @@ use URI::Escape qw/uri_escape_utf8/;
use PublicInbox::Hval qw(ascii_html obfuscate_addrs mid_href);
use PublicInbox::Address;
use PublicInbox::MID qw(mid_clean);
+use PublicInbox::Config;
-sub squote_maybe ($) {
- my ($val) = @_;
- if ($val =~ m{([^\w@\./,\%\+\-])}) {
- $val =~ s/(['!])/'\\$1'/g; # '!' for csh
- return "'$val'";
- }
- $val;
-}
+*squote_maybe = \&PublicInbox::Config::squote_maybe;
sub add_addrs {
my ($to, $cc, @addrs) = @_;
^ permalink raw reply related [flat|nested] 12+ messages in thread