* [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync
2021-08-17 8:52 [PATCH 0/3] lei: some mail sync stuff Eric Wong
@ 2021-08-17 8:52 ` Eric Wong
2021-08-17 8:52 ` [PATCH 2/3] ipc: remove WQ_MAX_WORKERS Eric Wong
2021-08-17 8:52 ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
We access this read-only in many places (and will in more),
so provide a shortcut to simplify callers.
---
lib/PublicInbox/LEI.pm | 9 +++++++++
lib/PublicInbox/LeiBlob.pm | 3 +--
lib/PublicInbox/LeiForgetMailSync.pm | 3 +--
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiLcat.pm | 10 ++++------
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
6 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 347dd280..e5232f1b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1425,4 +1425,13 @@ sub git_blob_id {
($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
}
+sub lms { # read-only LeiMailSync
+ my ($lei) = @_;
+ my $lse = $lei->{lse} // do {
+ my $sto = $lei->{sto} // _lei_store($lei);
+ $sto ? $sto->search : undef
+ };
+ $lse ? $lse->lms : undef;
+}
+
1;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 3158ca3b..21003894 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -133,8 +133,7 @@ sub lei_blob {
}
my $ce = $?;
return if $ce == 0;
- my $sto = $lei->_lei_store;
- my $lms = $sto ? $sto->search->lms : undef;
+ my $lms = $lei->lms;
if (my $bref = $lms ? $lms->local_blob($blob, 1) : undef) {
defined($lei->{-attach_idx}) and
return extract_attach($lei, $blob, $bref);
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 46dde1a7..c74ba25d 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -14,8 +14,7 @@ use PublicInbox::LeiExportKw;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index bf7a4836..2d2ff1a0 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -41,8 +41,7 @@ sub inspect_imap_uid ($$) {
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
- my $lse = $lei->{lse} or return $ent;
- my $lms = $lse->lms or return $ent;
+ my $lms = $lei->lms or return $ent;
my $folders = [ $folder ];
my $err = $lms->arg2folder($lei, $folders);
if ($err) {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 4a0c24a9..9d95e899 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -13,7 +13,7 @@ use PublicInbox::MID qw($MID_EXTRACT);
sub lcat_folder ($$$) {
my ($lei, $lms, $folder) = @_;
- $lms //= $lei->{lse}->lms // return;
+ $lms //= $lei->lms or return;
my $folders = [ $folder];
my $err = $lms->arg2folder($lei, $folders);
$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
@@ -29,7 +29,7 @@ sub lcat_folder ($$$) {
sub lcat_imap_uri ($$) {
my ($lei, $uri) = @_;
- my $lms = $lei->{lse}->lms or return;
+ my $lms = $lei->lms or return;
# cf. LeiXsearch->lcat_dump
if (defined $uri->uid) {
my $oidhex = $lms->imap_oid($lei, $uri);
@@ -129,8 +129,7 @@ sub lei_lcat {
my ($lei, @argv) = @_;
my $lxs = $lei->lxs_prepare or return;
$lei->ale->refresh_externals($lxs, $lei);
- my $sto = $lei->_lei_store(1);
- $lei->{lse} = $sto->search;
+ $lei->_lei_store(1);
my $opt = $lei->{opt};
my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
@@ -153,8 +152,7 @@ no args allowed on command-line with --stdin
sub _complete_lcat {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } $lms->folders;
}
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index cadc61ed..2d8913ac 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -103,8 +103,7 @@ sub _complete_ls_mail_source {
my $match_cb = $lei->complete_url_prepare(\@argv);
my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
my %f = map { $_ => 1 } @m;
- my $sto = $lei->_lei_store;
- if (my $lms = $sto ? $sto->search->lms : undef) {
+ if (my $lms = $lei->lms) {
@m = map { $match_cb->($_) } grep(
m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
@f{@m} = @m;
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 2/3] ipc: remove WQ_MAX_WORKERS
2021-08-17 8:52 [PATCH 0/3] lei: some mail sync stuff Eric Wong
2021-08-17 8:52 ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
@ 2021-08-17 8:52 ` Eric Wong
2021-08-17 8:52 ` [PATCH 3/3] lei forget-mail-sync: rely on lei/store process Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
We no longer rely on IO::FDPass, so there's no longer a reason
to limit this internally.
---
lib/PublicInbox/IPC.pm | 5 -----
1 file changed, 5 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index d909dc1c..9efe551b 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -20,7 +20,6 @@ use Socket qw(AF_UNIX MSG_EOR SOCK_STREAM);
my $MY_MAX_ARG_STRLEN = 4096 * 33; # extra 4K for serialization
my $SEQPACKET = eval { Socket::SOCK_SEQPACKET() }; # portable enough?
our @EXPORT_OK = qw(ipc_freeze ipc_thaw);
-my $WQ_MAX_WORKERS = 4096;
my ($enc, $dec);
# ->imports at BEGIN turns sereal_*_with_object into custom ops on 5.14+
# and eliminate method call overhead
@@ -354,7 +353,6 @@ sub wq_workers_start {
die "socketpair: $!";
$self->ipc_atfork_prepare;
$nr_workers //= $self->{-wq_nr_workers};
- $nr_workers = $WQ_MAX_WORKERS if $nr_workers > $WQ_MAX_WORKERS;
my $sigset = $oldset // PublicInbox::DS::block_signals();
$self->{-wq_workers} = {};
$self->{-wq_ident} = $ident;
@@ -367,7 +365,6 @@ sub wq_worker_incr { # SIGTTIN handler
my ($self, $oldset, $fields) = @_;
$self->{-wq_s2} or return;
die "-wq_nr_workers locked" if defined $self->{-wq_nr_workers};
- return if wq_workers($self) >= $WQ_MAX_WORKERS;
$self->ipc_atfork_prepare;
my $sigset = $oldset // PublicInbox::DS::block_signals();
_wq_worker_start($self, $sigset, $fields);
@@ -443,8 +440,6 @@ sub wq_kill {
kill($sig // 'TERM', keys %$workers);
}
-sub WQ_MAX_WORKERS { $WQ_MAX_WORKERS }
-
sub DESTROY {
my ($self) = @_;
my $ppid = $self->{-wq_ppid};
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 3/3] lei forget-mail-sync: rely on lei/store process
2021-08-17 8:52 [PATCH 0/3] lei: some mail sync stuff Eric Wong
2021-08-17 8:52 ` [PATCH 1/3] lei: add ->lms shortcut for LeiMailSync Eric Wong
2021-08-17 8:52 ` [PATCH 2/3] ipc: remove WQ_MAX_WORKERS Eric Wong
@ 2021-08-17 8:52 ` Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2021-08-17 8:52 UTC (permalink / raw)
To: meta
As implied in commit 6ff03ba2be9247f1
("lei export-kw: do not write directly to mail_sync.sqlite3"),
modifying mail_sync.sqlite3 directly can lead to conflicts
and making everything go through lei/store is easier.
---
lib/PublicInbox/LeiForgetMailSync.pm | 8 ++++----
lib/PublicInbox/LeiStore.pm | 6 ++++++
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index c74ba25d..940ca1b6 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -15,13 +15,13 @@ use PublicInbox::LeiExportKw;
sub lei_forget_mail_sync {
my ($lei, @folders) = @_;
my $lms = $lei->lms or return;
+ my $sto = $lei->_lei_store or return; # may disappear due to race
+ $sto->write_prepare;
my $err = $lms->arg2folder($lei, \@folders);
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
return $lei->fail($err->{fail}) if $err->{fail};
- delete $lms->{dbh};
- $lms->lms_begin;
- $lms->forget_folder($_) for @folders;
- $lms->lms_commit;
+ $sto->ipc_do('lms_forget_folders', @folders);
+ my $wait = $sto->ipc_do('done');
}
*_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 3f33d114..e8334187 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -216,6 +216,12 @@ sub lms_mv_src {
_lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn);
}
+sub lms_forget_folders {
+ my ($self, @folders) = @_;
+ my $lms = _lms_rw($self);
+ for my $f (@folders) { $lms->forget_folder($f) }
+}
+
sub set_sync_info {
my ($self, $oidhex, $folder, $id) = @_;
_lms_rw($self)->set_src($oidhex, $folder, $id);
^ permalink raw reply related [flat|nested] 4+ messages in thread