* [PATCH 1/2] lei: avoid deadlock on inotify/EVFILT_VNODE wakeups
2022-07-19 22:42 [PATCH 0/2] lei inotify/EVFILT_VNODE deadlock fix Eric Wong
@ 2022-07-19 22:42 ` Eric Wong
2022-07-19 22:42 ` [PATCH 2/2] lei note-event: inline note_event_arm_done Eric Wong
1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2022-07-19 22:42 UTC (permalink / raw)
To: meta
Enqueuing "note-event" requests from the DS event loop must
not wait on workers being able to drain the queue quickly
enough. Thus we make the SOCK_SEQPACKET writes nonblocking
and rely on the lei-daemon event loop to enqueue writes.
This is a unique problem for "note-event" since it reuses
workers in between commands, while most lei commands currently
fork off new workers.
---
MANIFEST | 1 +
lib/PublicInbox/IPC.pm | 26 +++++++++++++++--
lib/PublicInbox/LeiNoteEvent.pm | 5 ++--
lib/PublicInbox/WQBlocked.pm | 49 +++++++++++++++++++++++++++++++++
4 files changed, 77 insertions(+), 4 deletions(-)
create mode 100644 lib/PublicInbox/WQBlocked.pm
diff --git a/MANIFEST b/MANIFEST
index 2cbe66b7..923f5147 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -337,6 +337,7 @@ lib/PublicInbox/V2Writable.pm
lib/PublicInbox/View.pm
lib/PublicInbox/ViewDiff.pm
lib/PublicInbox/ViewVCS.pm
+lib/PublicInbox/WQBlocked.pm
lib/PublicInbox/WQWorker.pm
lib/PublicInbox/WWW.pm
lib/PublicInbox/WWW.pod
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 67e86a43..74862673 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# base class for remote IPC calls and workqueues, requires Storable or Sereal
@@ -43,7 +43,7 @@ if ($enc && $dec) { # should be custom ops
}
my $recv_cmd = PublicInbox::Spawn->can('recv_cmd4');
-my $send_cmd = PublicInbox::Spawn->can('send_cmd4') // do {
+our $send_cmd = PublicInbox::Spawn->can('send_cmd4') // do {
require PublicInbox::CmdIPC4;
$recv_cmd //= PublicInbox::CmdIPC4->can('recv_cmd4');
PublicInbox::CmdIPC4->can('send_cmd4');
@@ -348,6 +348,24 @@ sub wq_do {
}
}
+sub prepare_nonblock {
+ ($_[0]->{-wq_s1} // die 'BUG: no {-wq_s1}')->blocking(0);
+ $_[0]->{-reap_async} or die 'BUG: {-reap_async} needed for nonblock';
+ require PublicInbox::WQBlocked;
+}
+
+sub wq_nonblock_do { # always async
+ my ($self, $sub, @args) = @_;
+ my $buf = ipc_freeze([$sub, @args]);
+ if ($self->{wqb}) { # saturated once, assume saturated forever
+ $self->{wqb}->flush_send($buf);
+ } else {
+ $send_cmd->($self->{-wq_s1}, [], $buf, MSG_EOR) //
+ ($!{EAGAIN} ? PublicInbox::WQBlocked->new($self, $buf)
+ : croak("sendmsg: $!"));
+ }
+}
+
sub _wq_worker_start ($$$$) {
my ($self, $oldset, $fields, $one) = @_;
my ($bcast1, $bcast2);
@@ -405,6 +423,10 @@ sub wq_workers_start {
sub wq_close {
my ($self) = @_;
+ if (my $wqb = delete $self->{wqb}) {
+ $self->{-reap_async} or die 'BUG: {-reap_async} unset';
+ $wqb->enq_close;
+ }
delete @$self{qw(-wq_s1 -wq_s2)} or return;
return if $self->{-reap_async};
my @pids = keys %{$self->{-wq_workers}};
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index db387633..93f80116 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -58,7 +58,7 @@ sub eml_event ($$$$) {
}
}
-sub maildir_event { # via wq_io_do
+sub maildir_event { # via wq_nonblock_do
my ($self, $fn, $vmd, $state) = @_;
if (my $eml = PublicInbox::InboxWritable::eml_from_path($fn)) {
eml_event($self, $eml, $vmd, $state);
@@ -93,6 +93,7 @@ sub lei_note_event {
my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
$lei->wait_wq_events($op_c, $ops);
note_event_arm_done($lei);
+ $wq->prepare_nonblock;
$lei->{lne} = $wq;
};
if ($folder =~ /\Amaildir:/i) {
@@ -101,7 +102,7 @@ sub lei_note_event {
return if index($fl, 'T') >= 0;
my $kw = PublicInbox::MdirReader::flags2kw($fl);
my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] };
- $self->wq_do('maildir_event', $fn, $vmd, $state);
+ $self->wq_nonblock_do('maildir_event', $fn, $vmd, $state);
} # else: TODO: imap
}
diff --git a/lib/PublicInbox/WQBlocked.pm b/lib/PublicInbox/WQBlocked.pm
new file mode 100644
index 00000000..fbb43600
--- /dev/null
+++ b/lib/PublicInbox/WQBlocked.pm
@@ -0,0 +1,49 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# non-blocking workqueues, currently used by LeiNoteEvent to track renames
+package PublicInbox::WQBlocked;
+use v5.12;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLOUT EPOLLONESHOT);
+use PublicInbox::IPC;
+use Carp ();
+use Socket qw(MSG_EOR);
+
+sub new {
+ my ($cls, $wq, $buf) = @_;
+ my $self = bless { msgq => [$buf], }, $cls;
+ $wq->{wqb} = $self->SUPER::new($wq->{-wq_s1}, EPOLLOUT|EPOLLONESHOT);
+}
+
+sub flush_send {
+ my ($self) = @_;
+ push(@{$self->{msgq}}, $_[1]) if defined($_[1]);
+ while (defined(my $buf = shift @{$self->{msgq}})) {
+ if (ref($buf) eq 'CODE') {
+ $buf->($self); # could be \&PublicInbox::DS::close
+ } else {
+ my $wq_s1 = $self->{sock};
+ my $n = $PublicInbox::IPC::send_cmd->($wq_s1, [], $buf,
+ MSG_EOR);
+ next if defined($n);
+ Carp::croak("sendmsg: $!") unless $!{EAGAIN};
+ PublicInbox::DS::epwait($wq_s1, EPOLLOUT|EPOLLONESHOT);
+ unshift @{$self->{msgq}}, $buf;
+ last; # wait for ->event_step
+ }
+ }
+}
+
+sub enq_close { flush_send($_[0], $_[0]->can('close')) }
+
+sub event_step { # called on EPOLLOUT wakeup
+ my ($self) = @_;
+ eval { flush_send($self) } if $self->{sock};
+ if ($@) {
+ warn $@;
+ $self->close;
+ }
+}
+
+1;
^ permalink raw reply related [flat|nested] 3+ messages in thread* [PATCH 2/2] lei note-event: inline note_event_arm_done
2022-07-19 22:42 [PATCH 0/2] lei inotify/EVFILT_VNODE deadlock fix Eric Wong
2022-07-19 22:42 ` [PATCH 1/2] lei: avoid deadlock on inotify/EVFILT_VNODE wakeups Eric Wong
@ 2022-07-19 22:42 ` Eric Wong
1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2022-07-19 22:42 UTC (permalink / raw)
To: meta
This was a single-caller sub since 47d4e53734820b4e
(lei_mail_sync: rely on flock(2), avoid IPC, 2021-09-18)
and unlikely to be used further, so inline it and save
a few KB of memory.
---
lib/PublicInbox/LeiNoteEvent.pm | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 93f80116..8581bd9a 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -27,13 +27,6 @@ sub flush_task { # PublicInbox::DS timer callback
for my $lei (values %$todo) { flush_lei($lei) }
}
-# sets a timer to flush
-sub note_event_arm_done ($) {
- my ($lei) = @_;
- PublicInbox::DS::add_uniq_timer('flush_timer', 5, \&flush_task);
- $to_flush->{$lei->{cfg}->{'-f'}} //= $lei;
-}
-
sub eml_event ($$$$) {
my ($self, $eml, $vmd, $state) = @_;
my $sto = $self->{lei}->{sto};
@@ -92,7 +85,8 @@ sub lei_note_event {
$jobs = 4 if $jobs > 4; # same default as V2Writable
my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
$lei->wait_wq_events($op_c, $ops);
- note_event_arm_done($lei);
+ PublicInbox::DS::add_uniq_timer('flush_timer', 5, \&flush_task);
+ $to_flush->{$lei->{cfg}->{'-f'}} //= $lei;
$wq->prepare_nonblock;
$lei->{lne} = $wq;
};
^ permalink raw reply related [flat|nested] 3+ messages in thread