From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 0F66D1FBFB for ; Wed, 10 Jun 2020 07:06:27 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 42/82] imap: start doing iterative config reloading Date: Wed, 10 Jun 2020 07:04:39 +0000 Message-Id: <20200610070519.18252-43-e@yhbt.net> In-Reply-To: <20200610070519.18252-1-e@yhbt.net> References: <20200610070519.18252-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This will be used to prevent reloading a giant config with tens/hundreds of thousands of inboxes from blocking the event loop. --- lib/PublicInbox/Config.pm | 15 ++++++++++ lib/PublicInbox/IMAPD.pm | 61 +++++++++++++++++++++++++++++++++++---- script/public-inbox-imapd | 2 +- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index 458f29b2964..c18c9c75b4f 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -99,6 +99,21 @@ sub each_inbox { } } +sub iterate_start { + my ($self, $cb, $arg) = @_; + my $i = 0; + $self->{-iter} = [ \$i, $cb, $arg ]; +} + +# for PublicInbox::DS::next_tick +sub event_step { + my ($self) = @_; + my ($i, $cb, $arg) = @{$self->{-iter}}; + my $section = $self->{-section_order}->[$$i++]; + delete($self->{-iter}) unless defined($section); + $cb->($self, $section, $arg); +} + sub lookup_newsgroup { my ($self, $ng) = @_; _lookup_fill($self, '-by_newsgroup', lc($ng)); diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm index 966879d8ea5..a10fabffb1f 100644 --- a/lib/PublicInbox/IMAPD.pm +++ b/lib/PublicInbox/IMAPD.pm @@ -7,6 +7,8 @@ package PublicInbox::IMAPD; use strict; use parent qw(PublicInbox::NNTPD); use PublicInbox::InboxIdle; +use PublicInbox::IMAP; +# *UID_BLOCK = \&PublicInbox::IMAP::UID_BLOCK; sub new { my ($class) = @_; @@ -51,17 +53,64 @@ sub refresh_inboxlist ($) { $self->{inboxlist} = \@names; } -sub refresh_groups { - my ($self) = @_; - my $pi_config = $self->{pi_config} = PublicInbox::Config->new; - $self->SUPER::refresh_groups($pi_config); - refresh_inboxlist($self); +sub imapd_refresh_ibx { # pi_config->each_inbox cb + my ($ibx, $imapd) = @_; + my $ngname = $ibx->{newsgroup} or return; + if (ref $ngname) { + warn 'multiple newsgroups not supported: '. + join(', ', @$ngname). "\n"; + } elsif ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! || + $ngname =~ /\.[0-9]+-[0-9]+\z/) { + warn "mailbox name invalid: `$ngname'\n"; + } + + my $mm = $ibx->mm or return; + $ibx->{mm} = undef; + defined($ibx->{uidvalidity} = $mm->created_at) or return; + $imapd->{tmp_groups}->{$ngname} = $ibx; - if (my $idler = $self->{idler}) { + # preload to avoid fragmentation: + $ibx->description; + $ibx->base_url; + # my $max = $mm->max // 0; + # my $uid_min = UID_BLOCK * int($max/UID_BLOCK) + 1; +} + +sub imapd_refresh_finalize { + my ($imapd, $pi_config) = @_; + $imapd->{groups} = delete $imapd->{tmp_groups}; + $imapd->{grouplist} = [ values %{$imapd->{groups}} ]; + refresh_inboxlist($imapd); + $imapd->{pi_config} = $pi_config; + if (my $idler = $imapd->{idler}) { $idler->refresh($pi_config); } } +sub imapd_refresh_step { # pi_config->iterate_start cb + my ($pi_config, $section, $imapd) = @_; + if (defined($section)) { + return if $section !~ m!\Apublicinbox\.([^/]+)\z!; + my $ibx = $pi_config->lookup_name($1) or return; + imapd_refresh_ibx($ibx, $imapd); + } else { # "EOF" + imapd_refresh_finalize($imapd, $pi_config); + } +} + +sub refresh_groups { + my ($self, $sig) = @_; + my $pi_config = PublicInbox::Config->new; + $self->{tmp_groups} = {}; + if (0 && $sig) { # SIGHUP + $pi_config->iterate_start(\&imapd_refresh_step, $self); + PublicInbox::DS::requeue($pi_config); # call event_step + } else { # initial start + $pi_config->each_inbox(\&imapd_refresh_ibx, $self); + imapd_refresh_finalize($self, $pi_config); + } +} + sub idler_start { $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_config}); } diff --git a/script/public-inbox-imapd b/script/public-inbox-imapd index 63f865f53f0..60f2e6d8a36 100644 --- a/script/public-inbox-imapd +++ b/script/public-inbox-imapd @@ -9,6 +9,6 @@ use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP use PublicInbox::IMAPD; my $imapd = PublicInbox::IMAPD->new; PublicInbox::Daemon::run('0.0.0.0:143', - sub { $imapd->refresh_groups }, # refresh + sub { $imapd->refresh_groups(@_) }, # refresh sub ($$$) { PublicInbox::IMAP->new($_[0], $imapd) }, # post_accept $imapd);