From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id E83A11F61A for ; Wed, 3 Aug 2022 20:03:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1659557037; bh=yb3eNAv6JH50wUSqY8dG/fOTz29h8T6K181Vwe3nbpM=; h=From:To:Subject:Date:In-Reply-To:References:From; b=5YnBix/l2+E7RyMZo6/cHFN58Nzd2ILrMLiMQtzgYVKpS0+NZtNs1PRUC1n4GUWC2 67Nya7MgUIMrtstZYMv4C6FpBFt7PjKzDK5/771W5yc0Q0V8HAzL4CdhhIeaIUaKqK +d1lgdn77wJkm6ZdgCTL5O08WRC1FScnVda7A33Q= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 4/4] imapd: use nntpd_cache to speed up startup/reload time Date: Wed, 3 Aug 2022 20:03:57 +0000 Message-Id: <20220803200357.1322670-5-e@80x24.org> In-Reply-To: <20220803200357.1322670-1-e@80x24.org> References: <20220803200357.1322670-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: ConfigIter was still too slow despite being fair. The addition of ART_MIN in ALL->misc means it can be used as a startup/reload cache for -imapd, too. This results in a ~3x faster startup for -imapd with 50K inboxes. --- lib/PublicInbox/IMAP.pm | 17 +++---- lib/PublicInbox/IMAPD.pm | 100 +++++++++++++-------------------------- 2 files changed, 41 insertions(+), 76 deletions(-) diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index 19ead70c..9955984b 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -350,12 +350,12 @@ sub idle_done ($$) { "$idle_tag OK Idle done\r\n"; } -sub ensure_slices_exist ($$$) { - my ($imapd, $ibx, $max) = @_; - defined(my $mb_top = $ibx->{newsgroup}) or return; +sub ensure_slices_exist ($$) { + my ($imapd, $ibx) = @_; + my $mb_top = $ibx->{newsgroup} // return; my $mailboxes = $imapd->{mailboxes}; my @created; - for (my $i = int($max/UID_SLICE); $i >= 0; --$i) { + for (my $i = int($ibx->art_max/UID_SLICE); $i >= 0; --$i) { my $sub_mailbox = "$mb_top.$i"; last if exists $mailboxes->{$sub_mailbox}; $mailboxes->{$sub_mailbox} = $ibx; @@ -387,7 +387,8 @@ sub inbox_lookup ($$;$) { my $uid_end = $uid_base + UID_SLICE; $exists = $over->imap_exists($uid_base, $uid_end); } - ensure_slices_exist($self->{imapd}, $ibx, $over->max); + delete $ibx->{-art_max}; + ensure_slices_exist($self->{imapd}, $ibx); } else { if ($examine) { $self->{uid_base} = $uid_base; @@ -396,9 +397,9 @@ sub inbox_lookup ($$;$) { } # if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0", # check for new UID ranges (e.g. "INBOX.foo.bar.1") - if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) { - ensure_slices_exist($self->{imapd}, $z, - $z->over(1)->max); + if (my $ibx = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) { + delete $ibx->{-art_max}; + ensure_slices_exist($self->{imapd}, $ibx); } } ($ibx, $exists, $uidmax + 1, $uid_base); diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm index 6038fd88..5368ff04 100644 --- a/lib/PublicInbox/IMAPD.pm +++ b/lib/PublicInbox/IMAPD.pm @@ -6,7 +6,6 @@ package PublicInbox::IMAPD; use strict; use v5.10.1; use PublicInbox::Config; -use PublicInbox::ConfigIter; use PublicInbox::InboxIdle; use PublicInbox::IMAP; use PublicInbox::DummyInbox; @@ -15,7 +14,7 @@ my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox'; sub new { my ($class) = @_; bless { - mailboxes => {}, + # mailboxes => {}, err => \*STDERR, out => \*STDOUT, # ssl_ctx_opt => { SSL_cert_file => ..., SSL_key_file => ... } @@ -25,53 +24,45 @@ sub new { } sub imapd_refresh_ibx { # pi_cfg->each_inbox cb - my ($ibx, $imapd) = @_; - my $ngname = $ibx->{newsgroup} or return; + my ($ibx, $imapd, $cache, $dummies) = @_; + my $ngname = $ibx->{newsgroup} // return; # We require lower-case since IMAP mailbox names are # case-insensitive (but -nntpd matches INN in being - # case-sensitive + # case-sensitive) if ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! || # don't confuse with 50K slices $ngname =~ /\.[0-9]+\z/) { warn "mailbox name invalid: newsgroup=`$ngname'\n"; return; } - $ibx->over or return; - $ibx->{over} = undef; - - # RFC 3501 2.3.1.1 - "A good UIDVALIDITY value to use in - # this case is a 32-bit representation of the creation - # date/time of the mailbox" - eval { $ibx->uidvalidity }; - my $mm = delete($ibx->{mm}) or return; - defined($ibx->{uidvalidity}) or return; - PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx, $mm->max); - - # preload to avoid fragmentation: - $ibx->description; - $ibx->base_url; - - # ensure dummies are selectable - my $dummies = $imapd->{dummies}; - do { - $dummies->{$ngname} = $dummy; - } while ($ngname =~ s/\.[^\.]+\z//); + my $ce = $cache->{$ngname}; + %$ibx = (%$ibx, %$ce) if $ce; + # only valid if msgmap and over works: + if (defined($ibx->uidvalidity)) { + # fill ->{mailboxes}: + PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx); + # preload to avoid fragmentation: + $ibx->description; + $ibx->base_url; + # ensure dummies are selectable: + do { + $dummies->{$ngname} = $dummy; + } while ($ngname =~ s/\.[^\.]+\z//); + } + delete @$ibx{qw(mm over)}; } -sub imapd_refresh_finalize { - my ($imapd, $pi_cfg) = @_; - my $mailboxes; - if (my $next = delete $imapd->{imapd_next}) { - $imapd->{mailboxes} = delete $next->{mailboxes}; - $mailboxes = delete $next->{dummies}; - } else { - $mailboxes = delete $imapd->{dummies}; - } - %$mailboxes = (%$mailboxes, %{$imapd->{mailboxes}}); - $imapd->{mailboxes} = $mailboxes; - $imapd->{mailboxlist} = [ - map { $_->[2] } +sub refresh_groups { + my ($self, $sig) = @_; + my $pi_cfg = PublicInbox::Config->new; + my $mailboxes = $self->{mailboxes} = {}; + my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {}; + my $dummies = {}; + $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self, $cache, $dummies); + %$dummies = (%$dummies, %$mailboxes); + $mailboxes = $self->{mailboxes} = $dummies; + @{$self->{mailboxlist}} = map { $_->[2] } sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] } map { my $u = $_; # capitalize "INBOX" for user-familiarity @@ -85,40 +76,13 @@ sub imapd_refresh_finalize { [ $1, $2 + 0, qq[* LIST (\\HasNoChildren) "." $u\r\n] ] } - } keys %$mailboxes - ]; - $imapd->{pi_cfg} = $pi_cfg; - if (my $idler = $imapd->{idler}) { + } keys %$mailboxes; + $self->{pi_cfg} = $pi_cfg; + if (my $idler = $self->{idler}) { $idler->refresh($pi_cfg); } } -sub imapd_refresh_step { # PublicInbox::ConfigIter cb - my ($pi_cfg, $section, $imapd) = @_; - if (defined($section)) { - return if $section !~ m!\Apublicinbox\.([^/]+)\z!; - my $ibx = $pi_cfg->lookup_name($1) or return; - imapd_refresh_ibx($ibx, $imapd->{imapd_next}); - } else { # undef == "EOF" - imapd_refresh_finalize($imapd, $pi_cfg); - } -} - -sub refresh_groups { - my ($self, $sig) = @_; - my $pi_cfg = PublicInbox::Config->new; - if ($sig) { # SIGHUP is handled through the event loop - $self->{imapd_next} = { dummies => {}, mailboxes => {} }; - my $iter = PublicInbox::ConfigIter->new($pi_cfg, - \&imapd_refresh_step, $self); - $iter->event_step; - } else { # initial start is synchronous - $self->{dummies} = {}; - $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self); - imapd_refresh_finalize($self, $pi_cfg); - } -} - sub idler_start { $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_cfg}); }