From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 6B9E21FA0C for ; Wed, 10 Jun 2020 07:08:33 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 75/82] imap: STATUS/EXAMINE: rely on SQLite overview Date: Wed, 10 Jun 2020 07:05:12 +0000 Message-Id: <20200610070519.18252-76-e@yhbt.net> In-Reply-To: <20200610070519.18252-1-e@yhbt.net> References: <20200610070519.18252-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We can get exact values for EXISTS, UIDNEXT using SQLite rather than calculating off $ibx->mm->max ourselves. Furthermore, $ibx->mm is less useful than $ibx->over for IMAP (and for our read-only daemons in general) so do not depend on $ibx->mm outside of startup/reload to save FDs and reduce kernel page cache footprint. --- lib/PublicInbox/DummyInbox.pm | 11 ++++++----- lib/PublicInbox/IMAP.pm | 30 ++++++++++++++++++------------ lib/PublicInbox/IMAPD.pm | 2 +- lib/PublicInbox/Over.pm | 30 ++++++++++++++++++++++++++++++ t/over.t | 3 +++ 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/lib/PublicInbox/DummyInbox.pm b/lib/PublicInbox/DummyInbox.pm index b6c48db1a0e..c3f4f5c6674 100644 --- a/lib/PublicInbox/DummyInbox.pm +++ b/lib/PublicInbox/DummyInbox.pm @@ -9,13 +9,14 @@ use strict; sub created_at { 0 } # Msgmap::created_at sub mm { shift } -sub max { undef } # Msgmap::max -sub msg_range { [] } # Msgmap::msg_range +sub uid_range { [] } # Over::uid_range +sub subscribe_unlock { undef }; no warnings 'once'; -*uid_range = *query_xover = \&msg_range; +*max = \&created_at; +*query_xover = \&uid_range; *over = \&mm; -*subscribe_unlock = *unsubscribe_unlock = - *get_art = *description = *base_url = \&max; +*unsubscribe_unlock = + *get_art = *description = *base_url = \&subscribe_unlock; 1; diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index d0683530988..41c8066446c 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -183,7 +183,7 @@ sub cmd_noop ($$) { "$_[1] OK Noop done\r\n" } # called by PublicInbox::InboxIdle sub on_inbox_unlock { my ($self, $ibx) = @_; - my $new = $ibx->mm->max; + my $new = $ibx->over->max; my $uid_base = $self->{uid_base} // 0; my $uid_end = $uid_base + UID_BLOCK; defined(my $old = $self->{-idle_max}) or die 'BUG: -idle_max unset'; @@ -223,7 +223,7 @@ sub cmd_idle ($$) { # IDLE seems allowed by dovecot w/o a mailbox selected *shrug* my $ibx = $self->{ibx} or return "$tag BAD no mailbox selected\r\n"; $self->{-idle_tag} = $tag; - my $max = $ibx->mm->max // 0; + my $max = $ibx->over->max; my $uid_end = $self->{uid_base} + UID_BLOCK; my $sock = $self->{sock} or return; my $fd = fileno($sock); @@ -283,14 +283,20 @@ sub inbox_lookup ($$) { my $mb_top = $1; $uid_base = $2 * UID_BLOCK; $ibx = $self->{imapd}->{mailboxes}->{lc $mailbox} or return; - $exists = $ibx->mm->max // 0; - ensure_ranges_exist($self->{imapd}, $ibx, $exists); - my $uid_end = $uid_base + UID_BLOCK; - $exists = $uid_end if $exists > $uid_end; - $uidnext = $exists + 1; - $exists -= $uid_base; + my $max; + ($exists, $uidnext, $max) = $ibx->over->imap_status($uid_base, + $uid_base + UID_BLOCK); + ensure_ranges_exist($self->{imapd}, $ibx, $max); } else { # check for dummy inboxes - $ibx = $self->{imapd}->{mailboxes}->{lc $mailbox} or return; + $mailbox = lc $mailbox; + $ibx = $self->{imapd}->{mailboxes}->{$mailbox} or return; + + # if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0", + # check for new UID ranges (e.g. "INBOX.foo.bar.1") + if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) { + ensure_ranges_exist($self->{imapd}, $z, $z->over->max); + } + $uid_base = $exists = 0; $uidnext = 1; } @@ -590,7 +596,7 @@ sub range_step ($$) { ($beg, $end) = ($1 + 0, $2 + 0); } elsif ($range =~ /\A([0-9]+):\*\z/) { $beg = $1 + 0; - $end = $self->{ibx}->mm->max // 0; + $end = $self->{ibx}->over->max; my $uid_end = $self->{uid_base} + UID_BLOCK; $end = $uid_end if $end > $uid_end; $beg = $end if $beg > $end; @@ -1093,14 +1099,14 @@ sub cmd_uid_search ($$$;) { if (!scalar(keys %$q)) { $self->msg_more('* SEARCH'); my $beg = 1; - my $end = $ibx->mm->max // 0; + my $end = $ibx->over->max; uid_clamp($self, \$beg, \$end); long_response($self, \&uid_search_uid_range, $tag, \$beg, $end, $sql); } elsif (my $uid = $q->{uid}) { if ($uid =~ /\A([0-9]+):([0-9]+|\*)\z/s) { my ($beg, $end) = ($1, $2); - $end = $ibx->mm->max if $end eq '*'; + $end = $ibx->over->max if $end eq '*'; uid_clamp($self, \$beg, \$end); $self->msg_more('* SEARCH'); long_response($self, \&uid_search_uid_range, diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm index 186ec7b0062..e4dc90e8d0e 100644 --- a/lib/PublicInbox/IMAPD.pm +++ b/lib/PublicInbox/IMAPD.pm @@ -44,7 +44,7 @@ sub imapd_refresh_ibx { # pi_config->each_inbox cb # this case is a 32-bit representation of the creation # date/time of the mailbox" defined($ibx->{uidvalidity} = $mm->created_at) or return; - PublicInbox::IMAP::ensure_ranges_exist($imapd, $ibx, $mm->max // 1); + PublicInbox::IMAP::ensure_ranges_exist($imapd, $ibx, $mm->max // 0); # preload to avoid fragmentation: $ibx->description; diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm index 1faeff418f4..e0f20ea6d3d 100644 --- a/lib/PublicInbox/Over.pm +++ b/lib/PublicInbox/Over.pm @@ -229,4 +229,34 @@ sub uid_range { $dbh->selectcol_arrayref($q, undef, $beg, $end); } +sub max { + my ($self) = @_; + my $sth = $self->connect->prepare_cached(<<'', undef, 1); +SELECT MAX(num) FROM over WHERE num > 0 + + $sth->execute; + $sth->fetchrow_array // 0; +} + +sub imap_status { + my ($self, $uid_base, $uid_end) = @_; + my $dbh = $self->connect; + my $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT COUNT(num) FROM over WHERE num > ? AND num <= ? + + $sth->execute($uid_base, $uid_end); + my $exists = $sth->fetchrow_array; + + $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT MAX(num) + 1 FROM over WHERE num <= ? + + $sth->execute($uid_end); + my $uidnext = $sth->fetchrow_array; + + $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT MAX(num) FROM over WHERE num > 0 + + ($exists, $uidnext, $sth->fetchrow_array // 0); +} + 1; diff --git a/t/over.t b/t/over.t index 5586aa310e6..734fdaa3604 100644 --- a/t/over.t +++ b/t/over.t @@ -10,6 +10,8 @@ use_ok 'PublicInbox::OverIdx'; my ($tmpdir, $for_destroy) = tmpdir(); my $over = PublicInbox::OverIdx->new("$tmpdir/over.sqlite3"); $over->connect; +is($over->max, 0, 'max is zero on new DB (scalar context)'); +is_deeply([$over->max], [0], 'max is zero on new DB (list context)'); my $x = $over->next_tid; is(int($x), $x, 'integer tid'); my $y = $over->next_tid; @@ -58,6 +60,7 @@ foreach my $mid (qw(a b)) { my $msgs = [ map { $_->{num} } @{$over->get_thread($mid)} ]; is_deeply([98, 99], $msgs, "linked messages by Message-ID: <$mid>"); } +isnt($over->max, 0, 'max is non-zero'); $over->rollback_lazy;