From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 377441FC06 for ; Wed, 10 Jun 2020 07:07:33 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 74/82] imap: FETCH: try to make fake MSNs sequentially Date: Wed, 10 Jun 2020 07:05:11 +0000 Message-Id: <20200610070519.18252-75-e@yhbt.net> In-Reply-To: <20200610070519.18252-1-e@yhbt.net> References: <20200610070519.18252-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This appears to significantly improve header caching behavior with mutt. With the current public-inbox.org/git mirror(*), mutt will only re-FETCH the last ~300 or so messages in the final "inbox.comp.version-control.git.7" mailbox, instead of ~49,000 messages every time. It's not perfect, but a 500ms query is better than a >10s query and mutt itself spends as much time loading its header cache. (*) there are many gaps in NNTP article numbers (UIDs) due to spam removal from public-inbox-learn. --- lib/PublicInbox/IMAP.pm | 22 +++++++++++----------- t/imapd.t | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index fe3c7d1a85b..d0683530988 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -475,12 +475,12 @@ sub requeue_once ($) { $self->requeue if $new_size == 1; } -# my ($uid_base, $UID) = @_; -sub fetch_msn_uid ($$) { '* '.($_[1] - $_[0]).' FETCH (UID '.$_[1] } +# my ($msn, $UID) = @_; +sub fetch_msn_uid ($$) { '* '.(${$_[0]}++).' FETCH (UID '.$_[1] } sub fetch_run_ops { - my ($self, $uid_base, $smsg, $bref, $ops, $partial) = @_; - $self->msg_more(fetch_msn_uid($uid_base, $smsg->{num})); + my ($self, $msn, $smsg, $bref, $ops, $partial) = @_; + $self->msg_more(fetch_msn_uid($msn, $smsg->{num})); my ($eml, $k); for (my $i = 0; $i < @$ops;) { $k = $ops->[$i++]; @@ -492,7 +492,7 @@ sub fetch_run_ops { sub fetch_blob_cb { # called by git->cat_async via git_async_cat my ($bref, $oid, $type, $size, $fetch_arg) = @_; - my ($self, undef, $msgs, undef, $ops, $partial) = @$fetch_arg; + my ($self, undef, $msgs, $range_info, $ops, $partial) = @$fetch_arg; my $smsg = shift @$msgs or die 'BUG: no smsg'; if (!defined($oid)) { # it's possible to have TOCTOU if an admin runs @@ -501,7 +501,7 @@ sub fetch_blob_cb { # called by git->cat_async via git_async_cat } else { $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; } - fetch_run_ops($self, $self->{uid_base}, $smsg, $bref, $ops, $partial); + fetch_run_ops($self, $range_info->[3], $smsg, $bref, $ops, $partial); requeue_once($self); } @@ -601,7 +601,8 @@ sub range_step ($$) { return 'BAD fetch range'; } uid_clamp($self, \$beg, \$end) if defined($range); - [ $beg, $end, $$range_csv ]; + my $msn = $beg - $self->{uid_base}; + [ $beg, $end, $$range_csv, \$msn ]; } sub refill_range ($$$) { @@ -638,8 +639,7 @@ sub fetch_smsg { # long_response return; } } - my $uid_base = $self->{uid_base}; - fetch_run_ops($self, $uid_base, $_, undef, $ops) for @$msgs; + fetch_run_ops($self, $range_info->[3], $_, undef, $ops) for @$msgs; @$msgs = (); 1; # more } @@ -665,10 +665,10 @@ sub fetch_uid { # long_response } # continue looping } - my $uid_base = $self->{uid_base}; my ($i, $k); + my $msn = $range_info->[3]; for (@$uids) { - $self->msg_more(fetch_msn_uid($uid_base, $_)); + $self->msg_more(fetch_msn_uid($msn, $_)); for ($i = 0; $i < @$ops;) { $k = $ops->[$i++]; $ops->[$i++]->($self, $k); diff --git a/t/imapd.t b/t/imapd.t index c691e1a96a7..fdab074a249 100644 --- a/t/imapd.t +++ b/t/imapd.t @@ -349,8 +349,8 @@ is(scalar keys %$ret, 3, 'got all 3 messages'); } my $r2 = $mic->fetch_hash('1:*', 'BODY.PEEK[]') or BAIL_OUT "FETCH $@"; is(scalar keys %$r2, 2, 'did not get all 3 messages'); -is($r2->{2}->{'BODY[]'}, $ret->{2}->{RFC822}, 'message 2 unchanged'); -is($r2->{3}->{'BODY[]'}, $ret->{3}->{RFC822}, 'message 3 unchanged'); +is($r2->{1}->{'BODY[]'}, $ret->{2}->{RFC822}, 'message 2 unchanged'); +is($r2->{2}->{'BODY[]'}, $ret->{3}->{RFC822}, 'message 3 unchanged'); $r2 = $mic->fetch_hash(2, 'BODY.PEEK[HEADER.FIELDS (message-id)]') or BAIL_OUT "FETCH $@"; is($r2->{2}->{'BODY[HEADER.FIELDS (MESSAGE-ID)]'},