From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B8FE11FBD7 for ; Wed, 10 Jun 2020 07:05:22 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 19/82] imap: support sequence number FETCH Date: Wed, 10 Jun 2020 07:04:16 +0000 Message-Id: <20200610070519.18252-20-e@yhbt.net> In-Reply-To: <20200610070519.18252-1-e@yhbt.net> References: <20200610070519.18252-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We'll return dummy messages for now when sequence numbers go missing, in case clients can't handle missing messages. --- lib/PublicInbox/IMAP.pm | 87 ++++++++++++++++++++++++++++++++++++----- t/imapd.t | 18 +++++++++ 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index 673e1646216..2aa7ab346c1 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -348,11 +348,34 @@ sub fetch_body ($;$) { join('', @hold); } +sub dummy_message ($$) { + my ($seqno, $ibx) = @_; + my $ret = <{newsgroup}>\r +Subject: dummy message #$seqno\r +\r +You're seeing this message because your IMAP client didn't use UIDs.\r +The message which used to use this sequence number was likely spam\r +and removed by the administrator.\r +EOF + \$ret; +} + sub uid_fetch_cb { # called by git->cat_async my ($bref, $oid, $type, $size, $fetch_m_arg) = @_; my ($self, undef, $ibx, undef, undef, $msgs, $want) = @$fetch_m_arg; my $smsg = shift @$msgs or die 'BUG: no smsg'; - $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; + if (!defined($oid)) { + # it's possible to have TOCTOU if an admin runs + # public-inbox-(edit|purge), just move onto the next message + return unless defined $want->{-seqno}; + $bref = dummy_message($smsg->{num}, $ibx); + } else { + $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; + } $$bref =~ s/(?{ibx} or return "$tag BAD No mailbox selected\r\n"; - if ($want[0] =~ s/\A\(//s) { - $want[-1] =~ s/\)\z//s or return "$tag BAD no rparen\r\n"; + if ($want->[0] =~ s/\A\(//s) { + $want->[-1] =~ s/\)\z//s or return "$tag BAD no rparen\r\n"; } my (%partial, %want); - while (defined(my $att = shift @want)) { + while (defined(my $att = shift @$want)) { $att = uc($att); my $x = $FETCH_ATT{$att}; if ($x) { %want = (%want, %$x); - } elsif (!partial_prepare(\%partial, \@want, $att)) { + } elsif (!partial_prepare(\%partial, $want, $att)) { return "$tag BAD param: $att\r\n"; } } @@ -629,8 +652,54 @@ sub cmd_uid_fetch ($$$;@) { } else { return "$tag BAD fetch range\r\n"; } - long_response($self, \&uid_fetch_m, $tag, $ibx, - \$beg, $end, $msgs, \%want); + [ $tag, $ibx, \$beg, $end, $msgs, \%want ]; +} + +sub cmd_uid_fetch ($$$;@) { + my ($self, $tag, $range, @want) = @_; + my $args = fetch_common($self, $tag, $range, \@want); + ref($args) eq 'ARRAY' ? + long_response($self, \&uid_fetch_m, @$args) : + $args; # error +} + +sub seq_fetch_m { # long_response + my ($self, $tag, $ibx, $beg, $end, $msgs, $want) = @_; + if (!@$msgs) { # refill + @$msgs = @{$ibx->over->query_xover($$beg, $end)}; + if (!@$msgs) { + $self->write(\"$tag OK Fetch done\r\n"); + return; + } + $$beg = $msgs->[-1]->{num} + 1; + } + my $seq = $want->{-seqno}++; + my $cur_num = $msgs->[0]->{num}; + if ($cur_num == $seq) { # as expected + my $git = $ibx->git; + $git->cat_async_begin; # TODO: actually make async + $git->cat_async($msgs->[0]->{blob}, \&uid_fetch_cb, \@_); + $git->cat_async_wait; + } elsif ($cur_num > $seq) { + # send dummy messages until $seq catches up to $cur_num + my $smsg = bless { num => $seq, ts => 0 }, 'PublicInbox::Smsg'; + unshift @$msgs, $smsg; + my $bref = dummy_message($seq, $ibx); + uid_fetch_cb($bref, undef, undef, undef, \@_); + } else { # should not happen + die "BUG: cur_num=$cur_num < seq=$seq"; + } + 1; # more messages on the way +} + +sub cmd_fetch ($$$;@) { + my ($self, $tag, $range, @want) = @_; + my $args = fetch_common($self, $tag, $range, \@want); + ref($args) eq 'ARRAY' ? do { + my $want = $args->[-1]; + $want->{-seqno} = ${$args->[2]}; # $$beg + long_response($self, \&seq_fetch_m, @$args) + } : $args; # error } sub uid_search_all { # long_response diff --git a/t/imapd.t b/t/imapd.t index fcbbdc09d31..fc90948e182 100644 --- a/t/imapd.t +++ b/t/imapd.t @@ -306,6 +306,24 @@ Content-Disposition: attachment; filename="embed2x\.eml"\r EOF }); # each_inbox +# message sequence numbers :< +is($mic->Uid(0), 0, 'disable UID on '.ref($mic)); +ok($mic->reconnect, 'reconnected'); +$ret = $mic->fetch_hash('1:*', 'RFC822') or BAIL_OUT "FETCH $@"; +is(scalar keys %$ret, 3, 'got all 3 messages'); +{ + my $rdr = { 0 => \($ret->{1}->{RFC822}) }; + my $env = { HOME => $ENV{HOME} }; + my @cmd = qw(-learn rm --all); + run_script(\@cmd, $env, $rdr) or BAIL_OUT('-learn rm'); +} +my $r2 = $mic->fetch_hash('1:*', 'RFC822') or BAIL_OUT "FETCH $@"; +is(scalar keys %$r2, 3, 'still got all 3 messages'); +like($r2->{1}->{RFC822}, qr/dummy message #1/, 'got dummy message 1'); +is($r2->{2}->{RFC822}, $ret->{2}->{RFC822}, 'message 2 unchanged'); +is($r2->{3}->{RFC822}, $ret->{3}->{RFC822}, 'message 3 unchanged'); +ok($mic->logout, 'logged out'); + $td->kill; $td->join; is($?, 0, 'no error in exited process');