From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 5B02D1FBC7 for ; Wed, 10 Jun 2020 07:08:34 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 80/82] imap: remove non-UID SEARCH for now Date: Wed, 10 Jun 2020 07:05:17 +0000 Message-Id: <20200610070519.18252-81-e@yhbt.net> In-Reply-To: <20200610070519.18252-1-e@yhbt.net> References: <20200610070519.18252-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Supporting MSNs in long-lived connections beyond the lifetime of a single request/response cycle is not scalable to a C10K scenario. It's probably not needed, since most clients seem to use UIDs. A somewhat efficient implementation I can come up uses pack("S*" ...) (AKA "uint16_t mapping[50000]") has an overhead of 100K per-client socket on a mailbox with 50K messages. The 100K is a contiguous scalar, so it could be swapped out for idle clients on most architectures if THP is disabled. An alternative could be to use a tempfile as an allocator partitioned into 100K chunks (or SQLite); but I'll only do that if somebody presents a compelling case to support MSN SEARCH. --- lib/PublicInbox/IMAP.pm | 33 +++++++++------------------------ t/imapd.t | 37 ------------------------------------- 2 files changed, 9 insertions(+), 61 deletions(-) diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index 1ba3a3ff1bb..7efe5387646 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -986,21 +986,14 @@ sub parse_date ($) { # 02-Oct-1993 timegm(0, 0, 0, $dd, $mm, $yyyy); } -sub msn_convert ($$) { - my ($self, $uids) = @_; - my $adj = $self->{uid_base}; - $_ -= $adj for @$uids; -} - sub search_uid_range { # long_response - my ($self, $tag, $sql, $range_info, $want_msn) = @_; + my ($self, $tag, $sql, $range_info) = @_; my $uids = []; if (defined(my $err = refill_uids($self, $uids, $range_info, $sql))) { $err ||= 'OK Search done'; $self->write("\r\n$tag $err\r\n"); return; } - msn_convert($self, $uids) if $want_msn; $self->msg_more(join(' ', '', @$uids)); 1; # more } @@ -1162,46 +1155,38 @@ sub refill_xap ($$$$) { } sub search_xap_range { # long_response - my ($self, $tag, $q, $range_info, $want_msn) = @_; + my ($self, $tag, $q, $range_info) = @_; my $uids = []; if (defined(my $err = refill_xap($self, $uids, $range_info, $q))) { $err ||= 'OK Search done'; $self->write("\r\n$tag $err\r\n"); return; } - msn_convert($self, $uids) if $want_msn; $self->msg_more(join(' ', '', @$uids)); 1; # more } -sub search_common { - my ($self, $tag, $rest, $want_msn) = @_; +sub cmd_uid_search ($$$;) { + my ($self, $tag) = splice(@_, 0, 2); my $ibx = $self->{ibx} or return "$tag BAD No mailbox selected\r\n"; - my $q = parse_query($self, $rest); + my $q = parse_query($self, \@_); return "$tag $q\r\n" if !ref($q); my ($sql, $range_info) = delete @$q{qw(sql range_info)}; if (!scalar(keys %$q)) { # overview.sqlite3 $self->msg_more('* SEARCH'); long_response($self, \&search_uid_range, - $tag, $sql, $range_info, $want_msn); + $tag, $sql, $range_info); } elsif ($q = $q->{xap}) { $self->msg_more('* SEARCH'); long_response($self, \&search_xap_range, - $tag, $q, $range_info, $want_msn); + $tag, $q, $range_info); } else { "$tag BAD Error\r\n"; } } -sub cmd_uid_search ($$$;) { - my ($self, $tag) = splice(@_, 0, 2); - search_common($self, $tag, \@_); -} - -sub cmd_search ($$$;) { - my ($self, $tag) = splice(@_, 0, 2); - search_common($self, $tag, \@_, 1); -} +# note: MSN SEARCH is NOT supported. Do any widely-used MUAs +# rely on MSNs from SEARCH results? Let us know at meta@public-inbox.org sub args_ok ($$) { # duplicated from PublicInbox::NNTP my ($cb, $argc) = @_; diff --git a/t/imapd.t b/t/imapd.t index c7e0baaa10b..f5ca8b7eb03 100644 --- a/t/imapd.t +++ b/t/imapd.t @@ -360,43 +360,6 @@ is(scalar keys %$ret, 3, 'got all 3 messages with comma-separated sequence'); $ret = $mic->fetch_hash('1:*', 'RFC822') or BAIL_OUT "FETCH $@"; is(scalar keys %$ret, 3, 'got all 3 messages'); -SKIP: { - # do any clients use non-UID IMAP SEARCH? - skip 'Xapian missing', 2 if $level eq 'basic'; - my $x = $mic->search('all'); - is_deeply($x, [1, 2, 3], 'MSN SEARCH works before rm'); - $x = $mic->search(qw(header subject embedded)); - is_deeply($x, [2], 'MSN SEARCH on Subject works before rm'); -} - -{ - my $rdr = { 0 => \($ret->{1}->{RFC822}) }; - my $env = { HOME => $ENV{HOME} }; - my @cmd = qw(-learn rm --all); - run_script(\@cmd, $env, $rdr) or BAIL_OUT('-learn rm'); -} - -SKIP: { - # do any clients use non-UID IMAP SEARCH? We only ensure - # MSN "SEARCH" can return a result which can be retrieved - # via MSN "FETCH" - skip 'Xapian missing', 3 if $level eq 'basic'; - my $x = $mic->search(qw(header subject embedded)); - is(scalar(@$x), 1, 'MSN SEARCH on Subject works after rm'); - $x = $mic->message_string($x->[0]); - is($x, $ret->{2}->{RFC822}, 'message 2 unchanged'); -} - -my $r2 = $mic->fetch_hash('1:*', 'BODY.PEEK[]') or BAIL_OUT "FETCH $@"; -is(scalar keys %$r2, 2, 'did not get all 3 messages'); -is($r2->{1}->{'BODY[]'}, $ret->{2}->{RFC822}, 'message 2 unchanged'); -is($r2->{2}->{'BODY[]'}, $ret->{3}->{RFC822}, 'message 3 unchanged'); -$r2 = $mic->fetch_hash(2, 'BODY.PEEK[HEADER.FIELDS (message-id)]') - or BAIL_OUT "FETCH $@"; -is($r2->{2}->{'BODY[HEADER.FIELDS (MESSAGE-ID)]'}, - 'Message-ID: <20200418222508.GA13918@dcvr>'."\r\n\r\n", - 'BODY.PEEK[HEADER.FIELDS ...] drops .PEEK'); - { my @new_list = $mic->list; # tag differs in [-1]