From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 224991FBDF for ; Wed, 10 Jun 2020 07:06:26 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 37/82] xt: add imapd-validate and imapd-mbsync-oimap Date: Wed, 10 Jun 2020 07:04:34 +0000 Message-Id: <20200610070519.18252-38-e@yhbt.net> In-Reply-To: <20200610070519.18252-1-e@yhbt.net> References: <20200610070519.18252-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: imapd-validate is a beefed up version of our nntpd-validate test which hammers the server with parallel connections over regular IMAP, IMAPS, IMAP+STARTTLS; and COMPRESS=DEFLATE variants of each of those. It uses $START_UID:$END_UID fetch ranges to reduce requests and slurp many responses at once to saturate "git cat-file --batch" processes. mbsync(1) also uses pipelining extensively (but IMHO unnecessarily), so it was able to shake out some bugs in the async git code. Finally, we remove xt/cmp-imapd-compress.t since it's redundant now that we have PublicInbox::IMAPClient to work around bugs in Mail::IMAPClient. --- MANIFEST | 3 +- xt/cmp-imapd-compress.t | 83 ------------------- xt/imapd-mbsync-oimap.t | 129 +++++++++++++++++++++++++++++ xt/imapd-validate.t | 175 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 306 insertions(+), 84 deletions(-) delete mode 100644 xt/cmp-imapd-compress.t create mode 100644 xt/imapd-mbsync-oimap.t create mode 100644 xt/imapd-validate.t diff --git a/MANIFEST b/MANIFEST index 6744a519efc..957228250a7 100644 --- a/MANIFEST +++ b/MANIFEST @@ -347,12 +347,13 @@ t/www_listing.t t/www_static.t t/x-unknown-alpine.eml t/xcpdb-reshard.t -xt/cmp-imapd-compress.t xt/cmp-msgstr.t xt/cmp-msgview.t xt/eml_check_limits.t xt/git-http-backend.t xt/git_async_cmp.t +xt/imapd-mbsync-oimap.t +xt/imapd-validate.t xt/mem-msgview.t xt/msgtime_cmp.t xt/nntpd-validate.t diff --git a/xt/cmp-imapd-compress.t b/xt/cmp-imapd-compress.t deleted file mode 100644 index b12cf74e2d5..00000000000 --- a/xt/cmp-imapd-compress.t +++ /dev/null @@ -1,83 +0,0 @@ -#!perl -w -# Copyright (C) 2020 all contributors -# License: AGPL-3.0+ -use strict; -use Test::More; -use PublicInbox::TestCommon; -require_mods('Data::Dumper'); -Data::Dumper->import('Dumper'); -my $inboxdir = $ENV{GIANT_INBOX_DIR}; -(defined($inboxdir) && -d $inboxdir) or - plan skip_all => "GIANT_INBOX_DIR not defined for $0"; -plan skip_all => "bad characters in $inboxdir" if $inboxdir =~ m![^\w\.\-/]!; -my ($tmpdir, $for_destroy) = tmpdir(); -my $cfg = "$tmpdir/cfg"; -my $mailbox = 'inbox.test'; -{ - open my $fh, '>', $cfg or BAIL_OUT "open: $!"; - print $fh < $cfg }; -my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT "-imapd: $?"; -my ($host, $port) = ($sock->sockhost, $sock->sockport); -my $c = tcp_connect($sock); -like(readline($c), qr/CAPABILITY /, 'got greeting'); -undef $c; - -SKIP: { - require_mods('Mail::IMAPClient', 3); - unless ($ENV{RT_132720_FIXED}) { - my $bug = 'https://rt.cpan.org/Ticket/Display.html?id=132720'; - skip "<$bug>, RT_132720_FIXED not defined", 3; - } - my %opt = (Server => $host, Port => $port, - User => 'u', Password => 'p', Clear => 1); - my $uc = Mail::IMAPClient->new(%opt); - my $c = Mail::IMAPClient->new(%opt); - ok($c->compress, 'enabled compression'); - ok $c->examine($mailbox), 'compressed EXAMINE-ed'; - ok $uc->examine($mailbox), 'uncompress EXAMINE-ed'; - my $range = $uc->search('all'); - for my $uid (@$range) { - my $A = $uc->fetch_hash($uid, 'BODY[]'); - my $B = $c->fetch_hash($uid, 'BODY[]'); - if (!is_deeply($A, $B, "$uid identical")) { - diag Dumper([$A, $B]); - diag Dumper([$uc, $c]); - last; - } - } - $uc->logout; - $c->logout; -} - -SKIP: { - require_mods('Mail::IMAPTalk', 3); - my %opt = (Server => $host, Port => $port, UseSSL => 0, - Username => 'u', Password => 'p', Uid => 1); - my $uc = Mail::IMAPTalk->new(%opt) or BAIL_OUT 'IMAPTalk->new'; - my $c = Mail::IMAPTalk->new(%opt, UseCompress => 1) or - BAIL_OUT 'IMAPTalk->new(UseCompress => 1)'; - ok $c->examine($mailbox), 'compressed EXAMINE-ed'; - ok $uc->examine($mailbox), 'uncompress EXAMINE-ed'; - my $range = $uc->search('all'); - for my $uid (@$range) { - my $A = $uc->fetch($uid, 'rfc822'); - my $B = $c->fetch($uid, 'rfc822'); - if (!is_deeply($A, $B, "$uid identical")) { - diag Dumper([$A, $B]); - diag Dumper([$uc, $c]); - last; - } - } -} -done_testing; diff --git a/xt/imapd-mbsync-oimap.t b/xt/imapd-mbsync-oimap.t new file mode 100644 index 00000000000..d2237a24bef --- /dev/null +++ b/xt/imapd-mbsync-oimap.t @@ -0,0 +1,129 @@ +#!perl -w +# Copyright (C) 2020 all contributors +# License: AGPL-3.0+ +# ensure mbsync and offlineimap compatibility +use strict; +use Test::More; +use File::Path qw(mkpath); +use PublicInbox::TestCommon; +use PublicInbox::Spawn qw(which spawn); +my $inboxdir = $ENV{GIANT_INBOX_DIR}; +(defined($inboxdir) && -d $inboxdir) or + plan skip_all => "GIANT_INBOX_DIR not defined for $0"; +plan skip_all => "bad characters in $inboxdir" if $inboxdir =~ m![^\w\.\-/]!; +my ($tmpdir, $for_destroy) = tmpdir(); +my $cfg = "$tmpdir/cfg"; +my $mailbox = 'inbox.test'; +{ + open my $fh, '>', $cfg or BAIL_OUT "open: $!"; + print $fh < $cfg }; +my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT "-imapd: $?"; +{ + my $c = tcp_connect($sock); + like(readline($c), qr/CAPABILITY /, 'got greeting'); +} +my ($host, $port) = ($sock->sockhost, $sock->sockport); +my %pids; + +SKIP: { + mkpath([map { "$tmpdir/oimapdir/$_" } qw(cur new tmp)]); + my $oimap = which('offlineimap') or skip 'no offlineimap(1)', 1; + open my $fh, '>', "$tmpdir/.offlineimaprc" or BAIL_OUT "open: $!"; + print $fh < +usecompression = no +EOF + close $fh or BAIL_OUT "close: $!"; + my $cmd = [ $oimap, qw(-o -q -u quiet) ]; + my $pid = spawn($cmd, { HOME => $tmpdir }, { 1 => 2 }); + $pids{$pid} = $cmd; +} + +SKIP: { + mkpath([map { "$tmpdir/mbsyncdir/test/$_" } qw(cur new tmp)]); + my $mbsync = which('mbsync') or skip 'no mbsync(1)', 1; + open my $fh, '>', "$tmpdir/.mbsyncrc" or BAIL_OUT "open: $!"; + print $fh < $tmpdir }, { 1 => 2 }); + $pids{$pid} = $cmd; +} + +while (scalar keys %pids) { + my $pid = waitpid(-1, 0) or next; + my $cmd = delete $pids{$pid} or next; + is($?, 0, join(' ', @$cmd, 'done')); +} + +if (my $sec = $ENV{TEST_PERSIST}) { + diag "sleeping ${sec}s, imap://$host:$port/$mailbox available"; + diag "tmpdir=$tmpdir (Maildirs available)"; + diag "stdout=$out"; + diag "stderr=$err"; + diag "pid=$td->{pid}"; + sleep $sec; +} +$td->kill; +$td->join; +is($?, 0, 'no error on -imapd exit'); +done_testing; diff --git a/xt/imapd-validate.t b/xt/imapd-validate.t new file mode 100644 index 00000000000..f96ec8791b9 --- /dev/null +++ b/xt/imapd-validate.t @@ -0,0 +1,175 @@ +#!perl -w +# Copyright (C) 2020 all contributors +# License: AGPL-3.0+ +# Expensive test to validate compression and TLS. +use strict; +use Test::More; +use Symbol qw(gensym); +use PublicInbox::DS qw(now); +use POSIX qw(_exit); +use PublicInbox::TestCommon; +my $inbox_dir = $ENV{GIANT_INBOX_DIR}; +plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir; +# how many emails to read into memory at once per-process +my $BATCH = $ENV{TEST_BATCH} // 100; +my $REPEAT = $ENV{TEST_REPEAT} // 1; + +require_mods(qw(Mail::IMAPClient)); +my $imap_client = 'Mail::IMAPClient'; +my $can_compress = $imap_client->can('compress'); +if ($can_compress) { # hope this gets fixed upstream, soon + require PublicInbox::IMAPClient; + $imap_client = 'PublicInbox::IMAPClient'; +} + +my $test_tls = $ENV{TEST_SKIP_TLS} ? 0 : eval { require IO::Socket::SSL }; +my ($cert, $key) = qw(certs/server-cert.pem certs/server-key.pem); +if ($test_tls && !-r $key || !-r $cert) { + plan skip_all => + "certs/ missing for $0, run $^X ./certs/create-certs.perl"; +} +my ($tmpdir, $for_destroy) = tmpdir(); +my %OPT = qw(User u Password p); +my (%STARTTLS_OPT, %IMAPS_OPT, $td, $mailbox, $make_local_server); +if (($ENV{IMAP_TEST_URL} // '') =~ m!\Aimap://([^/]+)/(.+)\z!) { + ($OPT{Server}, $mailbox) = ($1, $2); + $OPT{Server} =~ s/:([0-9]+)\z// and $OPT{Port} = $1 + 0; + %STARTTLS_OPT = %OPT; + %IMAPS_OPT = (%OPT, Port => 993) if $OPT{Port} == 143; +} else { + require_mods(qw(DBD::SQLite)); + $make_local_server->(); +} + +my %opts = (imap => \%OPT, 'imap+compress' => { %OPT, Compress => 1 }); +my $uid_max = do { + my $mic = $imap_client->new(%OPT) or BAIL_OUT "new $!"; + $mic->examine($mailbox) or BAIL_OUT "examine: $!"; + my $next = $mic->uidnext($mailbox) or BAIL_OUT "uidnext: $!"; + $next - 1; +}; + +if (scalar keys %STARTTLS_OPT) { + $opts{starttls} = \%STARTTLS_OPT; + $opts{'starttls+compress'} = { %STARTTLS_OPT, Compress => 1 }; +} +if (scalar keys %IMAPS_OPT) { + $opts{imaps} = \%IMAPS_OPT; + $opts{'imaps+compress'} = { %IMAPS_OPT, Compress => 1 }; +} + +my $do_get_all = sub { + my ($desc, $opt) = @_; + local $SIG{__DIE__} = sub { print STDERR $desc, ': ', @_; _exit(1) }; + my $t0 = now(); + my $dig = Digest::SHA->new(1); + my $mic = $imap_client->new(%$opt); + $mic->examine($mailbox) or die "examine: $!"; + my $uid_base = 1; + my $bytes = 0; + my $nr = 0; + until ($uid_base > $uid_max) { + my $end = $uid_base + $BATCH; + my $ret = $mic->fetch_hash("$uid_base:$end", 'BODY[]') or last; + for my $uid ($uid_base..$end) { + $dig->add($uid); + my $h = delete $ret->{$uid} or next; + my $body = delete $h->{'BODY[]'} or + die "no BODY[] for UID=$uid"; + $dig->add($body); + $bytes += length($body); + ++$nr; + } + $uid_base = $end + 1; + } + $mic->logout or die "logout failed: $!"; + my $elapsed = sprintf('%0.3f', now() - $t0); + my $res = $dig->hexdigest; + print STDERR "# $desc $res (${elapsed}s) $bytes bytes, NR=$nr\n"; + $res; +}; + +my (%pids, %res); +for (1..$REPEAT) { + while (my ($desc, $opt) = each %opts) { + pipe(my ($r, $w)) or die; + my $pid = fork; + if ($pid == 0) { + close $r or die; + my $res = $do_get_all->($desc, $opt); + print $w $res or die; + close $w or die; + _exit(0); + } + close $w or die; + $pids{$pid} = [ $desc, $r ]; + } +} + +while (scalar keys %pids) { + my $pid = waitpid(-1, 0) or next; + my $child = delete $pids{$pid} or next; + my ($desc, $rpipe) = @$child; + is($?, 0, "$desc done"); + my $sum = do { local $/; <$rpipe> }; + push @{$res{$sum}}, $desc; +} +is(scalar keys %res, 1, 'all got the same result'); +$td->kill; +$td->join; +is($?, 0, 'no error on -imapd exit'); +done_testing; + +BEGIN { + +$make_local_server = sub { + require PublicInbox::Inbox; + $mailbox = 'inbox.test'; + my $ibx = { inboxdir => $inbox_dir, newsgroup => $mailbox }; + $ibx = PublicInbox::Inbox->new($ibx); + my $pi_config = "$tmpdir/config"; + { + open my $fh, '>', $pi_config or die "open($pi_config): $!"; + print $fh <<"" or die "print $pi_config: $!"; +[publicinbox "test"] + newsgroup = $mailbox + inboxdir = $inbox_dir + address = test\@example.com + + close $fh or die "close($pi_config): $!"; + } + my ($out, $err) = ("$tmpdir/out", "$tmpdir/err"); + for ($out, $err) { + open my $fh, '>', $_ or die "truncate: $!"; + } + my $imap = tcp_server(); + my $rdr = { 3 => $imap }; + $OPT{Server} = $imap->sockhost; + $OPT{Port} = $imap->sockport; + + # not using multiple workers, here, since we want to increase + # the chance of tripping concurrency bugs within PublicInbox/IMAP*.pm + my $cmd = [ '-imapd', "--stdout=$out", "--stderr=$err", '-W0' ]; + push @$cmd, '-limap://'.$imap->sockhost.':'.$imap->sockport; + if ($test_tls) { + my $imaps = tcp_server(); + $rdr->{4} = $imaps; + push @$cmd, '-limaps://'.$imaps->sockhost.':'.$imaps->sockport; + push @$cmd, "--cert=$cert", "--key=$key"; + my $tls_opt = [ + SSL_hostname => 'server.local', + SSL_verifycn_name => 'server.local', + SSL_verify_mode => IO::Socket::SSL::SSL_VERIFY_PEER(), + SSL_ca_file => 'certs/test-ca.pem', + ]; + %STARTTLS_OPT = (%OPT, Starttls => $tls_opt); + %IMAPS_OPT = (%OPT, Ssl => $tls_opt, + Server => $imaps->sockhost, + Port => $imaps->sockport + ); + } + print STDERR "# CMD ". join(' ', @$cmd). "\n"; + my $env = { PI_CONFIG => $pi_config }; + $td = start_script($cmd, $env, $rdr); +}; +} # BEGIN