From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 11/11] imap+nntp: share COMPRESS implementation
Date: Sat, 23 Jul 2022 04:41:55 +0000 [thread overview]
Message-ID: <20220723044155.3733949-12-e@80x24.org> (raw)
In-Reply-To: <20220723044155.3733949-1-e@80x24.org>
Their code was nearly identical to begin with, so save some
memory in -netd and disk space for all of our tarball/distro
users, at least.
And I seem to have used multiple inheritance successfully, here,
maybe...
---
MANIFEST | 3 +-
lib/PublicInbox/DS.pm | 4 +-
.../{NNTPdeflate.pm => DSdeflate.pm} | 13 +-
lib/PublicInbox/IMAP.pm | 10 +-
lib/PublicInbox/IMAPD.pm | 2 +-
lib/PublicInbox/IMAPdeflate.pm | 126 ------------------
lib/PublicInbox/NNTP.pm | 10 +-
lib/PublicInbox/NNTPD.pm | 2 +-
xt/mem-imapd-tls.t | 4 +-
9 files changed, 26 insertions(+), 148 deletions(-)
rename lib/PublicInbox/{NNTPdeflate.pm => DSdeflate.pm} (92%)
delete mode 100644 lib/PublicInbox/IMAPdeflate.pm
diff --git a/MANIFEST b/MANIFEST
index 923f5147..10547351 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -169,6 +169,7 @@ lib/PublicInbox/ContentHash.pm
lib/PublicInbox/DS.pm
lib/PublicInbox/DSKQXS.pm
lib/PublicInbox/DSPoll.pm
+lib/PublicInbox/DSdeflate.pm
lib/PublicInbox/Daemon.pm
lib/PublicInbox/DirIdle.pm
lib/PublicInbox/DummyInbox.pm
@@ -206,7 +207,6 @@ lib/PublicInbox/IMAP.pm
lib/PublicInbox/IMAPClient.pm
lib/PublicInbox/IMAPD.pm
lib/PublicInbox/IMAPTracker.pm
-lib/PublicInbox/IMAPdeflate.pm
lib/PublicInbox/IMAPsearchqp.pm
lib/PublicInbox/IPC.pm
lib/PublicInbox/IdxStack.pm
@@ -295,7 +295,6 @@ lib/PublicInbox/Msgmap.pm
lib/PublicInbox/MultiGit.pm
lib/PublicInbox/NNTP.pm
lib/PublicInbox/NNTPD.pm
-lib/PublicInbox/NNTPdeflate.pm
lib/PublicInbox/NetNNTPSocks.pm
lib/PublicInbox/NetReader.pm
lib/PublicInbox/NetWriter.pm
diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index fee31e3d..ef483aac 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -648,8 +648,8 @@ sub shutdn ($) {
}
}
-sub zflush {} # overridden by NNTPdeflate and IMAPdeflate
-
+sub zflush {} # overridden by DSdeflate
+sub compressed {} # overridden by DSdeflate
sub long_response_done {} # overridden by Net::NNTP
sub long_step {
diff --git a/lib/PublicInbox/NNTPdeflate.pm b/lib/PublicInbox/DSdeflate.pm
similarity index 92%
rename from lib/PublicInbox/NNTPdeflate.pm
rename to lib/PublicInbox/DSdeflate.pm
index 352d4842..b5208e43 100644
--- a/lib/PublicInbox/NNTPdeflate.pm
+++ b/lib/PublicInbox/DSdeflate.pm
@@ -2,6 +2,7 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# RFC 8054 NNTP COMPRESS DEFLATE implementation
+# RFC 4978 IMAP COMPRESS=DEFLATE extension
#
# RSS usage for 10K idle-but-did-something NNTP clients on 64-bit:
# TLS + DEFLATE[a] : 1.8 GB (MemLevel=9, 1.2 GB with MemLevel=8)
@@ -14,14 +15,13 @@
# [b] - memory-optimized implementation using a global deflate context.
# It's less efficient in terms of compression, but way more
# efficient in terms of server memory usage.
-package PublicInbox::NNTPdeflate;
+package PublicInbox::DSdeflate;
use strict;
-use 5.010_001;
-use parent qw(PublicInbox::NNTP);
+use v5.10.1;
use Compress::Raw::Zlib;
my %IN_OPT = (
- -Bufsize => PublicInbox::NNTP::LINE_MAX,
+ -Bufsize => 1024,
-WindowBits => -15, # RFC 1951
-AppendOutput => 1,
);
@@ -42,21 +42,18 @@ my $zout;
$err == Z_OK or die "Failed to initialize zlib deflate stream: $err";
}
-
sub enable {
my ($class, $self) = @_;
my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT);
if ($err != Z_OK) {
$self->err("Inflate->new failed: $err");
- $self->write(\"403 Unable to activate compression\r\n");
return;
}
- $self->write(\"206 Compression active\r\n");
bless $self, $class;
$self->{zin} = $in;
}
-# overrides PublicInbox::NNTP::compressed
+# overrides PublicInbox::DS::compressed
sub compressed { 1 }
sub do_read ($$$$) {
diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index ce0dce0f..805f1102 100644
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -1212,8 +1212,6 @@ sub event_step {
$self->requeue unless $pending;
}
-sub compressed { undef }
-
# RFC 4978
sub cmd_compress ($$$) {
my ($self, $tag, $alg) = @_;
@@ -1223,7 +1221,9 @@ sub cmd_compress ($$$) {
# CRIME made TLS compression obsolete
# return "$tag NO [COMPRESSIONACTIVE]\r\n" if $self->tls_compressed;
- PublicInbox::IMAPdeflate->enable($self, $tag);
+ PublicInbox::IMAPdeflate->enable($self) or return
+ \"$tag BAD failed to activate compression\r\n";
+ PublicInbox::DS::write($self, \"$tag OK DEFLATE active\r\n");
$self->requeue;
undef
}
@@ -1269,4 +1269,8 @@ our @ISA = qw(PublicInbox::IMAP);
sub logged_in { 0 }
+package PublicInbox::IMAPdeflate;
+use PublicInbox::DSdeflate;
+our @ISA = qw(PublicInbox::DSdeflate PublicInbox::IMAP);
+
1;
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index d8814324..b24097a2 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -9,7 +9,7 @@ use v5.10.1;
use PublicInbox::Config;
use PublicInbox::ConfigIter;
use PublicInbox::InboxIdle;
-use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP
+use PublicInbox::IMAP;
use PublicInbox::DummyInbox;
my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox';
diff --git a/lib/PublicInbox/IMAPdeflate.pm b/lib/PublicInbox/IMAPdeflate.pm
deleted file mode 100644
index d5929ef2..00000000
--- a/lib/PublicInbox/IMAPdeflate.pm
+++ /dev/null
@@ -1,126 +0,0 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# TODO: reduce duplication from PublicInbox::NNTPdeflate
-
-# RFC 4978
-package PublicInbox::IMAPdeflate;
-use strict;
-use warnings;
-use 5.010_001;
-use base qw(PublicInbox::IMAP);
-use Compress::Raw::Zlib;
-
-my %IN_OPT = (
- -Bufsize => 1024,
- -WindowBits => -15, # RFC 1951
- -AppendOutput => 1,
-);
-
-# global deflate context and buffer
-my $zbuf = \(my $buf = '');
-my $zout;
-{
- my $err;
- ($zout, $err) = Compress::Raw::Zlib::Deflate->new(
- # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9,
- # the zlib C library and git use MemLevel=8 as the default
- # -MemLevel => 9,
- -Bufsize => 65536, # same as nnrpd
- -WindowBits => -15, # RFC 1951
- -AppendOutput => 1,
- );
- $err == Z_OK or die "Failed to initialize zlib deflate stream: $err";
-}
-
-sub enable {
- my ($class, $self, $tag) = @_;
- my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT);
- if ($err != Z_OK) {
- $self->err("Inflate->new failed: $err");
- $self->write(\"$tag BAD failed to activate compression\r\n");
- return;
- }
- $self->write(\"$tag OK DEFLATE active\r\n");
- bless $self, $class;
- $self->{zin} = $in;
-}
-
-# overrides PublicInbox::NNTP::compressed
-sub compressed { 1 }
-
-sub do_read ($$$$) {
- my ($self, $rbuf, $len, $off) = @_;
-
- my $zin = $self->{zin} or return; # closed
- my $doff;
- my $dbuf = delete($self->{dbuf}) // '';
- $doff = length($dbuf);
- my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return;
-
- # Workaround inflate bug appending to OOK scalars:
- # <https://rt.cpan.org/Ticket/Display.html?id=132734>
- # We only have $off if the client is pipelining, and pipelining
- # is where our substr() OOK optimization in event_step makes sense.
- if ($off) {
- my $copy = $$rbuf;
- undef $$rbuf;
- $$rbuf = $copy;
- }
-
- # assert(length($$rbuf) == $off) as far as NNTP.pm is concerned
- # -ConsumeInput is true, so $dbuf is automatically emptied
- my $err = $zin->inflate($dbuf, $rbuf);
- if ($err == Z_OK) {
- $self->{dbuf} = $dbuf if $dbuf ne '';
- $r = length($$rbuf) and return $r;
- # nothing ready, yet, get more, later
- $self->requeue;
- } else {
- delete $self->{zin};
- $self->close;
- }
- 0;
-}
-
-# override PublicInbox::DS::msg_more
-sub msg_more ($$) {
- my $self = $_[0];
-
- # $_[1] may be a reference or not for ->deflate
- my $err = $zout->deflate($_[1], $zbuf);
- $err == Z_OK or die "->deflate failed $err";
- 1;
-}
-
-sub zflush ($) {
- my ($self) = @_;
-
- my $deflated = $zbuf;
- $zbuf = \(my $next = '');
-
- my $err = $zout->flush($deflated, Z_FULL_FLUSH);
- $err == Z_OK or die "->flush failed $err";
-
- # We can still let the lower socket layer do buffering:
- PublicInbox::DS::msg_more($self, $$deflated);
-}
-
-# compatible with PublicInbox::DS::write, so $_[1] may be a reference or not
-sub write ($$) {
- my $self = $_[0];
- return PublicInbox::DS::write($self, $_[1]) if ref($_[1]) eq 'CODE';
-
- my $deflated = $zbuf;
- $zbuf = \(my $next = '');
-
- # $_[1] may be a reference or not for ->deflate
- my $err = $zout->deflate($_[1], $deflated);
- $err == Z_OK or die "->deflate failed $err";
- $err = $zout->flush($deflated, Z_FULL_FLUSH);
- $err == Z_OK or die "->flush failed $err";
-
- # We can still let the socket layer do buffering:
- PublicInbox::DS::write($self, $deflated);
-}
-
-1;
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 3929f817..8ad7adc1 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -883,8 +883,6 @@ sub cmd_xover ($;$) {
$self->long_response(\&xover_i, @$r);
}
-sub compressed { undef }
-
sub cmd_starttls ($) {
my ($self) = @_;
my $sock = $self->{sock} or return;
@@ -903,7 +901,9 @@ sub cmd_compress ($$) {
my ($self, $alg) = @_;
return "503 Only DEFLATE is supported\r\n" if uc($alg) ne 'DEFLATE';
return r502 if $self->compressed;
- PublicInbox::NNTPdeflate->enable($self);
+ PublicInbox::NNTPdeflate->enable($self) or return
+ \"403 Unable to activate compression\r\n";
+ PublicInbox::DS::write($self, \"206 Compression active\r\n");
$self->requeue;
undef
}
@@ -985,4 +985,8 @@ sub busy { # for graceful shutdown in PublicInbox::Daemon:
defined($self->{rbuf}) || defined($self->{wbuf})
}
+package PublicInbox::NNTPdeflate;
+use PublicInbox::DSdeflate;
+our @ISA = qw(PublicInbox::DSdeflate PublicInbox::NNTP);
+
1;
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 6e79f0be..f31d4381 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -9,7 +9,7 @@ use v5.10.1;
use Sys::Hostname;
use PublicInbox::Config;
use PublicInbox::InboxIdle;
-use PublicInbox::NNTPdeflate; # loads PublicInbox::NNTP
+use PublicInbox::NNTP;
sub new {
my ($class) = @_;
diff --git a/xt/mem-imapd-tls.t b/xt/mem-imapd-tls.t
index 8992a6fc..d728ce32 100644
--- a/xt/mem-imapd-tls.t
+++ b/xt/mem-imapd-tls.t
@@ -1,5 +1,5 @@
#!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Idle client memory usage test, particularly after EXAMINE when
# Message Sequence Numbers are loaded
@@ -221,7 +221,7 @@ package IMAPCdeflate;
use strict;
our @ISA;
use Compress::Raw::Zlib;
-use PublicInbox::IMAPdeflate;
+use PublicInbox::IMAP;
my %ZIN_OPT;
BEGIN {
@ISA = qw(IMAPC);
prev parent reply other threads:[~2022-07-23 4:41 UTC|newest]
Thread overview: 12+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-07-23 4:41 [PATCH 00/11] IMAP, NNTP, POP3 golfing Eric Wong
2022-07-23 4:41 ` [PATCH 01/11] nntp: pass regexp to split() callers Eric Wong
2022-07-23 4:41 ` [PATCH 02/11] nntp: start adding CRLF to responses natively Eric Wong
2022-07-23 4:41 ` [PATCH 03/11] nntp: remove more() wrapper Eric Wong
2022-07-23 4:41 ` [PATCH 04/11] ds: support greeting protocols Eric Wong
2022-07-23 4:41 ` [PATCH 05/11] ds: move no-op ->zflush to common base class Eric Wong
2022-07-23 4:41 ` [PATCH 06/11] ds: move requeue_once Eric Wong
2022-07-23 4:41 ` [PATCH 07/11] nntp: listgroup_range_i: remove useless `map' op Eric Wong
2022-07-23 4:41 ` [PATCH 08/11] nntp: inline CRLF in all response lines Eric Wong
2022-07-23 4:41 ` [PATCH 09/11] ds: share long_step between NNTP and IMAP Eric Wong
2022-07-23 4:41 ` [PATCH 10/11] nntp: resolve inboxes immediately on group listings Eric Wong
2022-07-23 4:41 ` Eric Wong [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220723044155.3733949-12-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).