* [PATCH 0/2] allow OpenSSL SHA-(1|256) use if installed
@ 2023-01-29 10:30 Eric Wong
2023-01-29 10:30 ` [PATCH 1/2] use Net::SSLeay (OpenSSL) for SHA-(1|256) " Eric Wong
2023-01-29 10:30 ` [PATCH 2/2] content_digest_dbg: convert to arrayref and limit to lei Eric Wong
0 siblings, 2 replies; 3+ messages in thread
From: Eric Wong @ 2023-01-29 10:30 UTC (permalink / raw)
To: meta
OpenSSL has platform-specific asm, and the SHA-256 x86-64
implementation is nearly twice as fast as the portable C
implementation in Perl5. SHA-1 is a tad faster, too.
We still need to use Digest::SHA in a few places for
->clone to to bifurcate state.
Eric Wong (2):
use Net::SSLeay (OpenSSL) for SHA-(1|256) if installed
content_digest_dbg: convert to arrayref and limit to lei
MANIFEST | 2 +
lib/PublicInbox/ContentDigestDbg.pm | 12 +++---
lib/PublicInbox/ContentHash.pm | 11 +++---
lib/PublicInbox/Fetch.pm | 4 +-
lib/PublicInbox/Git.pm | 4 +-
lib/PublicInbox/LeiDedupe.pm | 6 +--
lib/PublicInbox/LeiMirror.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 4 +-
lib/PublicInbox/LeiSucks.pm | 12 +++---
lib/PublicInbox/Linkify.pm | 2 +-
lib/PublicInbox/MID.pm | 8 ++--
lib/PublicInbox/MailDiff.pm | 5 +--
lib/PublicInbox/MdirReader.pm | 4 +-
lib/PublicInbox/NNTP.pm | 2 +-
lib/PublicInbox/SHA.pm | 58 +++++++++++++++++++++++++++++
lib/PublicInbox/WwwAtomStream.pm | 2 +-
t/clone-coderepo.t | 2 +-
t/httpd-corner.psgi | 4 +-
t/httpd-corner.t | 2 +-
t/ipc.t | 6 +--
t/nntpd.t | 4 +-
t/sha.t | 25 +++++++++++++
t/www_listing.t | 2 +-
xt/git_async_cmp.t | 10 ++---
xt/imapd-validate.t | 5 ++-
xt/nntpd-validate.t | 5 ++-
26 files changed, 145 insertions(+), 58 deletions(-)
create mode 100644 lib/PublicInbox/SHA.pm
create mode 100644 t/sha.t
^ permalink raw reply [flat|nested] 3+ messages in thread
* [PATCH 1/2] use Net::SSLeay (OpenSSL) for SHA-(1|256) if installed
2023-01-29 10:30 [PATCH 0/2] allow OpenSSL SHA-(1|256) use if installed Eric Wong
@ 2023-01-29 10:30 ` Eric Wong
2023-01-29 10:30 ` [PATCH 2/2] content_digest_dbg: convert to arrayref and limit to lei Eric Wong
1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2023-01-29 10:30 UTC (permalink / raw)
To: meta
On my x86-64 machine, OpenSSL SHA-256 is nearly twice as fast as
the Digest::SHA implementation from Perl, most likely due to an
optimized assembly implementation. SHA-1 is a few percent
faster, too.
---
MANIFEST | 2 +
lib/PublicInbox/ContentDigestDbg.pm | 4 +-
lib/PublicInbox/ContentHash.pm | 11 +++---
lib/PublicInbox/Fetch.pm | 4 +-
lib/PublicInbox/Git.pm | 4 +-
lib/PublicInbox/LeiDedupe.pm | 6 +--
lib/PublicInbox/LeiMirror.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 4 +-
lib/PublicInbox/LeiSucks.pm | 12 +++---
lib/PublicInbox/Linkify.pm | 2 +-
lib/PublicInbox/MID.pm | 8 ++--
lib/PublicInbox/MdirReader.pm | 4 +-
lib/PublicInbox/NNTP.pm | 2 +-
lib/PublicInbox/SHA.pm | 58 +++++++++++++++++++++++++++++
lib/PublicInbox/WwwAtomStream.pm | 2 +-
t/clone-coderepo.t | 2 +-
t/httpd-corner.psgi | 4 +-
t/httpd-corner.t | 2 +-
t/ipc.t | 6 +--
t/nntpd.t | 4 +-
t/sha.t | 25 +++++++++++++
t/www_listing.t | 2 +-
xt/git_async_cmp.t | 10 ++---
xt/imapd-validate.t | 5 ++-
xt/nntpd-validate.t | 5 ++-
25 files changed, 139 insertions(+), 51 deletions(-)
create mode 100644 lib/PublicInbox/SHA.pm
create mode 100644 t/sha.t
diff --git a/MANIFEST b/MANIFEST
index c494d6f7..f22f60aa 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -312,6 +312,7 @@ lib/PublicInbox/Reply.pm
lib/PublicInbox/RepoAtom.pm
lib/PublicInbox/RepoSnapshot.pm
lib/PublicInbox/RepoTree.pm
+lib/PublicInbox/SHA.pm
lib/PublicInbox/SaPlugin/ListMirror.pm
lib/PublicInbox/SaPlugin/ListMirror.pod
lib/PublicInbox/Search.pm
@@ -559,6 +560,7 @@ t/run.perl
t/search-amsg.eml
t/search-thr-index.t
t/search.t
+t/sha.t
t/shared_kv.t
t/sigfd.t
t/solve/0001-simple-mod.patch
diff --git a/lib/PublicInbox/ContentDigestDbg.pm b/lib/PublicInbox/ContentDigestDbg.pm
index 425e8589..899afbbe 100644
--- a/lib/PublicInbox/ContentDigestDbg.pm
+++ b/lib/PublicInbox/ContentDigestDbg.pm
@@ -3,9 +3,9 @@
package PublicInbox::ContentDigestDbg; # cf. PublicInbox::ContentDigest
use v5.12;
use Data::Dumper;
-use Digest::SHA;
+use PublicInbox::SHA;
-sub new { bless { dig => Digest::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
+sub new { bless { dig => PublicInbox::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
sub add {
$_[0]->{dig}->add($_[1]);
diff --git a/lib/PublicInbox/ContentHash.pm b/lib/PublicInbox/ContentHash.pm
index 1afbb413..d3ff146a 100644
--- a/lib/PublicInbox/ContentHash.pm
+++ b/lib/PublicInbox/ContentHash.pm
@@ -15,7 +15,8 @@ use PublicInbox::MID qw(mids references);
use PublicInbox::MsgIter;
# not sure if less-widely supported hash families are worth bothering with
-use Digest::SHA;
+use PublicInbox::SHA; # faster, but no ->clone
+use Digest::SHA; # we still need this for ->clone
sub digest_addr ($$$) {
my ($dig, $h, $v) = @_;
@@ -93,15 +94,15 @@ sub content_digest ($;$) {
}
sub content_hash ($) {
- content_digest($_[0])->digest;
+ content_digest($_[0], PublicInbox::SHA->new(256))->digest;
}
+# don't clone the result of this
sub git_sha ($$) {
my ($n, $eml) = @_;
- my $dig = Digest::SHA->new($n);
+ my $dig = PublicInbox::SHA->new($n);
my $bref = ref($eml) eq 'SCALAR' ? $eml : \($eml->as_string);
- $dig->add('blob '.length($$bref)."\0");
- $dig->add($$bref);
+ $dig->add('blob '.length($$bref)."\0", $$bref);
$dig;
}
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 198e2a60..f93eeebe 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -92,9 +92,9 @@ sub do_manifest ($$$) {
sub get_fingerprint2 {
my ($git_dir) = @_;
- require Digest::SHA;
+ require PublicInbox::SHA;
my $rd = popen_rd([qw(git show-ref)], undef, { -C => $git_dir });
- Digest::SHA::sha256(do { local $/; <$rd> });
+ PublicInbox::SHA::sha256(do { local $/; <$rd> });
}
sub writable_dir ($) {
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 3e2b435c..fd7a0382 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -20,7 +20,7 @@ use PublicInbox::Spawn qw(popen_rd which);
use PublicInbox::Tmpfile;
use IO::Poll qw(POLLIN);
use Carp qw(croak carp);
-use Digest::SHA ();
+use PublicInbox::SHA ();
use PublicInbox::DS qw(awaitpid);
our @EXPORT_OK = qw(git_unquote git_quote);
our $PIPE_BUFSIZ = 65536; # Linux default
@@ -630,7 +630,7 @@ sub cloneurl {
sub manifest_entry {
my ($self, $epoch, $default_desc) = @_;
my $fh = $self->popen('show-ref');
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
while (read($fh, my $buf, 65536)) {
$dig->add($buf);
}
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index 32f99cd0..22864508 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -1,10 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
package PublicInbox::LeiDedupe;
use strict;
use v5.10.1;
use PublicInbox::ContentHash qw(content_hash git_sha);
-use Digest::SHA ();
+use PublicInbox::SHA ();
# n.b. mutt sets most of these headers not sure about Bytes
our @OID_IGNORE = qw(Status X-Status Content-Length Lines Bytes);
@@ -30,7 +30,7 @@ sub _oidbin ($) { defined($_[0]) ? pack('H*', $_[0]) : undef }
sub smsg_hash ($) {
my ($smsg) = @_;
- my $dig = Digest::SHA->new(256);
+ my $dig = PublicInbox::SHA->new(256);
my $x = join("\0", @$smsg{qw(from to cc ds subject references mid)});
utf8::encode($x);
$dig->add($x);
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index abf66315..31013360 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -18,7 +18,7 @@ use PublicInbox::Config;
use PublicInbox::Inbox;
use PublicInbox::LeiCurl;
use PublicInbox::OnDestroy;
-use Digest::SHA qw(sha256_hex sha1_hex);
+use PublicInbox::SHA qw(sha256_hex sha1_hex);
use POSIX qw(strftime);
our $LIVE; # pid => callback
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index ed92bfd1..e5396342 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# pretends to be like LeiDedupe and also PublicInbox::Inbox
@@ -13,7 +13,7 @@ use PublicInbox::Config;
use PublicInbox::Spawn qw(run_die);
use PublicInbox::ContentHash qw(git_sha);
use PublicInbox::MID qw(mids_for_index);
-use Digest::SHA qw(sha256_hex);
+use PublicInbox::SHA qw(sha256_hex);
our $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
# move this to PublicInbox::Config if other things use it:
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index 8e866fc9..35d0a8de 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Undocumented hidden command somebody might discover if they're
@@ -7,7 +7,7 @@
package PublicInbox::LeiSucks;
use strict;
use v5.10.1;
-use Digest::SHA ();
+use PublicInbox::SHA qw(sha1_hex);
use Config;
use POSIX ();
use PublicInbox::Config;
@@ -54,13 +54,13 @@ sub lei_sucks {
} else {
push @out, "Xapian not available: $@\n";
}
- my $dig = Digest::SHA->new(1);
push @out, "public-inbox blob OIDs of loaded features:\n";
for my $m (grep(m{^PublicInbox/}, sort keys %INC)) {
my $f = $INC{$m} // next; # lazy require failed (missing dep)
- $dig->add('blob '.(-s $f)."\0");
- $dig->addfile($f);
- push @out, ' '.$dig->hexdigest.' '.$m."\n";
+ open my $fh, '<', $f or do { warn "open($f): $!"; next };
+ my $hex = sha1_hex('blob '.(-s $fh)."\0".
+ (do { local $/; <$fh> } // die("read: $!")));
+ push @out, ' '.$hex.' '.$m."\n";
}
push @out, <<'EOM';
Let us know how it sucks! Please include the above and any other
diff --git a/lib/PublicInbox/Linkify.pm b/lib/PublicInbox/Linkify.pm
index 9fc3128f..306a57e7 100644
--- a/lib/PublicInbox/Linkify.pm
+++ b/lib/PublicInbox/Linkify.pm
@@ -12,7 +12,7 @@
package PublicInbox::Linkify;
use strict;
use v5.10.1;
-use Digest::SHA qw/sha1_hex/;
+use PublicInbox::SHA qw(sha1_hex);
use PublicInbox::Hval qw(ascii_html mid_href);
use PublicInbox::MID qw($MID_EXTRACT);
diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm
index 35b517e0..4819cc25 100644
--- a/lib/PublicInbox/MID.pm
+++ b/lib/PublicInbox/MID.pm
@@ -1,15 +1,15 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Various Message-ID-related functions.
package PublicInbox::MID;
use strict;
-use warnings;
-use base qw/Exporter/;
+use v5.10.1; # TODO: check unicode_strings compat for v5.12
+use parent qw(Exporter);
our @EXPORT_OK = qw(mid_clean id_compress mid2path mid_escape MID_ESC
mids references mids_for_index mids_in $MID_EXTRACT);
use URI::Escape qw(uri_escape_utf8);
-use Digest::SHA qw/sha1_hex/;
+use PublicInbox::SHA qw(sha1_hex);
require PublicInbox::Address;
use constant {
ID_MAX => 40, # SHA-1 hex length for HTML id anchors
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index dbb74d6d..db5f4545 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Maildirs for now, MH eventually
@@ -8,7 +8,7 @@ package PublicInbox::MdirReader;
use strict;
use v5.10.1;
use PublicInbox::InboxWritable qw(eml_from_path);
-use Digest::SHA qw(sha256_hex);
+use PublicInbox::SHA qw(sha256_hex);
# returns Maildir flags from a basename ('' for no flags, undef for invalid)
sub maildir_basename_flags {
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index dd33a232..7a91e7eb 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -15,7 +15,7 @@ use PublicInbox::MID qw(mid_escape $MID_EXTRACT);
use PublicInbox::Eml;
use POSIX qw(strftime);
use PublicInbox::DS qw(now);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
use Time::Local qw(timegm timelocal);
use PublicInbox::GitAsyncCat;
use PublicInbox::Address;
diff --git a/lib/PublicInbox/SHA.pm b/lib/PublicInbox/SHA.pm
new file mode 100644
index 00000000..da70beef
--- /dev/null
+++ b/lib/PublicInbox/SHA.pm
@@ -0,0 +1,58 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# OpenSSL exception added in commit 22711f81f4e79da6b796820e37803a05cae14645
+# (README: add OpenSSL exception, 2015-10-05)
+
+# Replaces most uses of Digest::SHA with OpenSSL via Net::SSLeay if
+# possible. OpenSSL SHA-256 is nearly twice as fast as Digest::SHA on
+# x86-64, and SHA-1 is a bit faster as well.
+# I don't think we can implement Digest::SHA->clone with what Net::SSLeay
+# gives us... (maybe EVP_MD_CTX_copy+EVP_MD_CTX_copy_ex need to be added
+# to Net::SSLeay?)
+package PublicInbox::SHA;
+use v5.12;
+require Exporter;
+our @EXPORT_OK = qw(sha1_hex sha256_hex sha256);
+our @ISA;
+
+BEGIN {
+ push @ISA, 'Exporter';
+ unless (eval(<<'EOM')) {
+use Net::SSLeay 1.43;
+my %SHA = (
+ 1 => Net::SSLeay::EVP_get_digestbyname('sha1'),
+ 256 => Net::SSLeay::EVP_get_digestbyname('sha256'),
+);
+
+sub new {
+ my ($cls, $n) = @_;
+ my $mdctx = Net::SSLeay::EVP_MD_CTX_create();
+ Net::SSLeay::EVP_DigestInit($mdctx, $SHA{$n}) or
+ die "EVP_DigestInit $n: $!";
+ bless \$mdctx, $cls;
+}
+
+sub add {
+ my $self = shift;
+ Net::SSLeay::EVP_DigestUpdate($$self, $_) for @_;
+ $self;
+}
+
+sub digest { Net::SSLeay::EVP_DigestFinal(${$_[0]}) };
+sub hexdigest { unpack('H*', Net::SSLeay::EVP_DigestFinal(${$_[0]})) }
+sub DESTROY { Net::SSLeay::EVP_MD_CTX_destroy(${$_[0]}) };
+
+sub sha1_hex { unpack('H*', Net::SSLeay::SHA1($_[0])) };
+sub sha256_hex { unpack('H*', Net::SSLeay::SHA256($_[0])) };
+*sha256 = \&Net::SSLeay::SHA256;
+# end of eval
+EOM
+ require Digest::SHA; # stdlib fallback
+ push @ISA, 'Digest::SHA';
+ *sha1_hex = \&Digest::SHA::sha1_hex;
+ *sha256_hex = \&Digest::SHA::sha256_hex;
+ *sha256 = \&Digest::SHA::sha256;
+}
+
+} # /BEGIN
+1;
diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm
index 83a8818e..737cc6cb 100644
--- a/lib/PublicInbox/WwwAtomStream.pm
+++ b/lib/PublicInbox/WwwAtomStream.pm
@@ -8,7 +8,7 @@ use strict;
use parent 'PublicInbox::GzipFilter';
use POSIX qw(strftime);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
use PublicInbox::Address;
use PublicInbox::Hval qw(ascii_html mid_href);
use PublicInbox::MsgTime qw(msg_timestamp);
diff --git a/t/clone-coderepo.t b/t/clone-coderepo.t
index 2e628963..1f33a6d7 100644
--- a/t/clone-coderepo.t
+++ b/t/clone-coderepo.t
@@ -6,7 +6,7 @@ use PublicInbox::TestCommon;
use PublicInbox::Import;
use File::Temp;
use File::Path qw(remove_tree);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
require_mods(qw(json Plack::Builder HTTP::Date HTTP::Status));
require_git '1.8.5';
require_ok 'PublicInbox::LeiMirror';
diff --git a/t/httpd-corner.psgi b/t/httpd-corner.psgi
index ea643024..1e96d7b1 100644
--- a/t/httpd-corner.psgi
+++ b/t/httpd-corner.psgi
@@ -4,7 +4,7 @@
# Usage: plackup [OPTIONS] /path/to/this/file
use v5.12;
use Plack::Builder;
-require Digest::SHA;
+require PublicInbox::SHA;
if (defined(my $f = $ENV{TEST_OPEN_FIFO})) {
open my $fh, '>', $f or die "open($f): $!";
say $fh 'hi';
@@ -29,7 +29,7 @@ my $app = sub {
my $h = [ 'Content-Type' => 'text/plain' ];
my $body = [];
if ($path eq '/sha1') {
- my $sha1 = Digest::SHA->new('SHA-1');
+ my $sha1 = PublicInbox::SHA->new(1);
my $buf;
while (1) {
my $r = $in->read($buf, 4096);
diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index 88820270..7600c2b9 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -7,7 +7,7 @@ use strict; use v5.10.1; use PublicInbox::TestCommon;
use Time::HiRes qw(gettimeofday tv_interval);
use PublicInbox::Spawn qw(spawn popen_rd);
require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
use IO::Handle ();
use IO::Socket::UNIX;
use Fcntl qw(:seek);
diff --git a/t/ipc.t b/t/ipc.t
index ce89f94b..fd4d5599 100644
--- a/t/ipc.t
+++ b/t/ipc.t
@@ -1,19 +1,19 @@
#!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use v5.10.1;
use Test::More;
use PublicInbox::TestCommon;
use Fcntl qw(SEEK_SET);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
require_mods(qw(Storable||Sereal));
require_ok 'PublicInbox::IPC';
my ($tmpdir, $for_destroy) = tmpdir();
state $once = eval <<'';
package PublicInbox::IPC;
use strict;
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
sub test_array { qw(test array) }
sub test_scalar { 'scalar' }
sub test_scalarref { \'scalarref' }
diff --git a/t/nntpd.t b/t/nntpd.t
index bebf4203..84a60574 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -7,7 +7,7 @@ use PublicInbox::Eml;
use Socket qw(IPPROTO_TCP TCP_NODELAY);
use Sys::Hostname;
use POSIX qw(_exit);
-use Digest::SHA;
+use PublicInbox::SHA;
# t/nntpd-v2.t wraps this for v2
my $version = $ENV{PI_TEST_VERSION} || 1;
@@ -304,7 +304,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
my %sums;
for (1..$nart) {
<$s> =~ /\A220 / or _exit(4);
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
while (my $l = <$s>) {
last if $l eq ".\r\n";
$dig->add($l);
diff --git a/t/sha.t b/t/sha.t
new file mode 100644
index 00000000..2e2d5636
--- /dev/null
+++ b/t/sha.t
@@ -0,0 +1,25 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::SHA;
+use Test::More;
+
+{
+ my $dig = PublicInbox::SHA->new(1);
+ open my $fh, '<', 'COPYING' or die "open: $!";
+ $dig->add(do { local $/; <$fh> });
+ is($dig->hexdigest, '78e50e186b04c8fe1defaa098f1c192181b3d837',
+ 'AGPL-3 matches');
+}
+
+SKIP: {
+ my $n = $ENV{TEST_LEAK_NR} or skip 'TEST_LEAK_NR unset', 1;
+ for (1..$n) {
+ PublicInbox::SHA->new(1)->add('hello')->digest;
+ PublicInbox::SHA->new(1)->add('hello');
+ PublicInbox::SHA->new(1);
+ }
+}
+
+done_testing;
diff --git a/t/www_listing.t b/t/www_listing.t
index 6166b94e..4784c39d 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -5,7 +5,7 @@
use strict; use v5.10.1; use PublicInbox::TestCommon;
use PublicInbox::Import;
use IO::Uncompress::Gunzip qw(gunzip);
-require_mods(qw(json URI::Escape Plack::Builder Digest::SHA HTTP::Tiny));
+require_mods(qw(json URI::Escape Plack::Builder HTTP::Tiny));
require PublicInbox::WwwListing;
require PublicInbox::ManifestJsGz;
use PublicInbox::Config;
diff --git a/xt/git_async_cmp.t b/xt/git_async_cmp.t
index d66b371f..9edc1f37 100644
--- a/xt/git_async_cmp.t
+++ b/xt/git_async_cmp.t
@@ -1,10 +1,10 @@
#!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use Test::More;
use Benchmark qw(:all);
-use Digest::SHA;
+use PublicInbox::SHA;
use PublicInbox::TestCommon;
my $git_dir = $ENV{GIANT_GIT_DIR};
plan 'skip_all' => "GIANT_GIT_DIR not defined for $0" unless defined($git_dir);
@@ -20,7 +20,7 @@ my @dig;
my $nr = $ENV{NR} || 1;
diag "NR=$nr";
my $async = timeit($nr, sub {
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $cb = sub {
my ($bref) = @_;
$dig->add($$bref);
@@ -37,7 +37,7 @@ my $async = timeit($nr, sub {
});
my $sync = timeit($nr, sub {
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $cat = $git->popen(@cat);
while (<$cat>) {
my ($oid, undef, undef) = split(/ /);
@@ -51,7 +51,7 @@ my $sync = timeit($nr, sub {
ok(scalar(@dig) >= 2, 'got some digests');
my $ref = shift @dig;
my $exp = $ref->[1];
-isnt($exp, Digest::SHA->new(1)->hexdigest, 'not empty');
+isnt($exp, PublicInbox::SHA->new(1)->hexdigest, 'not empty');
foreach (@dig) {
is($_->[1], $exp, "digest matches $_->[0] <=> $ref->[0]");
}
diff --git a/xt/imapd-validate.t b/xt/imapd-validate.t
index 5d27d2a0..5d665fa9 100644
--- a/xt/imapd-validate.t
+++ b/xt/imapd-validate.t
@@ -1,11 +1,12 @@
#!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Expensive test to validate compression and TLS.
use strict;
use v5.10.1;
use Symbol qw(gensym);
use PublicInbox::DS qw(now);
+use PublicInbox::SHA;
use POSIX qw(_exit);
use PublicInbox::TestCommon;
my $inbox_dir = $ENV{GIANT_INBOX_DIR};
@@ -64,7 +65,7 @@ my $do_get_all = sub {
my ($desc, $opt) = @_;
local $SIG{__DIE__} = sub { print STDERR $desc, ': ', @_; _exit(1) };
my $t0 = now();
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $mic = $imap_client->new(%$opt);
$mic->examine($mailbox) or die "examine: $!";
my $uid_base = 1;
diff --git a/xt/nntpd-validate.t b/xt/nntpd-validate.t
index 83f024f9..a6f3980e 100644
--- a/xt/nntpd-validate.t
+++ b/xt/nntpd-validate.t
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Integration test to validate compression.
@@ -9,6 +9,7 @@ use Symbol qw(gensym);
use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
use POSIX qw(_exit);
use PublicInbox::TestCommon;
+use PublicInbox::SHA;
my $inbox_dir = $ENV{GIANT_INBOX_DIR};
plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
my $mid = $ENV{TEST_MID};
@@ -55,7 +56,7 @@ sub do_get_all {
my ($methods) = @_;
my $desc = join(',', @$methods);
my $t0 = clock_gettime(CLOCK_MONOTONIC);
- my $dig = Digest::SHA->new(1);
+ my $dig = PublicInbox::SHA->new(1);
my $digfh = gensym;
my $tmpfh;
if ($File::Temp::KEEP_ALL) {
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [PATCH 2/2] content_digest_dbg: convert to arrayref and limit to lei
2023-01-29 10:30 [PATCH 0/2] allow OpenSSL SHA-(1|256) use if installed Eric Wong
2023-01-29 10:30 ` [PATCH 1/2] use Net::SSLeay (OpenSSL) for SHA-(1|256) " Eric Wong
@ 2023-01-29 10:30 ` Eric Wong
1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2023-01-29 10:30 UTC (permalink / raw)
To: meta
Since it's an extremely small class and not subclassed or
anything, we'll make it even smaller as an arrayref.
We also don't load this for PublicInbox::WWW or anything that
runs in public-facing daemons.
---
lib/PublicInbox/ContentDigestDbg.pm | 10 ++++++----
lib/PublicInbox/MailDiff.pm | 5 +----
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/ContentDigestDbg.pm b/lib/PublicInbox/ContentDigestDbg.pm
index 899afbbe..5de0ee8a 100644
--- a/lib/PublicInbox/ContentDigestDbg.pm
+++ b/lib/PublicInbox/ContentDigestDbg.pm
@@ -1,17 +1,19 @@
# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# only loaded in lei
package PublicInbox::ContentDigestDbg; # cf. PublicInbox::ContentDigest
use v5.12;
use Data::Dumper;
use PublicInbox::SHA;
+$Data::Dumper::Useqq = $Data::Dumper::Terse = 1;
-sub new { bless { dig => PublicInbox::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
+sub new { bless [ PublicInbox::SHA->new(256), $_[1] ], __PACKAGE__ }
sub add {
- $_[0]->{dig}->add($_[1]);
- print { $_[0]->{fh} } Dumper([split(/^/sm, $_[1])]) or die "print $!";
+ $_[0]->[0]->add($_[1]);
+ print { $_[0]->[1] } Dumper([split(/^/sm, $_[1])]) or die "print $!";
}
-sub hexdigest { $_[0]->{dig}->hexdigest; }
+sub hexdigest { $_[0]->[0]->hexdigest }
1;
diff --git a/lib/PublicInbox/MailDiff.pm b/lib/PublicInbox/MailDiff.pm
index 0ed06f9a..a0ecef9f 100644
--- a/lib/PublicInbox/MailDiff.pm
+++ b/lib/PublicInbox/MailDiff.pm
@@ -4,8 +4,6 @@ package PublicInbox::MailDiff;
use v5.12;
use File::Temp 0.19 (); # 0.19 for ->newdir
use PublicInbox::ContentHash qw(content_digest);
-use PublicInbox::ContentDigestDbg;
-use Data::Dumper ();
use PublicInbox::MsgIter qw(msg_part_text);
use PublicInbox::ViewDiff qw(flush_diff);
use PublicInbox::GitAsyncCat;
@@ -34,12 +32,11 @@ sub dump_eml ($$$) {
$eml->each_part(\&write_part, $self);
return if $self->{ctx}; # don't need content_digest noise in WWW UI
+ require PublicInbox::ContentDigestDbg;
# XXX is this even useful? perhaps hide it behind a CLI switch
open my $fh, '>', "$dir/content_digest" or die "open: $!";
my $dig = PublicInbox::ContentDigestDbg->new($fh);
- local $Data::Dumper::Useqq = 1;
- local $Data::Dumper::Terse = 1;
content_digest($eml, $dig);
print $fh "\n", $dig->hexdigest, "\n" or die "print $!";
close $fh or die "close: $!";
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2023-01-29 10:30 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-01-29 10:30 [PATCH 0/2] allow OpenSSL SHA-(1|256) use if installed Eric Wong
2023-01-29 10:30 ` [PATCH 1/2] use Net::SSLeay (OpenSSL) for SHA-(1|256) " Eric Wong
2023-01-29 10:30 ` [PATCH 2/2] content_digest_dbg: convert to arrayref and limit to lei Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).