From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 17/30] syscall: common $F_SETPIPE_SZ definition
Date: Tue, 17 Oct 2023 23:38:02 +0000 [thread overview]
Message-ID: <20231017233815.1637932-18-e@80x24.org> (raw)
In-Reply-To: <20231017233815.1637932-1-e@80x24.org>
We use this in various places to minimize or maximize pipe
size on Linux. So keep it all in one place.
---
lib/PublicInbox/CidxLogP.pm | 4 ++--
lib/PublicInbox/EOFpipe.pm | 6 +++---
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/SearchIdxShard.pm | 14 +++++++-------
lib/PublicInbox/Syscall.pm | 16 ++++++++--------
t/gcf2.t | 5 +++--
t/lei-sigpipe.t | 7 +++----
7 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/lib/PublicInbox/CidxLogP.pm b/lib/PublicInbox/CidxLogP.pm
index 7877d5ac..34f7201d 100644
--- a/lib/PublicInbox/CidxLogP.pm
+++ b/lib/PublicInbox/CidxLogP.pm
@@ -10,12 +10,12 @@
package PublicInbox::CidxLogP;
use v5.12;
use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT $F_SETPIPE_SZ);
sub new {
my ($cls, $rd, $cidx, $git, $roots) = @_;
my $self = bless { cidx => $cidx, git => $git, roots => $roots }, $cls;
- fcntl($rd, 1031, 1048576) if $^O eq 'linux'; # fatter pipes
+ fcntl($rd, $F_SETPIPE_SZ, 1048576) if $F_SETPIPE_SZ;
$self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
}
diff --git a/lib/PublicInbox/EOFpipe.pm b/lib/PublicInbox/EOFpipe.pm
index 628e9366..3474874f 100644
--- a/lib/PublicInbox/EOFpipe.pm
+++ b/lib/PublicInbox/EOFpipe.pm
@@ -4,13 +4,13 @@
package PublicInbox::EOFpipe;
use v5.12;
use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT $F_SETPIPE_SZ);
sub new {
my (undef, $rd, $cb) = @_;
my $self = bless { cb => $cb }, __PACKAGE__;
- # 1031: F_SETPIPE_SZ, 4096: page size
- fcntl($rd, 1031, 4096) if $^O eq 'linux';
+ # 4096: page size
+ fcntl($rd, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
$self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d83a403c..25b66b3b 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -21,6 +21,7 @@ use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
use PublicInbox::ContentHash qw(git_sha);
use POSIX qw(strftime);
use autodie qw(open read seek truncate);
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
sub new {
my ($class) = @_;
@@ -536,7 +537,6 @@ sub do_query {
if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {
$lei->start_mua;
}
- my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
if ($l2m->{-wq_nr_workers} > 1 &&
$l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
# setup two barriers to coordinate ->has_entries
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 21bd56c2..1630eb4a 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -7,6 +7,7 @@ package PublicInbox::SearchIdxShard;
use v5.12;
use parent qw(PublicInbox::SearchIdx PublicInbox::IPC);
use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
sub new {
my ($class, $v2w, $shard) = @_; # v2w may be ExtSearchIdx
@@ -20,13 +21,12 @@ sub new {
if ($v2w->{parallel}) {
local $self->{-v2w_afc} = $v2w;
$self->ipc_worker_spawn("shard[$shard]");
- # F_SETPIPE_SZ = 1031 on Linux; increasing the pipe size for
- # inputs speeds V2Writable batch imports across 8 cores by
- # nearly 20%. Since any of our responses are small, make
- # the response pipe as small as possible
- if ($^O eq 'linux') {
- fcntl($self->{-ipc_req}, 1031, 1048576);
- fcntl($self->{-ipc_res}, 1031, 4096);
+ # Increasing the pipe size for requests speeds V2 batch imports
+ # across 8 cores by nearly 20%. Since many of our responses
+ # are small, make the response pipe as small as possible
+ if ($F_SETPIPE_SZ) {
+ fcntl($self->{-ipc_req}, $F_SETPIPE_SZ, 1048576);
+ fcntl($self->{-ipc_res}, $F_SETPIPE_SZ, 4096);
}
}
$self;
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index e83beb6a..78181bb6 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -28,7 +28,7 @@ our @EXPORT_OK = qw(epoll_ctl epoll_create epoll_wait
EPOLLIN EPOLLOUT EPOLLET
EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
EPOLLONESHOT EPOLLEXCLUSIVE
- signalfd rename_noreplace %SIGNUM);
+ signalfd rename_noreplace %SIGNUM $F_SETPIPE_SZ);
use constant {
EPOLLIN => 1,
EPOLLOUT => 4,
@@ -55,13 +55,12 @@ use constant {
my @BYTES_4_hole = BYTES_4_hole ? (0) : ();
-our (
- $SYS_epoll_create,
- $SYS_epoll_ctl,
- $SYS_epoll_wait,
- $SYS_signalfd4,
- $SYS_renameat2,
- );
+our ($SYS_epoll_create,
+ $SYS_epoll_ctl,
+ $SYS_epoll_wait,
+ $SYS_signalfd4,
+ $SYS_renameat2,
+ $F_SETPIPE_SZ);
my ($SYS_sendmsg, $SYS_recvmsg);
my $SYS_fstatfs; # don't need fstatfs64, just statfs.f_type
@@ -70,6 +69,7 @@ my $SFD_CLOEXEC = 02000000; # Perl does not expose O_CLOEXEC
our $no_deprecated = 0;
if ($^O eq "linux") {
+ $F_SETPIPE_SZ = 1031;
my (undef, undef, $release, undef, $machine) = POSIX::uname();
my ($maj, $min) = ($release =~ /\A([0-9]+)\.([0-9]+)/);
$SYS_renameat2 = 0 if "$maj.$min" < 3.15;
diff --git a/t/gcf2.t b/t/gcf2.t
index d12a4420..33f3bbca 100644
--- a/t/gcf2.t
+++ b/t/gcf2.t
@@ -10,6 +10,7 @@ use POSIX qw(_exit);
use Cwd qw(abs_path);
require_mods('PublicInbox::Gcf2');
use_ok 'PublicInbox::Gcf2';
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
use PublicInbox::Import;
my ($tmpdir, $for_destroy) = tmpdir();
@@ -109,7 +110,7 @@ SKIP: {
for my $blk (1, 0) {
my ($r, $w);
pipe($r, $w) or BAIL_OUT $!;
- fcntl($w, 1031, 4096) or
+ fcntl($w, $F_SETPIPE_SZ, 4096) or
skip('Linux too old for F_SETPIPE_SZ', 14);
$w->blocking($blk);
seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
@@ -129,7 +130,7 @@ SKIP: {
$ck_copying->("pipe blocking($blk)");
pipe($r, $w) or BAIL_OUT $!;
- fcntl($w, 1031, 4096) or BAIL_OUT $!;
+ fcntl($w, $F_SETPIPE_SZ, 4096) or BAIL_OUT $!;
$w->blocking($blk);
close $r;
local $SIG{PIPE} = 'IGNORE';
diff --git a/t/lei-sigpipe.t b/t/lei-sigpipe.t
index 55c208e2..622598a4 100644
--- a/t/lei-sigpipe.t
+++ b/t/lei-sigpipe.t
@@ -6,6 +6,7 @@ use v5.10.1;
use PublicInbox::TestCommon;
use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
# undo systemd (and similar) ignoring SIGPIPE, since lei expects to be run
# from an interactive terminal:
@@ -21,10 +22,8 @@ test_lei(sub {
my $imported;
for my $out ([], [qw(-f mboxcl2)], [qw(-f text)]) {
pipe(my ($r, $w)) or BAIL_OUT $!;
- my $size = 65536;
- if ($^O eq 'linux' && fcntl($w, 1031, 4096)) {
- $size = 4096;
- }
+ my $size = $F_SETPIPE_SZ && fcntl($w, $F_SETPIPE_SZ, 4096) ?
+ 4096 : 65536;
unless (-f $f) {
open my $fh, '>', $f or xbail "open $f: $!";
print $fh <<'EOM' or xbail;
next prev parent reply other threads:[~2023-10-17 23:38 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-10-17 23:37 [PATCH 00/30] autodie-ification and code simplifications Eric Wong
2023-10-17 23:37 ` [PATCH 01/30] lei_mirror: start converting to autodie Eric Wong
2023-10-17 23:37 ` [PATCH 02/30] lei_mirror: autodie most `close' calls Eric Wong
2023-10-17 23:37 ` [PATCH 03/30] lei_mirror: use autodie for most `open' calls Eric Wong
2023-10-17 23:37 ` [PATCH 04/30] git: introduce read_all function Eric Wong
2023-10-17 23:37 ` [PATCH 05/30] import: use read_all to detect short reads Eric Wong
2023-10-17 23:37 ` [PATCH 06/30] lei_mirror: use read_all Eric Wong
2023-10-17 23:37 ` [PATCH 07/30] use read_all in more places to improve safety Eric Wong
2023-10-17 23:37 ` [PATCH 08/30] xap_helper*: use autodie in more places Eric Wong
2023-10-17 23:37 ` [PATCH 09/30] xap_helper: die more easily in both implementations Eric Wong
2023-10-17 23:37 ` [PATCH 10/30] xap_helper: simplify SIGTERM exit checks Eric Wong
2023-10-17 23:37 ` [PATCH 11/30] xap_helper: autodie for getsockopt Eric Wong
2023-10-17 23:37 ` [PATCH 12/30] xap_client: autodie for pipe and socketpair Eric Wong
2023-10-17 23:37 ` [PATCH 13/30] xt/git-http-backend: remove Net::HTTP usage Eric Wong
2023-10-17 23:37 ` [PATCH 14/30] ds: introduce and use do_fork helper Eric Wong
2023-10-17 23:38 ` [PATCH 15/30] ds: get rid of SetLoopTimeout Eric Wong
2023-10-17 23:38 ` [PATCH 16/30] cindex: drop some unused functions Eric Wong
2023-10-17 23:38 ` Eric Wong [this message]
2023-10-17 23:38 ` [PATCH 18/30] t/lei-up: additional diagnostics for match failures Eric Wong
2023-10-17 23:38 ` [PATCH 19/30] test_common: use autodie and read_all where possible Eric Wong
2023-10-17 23:38 ` [PATCH 20/30] test_common: only hide TCP port in messages Eric Wong
2023-10-17 23:38 ` [PATCH 21/30] test_common: use $cwdfh for every run_script command Eric Wong
2023-10-17 23:38 ` [PATCH 22/30] init: drop extraneous `+' Eric Wong
2023-10-17 23:38 ` [PATCH 23/30] init: use autodie to reduce distractions Eric Wong
2023-10-17 23:38 ` [PATCH 24/30] xt/mem-imapd-tls: remove unused/broken epoll imports Eric Wong
2023-10-17 23:38 ` [PATCH 25/30] xt/mem-imapd-tls: reduce FDs for lsof use Eric Wong
2023-10-17 23:38 ` [PATCH 26/30] lei: use autodie where appropriate Eric Wong
2023-10-17 23:38 ` [PATCH 27/30] lei_auth: update comments and use v5.12 Eric Wong
2023-10-17 23:38 ` [PATCH 28/30] lei_config: drop redundant open check Eric Wong
2023-10-17 23:38 ` [PATCH 29/30] convert: use read_all to simplify error checks Eric Wong
2023-10-17 23:38 ` [PATCH 30/30] idx_stack: use autodie + read_all Eric Wong
2023-10-19 1:14 ` [PATCH 31/30] lei: simplify startq/au_done wakeup notifications Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20231017233815.1637932-18-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).