unofficial mirror of meta@public-inbox.org
 help / color / mirror / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 17/30] syscall: common $F_SETPIPE_SZ definition
Date: Tue, 17 Oct 2023 23:38:02 +0000	[thread overview]
Message-ID: <20231017233815.1637932-18-e@80x24.org> (raw)
In-Reply-To: <20231017233815.1637932-1-e@80x24.org>

We use this in various places to minimize or maximize pipe
size on Linux.  So keep it all in one place.
---
 lib/PublicInbox/CidxLogP.pm       |  4 ++--
 lib/PublicInbox/EOFpipe.pm        |  6 +++---
 lib/PublicInbox/LeiXSearch.pm     |  2 +-
 lib/PublicInbox/SearchIdxShard.pm | 14 +++++++-------
 lib/PublicInbox/Syscall.pm        | 16 ++++++++--------
 t/gcf2.t                          |  5 +++--
 t/lei-sigpipe.t                   |  7 +++----
 7 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/lib/PublicInbox/CidxLogP.pm b/lib/PublicInbox/CidxLogP.pm
index 7877d5ac..34f7201d 100644
--- a/lib/PublicInbox/CidxLogP.pm
+++ b/lib/PublicInbox/CidxLogP.pm
@@ -10,12 +10,12 @@
 package PublicInbox::CidxLogP;
 use v5.12;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT $F_SETPIPE_SZ);
 
 sub new {
 	my ($cls, $rd, $cidx, $git, $roots) = @_;
 	my $self = bless { cidx => $cidx, git => $git, roots => $roots }, $cls;
-	fcntl($rd, 1031, 1048576) if $^O eq 'linux'; # fatter pipes
+	fcntl($rd, $F_SETPIPE_SZ, 1048576) if $F_SETPIPE_SZ;
 	$self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
 }
 
diff --git a/lib/PublicInbox/EOFpipe.pm b/lib/PublicInbox/EOFpipe.pm
index 628e9366..3474874f 100644
--- a/lib/PublicInbox/EOFpipe.pm
+++ b/lib/PublicInbox/EOFpipe.pm
@@ -4,13 +4,13 @@
 package PublicInbox::EOFpipe;
 use v5.12;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT $F_SETPIPE_SZ);
 
 sub new {
 	my (undef, $rd, $cb) = @_;
 	my $self = bless { cb => $cb }, __PACKAGE__;
-	# 1031: F_SETPIPE_SZ, 4096: page size
-	fcntl($rd, 1031, 4096) if $^O eq 'linux';
+	# 4096: page size
+	fcntl($rd, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
 	$self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
 }
 
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d83a403c..25b66b3b 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -21,6 +21,7 @@ use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
 use PublicInbox::ContentHash qw(git_sha);
 use POSIX qw(strftime);
 use autodie qw(open read seek truncate);
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 
 sub new {
 	my ($class) = @_;
@@ -536,7 +537,6 @@ sub do_query {
 		if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {
 			$lei->start_mua;
 		}
-		my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
 		if ($l2m->{-wq_nr_workers} > 1 &&
 				$l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
 			# setup two barriers to coordinate ->has_entries
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 21bd56c2..1630eb4a 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -7,6 +7,7 @@ package PublicInbox::SearchIdxShard;
 use v5.12;
 use parent qw(PublicInbox::SearchIdx PublicInbox::IPC);
 use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 
 sub new {
 	my ($class, $v2w, $shard) = @_; # v2w may be ExtSearchIdx
@@ -20,13 +21,12 @@ sub new {
 	if ($v2w->{parallel}) {
 		local $self->{-v2w_afc} = $v2w;
 		$self->ipc_worker_spawn("shard[$shard]");
-		# F_SETPIPE_SZ = 1031 on Linux; increasing the pipe size for
-		# inputs speeds V2Writable batch imports across 8 cores by
-		# nearly 20%.  Since any of our responses are small, make
-		# the response pipe as small as possible
-		if ($^O eq 'linux') {
-			fcntl($self->{-ipc_req}, 1031, 1048576);
-			fcntl($self->{-ipc_res}, 1031, 4096);
+		# Increasing the pipe size for requests speeds V2 batch imports
+		# across 8 cores by nearly 20%.  Since many of our responses
+		# are small, make the response pipe as small as possible
+		if ($F_SETPIPE_SZ) {
+			fcntl($self->{-ipc_req}, $F_SETPIPE_SZ, 1048576);
+			fcntl($self->{-ipc_res}, $F_SETPIPE_SZ, 4096);
 		}
 	}
 	$self;
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index e83beb6a..78181bb6 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -28,7 +28,7 @@ our @EXPORT_OK = qw(epoll_ctl epoll_create epoll_wait
                   EPOLLIN EPOLLOUT EPOLLET
                   EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
                   EPOLLONESHOT EPOLLEXCLUSIVE
-                  signalfd rename_noreplace %SIGNUM);
+                  signalfd rename_noreplace %SIGNUM $F_SETPIPE_SZ);
 use constant {
 	EPOLLIN => 1,
 	EPOLLOUT => 4,
@@ -55,13 +55,12 @@ use constant {
 
 my @BYTES_4_hole = BYTES_4_hole ? (0) : ();
 
-our (
-     $SYS_epoll_create,
-     $SYS_epoll_ctl,
-     $SYS_epoll_wait,
-     $SYS_signalfd4,
-     $SYS_renameat2,
-     );
+our ($SYS_epoll_create,
+	$SYS_epoll_ctl,
+	$SYS_epoll_wait,
+	$SYS_signalfd4,
+	$SYS_renameat2,
+	$F_SETPIPE_SZ);
 
 my ($SYS_sendmsg, $SYS_recvmsg);
 my $SYS_fstatfs; # don't need fstatfs64, just statfs.f_type
@@ -70,6 +69,7 @@ my $SFD_CLOEXEC = 02000000; # Perl does not expose O_CLOEXEC
 our $no_deprecated = 0;
 
 if ($^O eq "linux") {
+	$F_SETPIPE_SZ = 1031;
     my (undef, undef, $release, undef, $machine) = POSIX::uname();
     my ($maj, $min) = ($release =~ /\A([0-9]+)\.([0-9]+)/);
     $SYS_renameat2 = 0 if "$maj.$min" < 3.15;
diff --git a/t/gcf2.t b/t/gcf2.t
index d12a4420..33f3bbca 100644
--- a/t/gcf2.t
+++ b/t/gcf2.t
@@ -10,6 +10,7 @@ use POSIX qw(_exit);
 use Cwd qw(abs_path);
 require_mods('PublicInbox::Gcf2');
 use_ok 'PublicInbox::Gcf2';
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 use PublicInbox::Import;
 my ($tmpdir, $for_destroy) = tmpdir();
 
@@ -109,7 +110,7 @@ SKIP: {
 	for my $blk (1, 0) {
 		my ($r, $w);
 		pipe($r, $w) or BAIL_OUT $!;
-		fcntl($w, 1031, 4096) or
+		fcntl($w, $F_SETPIPE_SZ, 4096) or
 			skip('Linux too old for F_SETPIPE_SZ', 14);
 		$w->blocking($blk);
 		seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
@@ -129,7 +130,7 @@ SKIP: {
 		$ck_copying->("pipe blocking($blk)");
 
 		pipe($r, $w) or BAIL_OUT $!;
-		fcntl($w, 1031, 4096) or BAIL_OUT $!;
+		fcntl($w, $F_SETPIPE_SZ, 4096) or BAIL_OUT $!;
 		$w->blocking($blk);
 		close $r;
 		local $SIG{PIPE} = 'IGNORE';
diff --git a/t/lei-sigpipe.t b/t/lei-sigpipe.t
index 55c208e2..622598a4 100644
--- a/t/lei-sigpipe.t
+++ b/t/lei-sigpipe.t
@@ -6,6 +6,7 @@ use v5.10.1;
 use PublicInbox::TestCommon;
 use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
 use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 
 # undo systemd (and similar) ignoring SIGPIPE, since lei expects to be run
 # from an interactive terminal:
@@ -21,10 +22,8 @@ test_lei(sub {
 	my $imported;
 	for my $out ([], [qw(-f mboxcl2)], [qw(-f text)]) {
 		pipe(my ($r, $w)) or BAIL_OUT $!;
-		my $size = 65536;
-		if ($^O eq 'linux' && fcntl($w, 1031, 4096)) {
-			$size = 4096;
-		}
+		my $size = $F_SETPIPE_SZ && fcntl($w, $F_SETPIPE_SZ, 4096) ?
+			4096 : 65536;
 		unless (-f $f) {
 			open my $fh, '>', $f or xbail "open $f: $!";
 			print $fh <<'EOM' or xbail;

  parent reply	other threads:[~2023-10-17 23:38 UTC|newest]

Thread overview: 32+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-10-17 23:37 [PATCH 00/30] autodie-ification and code simplifications Eric Wong
2023-10-17 23:37 ` [PATCH 01/30] lei_mirror: start converting to autodie Eric Wong
2023-10-17 23:37 ` [PATCH 02/30] lei_mirror: autodie most `close' calls Eric Wong
2023-10-17 23:37 ` [PATCH 03/30] lei_mirror: use autodie for most `open' calls Eric Wong
2023-10-17 23:37 ` [PATCH 04/30] git: introduce read_all function Eric Wong
2023-10-17 23:37 ` [PATCH 05/30] import: use read_all to detect short reads Eric Wong
2023-10-17 23:37 ` [PATCH 06/30] lei_mirror: use read_all Eric Wong
2023-10-17 23:37 ` [PATCH 07/30] use read_all in more places to improve safety Eric Wong
2023-10-17 23:37 ` [PATCH 08/30] xap_helper*: use autodie in more places Eric Wong
2023-10-17 23:37 ` [PATCH 09/30] xap_helper: die more easily in both implementations Eric Wong
2023-10-17 23:37 ` [PATCH 10/30] xap_helper: simplify SIGTERM exit checks Eric Wong
2023-10-17 23:37 ` [PATCH 11/30] xap_helper: autodie for getsockopt Eric Wong
2023-10-17 23:37 ` [PATCH 12/30] xap_client: autodie for pipe and socketpair Eric Wong
2023-10-17 23:37 ` [PATCH 13/30] xt/git-http-backend: remove Net::HTTP usage Eric Wong
2023-10-17 23:37 ` [PATCH 14/30] ds: introduce and use do_fork helper Eric Wong
2023-10-17 23:38 ` [PATCH 15/30] ds: get rid of SetLoopTimeout Eric Wong
2023-10-17 23:38 ` [PATCH 16/30] cindex: drop some unused functions Eric Wong
2023-10-17 23:38 ` Eric Wong [this message]
2023-10-17 23:38 ` [PATCH 18/30] t/lei-up: additional diagnostics for match failures Eric Wong
2023-10-17 23:38 ` [PATCH 19/30] test_common: use autodie and read_all where possible Eric Wong
2023-10-17 23:38 ` [PATCH 20/30] test_common: only hide TCP port in messages Eric Wong
2023-10-17 23:38 ` [PATCH 21/30] test_common: use $cwdfh for every run_script command Eric Wong
2023-10-17 23:38 ` [PATCH 22/30] init: drop extraneous `+' Eric Wong
2023-10-17 23:38 ` [PATCH 23/30] init: use autodie to reduce distractions Eric Wong
2023-10-17 23:38 ` [PATCH 24/30] xt/mem-imapd-tls: remove unused/broken epoll imports Eric Wong
2023-10-17 23:38 ` [PATCH 25/30] xt/mem-imapd-tls: reduce FDs for lsof use Eric Wong
2023-10-17 23:38 ` [PATCH 26/30] lei: use autodie where appropriate Eric Wong
2023-10-17 23:38 ` [PATCH 27/30] lei_auth: update comments and use v5.12 Eric Wong
2023-10-17 23:38 ` [PATCH 28/30] lei_config: drop redundant open check Eric Wong
2023-10-17 23:38 ` [PATCH 29/30] convert: use read_all to simplify error checks Eric Wong
2023-10-17 23:38 ` [PATCH 30/30] idx_stack: use autodie + read_all Eric Wong
2023-10-19  1:14 ` [PATCH 31/30] lei: simplify startq/au_done wakeup notifications Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20231017233815.1637932-18-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).