[PATCH 0/3] some portability and dependency fixes

[PATCH 0/3] some portability and dependency fixes

[Eric Wong]

The first two patches fix some dependency problems I noticed
while working on ci/* stuff.  3/3 improves portability to
NetBSD and OpenBSD while I got to play with CPP macros a bit :P

Thinking out loud:

  While syscall.ph and friends aren't usable given the use of
  macros on most systemd and time-of-generation-to-time-of-use
  differences on *BSD systems; perhaps something like:

    eval %x{$^X -MPublicInbox::CompileSys}

  Could be usable without requiring Inline::C for sendmsg/recvmsg
  (but still require a C compiler, as `cc' is commonly-installed
  while Inline::C is not).

Eric Wong (3):
  fix some tests when `curl' is missing
  imapd: lazy-load IMAPsearchqp for Parse::RecDescent
  pop3d: support fcntl locks on NetBSD and OpenBSD

 devel/sysdefs-list       | 81 ++++++++++++++++++++++++++++++++++++++--
 lib/PublicInbox/IMAP.pm  |  2 +-
 lib/PublicInbox/IMAPD.pm |  1 +
 lib/PublicInbox/POP3D.pm |  3 +-
 t/clone-coderepo.t       |  1 +
 t/pop3d.t                |  5 ++-
 t/www_listing.t          |  3 +-
 7 files changed, 88 insertions(+), 8 deletions(-)

[PATCH 1/3] fix some tests when `curl' is missing

[Eric Wong]

Debian-based distros allow uninstalling the `curl' CLI tool
independently of libcurl* packages which git depends on.
---
 t/clone-coderepo.t | 1 +
 t/www_listing.t    | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/t/clone-coderepo.t b/t/clone-coderepo.t
index e28041e9..bce4ecd9 100644
--- a/t/clone-coderepo.t
+++ b/t/clone-coderepo.t
@@ -9,6 +9,7 @@ use File::Path qw(remove_tree);
 use PublicInbox::SHA qw(sha1_hex);
 require_mods(qw(json Plack::Builder HTTP::Date HTTP::Status));
 require_git '1.8.5';
+require_cmd 'curl';
 require_ok 'PublicInbox::LeiMirror';
 my ($tmpdir, $for_destroy) = tmpdir();
 my $pa = "$tmpdir/src/a.git";
diff --git a/t/www_listing.t b/t/www_listing.t
index 4784c39d..709dbd05 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -2,10 +2,11 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # manifest.js.gz generation and grok-pull integration test
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use PublicInbox::Import;
 use IO::Uncompress::Gunzip qw(gunzip);
 require_mods(qw(json URI::Escape Plack::Builder HTTP::Tiny));
+require_cmd 'curl';
 require PublicInbox::WwwListing;
 require PublicInbox::ManifestJsGz;
 use PublicInbox::Config;

[PATCH 2/3] imapd: lazy-load IMAPsearchqp for Parse::RecDescent

[Eric Wong]

This enables the t/pop3d.t test to pass when Parse::RecDescent
is not available.
---
 lib/PublicInbox/IMAP.pm  | 2 +-
 lib/PublicInbox/IMAPD.pm | 1 +
 t/pop3d.t                | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 00f99ef7..3c64cefa 100644
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -39,7 +39,6 @@ use PublicInbox::DS qw(now);
 use PublicInbox::GitAsyncCat;
 use Text::ParseWords qw(parse_line);
 use Errno qw(EAGAIN);
-use PublicInbox::IMAPsearchqp;
 
 my $Address;
 for my $mod (qw(Email::Address::XS Mail::Address)) {
@@ -1088,6 +1087,7 @@ sub search_uid_range { # long_response
 
 sub parse_imap_query ($$) {
 	my ($self, $query) = @_;
+	# IMAPsearchqp gets loaded in IMAPD->refresh_groups
 	my $q = PublicInbox::IMAPsearchqp::parse($self, $query);
 	if (ref($q)) {
 		my $max = $self->{ibx}->over(1)->max;
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 78323e57..bdadb7a3 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -55,6 +55,7 @@ sub _refresh_ibx { # pi_cfg->each_inbox cb
 sub refresh_groups {
 	my ($self, $sig) = @_;
 	my $pi_cfg = PublicInbox::Config->new;
+	require PublicInbox::IMAPsearchqp;
 	$self->{mailboxes} = $pi_cfg->{-imap_mailboxes} // do {
 		my $mailboxes = $self->{mailboxes} = {};
 		my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {};
diff --git a/t/pop3d.t b/t/pop3d.t
index dc52b0cf..001ceda8 100644
--- a/t/pop3d.t
+++ b/t/pop3d.t
@@ -276,7 +276,7 @@ EOF
 		like($x, qr/\Adataready\0+\z/, 'got dataready accf for pop3s');
 		$x = getsockopt($stls, IPPROTO_TCP,
 				$PublicInbox::Daemon::SO_ACCEPTFILTER);
-		is($x, undef, 'no BSD accept filter for plain IMAP');
+		is($x, undef, 'no BSD accept filter for plain POP3');
 	};
 
 	$td->kill;

[PATCH 3/3] pop3d: support fcntl locks on NetBSD and OpenBSD

[Eric Wong]

MboxLock already supported it since it locked the whole file,
but POP3D requires more fine-grained locking at file offsets.

I wonder if "struct flock" is old enough for it to be the same
across all the BSDs, it certainly seems so.

I originally considered using C11 `_Generic' support for the
struct offset/type dumping as I have in other projects, but
I am not ready to depend on C11 for this project, yet.

While we're modifying devel/sysdefs-list, add some Linux-only
structs to verify our `pack' templates are correct and remain
so when we encounter new architectures.
---
 devel/sysdefs-list       | 81 ++++++++++++++++++++++++++++++++++++++--
 lib/PublicInbox/POP3D.pm |  3 +-
 t/pop3d.t                |  3 +-
 3 files changed, 82 insertions(+), 5 deletions(-)

diff --git a/devel/sysdefs-list b/devel/sysdefs-list
index 9764cc29..aa7806ae 100755
--- a/devel/sysdefs-list
+++ b/devel/sysdefs-list
@@ -27,25 +27,57 @@ open my $fh, '>', $f or die "open $f $!";
 print $fh $str or die "print $f $!";
 close $fh or die "close $f $!";
 system($cc, '-o', $x, $f, @cflags) == 0 or die "$cc failed \$?=$?";
+say '%Config', map { " $_=$Config{$_}" } qw(ptrsize sizesize lseeksize);
 exec($x);
 __DATA__
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
+#include <assert.h>
 #include <signal.h>
+#include <stddef.h>
 #include <sys/syscall.h>
 #include <sys/ioctl.h>
 #ifdef __linux__
-#include <linux/fs.h>
+#	include <linux/fs.h>
+#	include <sys/epoll.h>
+#	include <sys/inotify.h>
+#	include <sys/socket.h> // we don't care about this on *BSD
+#	include <sys/vfs.h>
 #endif
 #include <sys/types.h>
+#include <fcntl.h>
 #include <unistd.h>
 #include <stdio.h>
 
+#define STRUCT_BEGIN(t) do { t x; printf(#t" => %zu bytes\n", sizeof(x))
+#define STRUCT_END } while (0)
+
+// prints the struct field name, @offset, and signed/unsigned bit size
+#define PR_NUM(f) do { \
+	x.f = ~0; \
+	printf("\t.%s @%zu %c%zu\n", #f, \
+		offsetof(typeof(x),f), \
+		x.f > 0 ? 'u' : 's', \
+		sizeof(x.f) * 8); \
+} while (0)
+
+#define PR_PTR(f) do { \
+	assert(sizeof(x.f) == sizeof(void *)); \
+	printf("\t.%s @%zu ptr\n", #f, offsetof(typeof(x),f)); \
+} while (0)
+
+#define PR_OFF(f) do { \
+	printf("\t.%s @%zu\n", #f, offsetof(typeof(x),f)); \
+} while (0)
+
 #define D(x) printf("$" #x " = %ld;\n", (long)x)
 
 int main(void)
 {
+	// verify Config{(ptr|size|lseek)size} entries match:
+	printf("sizeof ptr=%zu size_t=%zu off_t=%zu\n",
+		sizeof(void *), sizeof(size_t), sizeof(off_t));
 #ifdef __linux__
 	D(SYS_epoll_create1);
 	D(SYS_epoll_ctl);
@@ -64,11 +96,54 @@ int main(void)
 		(unsigned long)FS_IOC_GETFLAGS, (unsigned long)FS_IOC_SETFLAGS);
 #endif
 	MAYBE SYS_renameat2
+
+	STRUCT_BEGIN(struct epoll_event);
+		PR_NUM(events);
+		PR_NUM(data.u64);
+	STRUCT_END;
+
+	STRUCT_BEGIN(struct inotify_event);
+		PR_NUM(wd);
+		PR_NUM(mask);
+		PR_NUM(cookie);
+		PR_NUM(len);
+		PR_OFF(name);
+	STRUCT_END;
+
+	/*
+	 * msghdr and cmsghdr are portable, but we only care about its layout
+	 * on OSes like Linux with stable syscall numbers
+	 */
+	STRUCT_BEGIN(struct msghdr);
+		PR_PTR(msg_name);
+		PR_NUM(msg_namelen);
+		PR_PTR(msg_iov);
+		PR_NUM(msg_iovlen);
+		PR_PTR(msg_control);
+		PR_NUM(msg_controllen);
+		PR_NUM(msg_flags);
+	STRUCT_END;
+
+	STRUCT_BEGIN(struct cmsghdr);
+		PR_NUM(cmsg_len);
+		PR_NUM(cmsg_level);
+		PR_NUM(cmsg_type);
+	STRUCT_END;
+
+	STRUCT_BEGIN(struct statfs);
+		PR_NUM(f_type);
+	STRUCT_END;
 #endif /* Linux, any other OSes with stable syscalls? */
-	printf("size_t=%zu off_t=%zu pid_t=%zu\n",
-		 sizeof(size_t), sizeof(off_t), sizeof(pid_t));
 	D(SIGWINCH);
 	MAYBE _SC_NPROCESSORS_ONLN
 
+	STRUCT_BEGIN(struct flock);
+		PR_NUM(l_start);
+		PR_NUM(l_len);
+		PR_NUM(l_pid);
+		PR_NUM(l_type);
+		PR_NUM(l_whence);
+	STRUCT_END;
+
 	return 0;
 }
diff --git a/lib/PublicInbox/POP3D.pm b/lib/PublicInbox/POP3D.pm
index 3fc85efc..580a26d3 100644
--- a/lib/PublicInbox/POP3D.pm
+++ b/lib/PublicInbox/POP3D.pm
@@ -14,6 +14,7 @@ use PublicInbox::Syscall;
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use Fcntl qw(F_SETLK F_UNLCK F_WRLCK SEEK_SET);
 my @FLOCK;
+# are all BSDs the same "struct flock"? tested Free+Net+Open...
 if ($^O eq 'linux' || $^O =~ /bsd/) {
 	require Config;
 	my $off_t;
@@ -27,7 +28,7 @@ if ($^O eq 'linux' || $^O =~ /bsd/) {
 		if ($^O eq 'linux') {
 			@FLOCK = ("ss\@8$off_t$off_t\@32",
 				qw(l_type l_whence l_start l_len));
-		} elsif ($^O =~ /bsd/) {
+		} elsif ($^O =~ /bsd/) { # @32 may be enough
 			@FLOCK = ("${off_t}${off_t}lss\@256",
 				qw(l_start l_len l_pid l_type l_whence));
 		}
diff --git a/t/pop3d.t b/t/pop3d.t
index 001ceda8..2a7a7c3f 100644
--- a/t/pop3d.t
+++ b/t/pop3d.t
@@ -14,7 +14,8 @@ unless (-r $key && -r $cert) {
 # Net::POP3 is part of the standard library, but distros may split it off...
 require_mods(qw(DBD::SQLite Net::POP3 IO::Socket::SSL));
 require_git('2.6'); # for v2
-require_mods(qw(File::FcntlLock)) if $^O !~ /\A(?:linux|freebsd)\z/;
+$^O =~ /\A(?:linux|(?:free|net|open)bsd)\z/ or
+	require_mods(qw(File::FcntlLock));
 use_ok 'IO::Socket::SSL';
 use_ok 'PublicInbox::TLS';
 my ($tmpdir, $for_destroy) = tmpdir();