[PATCH 0/4] doc and ci scripts updates

[PATCH 0/4] doc and ci scripts updates

[Eric Wong]

Hoping these can keep minimal, full, and in-between
installations working reasonably well on all platforms.
OpenBSD pkg_add support has started (not sure what to
call it to not conflict with FreeBSD `pkg')

ci/profiles.sh will probably be rewritten in Perl,
and NetBSD (`pkgin') updates will be coming, too.

Eric Wong (4):
  update docs + tests for xapian-delve use
  ci/deps: add IMAP-related optional packages
  ci/deps: drop unnecessary mappings and add Inline
  ci: updates for OpenBSD

 Documentation/txt2pre |   2 +
 INSTALL               |   5 +-
 ci/deps.perl          | 119 +++++++++++++++++++++++++++---------------
 ci/profiles.sh        |  23 +++++++-
 t/cindex.t            |  14 +++--
 5 files changed, 114 insertions(+), 49 deletions(-)

[PATCH 1/4] update docs + tests for xapian-delve use

[Eric Wong]

Since -cindex uses the xapian-delve(1) command for `--prune'
functionality, we'll rename our `xapian-compact' dependency to
the Debian package name (xapian-tools) since `xapian-delve' is
in the same package.
---
 Documentation/txt2pre |  2 ++
 INSTALL               |  5 +++--
 ci/deps.perl          | 12 ++++++------
 t/cindex.t            | 14 ++++++++++----
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 62175f34..c258a90e 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -147,6 +147,8 @@ $xurls{'copydatabase(1)'} =
  'https://manpages.debian.org/stable/xapian-tools/copydatabase.1.en.html';
 $xurls{'xapian-compact(1)'} =
  'https://manpages.debian.org/stable/xapian-tools/xapian-compact.1.en.html';
+$xurls{'xapian-delve(1)'} =
+ 'https://manpages.debian.org/stable/xapian-tools/xapian-delve.1.en.html';
 $xurls{'gzip(1)'} = 'https://manpages.debian.org/stable/gzip/gzip.1.en.html';
 $xurls{'chmod(1)'} =
 	'https://manpages.debian.org/stable/coreutils/chmod.1.en.html';
diff --git a/INSTALL b/INSTALL
index 2e1c7ef7..617801cc 100644
--- a/INSTALL
+++ b/INSTALL
@@ -111,10 +111,11 @@ Numerous optional modules are likely to be useful as well:
 * highlight                        deb: libhighlight-perl
                                    (for syntax highlighting with coderepo)
 
-* xapian-compact (tool)            deb: xapian-tools
+* xapian-tools                     deb: xapian-tools
                                    pkg: xapian-core
                                    rpm: xapian-core
-                                   (only for public-inbox-compact(1))
+                                   (for public-inbox-compact(1) and
+				    public-inbox-cindex(1))
 
 * curl (tool)                      deb, pkg, rpm: curl
                                    (for HTTP(S) externals with curl)
diff --git a/ci/deps.perl b/ci/deps.perl
index ae85986d..94266029 100755
--- a/ci/deps.perl
+++ b/ci/deps.perl
@@ -1,10 +1,10 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Helper script for installing/uninstalling packages for CI use
 # Intended for use on non-production chroots or VMs since it
 # changes installed packages
-use strict;
+use v5.12;
 my $usage = "$0 PKG_FMT PROFILE [PROFILE_MOD]";
 my $pkg_fmt = shift;
 @ARGV or die $usage, "\n";
@@ -38,7 +38,7 @@ my $profiles = {
 		Search::Xapian
 		Socket6
 		highlight.pm
-		xapian-compact
+		xapian-tools
 		) ],
 
 	# optional developer stuff
@@ -119,9 +119,9 @@ my $non_auto = {
 		rpm => [],
 	},
 
-	# we call xapian-compact(1) in public-inbox-compact(1)
-	'xapian-compact' => {
-		deb => 'xapian-tools',
+	# we call xapian-compact(1) in public-inbox-compact(1) and
+	# xapian-delve(1) in public-inbox-cindex(1)
+	'xapian-tools' => {
 		pkg => 'xapian-core',
 		rpm => 'xapian-core', # ???
 	},
diff --git a/t/cindex.t b/t/cindex.t
index eca6cda2..9f57bd83 100644
--- a/t/cindex.t
+++ b/t/cindex.t
@@ -79,7 +79,7 @@ ok(run_script([qw(-cindex -L medium --dangerous -q -d),
 
 
 SKIP: {
-	have_xapian_compact;
+	have_xapian_compact 2;
 	ok(run_script([qw(-compact -q), "$tmp/ext"]), 'compact on full');
 	ok(run_script([qw(-compact -q), "$tmp/med"]), 'compact on medium');
 }
@@ -159,7 +159,8 @@ EOM
 	is(scalar($mset->items), 1, 'same result after reindex');
 }
 
-if ('--prune') {
+SKIP: { # --prune
+	require_cmd($ENV{XAPIAN_DELVE} || 'xapian-delve', 1);
 	my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
 	is(scalar($csrch->mset('s:hi')->items), 1, 'got hit');
 
@@ -188,7 +189,12 @@ ok(run_script([qw(-cindex --dangerous -q -d), "$tmp/ext", $zp]),
 
 ok(run_script([qw(-xcpdb), "$tmp/ext"]), 'xcpdb upgrade');
 ok(run_script([qw(-xcpdb -R4), "$tmp/ext"]), 'xcpdb reshard');
-ok(run_script([qw(-xcpdb -R2 --compact), "$tmp/ext"]), 'xcpdb reshard+compact');
-ok(run_script([qw(-xcpdb --compact), "$tmp/ext"]), 'xcpdb compact');
+
+SKIP: {
+	have_xapian_compact 2;
+	ok(run_script([qw(-xcpdb -R2 --compact), "$tmp/ext"]),
+		'xcpdb reshard+compact');
+	ok(run_script([qw(-xcpdb --compact), "$tmp/ext"]), 'xcpdb compact');
+};
 
 done_testing;

[PATCH 2/4] ci/deps: add IMAP-related optional packages

[Eric Wong]

Mail::IMAPClient and Parse::RecDescent are widely available in
all relevant OS package/ports systems.
---
 ci/deps.perl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/deps.perl b/ci/deps.perl
index 94266029..119ef07f 100755
--- a/ci/deps.perl
+++ b/ci/deps.perl
@@ -31,7 +31,9 @@ my $profiles = {
 		DBD::SQLite
 		DBI
 		Inline::C
+		Mail::IMAPClient
 		Net::Server
+		Parse::RecDescent
 		Plack
 		Plack::Test
 		Plack::Middleware::ReverseProxy

[PATCH 3/4] ci/deps: drop unnecessary mappings and add Inline

[Eric Wong]

The automatic mapping can work for more packages, so redundant
entries in $non_auto are just clutter.

Unfortunately, `Inline::C' is part of `Inline' on CentOS 7.x and
OpenBSD 7.3, so we'll add $non_auto mappings for those.

We'll also depend on `IO::Compress' to simplify mappings since
that's the CPAN distribution which holds both IO::Compress::Gzip
and IO::Compress::Gunzip and I'm not aware of any packagers who
split them.
---
 ci/deps.perl | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/ci/deps.perl b/ci/deps.perl
index 119ef07f..e7d43cd9 100755
--- a/ci/deps.perl
+++ b/ci/deps.perl
@@ -11,7 +11,9 @@ my $pkg_fmt = shift;
 
 my @test_essential = qw(Test::Simple); # we actually use Test::More
 
-# package profiles
+# package profiles.  Note we specify packages at maximum granularity,
+# which is typically deb for most things, but rpm seems to have the
+# highest granularity for things in the Prl standard library.
 my $profiles = {
 	# the smallest possible profile for testing
 	essential => [ qw(
@@ -20,7 +22,7 @@ my $profiles = {
 		Digest::SHA
 		Encode
 		ExtUtils::MakeMaker
-		IO::Compress::Gzip
+		IO::Compress
 		URI
 		), @test_essential ],
 
@@ -55,7 +57,7 @@ my $profiles = {
 # account for granularity differences between package systems and OSes
 my @precious;
 if ($^O eq 'freebsd') {
-	@precious = qw(perl curl Socket6 IO::Compress::Gzip);
+	@precious = qw(perl curl Socket6 IO::Compress);
 } elsif ($pkg_fmt eq 'rpm') {
 	@precious = qw(perl curl);
 }
@@ -87,33 +89,26 @@ my $non_auto = {
 	'Encode' => {
 		deb => 'perl', # libperl5.XX, but the XX varies
 		pkg => 'perl5',
-		rpm => 'perl-Encode',
 	},
 	'ExtUtils::MakeMaker' => {
 		deb => 'perl', # perl-modules-5.xx
 		pkg => 'perl5',
-		rpm => 'perl-ExtUtils-MakeMaker',
 	},
-	'IO::Compress::Gzip' => {
+	'IO::Compress' => {
 		deb => 'perl', # perl-modules-5.xx
 		pkg => 'perl5',
-		rpm => 'perl-IO-Compress',
+	},
+	'Inline::C' => {
+		rpm => 'perl-Inline', # for CentOS 7.x, at least
 	},
 	'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' },
 	'Plack::Test' => {
 		deb => 'libplack-perl',
 		pkg => 'p5-Plack',
-		rpm => 'perl-Plack-Test',
-	},
-	'URI' => {
-		deb => 'liburi-perl',
-		pkg => 'p5-URI',
-		rpm => 'perl-URI',
 	},
 	'Test::Simple' => {
 		deb => 'perl', # perl-modules-5.XX, but the XX varies
 		pkg => 'perl5',
-		rpm => 'perl-Test-Simple',
 	},
 	'highlight.pm' => {
 		deb => 'libhighlight-perl',
@@ -131,7 +126,6 @@ my $non_auto = {
 	# OS-specific
 	'IO::KQueue' => {
 		deb => [],
-		pkg => 'p5-IO-KQueue',
 		rpm => [],
 	},
 };

[PATCH 4/4] ci: updates for OpenBSD

[Eric Wong]

Still a work-in-progress, but OpenBSD's pkg_add/pkg_delete seem
to be working somewhat.  The dependency system seems to need some
extra help to ensure leaf packages with their own dependencies
(e.g.  `xapian-bindings-perl') get uninstalled before their
dependencies (`xapian-core').

Deduplicating the command-line is also required since both
pkg_add and pkg_delete will repeat the installation/removal if
a package is specified multiple times in the same invocation.
---
 ci/deps.perl   | 85 +++++++++++++++++++++++++++++++++++++-------------
 ci/profiles.sh | 23 ++++++++++++--
 2 files changed, 84 insertions(+), 24 deletions(-)

diff --git a/ci/deps.perl b/ci/deps.perl
index e7d43cd9..ea46b789 100755
--- a/ci/deps.perl
+++ b/ci/deps.perl
@@ -55,12 +55,15 @@ my $profiles = {
 };
 
 # account for granularity differences between package systems and OSes
-my @precious;
-if ($^O eq 'freebsd') {
-	@precious = qw(perl curl Socket6 IO::Compress);
-} elsif ($pkg_fmt eq 'rpm') {
-	@precious = qw(perl curl);
-}
+# curl, TimeDate (Date::Parse) and Socket6 are dependencies of git on
+# some/most OSes
+my @precious = do {
+	if ($^O eq 'freebsd') { qw(curl Socket6) }
+	elsif ($pkg_fmt eq 'rpm') { qw(curl) }
+	# OpenBSD git depends on Date::Parse (p5-Time-TimeDate) indirectly,
+	elsif ($^O eq 'openbsd') { qw(curl Socket6 Date::Parse) }
+	else { () }
+};
 
 if (@precious) {
 	my $re = join('|', map { quotemeta($_) } @precious);
@@ -76,11 +79,15 @@ $profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite DBI) ];
 
 # package names which can't be mapped automatically:
 my $non_auto = {
-	'perl' => { pkg => 'perl5' },
+	'perl' => {
+		pkg => 'perl5',
+		pkg_add => [], # Perl is part of OpenBSD base
+	},
 	'Date::Parse' => {
 		deb => 'libtimedate-perl',
 		pkg => 'p5-TimeDate',
 		rpm => 'perl-TimeDate',
+		pkg_add => 'p5-Time-TimeDate',
 	},
 	'Digest::SHA' => {
 		deb => 'perl', # libperl5.XX, but the XX varies
@@ -99,6 +106,7 @@ my $non_auto = {
 		pkg => 'perl5',
 	},
 	'Inline::C' => {
+		pkg_add => 'p5-Inline', # tested OpenBSD 7.3
 		rpm => 'perl-Inline', # for CentOS 7.x, at least
 	},
 	'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' },
@@ -106,6 +114,11 @@ my $non_auto = {
 		deb => 'libplack-perl',
 		pkg => 'p5-Plack',
 	},
+	'Search::Xapian' => {
+		pkg => [qw(xapian-core p5-Xapian)],
+		pkg_add => [qw(xapian-core xapian-bindings-perl)],
+		rpm => 'Search::Xapian', # 3rd-party repo
+	},
 	'Test::Simple' => {
 		deb => 'perl', # perl-modules-5.XX, but the XX varies
 		pkg => 'perl5',
@@ -130,11 +143,30 @@ my $non_auto = {
 	},
 };
 
+# OpenBSD and FreeBSD both use "p5-".($CPAN_NAME =~ s/::/-/gr)
+if ($pkg_fmt eq 'pkg_add') {
+	for my $name (keys %$non_auto) {
+		my $fbsd_pkg = $non_auto->{$name}->{pkg};
+		$fbsd_pkg = [] if ($fbsd_pkg // '') eq 'perl5';
+		$non_auto->{$name}->{pkg_add} //= $fbsd_pkg if $fbsd_pkg;
+	}
+}
+
+my %inst_check = (
+	pkg => sub { system(qw(pkg info -q), $_[0]) == 0 },
+	deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 },
+	pkg_add => sub { system(qw(pkg_info -q -e), "$_[0]->=0") == 0 },
+	rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 },
+);
+
+our $INST_CHECK = $inst_check{$pkg_fmt} || die <<"";
+don't know how to check install status for $pkg_fmt
+
 my (@pkg_install, @pkg_remove, %all);
 for my $ary (values %$profiles) {
 	$all{$_} = \@pkg_remove for @$ary;
 }
-if ($^O eq 'freebsd') {
+if ($^O =~ /\A(?:free|open)bsd\z/) {
 	$all{'IO::KQueue'} = \@pkg_remove;
 }
 $profiles->{all} = [ keys %all ]; # pseudo-profile for all packages
@@ -154,6 +186,10 @@ while (my ($pkg, $dst_pkg_list) = each %all) {
 	push @$dst_pkg_list, list(pkg2ospkg($pkg, $pkg_fmt));
 }
 
+my %inst = map { $_ => 1 } @pkg_install;
+@pkg_remove = grep { !$inst{$_} } @pkg_remove;
+@pkg_install = grep { !$INST_CHECK->($_) } @pkg_install;
+
 my @apt_opts =
 	qw(-o APT::Install-Recommends=false -o APT::Install-Suggests=false);
 
@@ -167,9 +203,8 @@ if ($pkg_fmt eq 'deb') {
 		# remove it in an "install" sub-command:
 		map { "$_-" } @pkg_remove);
 	root('apt-get', @apt_opts, qw(autoremove --purge -y), @quiet);
-} elsif ($pkg_fmt eq 'pkg') {
+} elsif ($pkg_fmt eq 'pkg') { # FreeBSD
 	my @quiet = $ENV{V} ? () : ('-q');
-	# FreeBSD, maybe other *BSDs are similar?
 
 	# don't remove stuff that isn't installed:
 	exclude_uninstalled(\@pkg_remove);
@@ -182,6 +217,21 @@ if ($pkg_fmt eq 'deb') {
 	exclude_uninstalled(\@pkg_remove);
 	root(qw(yum remove -y), @quiet, @pkg_remove) if @pkg_remove;
 	root(qw(yum install -y), @quiet, @pkg_install) if @pkg_install;
+} elsif ($pkg_fmt eq 'pkg_add') { # OpenBSD
+	exclude_uninstalled(\@pkg_remove);
+	my @quiet = $ENV{V} ? ('-'.('v'x$ENV{V})) : qw(-x); # -x : no progress
+	if (@pkg_remove) {
+		my @lifo = qw(xapian-bindings-perl);
+		for my $dep (@lifo) {
+			grep(/\A\Q$dep\E\z/, @pkg_remove) or next;
+			root(qw(pkg_delete -I), @quiet, $dep);
+			@pkg_remove = grep(!/\A\Q$dep\E\z/, @pkg_remove);
+		}
+		root(qw(pkg_delete -I), @quiet, @pkg_remove);
+	}
+	root(qw(pkg_delete -a), @quiet);
+	@pkg_install = map { "$_--" } @pkg_install; # disambiguate w3m
+	root(qw(pkg_add), @quiet, @pkg_install) if @pkg_install;
 } else {
 	die "unsupported package format: $pkg_fmt\n";
 }
@@ -206,7 +256,7 @@ sub pkg2ospkg {
 		} elsif ($fmt eq 'rpm') {
 			$pkg =~ s/::/-/g;
 			return "perl-$pkg"
-		} elsif ($fmt eq 'pkg') {
+		} elsif ($fmt =~ /\Apkg(?:_add)?\z/) {
 			$pkg =~ s/::/-/g;
 			return "p5-$pkg"
 		} else {
@@ -232,18 +282,9 @@ sub profile2dst {
 
 sub exclude_uninstalled {
 	my ($list) = @_;
-	my %inst_check = (
-		pkg => sub { system(qw(pkg info -q), $_[0]) == 0 },
-		deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 },
-		rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 },
-	);
-
-	my $cb = $inst_check{$pkg_fmt} || die <<"";
-don't know how to check install status for $pkg_fmt
-
-	my @tmp;
+	my (@tmp, %seen);
 	for my $pkg (@$list) {
-		push @tmp, $pkg if $cb->($pkg);
+		push @tmp, $pkg if !$seen{$pkg}++ && $INST_CHECK->($pkg);
 	}
 	@$list = @tmp;
 }
diff --git a/ci/profiles.sh b/ci/profiles.sh
index 55b998d7..04cefa15 100755
--- a/ci/profiles.sh
+++ b/ci/profiles.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Prints OS-specific package profiles to stdout (one per line) to use
@@ -7,7 +7,7 @@
 
 # set by os-release(5) or similar
 ID= VERSION_ID=
-case $(uname -o) in
+case $(uname -o 2>/dev/null || uname -s) in
 GNU/Linux)
 	for f in /etc/os-release /usr/lib/os-release
 	do
@@ -42,12 +42,22 @@ FreeBSD)
 		echo >&2 "ID=$ID $(uname -r) too old to support";
 		exit 1
 	}
+	;;
+OpenBSD)
+	ID=openbsd
+	VERSION_ID=$(uname -r | cut -d . -f 1)
+	test "$VERSION_ID" -lt 7 && {
+		echo >&2 "ID=$ID $(uname -r) too old to support";
+		exit 1
+	}
+	;;
 esac
 
 case $ID in
 freebsd) PKG_FMT=pkg ;;
 debian|ubuntu) PKG_FMT=deb ;;
 centos|redhat|fedora) PKG_FMT=rpm ;;
+openbsd) PKG_FMT=pkg_add ;; # unsure about name, but it's not FreeBSD `pkg'
 *) echo >&2 "PKG_FMT undefined for ID=$ID in $0"
 esac
 
@@ -74,6 +84,15 @@ centos-7) sed "s/^/$PKG_FMT /" <<EOF
 v2essential devtest
 essential devtest
 all Search::Xapian-
+EOF
+	;;
+openbsd-7) sed "s/^/$PKG_FMT /" <<EOF
+all devtest-
+all devtest Inline::C-
+all devtest Inline::C
+v2essential
+essential
+essential devtest-
 EOF
 	;;
 esac