From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 06/26] admin: hoist out resolve_inboxes for -compact and -index
Date: Thu, 23 May 2019 09:36:44 +0000 [thread overview]
Message-ID: <20190523093704.18367-7-e@80x24.org> (raw)
In-Reply-To: <20190523093704.18367-1-e@80x24.org>
Both of these index-affecting commands should work similarly
on the command-line.
public-inbox-index no longer complains about unconfigured
~/.public-inbox/config; but often I found myself being
annoyed by that, anyways...
---
lib/PublicInbox/Admin.pm | 34 ++++++++++++++++++++++++
script/public-inbox-compact | 35 ++++++------------------
script/public-inbox-index | 53 +++++++------------------------------
3 files changed, 52 insertions(+), 70 deletions(-)
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 3eff5cd..2784820 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -41,6 +41,40 @@ sub resolve_repo_dir {
}
}
+sub resolve_inboxes {
+ my ($argv, $warn_on_unconfigured) = @_;
+ require PublicInbox::Config;
+ require PublicInbox::Inbox;
+
+ my @ibxs = map { resolve_repo_dir($_) } @$argv;
+ push(@ibxs, resolve_repo_dir()) unless @ibxs;
+
+ my %dir2ibx;
+ if (my $config = eval { PublicInbox::Config->new }) {
+ $config->each_inbox(sub {
+ my ($ibx) = @_;
+ $dir2ibx{abs_path($ibx->{mainrepo})} = $ibx;
+ });
+ } elsif ($warn_on_unconfigured) {
+ # do we really care about this? It's annoying...
+ warn $warn_on_unconfigured, "\n";
+ }
+ for my $i (0..$#ibxs) {
+ my $dir = $ibxs[$i];
+ $ibxs[$i] = $dir2ibx{$dir} ||= do {
+ my $name = "unconfigured-$i";
+ PublicInbox::Inbox->new({
+ name => $name,
+ address => [ "$name\@example.com" ],
+ mainrepo => $dir,
+ # TODO: consumers may want to warn on this:
+ #-unconfigured => 1,
+ });
+ };
+ }
+ @ibxs;
+}
+
# TODO: make Devel::Peek optional, only used for daemon
my @base_mod = qw(Email::MIME Date::Parse Devel::Peek);
my @over_mod = qw(DBD::SQLite DBI);
diff --git a/script/public-inbox-compact b/script/public-inbox-compact
index 4aa6273..709fb92 100755
--- a/script/public-inbox-compact
+++ b/script/public-inbox-compact
@@ -1,35 +1,16 @@
#!/usr/bin/perl -w
-# Copyright (C) 2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2018-2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use warnings;
-use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
-use PublicInbox::Search;
-use PublicInbox::Config;
use PublicInbox::InboxWritable;
-use Cwd 'abs_path';
use PublicInbox::Xapcmd;
+use PublicInbox::Admin;
+PublicInbox::Admin::require_or_die('-index');
my $usage = "Usage: public-inbox-compact REPO_DIR\n";
-my $dir = shift or die $usage;
-my $config = eval { PublicInbox::Config->new };
-my $ibx;
-$dir = abs_path($dir);
-if ($config) {
- $config->each_inbox(sub {
- $ibx = $_[0] if abs_path($_[0]->{mainrepo}) eq $dir
- });
+my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage;
+foreach (@ibxs) {
+ my $ibx = PublicInbox::InboxWritable->new($_);
+ # we rely on --no-renumber to keep docids synched to NNTP
+ PublicInbox::Xapcmd::run($ibx, [qw(xapian-compact --no-renumber)]);
}
-unless ($ibx) {
- warn "W: $dir not configured in ".
- PublicInbox::Config::default_file() . "\n";
- $ibx = {
- mainrepo => $dir,
- name => 'ignored',
- address => [ 'old@example.com' ],
- };
- $ibx = PublicInbox::Inbox->new($ibx);
-}
-$ibx = PublicInbox::InboxWritable->new($ibx);
-
-# we rely on --no-renumber to keep docids synched to NNTP
-PublicInbox::Xapcmd::run($ibx, [qw(xapian-compact --no-renumber)]);
diff --git a/script/public-inbox-index b/script/public-inbox-index
index cf001cc..9399c27 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -1,5 +1,5 @@
#!/usr/bin/perl -w
-# Copyright (C) 2015-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2015-2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Basic tool to create a Xapian search index for a git repository
# configured for public-inbox.
@@ -10,14 +10,8 @@ use strict;
use warnings;
use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
my $usage = "public-inbox-index REPO_DIR";
-use PublicInbox::Admin qw(resolve_repo_dir);
+use PublicInbox::Admin;
PublicInbox::Admin::require_or_die('-index');
-require PublicInbox::Config;
-
-my $config = eval { PublicInbox::Config->new } || eval {
- warn "public-inbox unconfigured for serving, indexing anyways...\n";
- undef;
-};
my $reindex;
my $prune;
@@ -32,53 +26,26 @@ my %opts = (
GetOptions(%opts) or die "bad command-line args\n$usage";
die "--jobs must be positive\n" if defined $jobs && $jobs < 0;
-my @dirs;
-
-if (@ARGV) {
- @dirs = map { resolve_repo_dir($_) } @ARGV;
-} else {
- @dirs = (resolve_repo_dir());
-}
-
sub usage { print STDERR "Usage: $usage\n"; exit 1 }
-usage() unless @dirs;
-
-defined($config) and $config->each_inbox(sub {
- my ($ibx) = @_;
- for my $i (0..$#dirs) {
- next if $dirs[$i] ne $ibx->{mainrepo};
- $dirs[$i] = $ibx;
- }
-});
-
-my @inboxes;
+# do we really care about this message? It's annoying...
+my $warn = 'public-inbox unconfigured for serving, indexing anyways...';
+my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $warn);
+PublicInbox::Admin::require_or_die('-index');
+usage() unless @ibxs;
my $mods = {};
-
-foreach my $dir (@dirs) {
- my $ibx = $dir;
- if (!ref($ibx)) {
- unless (-d $dir) {
- die "$dir does not appear to be an inbox repository\n";
- }
- $ibx = PublicInbox::Inbox->new({
- mainrepo => $dir,
- name => 'unnamed',
- indexlevel => $indexlevel,
- version => -f "$dir/inbox.lock" ? 2 : 1,
- });
- } elsif (defined $indexlevel && !defined($ibx->{indexlevel})) {
+foreach my $ibx (@ibxs) {
+ if (defined $indexlevel && !defined($ibx->{indexlevel})) {
# XXX: users can shoot themselves in the foot, with this...
$ibx->{indexlevel} = $indexlevel;
}
- push @inboxes, $ibx;
PublicInbox::Admin::scan_ibx_modules($mods, $ibx);
}
PublicInbox::Admin::require_or_die(keys %$mods);
require PublicInbox::SearchIdx;
-index_inbox($_) for @inboxes;
+index_inbox($_) for @ibxs;
sub index_inbox {
my ($repo) = @_;
--
EW
next prev parent reply other threads:[~2019-05-23 9:37 UTC|newest]
Thread overview: 28+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-05-23 9:36 [PATCH 00/26] xcpdb: ease Xapian DB format migrations Eric Wong
2019-05-23 9:36 ` [PATCH 01/26] t/convert-compact: skip on missing xapian-compact(1) Eric Wong
2019-05-23 9:36 ` [PATCH 02/26] v1writable: retire in favor of InboxWritable Eric Wong
2019-05-23 9:36 ` [PATCH 03/26] doc: document the reason for --no-renumber Eric Wong
2019-05-23 9:36 ` [PATCH 04/26] search: reenable phrase search on non-chert Xapian Eric Wong
2019-05-23 9:36 ` [PATCH 05/26] xapcmd: new module for wrapping Xapian commands Eric Wong
2019-05-23 9:36 ` Eric Wong [this message]
2019-05-23 9:36 ` [PATCH 07/26] xapcmd: support spawn options Eric Wong
2019-05-23 9:36 ` [PATCH 08/26] xcpdb: new tool which wraps Xapian's copydatabase(1) Eric Wong
2019-05-23 9:36 ` [PATCH 09/26] xapcmd: do not cleanup on errors Eric Wong
2019-05-23 9:36 ` [PATCH 10/26] admin: move index_inbox over Eric Wong
2019-05-23 9:36 ` [PATCH 11/26] xcpdb: implement using Perl bindings Eric Wong
2019-05-23 9:36 ` [PATCH 12/26] xapcmd: xcpdb supports compaction Eric Wong
2019-05-23 9:36 ` [PATCH 13/26] v2writable: hoist out log_range sub for readability Eric Wong
2019-05-23 9:36 ` [PATCH 14/26] xcpdb: use fine-grained locking Eric Wong
2019-05-23 9:36 ` [PATCH 15/26] xcpdb: implement progress reporting Eric Wong
2019-05-23 9:36 ` [PATCH 16/26] xcpdb: cleanup error handling and diagnosis Eric Wong
2019-05-23 9:36 ` [PATCH 17/26] xapcmd: avoid EXDEV when finalizing changes Eric Wong
2019-05-23 9:36 ` [PATCH 18/26] doc: xcpdb: update to reflect the current state Eric Wong
2019-05-23 9:36 ` [PATCH 19/26] xapcmd: use "print STDERR" for progress reporting Eric Wong
2019-05-23 9:36 ` [PATCH 20/26] xcpdb: show re-indexing progress Eric Wong
2019-05-23 9:36 ` [PATCH 21/26] xcpdb: remove temporary directories on aborts Eric Wong
2019-05-23 9:37 ` [PATCH 22/26] compact: reuse infrastructure from xcpdb Eric Wong
2019-05-23 9:37 ` [PATCH 23/26] xcpdb|compact: support some xapian-compact switches Eric Wong
2019-05-23 9:37 ` [PATCH 24/26] xapcmd: cleanup on interrupted xcpdb "--compact" Eric Wong
2019-05-23 9:37 ` [PATCH 25/26] xcpdb|compact: support --jobs/-j flag like gmake(1) Eric Wong
2019-05-23 9:37 ` [PATCH 26/26] xapcmd: do not reset %SIG until last Xtmpdir is done Eric Wong
2019-05-23 10:37 ` [PATCH 27/26] doc: various updates to reflect current state Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190523093704.18367-7-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).