From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B5E3A1F9F4 for ; Sun, 12 Sep 2021 07:47:16 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 4/7] clone|lei_mirror: write description in mirrors Date: Sun, 12 Sep 2021 07:47:13 +0000 Message-Id: <20210912074716.4713-5-e@80x24.org> In-Reply-To: <20210912074716.4713-1-e@80x24.org> References: <20210912074716.4713-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Instead of generic "Unnamed repository" or "missing" messages, show "mirror of $URL" since it seems like a better default when creating a mirror. --- lib/PublicInbox/LeiMirror.pm | 63 +++++++++++++++++++++++++----------- t/lei-mirror.t | 23 +++++++++++++ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index c128d13d..fe1cefe2 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -9,6 +9,7 @@ use parent qw(PublicInbox::IPC); use IO::Uncompress::Gunzip qw(gunzip $GunzipError); use PublicInbox::Spawn qw(popen_rd spawn); use File::Temp (); +use Fcntl qw(SEEK_SET); sub do_finish_mirror { # dwaitpid callback my ($arg, $pid) = @_; @@ -87,6 +88,27 @@ sub clone_cmd { @cmd; } +sub _get_txt { # non-fatal + my ($self, $endpoint, $file) = @_; + my $uri = URI->new($self->{src}); + my $lei = $self->{lei}; + my $path = $uri->path; + chop($path) eq '/' or die "BUG: $uri not canonicalized"; + $uri->path("$path/$endpoint"); + my $cmd = $self->{curl}->for_uri($lei, $uri, '--compressed'); + my $ce = "$self->{dst}/$file"; + my $ft = File::Temp->new(TEMPLATE => "$file-XXXX", + UNLINK => 1, DIR => $self->{dst}); + my $opt = { 0 => $lei->{0}, 1 => $ft, 2 => $lei->{2} }; + my $cerr = run_reap($lei, $cmd, $opt); + return "$uri missing" if ($cerr >> 8) == 22; + return "# @$cmd failed (non-fatal)" if $cerr; + my $f = $ft->filename; + rename($f, $ce) or return "rename($f, $ce): $! (non-fatal)"; + $ft->unlink_on_destroy(0); + undef; # success +} + # tries the relatively new /$INBOX/_/text/config/raw endpoint sub _try_config { my ($self) = @_; @@ -96,24 +118,10 @@ sub _try_config { File::Path::mkpath($dst); -d $dst or die "mkpath($dst): $!\n"; } - my $uri = URI->new($self->{src}); - my $lei = $self->{lei}; - my $path = $uri->path; - chop($path) eq '/' or die "BUG: $uri not canonicalized"; - $uri->path($path . '/_/text/config/raw'); - my $cmd = $self->{curl}->for_uri($lei, $uri, '--compressed'); - my $ce = "$dst/inbox.config.example"; - my $f = "$ce-$$.tmp"; - open(my $fh, '+>', $f) or return $lei->err("open $f: $! (non-fatal)"); - my $opt = { 0 => $lei->{0}, 1 => $fh, 2 => $lei->{2} }; - my $cerr = run_reap($lei, $cmd, $opt); - if (($cerr >> 8) == 22) { # 404 missing - unlink($f) if -s $fh == 0; - return; - } - return $lei->err("# @$cmd failed (non-fatal)") if $cerr; - rename($f, $ce) or return $lei->err("rename($f, $ce): $! (non-fatal)"); - my $cfg = PublicInbox::Config->git_config_dump($f, $lei->{2}); + my $err = _get_txt($self, qw(_/text/config/raw inbox.config.example)); + return $self->{lei}->err($err) if $err; + my $f = "$self->{dst}/inbox.config.example"; + my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2}); my $ibx = $self->{ibx} = {}; for my $sec (grep(/\Apublicinbox\./, @{$cfg->{-section_order}})) { for (qw(address newsgroup nntpmirror)) { @@ -122,9 +130,28 @@ sub _try_config { } } +sub set_description ($) { + my ($self) = @_; + my $f = "$self->{dst}/description"; + open my $fh, '+>>', $f or die "open($f): $!"; + seek($fh, 0, SEEK_SET) or die "seek($f): $!"; + chomp(my $d = do { local $/; <$fh> } // die "read($f): $!"); + if ($d eq '($INBOX_DIR/description missing)' || + $d =~ /^Unnamed repository/ || $d !~ /\S/) { + seek($fh, 0, SEEK_SET) or die "seek($f): $!"; + truncate($fh, 0) or die "truncate($f): $!"; + print $fh "mirror of $self->{src}\n" or die "print($f): $!"; + close $fh or die "close($f): $!"; + } +} + sub index_cloned_inbox { my ($self, $iv) = @_; my $lei = $self->{lei}; + my $err = _get_txt($self, qw(description description)); + $lei->err($err) if $err; # non fatal + eval { set_description($self) }; + warn $@ if $@; # n.b. public-inbox-clone works w/o (SQLite || Xapian) # lei is useless without Xapian + SQLite diff --git a/t/lei-mirror.t b/t/lei-mirror.t index 75e25b3f..35b77cf7 100644 --- a/t/lei-mirror.t +++ b/t/lei-mirror.t @@ -2,6 +2,7 @@ # Copyright (C) 2020-2021 all contributors # License: AGPL-3.0+ use strict; use v5.10.1; use PublicInbox::TestCommon; +use PublicInbox::Inbox; require_mods(qw(-httpd lei)); my $sock = tcp_server(); my ($tmpdir, $for_destroy) = tmpdir(); @@ -15,6 +16,8 @@ test_lei({ tmpdir => $tmpdir }, sub { my $t1 = "$home/t1-mirror"; lei_ok('add-external', $t1, '--mirror', "$http/t1/", \'--mirror v1'); ok(-f "$t1/public-inbox/msgmap.sqlite3", 't1-mirror indexed'); + is(PublicInbox::Inbox::try_cat("$t1/description"), + "mirror of $http/t1/\n", 'description set'); lei_ok('ls-external'); like($lei_out, qr!\Q$t1\E!, 't1 added to ls-externals'); @@ -22,6 +25,9 @@ test_lei({ tmpdir => $tmpdir }, sub { my $t2 = "$home/t2-mirror"; lei_ok('add-external', $t2, '--mirror', "$http/t2/", \'--mirror v2'); ok(-f "$t2/msgmap.sqlite3", 't2-mirror indexed'); + ok(-f "$t2/description", 't2 description'); + is(PublicInbox::Inbox::try_cat("$t2/description"), + "mirror of $http/t2/\n", 'description set'); lei_ok('ls-external'); like($lei_out, qr!\Q$t2\E!, 't2 added to ls-externals'); @@ -109,4 +115,21 @@ SKIP: { ok($td->kill, 'killed -httpd'); $td->join; +{ + require_ok 'PublicInbox::LeiMirror'; + my $mrr = { src => 'https://example.com/src/', dst => $tmpdir }; + my $exp = "mirror of https://example.com/src/\n"; + my $f = "$tmpdir/description"; + PublicInbox::LeiMirror::set_description($mrr); + is(PublicInbox::Inbox::try_cat($f), $exp, 'description set on ENOENT'); + + my $fh; + (open($fh, '>', $f) and close($fh)) or xbail $!; + PublicInbox::LeiMirror::set_description($mrr); + is(PublicInbox::Inbox::try_cat($f), $exp, 'description set on empty'); + (open($fh, '>', $f) and print $fh "x\n" and close($fh)) or xbail $!; + is(PublicInbox::Inbox::try_cat($f), "x\n", + 'description preserved if non-default'); +} + done_testing;