From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 985462018D for ; Mon, 28 Nov 2022 05:32:40 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613560; bh=tBt3PsIB3KBLCVCNu9kB0g3MIG1q4puwlnOwGxKcTRs=; h=From:To:Subject:Date:In-Reply-To:References:From; b=pd+jF/NPiTOylaxrGOhi9CUWcdcv+hj70csK2hgS3ayGsOqMhfzbDF0EXWD2wRixu cuYtiJsuvvmtysKVwyxQjcW7AJYAXxYxLcTCnUgO0UiAkrvuV4xtDkT0n2/PKjFuxs RBHXkMB72FxhotCwBoHbrbnQPfgPV3jQ4Hn5XPGk= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 38/95] lei_mirror: ensure curl exits 22 on HTTP 404 responses Date: Mon, 28 Nov 2022 05:31:35 +0000 Message-Id: <20221128053232.291618-39-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Oops, this is actually a long-standing bug :x --- lib/PublicInbox/LeiMirror.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 279ce30e..acf08665 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -46,7 +46,7 @@ sub try_scrape { my $uri = URI->new($self->{src}); my $lei = $self->{lei}; my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return; - my $cmd = $curl->for_uri($lei, $uri, '--compressed'); + my $cmd = $curl->for_uri($lei, $uri, qw(-f --compressed)); my $opt = { 0 => $lei->{0}, 2 => $lei->{2} }; my $fh = popen_rd($cmd, undef, $opt); my $html = do { local $/; <$fh> } // die "read(curl $uri): $!"; @@ -132,7 +132,7 @@ sub _get_txt_start { # non-fatal my $f = (split(m!/!, $endpoint))[-1]; my $ft = File::Temp->new(TEMPLATE => "$f-XXXX", TMPDIR => 1); my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} }; - my $cmd = $self->{curl}->for_uri($lei, $uri, qw(--compressed -R -o), + my $cmd = $self->{curl}->for_uri($lei, $uri, qw(-f --compressed -R -o), $ft->filename); my $jobs = $lei->{opt}->{jobs} // 1; reap_live() while keys(%$LIVE) >= $jobs; @@ -607,7 +607,7 @@ sub try_manifest { my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', UNLINK => 1, TMPDIR => 1, SUFFIX => '.tmp'); my $fn = $ft->filename; - my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $fn); + my $cmd = $curl->for_uri($lei, $uri, qw(-f -R -o), $fn); my %opt = map { $_ => $lei->{$_} } (0..2); my $cerr = run_reap($lei, $cmd, \%opt); if ($cerr) {