From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id E4CBA1FA00 for ; Sun, 28 Feb 2021 12:25:28 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/3] lei q: improve early aborts w/ remote externals Date: Sun, 28 Feb 2021 18:25:28 +0600 Message-Id: <20210228122528.18552-4-e@80x24.org> In-Reply-To: <20210228122528.18552-1-e@80x24.org> References: <20210228122528.18552-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We must issue LeiStore->done if a client disconnects while we're streaming from a remote external. This can happen via SIGPIPE, or if a client process is interrupted by any other means. --- lib/PublicInbox/LEI.pm | 3 +++ lib/PublicInbox/LeiImport.pm | 3 ++- lib/PublicInbox/LeiXSearch.pm | 4 +-- t/lei-externals.t | 49 ++++++++++++++++++++++++++++++----- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index f5e42869..834e399f 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -970,6 +970,9 @@ sub dclose { } } close(delete $self->{1}) if $self->{1}; # may reap_compress + if (my $sto = delete $self->{sto}) { + $sto->ipc_do('done'); + } $self->close if $self->{sock}; # PublicInbox::DS::close } diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index c2c98030..23cecd53 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -18,7 +18,8 @@ sub import_done_wait { # dwaitpid callback my ($arg, $pid) = @_; my ($imp, $lei) = @$arg; $lei->child_error($?, 'non-fatal errors during import') if $?; - my $ign = $lei->{sto}->ipc_do('done'); # PublicInbox::LeiStore::done + my $sto = delete $lei->{sto}; + my $wait = $sto->ipc_do('done') if $sto; # PublicInbox::LeiStore::done $lei->dclose; } diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index 9a6457d7..d4607e16 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -349,7 +349,7 @@ Error closing $lei->{ovv}->{dst}: $! sub do_post_augment { my ($lei) = @_; - my $l2m = $lei->{l2m} or die 'BUG: unexpected do_post_augment'; + my $l2m = $lei->{l2m} or return; # client disconnected my $err; eval { $l2m->post_augment($lei) }; $err = $@; @@ -368,7 +368,7 @@ sub do_post_augment { sub incr_post_augment { # called whenever an l2m shard finishes augment my ($lei) = @_; - my $l2m = $lei->{l2m} or die 'BUG: unexpected incr_post_augment'; + my $l2m = $lei->{l2m} or return; # client disconnected return if ++$lei->{nr_post_augment} != $l2m->{-wq_nr_workers}; do_post_augment($lei); } diff --git a/t/lei-externals.t b/t/lei-externals.t index d422a9d1..b78b5580 100644 --- a/t/lei-externals.t +++ b/t/lei-externals.t @@ -6,7 +6,8 @@ use Fcntl qw(SEEK_SET); use PublicInbox::Spawn qw(which); use PublicInbox::OnDestroy; require_git 2.6; -require_mods(qw(DBD::SQLite Search::Xapian)); +require_mods(qw(json DBD::SQLite Search::Xapian)); +use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE); my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/ http://czquwvybam4bgbro.onion/meta/ @@ -15,19 +16,55 @@ my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/ my $test_external_remote = sub { my ($url, $k) = @_; SKIP: { - my $nr = 5; - skip "$k unset", $nr if !$url; - which('curl') or skip 'no curl', $nr; - which('torsocks') or skip 'no torsocks', $nr if $url =~ m!\.onion/!; + skip "$k unset", 1 if !$url; + state $curl = which('curl'); + $curl or skip 'no curl', 1; + which('torsocks') or skip 'no torsocks', 1 if $url =~ m!\.onion/!; my $mid = '20140421094015.GA8962@dcvr.yhbt.net'; my @cmd = ('q', '--only', $url, '-q', "m:$mid"); lei_ok(@cmd, \"query $url"); is($lei_err, '', "no errors on $url"); my $res = json_utf8->decode($lei_out); - is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url"); + is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url") or + skip 'further remote tests', 1; lei_ok(@cmd, 'd:..20101002', \'no results, no error'); is($lei_err, '', 'no output on 404, matching local FS behavior'); is($lei_out, "[null]\n", 'got null results'); + my ($pid_before, $pid_after); + if (-d $ENV{XDG_RUNTIME_DIR} && -w _) { + lei_ok 'daemon-pid'; + chomp($pid_before = $lei_out); + ok($pid_before, 'daemon is live'); + } + for my $out ([], [qw(-f mboxcl2)]) { + pipe(my ($r, $w)) or BAIL_OUT $!; + open my $err, '+>', undef or BAIL_OUT $!; + my $opt = { run_mode => 0, 1 => $w, 2 => $err }; + my $cmd = [qw(lei q -qt), @$out, 'bytes:1..']; + my $tp = start_script($cmd, undef, $opt); + close $w; + sysread($r, my $buf, 1); + close $r; # trigger SIGPIPE + $tp->join; + ok(WIFSIGNALED($?), "signaled @$out"); + is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out"); + seek($err, 0, 0); + my @err = grep(!m{mkdir .*sun_path\b}, <$err>); + is_deeply(\@err, [], "no errors @$out"); + } + if (-d $ENV{XDG_RUNTIME_DIR} && -w _) { + lei_ok 'daemon-pid'; + chomp(my $pid_after = $lei_out); + is($pid_after, $pid_before, 'pid unchanged') or + skip 'daemon died', 1; + lei_ok 'daemon-kill'; + my $alive = 1; + for (1..100) { + $alive = kill(0, $pid_after) or last; + tick(); + } + ok(!$alive, 'daemon-kill worked'); + } } # /SKIP }; # /sub