[PATCH 0/3] lei p2q (patch-to-query)

[PATCH 0/3] lei p2q (patch-to-query)

[Eric Wong]

Pipes (the *nix kind) are good.

Eric Wong (3):
  lei p2q: patch-to-query generator for "lei q --stdin"
  lei q: fix "-" shortcut for --stdin
  lei q: improve early aborts w/ remote externals

 MANIFEST                      |   2 +
 lib/PublicInbox/LEI.pm        |  40 ++++++-
 lib/PublicInbox/LeiImport.pm  |   3 +-
 lib/PublicInbox/LeiP2q.pm     | 197 ++++++++++++++++++++++++++++++++++
 lib/PublicInbox/LeiXSearch.pm |   4 +-
 t/lei-externals.t             |  49 +++++++--
 t/lei-p2q.t                   |  29 +++++
 t/lei-q-thread.t              |   3 +-
 8 files changed, 313 insertions(+), 14 deletions(-)
 create mode 100644 lib/PublicInbox/LeiP2q.pm
 create mode 100644 t/lei-p2q.t

[PATCH 1/3] lei p2q: patch-to-query generator for "lei q --stdin"

[Eric Wong]

Instead of teaching the to-be-implemented "lei show" to search
threads/messages based commits, this orthogonal sub-command is
designed to generate queries for use with "lei q --stdin".

URI-escaped query parameters may be generated with --uri for
HTTP(S) public-inbox instances, but otherwise the output is
designed for "lei q --stdin".

To find threads for a given git commit from a git worktree:

	lei p2q $COMMIT_OID | lei q --stdin -t ...

It can also read via --stdin|-

	curl $INBOX_URL/$MSGID/raw | lei p2q - | lei q --stdin -t

Or from the filesystem:

	lei p2q $(git format-patch -1) | lei q --stdin -t

This defaults to only generating "dfpost:"-prefixed terms since
I've found those most useful for finding messages relating to a
commit.  This is subject to change.

--want=s@ is a comma-separated or multi-value list of prefixes
that defaults to "dfpost7".  Not all are implemented, yet, but
s, dfn, dfpre, and dfpost all seem to mostly work.  Phrase
handling may need to be tweaked to work with Xapian.

OR, NEAR, ADJ, AND, NOT may be used with --want
(e.g. --want=dfpost,OR,dfn)

Prefixing the field prefix with '+' or '-' (e.g. --want=+dfpost)
generates "+dfpost:$EXTRACTED_OID" for Xapian.   For non-boolean
search prefixes, wildcard (*) may also be supplied: (--want=dfn*)

For boolean search prefixes, suffixing the field prefix with a
digit (e.g. --want=dfpost7) provides a minimum length, allowing
truncated variations to be searched.  This is helpful for
finding older messages as git chooses longer dfpost|dfpre
abbreviations as repos get larger.

Automatic date range generation is not implemented, yet.
---
 MANIFEST                  |   2 +
 lib/PublicInbox/LEI.pm    |  30 +++++-
 lib/PublicInbox/LeiP2q.pm | 197 ++++++++++++++++++++++++++++++++++++++
 t/lei-p2q.t               |  29 ++++++
 4 files changed, 257 insertions(+), 1 deletion(-)
 create mode 100644 lib/PublicInbox/LeiP2q.pm
 create mode 100644 t/lei-p2q.t

diff --git a/MANIFEST b/MANIFEST
index 11ec5c01..5044e21c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -187,6 +187,7 @@ lib/PublicInbox/LeiHelp.pm
 lib/PublicInbox/LeiImport.pm
 lib/PublicInbox/LeiMirror.pm
 lib/PublicInbox/LeiOverview.pm
+lib/PublicInbox/LeiP2q.pm
 lib/PublicInbox/LeiQuery.pm
 lib/PublicInbox/LeiSearch.pm
 lib/PublicInbox/LeiStore.pm
@@ -373,6 +374,7 @@ t/lei-import-maildir.t
 t/lei-import-nntp.t
 t/lei-import.t
 t/lei-mirror.t
+t/lei-p2q.t
 t/lei-q-remote-import.t
 t/lei-q-thread.t
 t/lei.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0da24499..a2f8ffe7 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -179,6 +179,9 @@ our %CMD = ( # sorted in order of importance/use:
 	qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s quiet|q
 	lock=s@ kw|keywords|flags! C=s@),
 	],
+'p2q' => [ 'FILE|COMMIT_OID|--stdin',
+	"use a patch to generate a query for `lei q --stdin'",
+	qw(stdin| want|w=s@ uri debug) ],
 'config' => [ '[...]', sub {
 		'git-config(1) wrapper for '._config_path($_[0]);
 	}, qw(config-file|system|global|file|f=s), # for conflict detection
@@ -238,6 +241,10 @@ my %OPTDESC = (
 'show	threads|t' => 'display entire thread a message belongs to',
 'q	threads|t+' =>
 	'return all messages in the same threads as the actual match(es)',
+
+'want|w=s@' => [ 'PREFIX|dfpost|dfn', # common ones in help...
+		'search prefixes to extract (default: dfpost7)' ],
+
 'alert=s@' => ['CMD,:WINCH,:bell,<any command>',
 	'run command(s) or perform ops when done writing to output ' .
 	'(default: ":WINCH,:bell" with --mua and Maildir/IMAP output, ' .
@@ -331,7 +338,7 @@ my %CONFIG_KEYS = (
 	'leistore.dir' => 'top-level storage location',
 );
 
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv); # internal workers
+my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q); # internal workers
 
 # pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
 sub x_it ($$) {
@@ -673,6 +680,11 @@ sub lei_convert {
 	PublicInbox::LeiConvert->call(@_);
 }
 
+sub lei_p2q {
+	require PublicInbox::LeiP2q;
+	PublicInbox::LeiP2q->call(@_);
+}
+
 sub lei_init {
 	my ($self, $dir) = @_;
 	my $cfg = _lei_cfg($self, 1);
@@ -854,6 +866,22 @@ sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
 	}
 }
 
+my %path_to_fd = ('/dev/stdin' => 0, '/dev/stdout' => 1, '/dev/stderr' => 2);
+$path_to_fd{"/dev/fd/$_"} = $path_to_fd{"/proc/self/fd/$_"} for (0..2);
+sub fopen {
+	my ($self, $mode, $path) = @_;
+	rel2abs($self, $path);
+	$path =~ tr!/!/!s;
+	if (defined(my $fd = $path_to_fd{$path})) {
+		return $self->{$fd};
+	}
+	if ($path =~ m!\A/(?:dev|proc/self)/fd/[0-9]+\z!) {
+		return fail($self, "cannot open $path from daemon");
+	}
+	open my $fh, $mode, $path or return;
+	$fh;
+}
+
 # caller needs to "-t $self->{1}" to check if tty
 sub start_pager {
 	my ($self) = @_;
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
new file mode 100644
index 00000000..d1dd125e
--- /dev/null
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -0,0 +1,197 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# front-end for the "lei patch-to-query" sub-command
+package PublicInbox::LeiP2q;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+use PublicInbox::Eml;
+use PublicInbox::Smsg;
+use PublicInbox::MsgIter qw(msg_part_text);
+use PublicInbox::Git qw(git_unquote);
+use PublicInbox::Spawn qw(popen_rd);
+use URI::Escape qw(uri_escape_utf8);
+
+sub xphrase ($) {
+	my ($s) = @_;
+	return () unless $s =~ /\S/;
+	# cf. xapian-core/queryparser/queryparser.lemony
+	# [\./:\\\@] - is_phrase_generator (implicit phrase search)
+	# FIXME not really sure about these..., we basically want to
+	# extract the longest phrase possible that Xapian can handle
+	map {
+		s/\A\s*//;
+		s/\s+\z//;
+		/[\|=><,\sA-Z]/ && !m![\./:\\\@]! ? qq("$_") : $_;
+	} ($s =~ m!(\w[\|=><,\./:\\\@\-\w\s]+)!g);
+}
+
+sub extract_terms { # eml->each_part callback
+	my ($p, $lei) = @_;
+	my $part = $p->[0]; # ignore $depth and @idx;
+	my $ct = $part->content_type || 'text/plain';
+	my ($s, undef) = msg_part_text($part, $ct);
+	defined $s or return;
+	my $in_diff;
+	# TODO: b: nq: q:
+	for (split(/\n/, $s)) {
+		if ($in_diff && s/^ //) { # diff context
+			push @{$lei->{qterms}->{dfctx}}, xphrase($_);
+		} elsif (/^-- $/) { # email signature begins
+			$in_diff = undef;
+		} elsif (m!^diff --git "?[^/]+/.+ "?[^/]+/.+\z!) {
+			# wait until "---" and "+++" to capture filenames
+			$in_diff = 1;
+		} elsif (/^index ([a-f0-9]+)\.\.([a-f0-9]+)\b/) {
+			my ($oa, $ob) = ($1, $2);
+			push @{$lei->{qterms}->{dfpre}}, $oa;
+			push @{$lei->{qterms}->{dfpost}}, $ob;
+			# who uses dfblob?
+		} elsif (m!^(?:---|\+{3}) ("?[^/]+/.+)!) {
+			my $fn = (split(m!/!, git_unquote($1.''), 2))[1];
+			push @{$lei->{qterms}->{dfn}}, xphrase($fn);
+		} elsif ($in_diff && s/^\+//) { # diff added
+			push @{$lei->{qterms}->{dfb}}, xphrase($_);
+		} elsif ($in_diff && s/^-//) { # diff removed
+			push @{$lei->{qterms}->{dfa}}, xphrase($_);
+		} elsif (/^@@ (?:\S+) (?:\S+) @@\s*(\S+.*)/) {
+			push @{$lei->{qterms}->{dfhh}}, xphrase($1);
+		} elsif (/^(?:dis)similarity index/ ||
+				/^(?:old|new) mode/ ||
+				/^(?:deleted|new) file mode/ ||
+				/^(?:copy|rename) (?:from|to) / ||
+				/^(?:dis)?similarity index / ||
+				/^\\ No newline at end of file/ ||
+				/^Binary files .* differ/) {
+		} elsif ($_ eq '') {
+			# possible to be in diff context, some mail may be
+			# stripped by MUA or even GNU diff(1).  "git apply"
+			# treats a bare "\n" as diff context, too
+		} else {
+			$in_diff = undef;
+		}
+	}
+}
+
+my %pfx2smsg = (
+	t => [ qw(to) ],
+	c => [ qw(cc) ],
+	f => [ qw(from) ],
+	tc => [ qw(to cc) ],
+	tcf => [ qw(to cc from) ],
+	a => [ qw(to cc from) ],
+	s => [ qw(subject) ],
+	bs => [ qw(subject) ], # body handled elsewhere
+	d => [ qw(ds) ], # nonsense?
+	dt => [ qw(ds) ], # ditto...
+	rt => [ qw(ts) ], # ditto...
+);
+
+sub do_p2q { # via wq_do
+	my ($self) = @_;
+	my $lei = $self->{lei};
+	my $want = $lei->{opt}->{want} // [ qw(dfpost7) ];
+	my @want = split(/[, ]+/, "@$want");
+	for (@want) {
+		/\A(?:(d|dt|rt):)?([0-9]+)(\.(?:day|weeks)s?)?\z/ or next;
+		my ($pfx, $n, $unit) = ($1, $2, $3);
+		$n *= 86400 * ($unit =~ /week/i ? 7 : 1);
+		$_ = [ $pfx, $n ];
+	}
+	my $smsg = bless {}, 'PublicInbox::Smsg';
+	my $in = $self->{0};
+	unless ($in) {
+		my $input = $self->{input};
+		if (-e $input) {
+			$in = $lei->fopen('<', $input) or
+				return $lei->fail("open < $input: $!");
+		} else {
+			my @cmd = (qw(git format-patch --stdout -1), $input);
+			$in = popen_rd(\@cmd, undef, { 2 => $lei->{2} });
+		}
+	};
+	my $eml = PublicInbox::Eml->new(\(do { local $/; <$in> }));
+	$lei->{diff_want} = +{ map { $_ => 1 } @want };
+	$smsg->populate($eml);
+	while (my ($pfx, $fields) = each %pfx2smsg) {
+		next unless $lei->{diff_want}->{$pfx};
+		for my $f (@$fields) {
+			my $v = $smsg->{$f} // next;
+			push @{$lei->{qterms}->{$pfx}}, xphrase($v);
+		}
+	}
+	$eml->each_part(\&extract_terms, $lei, 1);
+	if ($lei->{opt}->{debug}) {
+		my $json = ref(PublicInbox::Config->json)->new;
+		$json->utf8->canonical->pretty;
+		$lei->err($json->encode($lei->{qterms}));
+	}
+	my (@q, %seen);
+	for my $pfx (@want) {
+		if (ref($pfx) eq 'ARRAY') {
+			my ($p, $t_range) = @$pfx; # TODO
+
+		} elsif ($pfx =~ m!\A(?:OR|XOR|AND|NOT)\z! ||
+				$pfx =~ m!\A(?:ADJ|NEAR)(?:/[0-9]+)?\z!) {
+			push @q, $pfx;
+		} else {
+			my $plusminus = ($pfx =~ s/\A([\+\-])//) ? $1 : '';
+			my $end = ($pfx =~ s/([0-9\*]+)\z//) ? $1 : '';
+			my $x = delete($lei->{qterms}->{$pfx}) or next;
+			my $star = $end =~ tr/*//d ? '*' : '';
+			my $min_len = ($end // 0) + 0;
+
+			# no wildcards for bool_pfx_external
+			$star = '' if $pfx =~ /\A(dfpre|dfpost|mid)\z/;
+			$pfx = "$plusminus$pfx:";
+			if ($min_len) {
+				push @q, map {
+					my @t = ($pfx.$_.$star);
+					while (length > $min_len) {
+						chop $_;
+						push @t, 'OR', $pfx.$_.$star;
+					}
+					@t;
+				} @$x;
+			} else {
+				push @q, map {
+					my $k = $pfx.$_.$star;
+					$seen{$k}++ ? () : $k
+				} @$x;
+			}
+		}
+	}
+	if ($lei->{opt}->{uri}) {
+		@q = (join('+', map { uri_escape_utf8($_) } @q));
+	} else {
+		@q = (join(' ', @q));
+	}
+	$lei->out(@q, "\n");
+}
+
+sub call { # the "lei patch-to-query" entry point
+	my ($cls, $lei, $input) = @_;
+	my $self = $lei->{p2q} = bless {}, $cls;
+	if ($lei->{opt}->{stdin}) {
+		$self->{0} = delete $lei->{0}; # guard from lei_atfork_child
+	} else {
+		$self->{input} = $input;
+	}
+	my $op = $lei->workers_start($self, 'lei patch2query', 1, {
+		'' => [ $lei->{p2q_done} // $lei->can('dclose'), $lei ]
+	});
+	$self->wq_io_do('do_p2q', []);
+	$self->wq_close(1);
+	while ($op && $op->{sock}) { $op->event_step }
+}
+
+sub ipc_atfork_child {
+	my ($self) = @_;
+	my $lei = $self->{lei};
+	$lei->lei_atfork_child;
+	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
+	$self->SUPER::ipc_atfork_child;
+}
+
+1;
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
new file mode 100644
index 00000000..1a2c2e4f
--- /dev/null
+++ b/t/lei-p2q.t
@@ -0,0 +1,29 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_git 2.6;
+require_mods(qw(json DBD::SQLite Search::Xapian));
+
+test_lei(sub {
+	lei_ok(qw(p2q -w dfpost t/data/0001.patch));
+	is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname');
+	open my $fh, '+<', 't/data/0001.patch';
+	lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
+	is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin');
+
+	lei_ok(qw(p2q --uri t/data/0001.patch -w), 'dfpost,dfn');
+	is($lei_out, "dfpost%3A6e006fd73b1d+".
+		"dfn%3Alib%2FPublicInbox%2FSearch.pm\n",
+		'--uri -w dfpost,dfn');
+	lei_ok(qw(p2q t/data/0001.patch), '--want=dfpost,OR,dfn');
+	is($lei_out, "dfpost:6e006fd73b1d OR dfn:lib/PublicInbox/Search.pm\n",
+		'--want=OR');
+	lei_ok(qw(p2q t/data/0001.patch --want=dfpost9));
+	is($lei_out, "dfpost:6e006fd73b1d OR " .
+			"dfpost:6e006fd73b1 OR " .
+			"dfpost:6e006fd73b OR " .
+			"dfpost:6e006fd73\n",
+		'3-byte chop');
+});
+done_testing;

[PATCH 2/3] lei q: fix "-" shortcut for --stdin

[Eric Wong]

Due to the way our option parser handles this special case, it
must be the first option spec.  This helps us document things
better, even, since many command accept either a pathname or
--stdin|-.
---
 lib/PublicInbox/LEI.pm | 7 ++++---
 t/lei-q-thread.t       | 3 ++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a2f8ffe7..f5e42869 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -108,10 +108,11 @@ sub index_opt {
 # see lei__complete() and PublicInbox::LeiHelp
 # command => [ positional_args, 1-line description, Getopt::Long option spec ]
 our %CMD = ( # sorted in order of importance/use:
-'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', qw(
-	save-as=s output|mfolder|o=s format|f=s dedupe|d=s threads|t+ augment|a
+'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms',
+	'stdin|', # /|\z/ must be first for lone dash
+	qw(save-as=s output|mfolder|o=s format|f=s dedupe|d=s threads|t+
 	sort|s=s reverse|r offset=i remote! local! external! pretty
-	include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g stdin|
+	include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g augment|a
 	import-remote! lock=s@
 	alert=s@ mua=s no-torsocks torsocks=s verbose|v+ quiet|q C=s@),
 	PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
diff --git a/t/lei-q-thread.t b/t/lei-q-thread.t
index 66db28a9..0ddf47a6 100644
--- a/t/lei-q-thread.t
+++ b/t/lei-q-thread.t
@@ -21,7 +21,8 @@ test_lei(sub {
 	$buf = PublicInbox::LeiToMail::eml2mboxrd($eml, { kw => ['draft'] });
 	lei_ok([qw(import -F mboxrd -)], undef, { 0 => $buf, %$lei_opt });
 
-	lei_ok qw(q -t m:testmessage@example.com);
+	lei_ok([qw(q - -t)], undef,
+		{ 0 => \'m:testmessage@example.com', %$lei_opt });
 	$res = json_utf8->decode($lei_out);
 	is(scalar(@$res), 3, 'got 2 results');
 	pop @$res;

[PATCH 3/3] lei q: improve early aborts w/ remote externals

[Eric Wong]

We must issue LeiStore->done if a client disconnects
while we're streaming from a remote external.  This
can happen via SIGPIPE, or if a client process is
interrupted by any other means.
---
 lib/PublicInbox/LEI.pm        |  3 +++
 lib/PublicInbox/LeiImport.pm  |  3 ++-
 lib/PublicInbox/LeiXSearch.pm |  4 +--
 t/lei-externals.t             | 49 ++++++++++++++++++++++++++++++-----
 4 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f5e42869..834e399f 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -970,6 +970,9 @@ sub dclose {
 		}
 	}
 	close(delete $self->{1}) if $self->{1}; # may reap_compress
+	if (my $sto = delete $self->{sto}) {
+		$sto->ipc_do('done');
+	}
 	$self->close if $self->{sock}; # PublicInbox::DS::close
 }
 
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index c2c98030..23cecd53 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -18,7 +18,8 @@ sub import_done_wait { # dwaitpid callback
 	my ($arg, $pid) = @_;
 	my ($imp, $lei) = @$arg;
 	$lei->child_error($?, 'non-fatal errors during import') if $?;
-	my $ign = $lei->{sto}->ipc_do('done'); # PublicInbox::LeiStore::done
+	my $sto = delete $lei->{sto};
+	my $wait = $sto->ipc_do('done') if $sto; # PublicInbox::LeiStore::done
 	$lei->dclose;
 }
 
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 9a6457d7..d4607e16 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -349,7 +349,7 @@ Error closing $lei->{ovv}->{dst}: $!
 
 sub do_post_augment {
 	my ($lei) = @_;
-	my $l2m = $lei->{l2m} or die 'BUG: unexpected do_post_augment';
+	my $l2m = $lei->{l2m} or return; # client disconnected
 	my $err;
 	eval { $l2m->post_augment($lei) };
 	$err = $@;
@@ -368,7 +368,7 @@ sub do_post_augment {
 
 sub incr_post_augment { # called whenever an l2m shard finishes augment
 	my ($lei) = @_;
-	my $l2m = $lei->{l2m} or die 'BUG: unexpected incr_post_augment';
+	my $l2m = $lei->{l2m} or return; # client disconnected
 	return if ++$lei->{nr_post_augment} != $l2m->{-wq_nr_workers};
 	do_post_augment($lei);
 }
diff --git a/t/lei-externals.t b/t/lei-externals.t
index d422a9d1..b78b5580 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -6,7 +6,8 @@ use Fcntl qw(SEEK_SET);
 use PublicInbox::Spawn qw(which);
 use PublicInbox::OnDestroy;
 require_git 2.6;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Search::Xapian));
+use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
 
 my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
 	http://czquwvybam4bgbro.onion/meta/
@@ -15,19 +16,55 @@ my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
 my $test_external_remote = sub {
 	my ($url, $k) = @_;
 SKIP: {
-	my $nr = 5;
-	skip "$k unset", $nr if !$url;
-	which('curl') or skip 'no curl', $nr;
-	which('torsocks') or skip 'no torsocks', $nr if $url =~ m!\.onion/!;
+	skip "$k unset", 1 if !$url;
+	state $curl = which('curl');
+	$curl or skip 'no curl', 1;
+	which('torsocks') or skip 'no torsocks', 1 if $url =~ m!\.onion/!;
 	my $mid = '20140421094015.GA8962@dcvr.yhbt.net';
 	my @cmd = ('q', '--only', $url, '-q', "m:$mid");
 	lei_ok(@cmd, \"query $url");
 	is($lei_err, '', "no errors on $url");
 	my $res = json_utf8->decode($lei_out);
-	is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
+	is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url") or
+		skip 'further remote tests', 1;
 	lei_ok(@cmd, 'd:..20101002', \'no results, no error');
 	is($lei_err, '', 'no output on 404, matching local FS behavior');
 	is($lei_out, "[null]\n", 'got null results');
+	my ($pid_before, $pid_after);
+	if (-d $ENV{XDG_RUNTIME_DIR} && -w _) {
+		lei_ok 'daemon-pid';
+		chomp($pid_before = $lei_out);
+		ok($pid_before, 'daemon is live');
+	}
+	for my $out ([], [qw(-f mboxcl2)]) {
+		pipe(my ($r, $w)) or BAIL_OUT $!;
+		open my $err, '+>', undef or BAIL_OUT $!;
+		my $opt = { run_mode => 0, 1 => $w, 2 => $err };
+		my $cmd = [qw(lei q -qt), @$out, 'bytes:1..'];
+		my $tp = start_script($cmd, undef, $opt);
+		close $w;
+		sysread($r, my $buf, 1);
+		close $r; # trigger SIGPIPE
+		$tp->join;
+		ok(WIFSIGNALED($?), "signaled @$out");
+		is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out");
+		seek($err, 0, 0);
+		my @err = grep(!m{mkdir .*sun_path\b}, <$err>);
+		is_deeply(\@err, [], "no errors @$out");
+	}
+	if (-d $ENV{XDG_RUNTIME_DIR} && -w _) {
+		lei_ok 'daemon-pid';
+		chomp(my $pid_after = $lei_out);
+		is($pid_after, $pid_before, 'pid unchanged') or
+			skip 'daemon died', 1;
+		lei_ok 'daemon-kill';
+		my $alive = 1;
+		for (1..100) {
+			$alive = kill(0, $pid_after) or last;
+			tick();
+		}
+		ok(!$alive, 'daemon-kill worked');
+	}
 } # /SKIP
 }; # /sub
 

Re: [PATCH 1/3] lei p2q: patch-to-query generator for "lei q --stdin"

[Kyle Meyer]

Eric Wong writes:

> Instead of teaching the to-be-implemented "lei show" to search
> threads/messages based commits, this orthogonal sub-command is
> designed to generate queries for use with "lei q --stdin".
>
> URI-escaped query parameters may be generated with --uri for
> HTTP(S) public-inbox instances, but otherwise the output is
> designed for "lei q --stdin".
>
> To find threads for a given git commit from a git worktree:
>
> 	lei p2q $COMMIT_OID | lei q --stdin -t ...
>
> It can also read via --stdin|-
>
> 	curl $INBOX_URL/$MSGID/raw | lei p2q - | lei q --stdin -t
>
> Or from the filesystem:
>
> 	lei p2q $(git format-patch -1) | lei q --stdin -t

Very nice :)

> diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
[...]
> +		} elsif (m!^(?:---|\+{3}) ("?[^/]+/.+)!) {
> +			my $fn = (split(m!/!, git_unquote($1.''), 2))[1];
> +			push @{$lei->{qterms}->{dfn}}, xphrase($fn);
> +		} elsif ($in_diff && s/^\+//) { # diff added
> +			push @{$lei->{qterms}->{dfb}}, xphrase($_);
> +		} elsif ($in_diff && s/^-//) { # diff removed
> +			push @{$lei->{qterms}->{dfa}}, xphrase($_);

I noticed an unexpected term when trying dfa:

  $ curl -fSs \
    https://public-inbox.org/meta/20210228122528.18552-2-e@80x24.org/raw >msg
  $ lei p2q --want=dfa msg
  dfa:my @WQ_KEYS = qw dfa:"lxs l2m imp mrr cnv" dfa:"internal workers" dfa:dev/null

So I think the upstream "--- " filename regexp needs to be adjusted to
account for "/dev/null".

[PATCH 4/3] lei p2q: fix /dev/null filenames, fix phrase quoting rules

[Eric Wong]

Kyle Meyer <kyle@kyleam.com> wrote:
> I noticed an unexpected term when trying dfa:
> 
>   $ curl -fSs \
>     https://public-inbox.org/meta/20210228122528.18552-2-e@80x24.org/raw >msg
>   $ lei p2q --want=dfa msg
>   dfa:my @WQ_KEYS = qw dfa:"lxs l2m imp mrr cnv" dfa:"internal workers" dfa:dev/null
> 
> So I think the upstream "--- " filename regexp needs to be adjusted to
> account for "/dev/null".

Thanks.  Also, "my @WQ_KEYS = qw" needs to be quoted, at least,
(and maybe '(' and ')', need to check Xapian more closely....

And I'll have to fix them in SearchIdx (and probably switch to
use a common parser for indexing + term generation).

On a side note: I find myself mega-confused using public-inbox
patches as test data.  I thought Perl was choking and spitting
code back out at me :x

---8<---
Subject: [PATCH] lei p2q: fix /dev/null filenames, fix phrase quoting rules

/dev/null mis-handling was reported by Kyle Meyer.

Phrases quoting rules are also refined to avoid leaving spaces
unquoted when "phrase generator" characters exist.  Also,
context-free hunk headers no longer clobber the in_diff
state of the parser, since git can still generate those.

Link: https://public-inbox.org/meta/87k0qrrhve.fsf@kyleam.com/
---
 lib/PublicInbox/LeiP2q.pm | 10 +++++++---
 t/lei-p2q.t               |  3 +++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index d1dd125e..e7ddc852 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -12,6 +12,7 @@ use PublicInbox::MsgIter qw(msg_part_text);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::Spawn qw(popen_rd);
 use URI::Escape qw(uri_escape_utf8);
+my $FN = qr!((?:"?[^/\n]+/[^\r\n]+)|/dev/null)!;
 
 sub xphrase ($) {
 	my ($s) = @_;
@@ -23,7 +24,7 @@ sub xphrase ($) {
 	map {
 		s/\A\s*//;
 		s/\s+\z//;
-		/[\|=><,\sA-Z]/ && !m![\./:\\\@]! ? qq("$_") : $_;
+		m![^\./:\\\@\-\w]! ? qq("$_") : $_ ;
 	} ($s =~ m!(\w[\|=><,\./:\\\@\-\w\s]+)!g);
 }
 
@@ -40,7 +41,7 @@ sub extract_terms { # eml->each_part callback
 			push @{$lei->{qterms}->{dfctx}}, xphrase($_);
 		} elsif (/^-- $/) { # email signature begins
 			$in_diff = undef;
-		} elsif (m!^diff --git "?[^/]+/.+ "?[^/]+/.+\z!) {
+		} elsif (m!^diff --git $FN $FN!) {
 			# wait until "---" and "+++" to capture filenames
 			$in_diff = 1;
 		} elsif (/^index ([a-f0-9]+)\.\.([a-f0-9]+)\b/) {
@@ -48,13 +49,16 @@ sub extract_terms { # eml->each_part callback
 			push @{$lei->{qterms}->{dfpre}}, $oa;
 			push @{$lei->{qterms}->{dfpost}}, $ob;
 			# who uses dfblob?
-		} elsif (m!^(?:---|\+{3}) ("?[^/]+/.+)!) {
+		} elsif (m!^(?:---|\+{3}) ($FN)!) {
+			next if $1 eq '/dev/null';
 			my $fn = (split(m!/!, git_unquote($1.''), 2))[1];
 			push @{$lei->{qterms}->{dfn}}, xphrase($fn);
 		} elsif ($in_diff && s/^\+//) { # diff added
 			push @{$lei->{qterms}->{dfb}}, xphrase($_);
 		} elsif ($in_diff && s/^-//) { # diff removed
 			push @{$lei->{qterms}->{dfa}}, xphrase($_);
+		} elsif (/^@@ (?:\S+) (?:\S+) @@\s*$/) {
+			# traditional diff w/o -p
 		} elsif (/^@@ (?:\S+) (?:\S+) @@\s*(\S+.*)/) {
 			push @{$lei->{qterms}->{dfhh}}, xphrase($1);
 		} elsif (/^(?:dis)similarity index/ ||
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index 1a2c2e4f..87cf9fa7 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -25,5 +25,8 @@ test_lei(sub {
 			"dfpost:6e006fd73b OR " .
 			"dfpost:6e006fd73\n",
 		'3-byte chop');
+
+	lei_ok(qw(p2q t/data/message_embed.eml --want=dfb));
+	like($lei_out, qr/\bdfb:\S+/, 'got dfb off /dev/null file');
 });
 done_testing;