From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 811231FB05 for ; Sat, 23 Jan 2021 10:27:56 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 08/10] lei q: support a bunch of curl(1) options Date: Sat, 23 Jan 2021 10:27:53 +0000 Message-Id: <20210123102755.425-9-e@80x24.org> In-Reply-To: <20210123102755.425-1-e@80x24.org> References: <20210123102755.425-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Some of these options will make sense when on weird networks (behind firewalls, etc.) Some of these options may not make sense at all. This allows users who prefer to use the SOCKS5 proxy support in curl rather than torsocks(1), but we'll still support torsocks by default since some Tor instances aren't on the default 127.0.0.1:9050. --- lib/PublicInbox/LEI.pm | 4 ++-- lib/PublicInbox/LeiQuery.pm | 41 +++++++++++++++++++++++++++++++++++ lib/PublicInbox/LeiXSearch.pm | 13 +++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 890be575..a9123c6e 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -84,8 +84,8 @@ our %CMD = ( # sorted in order of importance/use: 'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw( save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a sort|s=s reverse|r offset=i remote local! external! pretty mua-cmd=s - verbose|v - since|after=s until|before=s), opt_dash('limit|n=i', '[0-9]+') ], + torsocks=s no-torsocks verbose|v since|after=s until|before=s), + PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ], 'show' => [ 'MID|OID', 'show a given object (Message-ID or object ID)', qw(type=s solve! format|f=s dedupe|d=s thread|t remote local!), diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index eebf217b..acab3c2c 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -66,4 +66,45 @@ sub lei_q { $lxs->do_query($self); } +# Stuff we may pass through to curl (as of 7.64.0), see curl manpage for +# details, so most options which make sense for HTTP/HTTPS (including proxy +# support for Tor and other methods of getting past weird networks). +# Most of these are untested by us, some may not make sense for our use case +# and typos below are likely. +# n.b. some short options (-$NUMBER) are not supported since they conflict +# with other "lei q" switches. +# FIXME: Getopt::Long doesn't easily let us support support options with +# '.' in them (e.g. --http1.1) +sub curl_opt { qw( + abstract-unix-socket=s anyauth basic cacert=s capath=s + cert-status cert-type cert|E=s ciphers=s config|K=s@ + connect-timeout=s connect-to=s cookie-jar|c=s cookie|b=s crlfile=s + digest disable dns-interface=s dns-ipv4-addr=s dns-ipv6-addr=s + dns-servers=s doh-url=s egd-file=s engine=s false-start + happy-eyeballs-timeout-ms=s haproxy-protocol header|H=s@ + http2-prior-knowledge http2 insecure|k + interface=s ipv4 ipv6 junk-session-cookies + key-type=s key=s limit-rate=s local-port=s location-trusted location|L + max-redirs=i max-time=s negotiate netrc-file=s netrc-optional netrc + no-alpn no-buffer|N no-npn no-sessionid noproxy=s ntlm-wb ntlm + pass=s pinnedpubkey=s post301 post302 post303 preproxy=s + proxy-anyauth proxy-basic proxy-cacert=s proxy-capath=s + proxy-cert-type=s proxy-cert=s proxy-ciphers=s proxy-crlfile=s + proxy-digest proxy-header=s@ proxy-insecure + proxy-key-type=s proxy-key proxy-negotiate proxy-ntlm proxy-pass=s + proxy-pinnedpubkey=s proxy-service-name=s proxy-ssl-allow-beast + proxy-tls13-ciphers=s proxy-tlsauthtype=s proxy-tlspassword=s + proxy-tlsuser=s proxy-tlsv1 proxy-user|U=s proxy=s + proxytunnel=s pubkey=s random-file=s referer=s resolve=s + retry-connrefused retry-delay=s retry-max-time=s retry=i + sasl-ir service-name=s socks4=s socks4a=s socks5-basic + socks5-gssapi-service-name=s socks5-gssapi socks5-hostname=s socks5=s + speed-limit|Y speed-type|y ssl-allow-beast sslv2 sslv3 + suppress-connect-headers tcp-fastopen tls-max=s + tls13-ciphers=s tlsauthtype=s tlspassword=s tlsuser=s + tlsv1 trace-ascii=s trace-time trace=s + unix-socket=s user-agent|A=s user|u=s +) +} + 1; diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index 8d36bca9..defe5e67 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -193,6 +193,7 @@ sub query_remote_mboxrd { my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing'; $dedupe->prepare_dedupe; my @cmd = qw(curl -XPOST -sSf); + $opt->{torsocks} = 'false' if $opt->{'no-torsocks'}; my $tor = $opt->{torsocks} //= 'auto'; if ($tor eq 'auto' && substr($uri->host, -6) eq '.onion' && (($lei->{env}->{LD_PRELOAD}//'') !~ /torsocks/)) { @@ -202,6 +203,18 @@ sub query_remote_mboxrd { } my $verbose = $opt->{verbose}; push @cmd, '-v' if $verbose; + for my $o ($lei->curl_opt) { + $o =~ s/\|[a-z0-9]\b//i; # remove single char short option + if ($o =~ s/=[is]@\z//) { + my $ary = $opt->{$o} or next; + push @cmd, map { ("--$o", $_) } @$ary; + } elsif ($o =~ s/=[is]\z//) { + my $val = $opt->{$o} // next; + push @cmd, "--$o", $val; + } elsif ($opt->{$o}) { + push @cmd, "--$o"; + } + } push @cmd, $uri->as_string; $lei->err("# @cmd") if $verbose; $? = 0;