From a32e6d440e38b97090c9ae3fbf607ec71a49277e Mon Sep 17 00:00:00 2001 From: "F. Jason Park" Date: Tue, 1 Mar 2022 01:38:33 -0800 Subject: [PATCH 3/3] [POC] Integrate the socks and url libraries FIXME add tests, and mention in doc/misc/url.texi that some `url-proxy-services' items can have full URLs, much like their env-var counterparts. Current API (only known to work on GNU/Linux): (require 'socks) (require 'url) (setopt url-proxy-services '(("https" . "socks5h://localhost:9050") ("http" . "socks5h://localhost:9050")) socks-server '("tor" "localhost" 9050 5) socks-username "" socks-password "") Then do something like M-x eww https://check.torproject.org RET or M-x list-packages RET. * lisp/url/url-gw.el (url-open-stream): Use presence and type of `url-using-proxy' to detect caller and massage input values according to legacy practices. * lisp/url/url-http.el: (url-http-find-free-connection): Don't call `url-open-stream' with host and port from active proxy. (url-http, url-http-async-sentinel): Only run `url-https-proxy-connect' for http proxies. * lisp/url/url-methods.el (url-scheme-register-proxy): When an environment variable's value is a full URL, include the scheme in the value of the new entry added to the `url-proxy-services' option if it appears in the variable `url-proxy-full-address-types'. * lisp/url/url-proxy.el (url-default-find-proxy-for-url): Preserve `url-proxy-services' entries whose value is a URL containing a scheme that appears in `url-proxy-full-address-type', and return that URL prefixed by the upcased scheme. (url-find-proxy-for-url): Recognize modified host/address value for socks entries of `url-proxy-services' and deal accordingly. (url-proxy): Handle a SOCKS proxy for http(s) connections only. * lisp/url/url-vars.el (url-proxy-full-address-types): New variable to specify types of URLs that should be preserved in full in the values of `url-proxy-services' entries. (url-proxy-services): Explain that values for certain gateways may need a leading scheme:// portion. (url-using-proxy): Add warning regarding expected type. --- lisp/url/url-gw.el | 8 +++++++- lisp/url/url-http.el | 19 ++++++++++--------- lisp/url/url-methods.el | 8 +++++--- lisp/url/url-proxy.el | 22 +++++++++++++++------- lisp/url/url-vars.el | 20 ++++++++++++++++++-- 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/lisp/url/url-gw.el b/lisp/url/url-gw.el index 568ce8679f5..a65245a58a3 100644 --- a/lisp/url/url-gw.el +++ b/lisp/url/url-gw.el @@ -28,7 +28,7 @@ (require 'url-vars) (require 'url-parse) -(autoload 'socks-open-network-stream "socks") +(autoload 'socks-open-network-stream "socks") ; FIXME remove this (defgroup url-gateway nil "URL gateway variables." @@ -226,6 +226,12 @@ url-open-stream Optional arg GATEWAY-METHOD specifies the gateway to be used, overriding the value of `url-gateway-method'." (unless url-gateway-unplugged + (when (url-p url-using-proxy) + (if (or (eq 'socks url-gateway-method) + (string-prefix-p "socks" (url-type url-using-proxy))) + (setq gateway-method 'socks) + (setq host (url-host url-using-proxy) + service (url-port url-using-proxy)))) (let* ((gwm (or gateway-method url-gateway-method)) (gw-method (if (and url-gateway-local-host-regexp (not (eq 'tls gwm)) diff --git a/lisp/url/url-http.el b/lisp/url/url-http.el index 947c6517ed1..abe2649a474 100644 --- a/lisp/url/url-http.el +++ b/lisp/url/url-http.el @@ -195,12 +195,7 @@ url-http-find-free-connection ;; like authentication. But we use another buffer afterwards. (unwind-protect (let ((proc (url-open-stream host buf - (if url-using-proxy - (url-host url-using-proxy) - host) - (if url-using-proxy - (url-port url-using-proxy) - port) + host port gateway-method))) ;; url-open-stream might return nil. (when (processp proc) @@ -1392,8 +1387,12 @@ url-http (error "Could not create connection to %s:%d" (url-host url) (url-port url))) (_ - (if (and url-http-proxy (string= "https" - (url-type url-current-object))) + (if (and url-http-proxy + ;; Set to "http" by `url-find-proxy-for-url' for + ;; any matching non-blacklisted, non-SOCKS scheme + ;; in `url-proxy-services', including "https". + (equal "http" (url-type url-http-proxy)) + (string= "https" (url-type url-current-object))) (url-https-proxy-connect connection) (set-process-sentinel connection #'url-http-end-of-document-sentinel) @@ -1475,7 +1474,9 @@ url-http-async-sentinel (url-http-end-of-document-sentinel proc why)) ((string= (substring why 0 4) "open") (setq url-http-connection-opened t) - (if (and url-http-proxy (string= "https" (url-type url-current-object))) + (if (and url-http-proxy + (equal "http" (url-type url-http-proxy)) + (string= "https" (url-type url-current-object))) (url-https-proxy-connect proc) (condition-case error (process-send-string proc (url-http-create-request)) diff --git a/lisp/url/url-methods.el b/lisp/url/url-methods.el index 9643e992044..9592307aea8 100644 --- a/lisp/url/url-methods.el +++ b/lisp/url/url-methods.el @@ -92,7 +92,6 @@ url-scheme-register-proxy ;; Then check if its a fully specified URL ((string-match url-nonrelative-link env-proxy) (setq urlobj (url-generic-parse-url env-proxy)) - (setf (url-type urlobj) "http") (setf (url-target urlobj) nil)) ;; Finally, fall back on the assumption that its just a hostname (t @@ -103,8 +102,11 @@ url-scheme-register-proxy (if (and (not cur-proxy) urlobj) (progn (setq url-proxy-services - (cons (cons scheme (format "%s:%d" (url-host urlobj) - (url-port urlobj))) + (cons (cons scheme (if (member (url-type urlobj) + url-proxy-full-address-types) + (url-recreate-url urlobj) + (format "%s:%d" (url-host urlobj) + (url-port urlobj)))) url-proxy-services)) (message "Using a proxy for %s..." scheme))))) diff --git a/lisp/url/url-proxy.el b/lisp/url/url-proxy.el index 0c330069789..b1583523cc6 100644 --- a/lisp/url/url-proxy.el +++ b/lisp/url/url-proxy.el @@ -34,11 +34,13 @@ url-default-find-proxy-for-url host)) (equal "www" (url-type urlobj))) "DIRECT") - ((cdr (assoc (url-type urlobj) url-proxy-services)) - (concat "PROXY " (cdr (assoc (url-type urlobj) url-proxy-services)))) - ;; - ;; Should check for socks - ;; + ((and-let* ((found (assoc (url-type urlobj) url-proxy-services))) + (concat (if-let ((non-scheme (string-search "://" (cdr found))) + (scheme (substring (cdr found) 0 non-scheme)) + ((member scheme url-proxy-full-address-types))) + (concat scheme " ") + "PROXY ") + (cdr found)))) (t "DIRECT"))) @@ -56,8 +58,11 @@ url-find-proxy-for-url ((string-match "^DIRECT" proxy) nil) ((string-match "^PROXY +" proxy) (concat "http://" (substring proxy (match-end 0)) "/")) - ((string-match "^SOCKS +" proxy) - (concat "socks://" (substring proxy (match-end 0)))) + ((string-match (rx bot "SOCKS" (** 0 2 alnum) " ") proxy) + (if-let ((m (substring proxy (match-end 0))) + ((string-search "://" m))) + m + (concat "socks://" m))) (t (display-warning 'url (format "Unknown proxy directive: %s" proxy) :error) nil)))) @@ -72,6 +77,9 @@ url-proxy (cond ((string= (url-type url-using-proxy) "http") (url-http url callback cbargs)) + ((and (string-prefix-p "socks" (url-type url-using-proxy)) + (string-prefix-p "http" (url-type url))) + (url-http url callback cbargs)) (t (error "Don't know how to use proxy `%s'" url-using-proxy)))) diff --git a/lisp/url/url-vars.el b/lisp/url/url-vars.el index 6d7d0d3c94c..ba219905fde 100644 --- a/lisp/url/url-vars.el +++ b/lisp/url/url-vars.el @@ -194,10 +194,24 @@ url-mail-command :type 'function :group 'url) +(defvar url-proxy-full-address-types + '("socks" "socks5" "socks5h" "socks4" "socks4a") + "Schemes for URL types preserved in `url-proxy-services' entries. +When dynamically adding a new `url-proxy-services' entry derived +from the environment, Emacs only retains the host and port +portions unless the URL's scheme appears in this variable's +value.") + (defcustom url-proxy-services nil "An alist of schemes and proxy servers that gateway them. Looks like ((\"http\" . \"hostname:portnumber\") ...). This is set up -from the ACCESS_proxy environment variables." +from the ACCESS_proxy environment variables. Certain gateway +types need server values to take the form of full URLs in order +to convey addtional information about for the proxy connection +itself, for example, SCHEME://USER@HOSTNAME:PORTNUMBER, in which +SCHEME is something like \"socks5\". As of Emacs 30.1, this only +applies to SCHEMEs appearing in the variable +`url-proxy-full-address-types'." :type '(repeat (cons :format "%v" (string :tag "Protocol") (string :tag "Proxy"))) @@ -315,7 +329,9 @@ url-show-status (defvar url-using-proxy nil "Either nil or the fully qualified proxy URL in use, e.g. -https://www.example.com/") +https://www.example.com/. Beware that some functions, such as +`url-proxy' and `url-http-end-of-document-sentinel', set this to +a `url' struct object.") (defcustom url-news-server nil "The default news server from which to get newsgroups/articles. -- 2.42.0