Help understanding the URL code

["Clément Pit--Claudel" <clement.pit@gmail.com>]

[-- Attachment #1.1.1: Type: text/plain, Size: 1729 bytes --]

Hi emacs-devel,

I'm looking into producing a patch to make it easier to customize the user-agent string that URL uses, but I'm hitting a number of walls. For example, what does this piece of code achieve?

    (defcustom url-user-agent (format "User-Agent: %sURL/%s\r\n"
                      (if url-package-name
                          (concat url-package-name "/"
                              url-package-version " ")
                        "") url-version)
      "User Agent used by the URL package for HTTP/HTTPS requests
    Should be a string or a function of no arguments returning a string."
      :type '(choice (string :tag "A static User-Agent string")
                     (function :tag "Call a function to get the User-Agent string"))
      :version "25.1"
      :group 'url)

This defcustom is evaluated the first time that url-vars.el is loaded, right? In that case, what's the point of the url-package-name and url-package-version variables? It seems that rebinding them won't ever do anything, since the definition is evaluated once and for all. Thus IIUC this bit of code will return different values depending on whether the url package had already been loaded before or whether it was loaded through url-retrieve-synchronously being autoloaded. Is that right?

    (let ((url-package-name "MyPackage"))
      (url-retrieve-synchronously "http://example.com")
      url-user-agent)

If that's correct, then the attached (draft) patch might help in fixing this issue.

I would also welcome help on other issues, such as #23140 (about being unable to let-bind url-mime-accept-string). Am I missing something in how these new parts of url.el are supposed to be used? 

Clément.

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.1.2: user-agent.patch --]
[-- Type: text/x-diff; name="user-agent.patch", Size: 3528 bytes --]

diff --git a/lisp/url/url-http.el b/lisp/url/url-http.el
index 5832e92..f775e72 100644
--- a/lisp/url/url-http.el
+++ b/lisp/url/url-http.el
@@ -211,15 +211,33 @@ url-http-find-free-connection
     (if connection
 	(url-http-mark-connection-as-busy host port connection))))
 
+(defun url-http--user-agent-default-string ()
+  "Compute a default User-Agent string based on `url-privacy-level'."
+  (let ((package-info (when url-package-name
+                        (format "%s/%s" url-package-name url-package-version)))
+        (emacs-info (unless (and (listp url-privacy-level)
+                                 (memq 'emacs url-privacy-level))
+                      (format "GNUEmacs/%s" emacs-version)))
+        (os-info (unless (and (listp url-privacy-level)
+                              (memq 'os url-privacy-level))
+                   (format "(%s; %s)" url-system-type url-os-type)))
+        (url-info (format "URL/%s" url-version)))
+    (string-join (delq nil (list package-info url-info
+                                 emacs-info os-info))
+                 " ")))
+
 ;; Building an HTTP request
 (defun url-http-user-agent-string ()
   (if (or (eq url-privacy-level 'paranoid)
 	  (and (listp url-privacy-level)
 	       (memq 'agent url-privacy-level)))
       ""
-    (if (functionp url-user-agent)
-        (funcall url-user-agent)
-      url-user-agent)))
+    (format "User-Agent: %s\r\n"
+            (string-trim
+             (cond
+              ((functionp url-user-agent) (funcall url-user-agent))
+              ((stringp url-user-agent) url-user-agent)
+              (t (url-http--user-agent-default-string)))))))
 
 (defun url-http-create-request (&optional ref-url)
   "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
diff --git a/lisp/url/url-vars.el b/lisp/url/url-vars.el
index 960a04a..48a4cbe 100644
--- a/lisp/url/url-vars.el
+++ b/lisp/url/url-vars.el
@@ -116,6 +116,7 @@ url-privacy-level
 Valid symbols are:
 email    -- the email address
 os       -- the operating system info
+emacs    -- the version of Emacs
 lastloc  -- the last location
 agent    -- do not send the User-Agent string
 cookies  -- never accept HTTP cookies
@@ -143,6 +144,7 @@ url-privacy-level
 		(checklist :tag "Custom"
 			   (const :tag "Email address" :value email)
 			   (const :tag "Operating system" :value os)
+			   (const :tag "Emacs version" :value emacs)
 			   (const :tag "Last location" :value lastloc)
 			   (const :tag "Browser identification" :value agent)
 			   (const :tag "No cookies" :value cookie)))
@@ -357,13 +359,11 @@ url-gateway-method
 		(const :tag "Direct connection" :value native))
   :group 'url-hairy)
 
-(defcustom url-user-agent (format "User-Agent: %sURL/%s\r\n"
-				  (if url-package-name
-				      (concat url-package-name "/"
-					      url-package-version " ")
-				    "") url-version)
-  "User Agent used by the URL package for HTTP/HTTPS requests
-Should be a string or a function of no arguments returning a string."
+(defcustom url-user-agent nil
+  "User Agent used by the URL package for HTTP/HTTPS requests.
+Should be a string not including the \"User-Agent:\", or a
+function of no arguments returning a such string.
+If nil, the value is calculated based on `url-privacy-level'."
   :type '(choice (string :tag "A static User-Agent string")
                  (function :tag "Call a function to get the User-Agent string"))
   :version "25.1"

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 836 bytes --]