[PATCH] import: pypi: read requirements from wheels.

[PATCH] import: pypi: read requirements from wheels.

[Cyril Roelandt]

* guix/import/pypi.scm (latest-wheel-release): New function.
---
 guix/import/pypi.scm | 112 ++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 85 insertions(+), 27 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index d54bb9f..2614ab5 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -72,6 +72,16 @@ or #f on failure."
         (raise (condition (&missing-source-error
                            (package pypi-package)))))))
 
+(define (latest-wheel-release pypi-package)
+  "Return the url of the wheel for the latest release of pypi-package, of #f if
+there isn't any."
+  (let ((releases (assoc-ref* pypi-package "releases"
+                              (assoc-ref* pypi-package "info" "version"))))
+    (or (find (lambda (release)
+                (string=? "bdist_wheel" (assoc-ref release "packagetype")))
+              releases)
+        #f)))
+
 (define (python->package-name name)
   "Given the NAME of a package on PyPI, return a Guix-compliant name for the
 package."
@@ -98,10 +108,10 @@ package definition."
     ((package-inputs ...)
      `((inputs (,'quasiquote ,package-inputs))))))
 
-(define (guess-requirements source-url tarball)
-  "Given SOURCE-URL and a TARBALL of the package, return a list of the required
-packages specified in the requirements.txt file. TARBALL will be extracted in
-the current directory, and will be deleted."
+(define (guess-requirements source-url wheel-url tarball)
+  "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of
+the required packages specified in the requirements.txt file. TARBALL will be
+extracted in the current directory, and will be deleted."
 
   (define (tarball-directory url)
     ;; Given the URL of the package's tarball, return the name of the directory
@@ -148,26 +158,73 @@ cannot determine package dependencies"))
                   (loop (cons (python->package-name (clean-requirement line))
                               result))))))))))
 
-  (let ((dirname (tarball-directory source-url)))
-    (if (string? dirname)
-        (let* ((req-file (string-append dirname "/requirements.txt"))
-               (exit-code (system* "tar" "xf" tarball req-file)))
-          ;; TODO: support more formats.
-          (if (zero? exit-code)
-              (dynamic-wind
-                (const #t)
-                (lambda ()
-                  (read-requirements req-file))
-                (lambda ()
-                  (delete-file req-file)
-                  (rmdir dirname)))
-              (begin
-                (warning (_ "'tar xf' failed with exit code ~a\n")
-                         exit-code)
-                '())))
-        '())))
+  (define (read-wheel-metadata wheel-archive)
+    ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
+    ;; requirements.
+    (let* ((dirname (string-append
+                     (string-join
+                      (list-head
+                       (string-split (last (string-split wheel-url #\/))  #\-) 2)
+                        "-")
+                     ".dist-info"))
+           (json-file (string-append dirname "/metadata.json")))
+      (and (system* "unzip" "-q" wheel-archive json-file)
+           (dynamic-wind
+             (const #t)
+             (lambda ()
+               (call-with-input-file json-file
+                 (lambda (port)
+                   (let* ((metadata (json->scm port))
+                          (run_requires (hash-ref metadata "run_requires"))
+                          (requirements (hash-ref (list-ref run_requires 0)
+                                                  "requires")))
+                     (map (lambda (r)
+                            (python->package-name (clean-requirement r)))
+                          requirements)))))
+             (lambda ()
+               (delete-file json-file)
+               (rmdir dirname))))))
+
+  (define (guess-requirements-from-wheel)
+    ;; Return the package's requirements using the wheel, or #f if an error
+    ;; occurs.
+    (call-with-temporary-output-file
+     (lambda (temp port)
+       (if wheel-url
+         (and (url-fetch wheel-url temp)
+              (read-wheel-metadata temp))
+         #f))))
+
+
+  (define (guess-requirements-from-source)
+    ;; Return the package's requirements by guessing them from the source.
+    (let ((dirname (tarball-directory source-url)))
+      (if (string? dirname)
+          (let* ((req-file (string-append dirname "/requirements.txt"))
+                 (exit-code (system* "tar" "xf" tarball req-file)))
+            ;; TODO: support more formats.
+            (if (zero? exit-code)
+                (dynamic-wind
+                  (const #t)
+                  (lambda ()
+                    (read-requirements req-file))
+                  (lambda ()
+                    (delete-file req-file)
+                    (rmdir dirname)))
+                (begin
+                  (warning (_ "'tar xf' failed with exit code ~a\n")
+                           exit-code)
+                  '())))
+          '())))
+
+  ;; First, try to compute the requirements using the wheel, since that is the
+  ;; most reliable option. If this does not work, try getting them by reading
+  ;; the "requirements.txt" file from the source.
+  (or (guess-requirements-from-wheel)
+      (guess-requirements-from-source)))
+
 
-(define (compute-inputs source-url tarball)
+(define (compute-inputs source-url wheel-url tarball)
   "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
 name/variable pairs describing the required inputs of this package."
   (sort
@@ -176,13 +233,13 @@ name/variable pairs describing the required inputs of this package."
          (append '("python-setuptools")
                  ;; Argparse has been part of Python since 2.7.
                  (remove (cut string=? "python-argparse" <>)
-                         (guess-requirements source-url tarball))))
+                         (guess-requirements source-url wheel-url tarball))))
     (lambda args
       (match args
         (((a _ ...) (b _ ...))
          (string-ci<? a b))))))
 
-(define (make-pypi-sexp name version source-url home-page synopsis
+(define (make-pypi-sexp name version source-url wheel-url home-page synopsis
                         description license)
   "Return the `package' s-expression for a python package with the given NAME,
 VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
@@ -199,7 +256,7 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
                         (base32
                          ,(guix-hash-url temp)))))
              (build-system python-build-system)
-             ,@(maybe-inputs (compute-inputs source-url temp))
+             ,@(maybe-inputs (compute-inputs source-url wheel-url temp))
              (home-page ,home-page)
              (synopsis ,synopsis)
              (description ,description)
@@ -218,11 +275,12 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
            (let ((name (assoc-ref* package "info" "name"))
                  (version (assoc-ref* package "info" "version"))
                  (release (assoc-ref (latest-source-release package) "url"))
+                 (wheel (assoc-ref (latest-wheel-release package) "url"))
                  (synopsis (assoc-ref* package "info" "summary"))
                  (description (assoc-ref* package "info" "summary"))
                  (home-page (assoc-ref* package "info" "home_page"))
                  (license (string->license (assoc-ref* package "info" "license"))))
-             (make-pypi-sexp name version release home-page synopsis
+             (make-pypi-sexp name version release wheel home-page synopsis
                              description license))))))
 
 (define (pypi-package? package)
-- 
2.6.2

Re: [PATCH] import: pypi: read requirements from wheels.

[Ludovic Courtès]

Cyril Roelandt <tipecaml@gmail.com> skribis:

> * guix/import/pypi.scm (latest-wheel-release): New function.

s/function/procedure/  :-)

Please also mention the changes in ‘guess-requirements’,
‘compute-inputs’, etc.

So do I get it right that pypi now provides packages both in Wheels and
in “traditional” format, but that Wheels provides more info about
dependencies?

IOW: What does this buy us?  :-)

> +(define (latest-wheel-release pypi-package)
> +  "Return the url of the wheel for the latest release of pypi-package, of #f if

Line a bit long.  s/of/or/

> +  (define (read-wheel-metadata wheel-archive)
> +    ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
> +    ;; requirements.
> +    (let* ((dirname (string-append
> +                     (string-join
> +                      (list-head
> +                       (string-split (last (string-split wheel-url #\/))  #\-) 2)
> +                        "-")

"-" should be aligned with (list-head.

I would be best to turn this transformation into a top-level procedure,
say ‘wheel-url->extracted-directory’.

> +                     ".dist-info"))
> +           (json-file (string-append dirname "/metadata.json")))
> +      (and (system* "unzip" "-q" wheel-archive json-file)
> +           (dynamic-wind
> +             (const #t)
> +             (lambda ()
> +               (call-with-input-file json-file
> +                 (lambda (port)
> +                   (let* ((metadata (json->scm port))
> +                          (run_requires (hash-ref metadata "run_requires"))
> +                          (requirements (hash-ref (list-ref run_requires 0)
> +                                                  "requires")))
> +                     (map (lambda (r)
> +                            (python->package-name (clean-requirement r)))
> +                          requirements)))))
> +             (lambda ()
> +               (delete-file json-file)
> +               (rmdir dirname))))))

Eventually I wonder if we should do this in a derivation instead of
hoping for ‘unzip’ & co. to be available there (“eventually”, because
the problem is already present with the ‘requirements.txt’ thingie.)

Could you add a test in tests/pypi.scm?

Thanks!

Ludo’.

Re: [PATCH] import: pypi: read requirements from wheels.

[Cyril Roelandt]

On 01/24/2016 09:08 PM, Ludovic Courtès wrote:
> Cyril Roelandt <tipecaml@gmail.com> skribis:
> 
>> * guix/import/pypi.scm (latest-wheel-release): New function.
> 
> s/function/procedure/  :-)
> 
> Please also mention the changes in ‘guess-requirements’,
> ‘compute-inputs’, etc.
> 
> So do I get it right that pypi now provides packages both in Wheels and
> in “traditional” format, but that Wheels provides more info about
> dependencies?
> 
> IOW: What does this buy us?  :-)

When uploading a package to PyPI, one may provide a wheel in addition to
a tarball. The wheel is built by setuptools, and contains metadata,
which includes *all* the requirements specified by the authors.

It is much better than reading "requirements.txt", because this file is
not mandatory.

Since wheels may not be provided, we need to keep the old way of
retrieving dependencies, even though it is not extremely reliable.

> 
>> +(define (latest-wheel-release pypi-package)
>> +  "Return the url of the wheel for the latest release of pypi-package, of #f if
> 
> Line a bit long.  s/of/or/
> 
>> +  (define (read-wheel-metadata wheel-archive)
>> +    ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
>> +    ;; requirements.
>> +    (let* ((dirname (string-append
>> +                     (string-join
>> +                      (list-head
>> +                       (string-split (last (string-split wheel-url #\/))  #\-) 2)
>> +                        "-")
> 
> "-" should be aligned with (list-head.
> 
> I would be best to turn this transformation into a top-level procedure,
> say ‘wheel-url->extracted-directory’.
> 
>> +                     ".dist-info"))
>> +           (json-file (string-append dirname "/metadata.json")))
>> +      (and (system* "unzip" "-q" wheel-archive json-file)
>> +           (dynamic-wind
>> +             (const #t)
>> +             (lambda ()
>> +               (call-with-input-file json-file
>> +                 (lambda (port)
>> +                   (let* ((metadata (json->scm port))
>> +                          (run_requires (hash-ref metadata "run_requires"))
>> +                          (requirements (hash-ref (list-ref run_requires 0)
>> +                                                  "requires")))
>> +                     (map (lambda (r)
>> +                            (python->package-name (clean-requirement r)))
>> +                          requirements)))))
>> +             (lambda ()
>> +               (delete-file json-file)
>> +               (rmdir dirname))))))
> 
> Eventually I wonder if we should do this in a derivation instead of
> hoping for ‘unzip’ & co. to be available there (“eventually”, because
> the problem is already present with the ‘requirements.txt’ thingie.)
> 

I think it is fine as is, seeing how it makes Python packaging much
easier :)

> Could you add a test in tests/pypi.scm?
> 

Will do!

Cyril.

Re: [PATCH] import: pypi: read requirements from wheels.

[Ludovic Courtès]

Cyril Roelandt <tipecaml@gmail.com> skribis:

> On 01/24/2016 09:08 PM, Ludovic Courtès wrote:
>> Cyril Roelandt <tipecaml@gmail.com> skribis:
>> 
>>> * guix/import/pypi.scm (latest-wheel-release): New function.
>> 
>> s/function/procedure/  :-)
>> 
>> Please also mention the changes in ‘guess-requirements’,
>> ‘compute-inputs’, etc.
>> 
>> So do I get it right that pypi now provides packages both in Wheels and
>> in “traditional” format, but that Wheels provides more info about
>> dependencies?
>> 
>> IOW: What does this buy us?  :-)
>
> When uploading a package to PyPI, one may provide a wheel in addition to
> a tarball. The wheel is built by setuptools, and contains metadata,
> which includes *all* the requirements specified by the authors.
>
> It is much better than reading "requirements.txt", because this file is
> not mandatory.
>
> Since wheels may not be provided, we need to keep the old way of
> retrieving dependencies, even though it is not extremely reliable.

OK, got it, that’s nice.  Could you add a comment at a relevant place in
the code to explain that?

>>> +      (and (system* "unzip" "-q" wheel-archive json-file)
>>> +           (dynamic-wind
>>> +             (const #t)
>>> +             (lambda ()
>>> +               (call-with-input-file json-file
>>> +                 (lambda (port)
>>> +                   (let* ((metadata (json->scm port))
>>> +                          (run_requires (hash-ref metadata "run_requires"))
>>> +                          (requirements (hash-ref (list-ref run_requires 0)
>>> +                                                  "requires")))
>>> +                     (map (lambda (r)
>>> +                            (python->package-name (clean-requirement r)))
>>> +                          requirements)))))
>>> +             (lambda ()
>>> +               (delete-file json-file)
>>> +               (rmdir dirname))))))
>> 
>> Eventually I wonder if we should do this in a derivation instead of
>> hoping for ‘unzip’ & co. to be available there (“eventually”, because
>> the problem is already present with the ‘requirements.txt’ thingie.)
>> 
>
> I think it is fine as is, seeing how it makes Python packaging much
> easier :)

I mean the functionality itself is perfect, but what I mean is that the
above process (taking an archive + unzip + guile-json as input,
producing a list of requirements as an output) could be modeled as a
derivation.  Anyway, no rush for that.

Thanks,
Ludo’.

[PATCH] import: pypi: read requirements from wheels.

[Cyril Roelandt]

* guix/import/pypi.scm (latest-wheel-release): New procedure.
---
 guix/import/pypi.scm | 116 +++++++++++++++++++++++++++++++++++++++------------
 tests/pypi.scm       |  74 ++++++++++++++++++++++++++++++++
 2 files changed, 163 insertions(+), 27 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index d54bb9f..0fea71a 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -72,6 +72,16 @@ or #f on failure."
         (raise (condition (&missing-source-error
                            (package pypi-package)))))))
 
+(define (latest-wheel-release pypi-package)
+  "Return the url of the wheel for the latest release of pypi-package,
+or #f if there isn't any."
+  (let ((releases (assoc-ref* pypi-package "releases"
+                              (assoc-ref* pypi-package "info" "version"))))
+    (or (find (lambda (release)
+                (string=? "bdist_wheel" (assoc-ref release "packagetype")))
+              releases)
+        #f)))
+
 (define (python->package-name name)
   "Given the NAME of a package on PyPI, return a Guix-compliant name for the
 package."
@@ -89,6 +99,14 @@ package on PyPI."
     ;; '/' + package name + '/' + ...
     (substring source-url 42 (string-rindex source-url #\/))))
 
+(define (wheel-url->extracted-directory wheel-url)
+  (string-append
+   (string-join
+    (list-head
+     (string-split (last (string-split wheel-url #\/))  #\-) 2)
+    "-")
+   ".dist-info"))
+
 (define (maybe-inputs package-inputs)
   "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a
 package definition."
@@ -98,10 +116,10 @@ package definition."
     ((package-inputs ...)
      `((inputs (,'quasiquote ,package-inputs))))))
 
-(define (guess-requirements source-url tarball)
-  "Given SOURCE-URL and a TARBALL of the package, return a list of the required
-packages specified in the requirements.txt file. TARBALL will be extracted in
-the current directory, and will be deleted."
+(define (guess-requirements source-url wheel-url tarball)
+  "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of
+the required packages specified in the requirements.txt file. TARBALL will be
+extracted in the current directory, and will be deleted."
 
   (define (tarball-directory url)
     ;; Given the URL of the package's tarball, return the name of the directory
@@ -148,26 +166,69 @@ cannot determine package dependencies"))
                   (loop (cons (python->package-name (clean-requirement line))
                               result))))))))))
 
-  (let ((dirname (tarball-directory source-url)))
-    (if (string? dirname)
-        (let* ((req-file (string-append dirname "/requirements.txt"))
-               (exit-code (system* "tar" "xf" tarball req-file)))
-          ;; TODO: support more formats.
-          (if (zero? exit-code)
-              (dynamic-wind
-                (const #t)
-                (lambda ()
-                  (read-requirements req-file))
-                (lambda ()
-                  (delete-file req-file)
-                  (rmdir dirname)))
-              (begin
-                (warning (_ "'tar xf' failed with exit code ~a\n")
-                         exit-code)
-                '())))
-        '())))
+  (define (read-wheel-metadata wheel-archive)
+    ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
+    ;; requirements.
+    (let* ((dirname (wheel-url->extracted-directory wheel-url))
+           (json-file (string-append dirname "/metadata.json")))
+      (and (system* "unzip" "-q" wheel-archive json-file)
+           (dynamic-wind
+             (const #t)
+             (lambda ()
+               (call-with-input-file json-file
+                 (lambda (port)
+                   (let* ((metadata (json->scm port))
+                          (run_requires (hash-ref metadata "run_requires"))
+                          (requirements (hash-ref (list-ref run_requires 0)
+                                                  "requires")))
+                     (map (lambda (r)
+                            (python->package-name (clean-requirement r)))
+                          requirements)))))
+             (lambda ()
+               (delete-file json-file)
+               (rmdir dirname))))))
+
+  (define (guess-requirements-from-wheel)
+    ;; Return the package's requirements using the wheel, or #f if an error
+    ;; occurs.
+    (call-with-temporary-output-file
+     (lambda (temp port)
+       (if wheel-url
+         (and (url-fetch wheel-url temp)
+              (read-wheel-metadata temp))
+         #f))))
+
+
+  (define (guess-requirements-from-source)
+    ;; Return the package's requirements by guessing them from the source.
+    (let ((dirname (tarball-directory source-url)))
+      (if (string? dirname)
+          (let* ((req-file (string-append dirname "/requirements.txt"))
+                 (exit-code (system* "tar" "xf" tarball req-file)))
+            ;; TODO: support more formats.
+            (if (zero? exit-code)
+                (dynamic-wind
+                  (const #t)
+                  (lambda ()
+                    (read-requirements req-file))
+                  (lambda ()
+                    (delete-file req-file)
+                    (rmdir dirname)))
+                (begin
+                  (warning (_ "'tar xf' failed with exit code ~a\n")
+                           exit-code)
+                  '())))
+          '())))
+
+  ;; First, try to compute the requirements using the wheel, since that is the
+  ;; most reliable option. If a wheel is not provided for this package, try
+  ;; getting them by reading the "requirements.txt" file from the source. Note
+  ;; that "requirements.txt" is not mandatory, so this is likely to fail.
+  (or (guess-requirements-from-wheel)
+      (guess-requirements-from-source)))
+
 
-(define (compute-inputs source-url tarball)
+(define (compute-inputs source-url wheel-url tarball)
   "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
 name/variable pairs describing the required inputs of this package."
   (sort
@@ -176,13 +237,13 @@ name/variable pairs describing the required inputs of this package."
          (append '("python-setuptools")
                  ;; Argparse has been part of Python since 2.7.
                  (remove (cut string=? "python-argparse" <>)
-                         (guess-requirements source-url tarball))))
+                         (guess-requirements source-url wheel-url tarball))))
     (lambda args
       (match args
         (((a _ ...) (b _ ...))
          (string-ci<? a b))))))
 
-(define (make-pypi-sexp name version source-url home-page synopsis
+(define (make-pypi-sexp name version source-url wheel-url home-page synopsis
                         description license)
   "Return the `package' s-expression for a python package with the given NAME,
 VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
@@ -199,7 +260,7 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
                         (base32
                          ,(guix-hash-url temp)))))
              (build-system python-build-system)
-             ,@(maybe-inputs (compute-inputs source-url temp))
+             ,@(maybe-inputs (compute-inputs source-url wheel-url temp))
              (home-page ,home-page)
              (synopsis ,synopsis)
              (description ,description)
@@ -218,11 +279,12 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
            (let ((name (assoc-ref* package "info" "name"))
                  (version (assoc-ref* package "info" "version"))
                  (release (assoc-ref (latest-source-release package) "url"))
+                 (wheel (assoc-ref (latest-wheel-release package) "url"))
                  (synopsis (assoc-ref* package "info" "summary"))
                  (description (assoc-ref* package "info" "summary"))
                  (home-page (assoc-ref* package "info" "home_page"))
                  (license (string->license (assoc-ref* package "info" "license"))))
-             (make-pypi-sexp name version release home-page synopsis
+             (make-pypi-sexp name version release wheel home-page synopsis
                              description license))))))
 
 (define (pypi-package? package)
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 960b8cd..e89fd64 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -42,6 +42,9 @@
       }, {
         \"url\": \"https://example.com/foo-1.0.0.tar.gz\",
         \"packagetype\": \"sdist\",
+      }, {
+        \"url\": \"https://example.com/foo-1.0.0-py2.py3-none-any.whl\",
+        \"packagetype\": \"bdist_wheel\",
       }
     ]
   }
@@ -56,6 +59,18 @@
 bar
 baz > 13.37")
 
+(define test-metadata
+  "{
+  \"run_requires\": [
+    {
+      \"requires\": [
+        \"bar\",
+        \"baz (>13.37)\"
+      ]
+    }
+  ]
+}")
+
 (test-begin "pypi")
 
 (test-assert "pypi->guix-package"
@@ -77,6 +92,65 @@ baz > 13.37")
                  (delete-file-recursively "foo-1.0.0")
                  (set! test-source-hash
                        (call-with-input-file file-name port-sha256))))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
+             (_ (error "Unexpected URL: " url)))))
+    (match (pypi->guix-package "foo")
+      (('package
+         ('name "python-foo")
+         ('version "1.0.0")
+         ('source ('origin
+                    ('method 'url-fetch)
+                    ('uri (pypi-uri "foo" version))
+                    ('sha256
+                     ('base32
+                      (? string? hash)))))
+         ('build-system 'python-build-system)
+         ('inputs
+          ('quasiquote
+           (("python-bar" ('unquote 'python-bar))
+            ("python-baz" ('unquote 'python-baz))
+            ("python-setuptools" ('unquote 'python-setuptools)))))
+         ('home-page "http://example.com")
+         ('synopsis "summary")
+         ('description "summary")
+         ('license 'lgpl2.0))
+       (string=? (bytevector->nix-base32-string
+                  test-source-hash)
+                 hash))
+      (x
+       (pk 'fail x #f)))))
+
+(test-assert "pypi->guix-package, wheels"
+  ;; Replace network resources with sample data.
+  (mock ((guix import utils) url-fetch
+         (lambda (url file-name)
+           (match url
+             ("https://pypi.python.org/pypi/foo/json"
+              (with-output-to-file file-name
+                (lambda ()
+                  (display test-json))))
+             ("https://example.com/foo-1.0.0.tar.gz"
+               (begin
+                 (mkdir "foo-1.0.0")
+                 (with-output-to-file "foo-1.0.0/requirements.txt"
+                   (lambda ()
+                     (display test-requirements)))
+                 (system* "tar" "czvf" file-name "foo-1.0.0/")
+                 (delete-file-recursively "foo-1.0.0")
+                 (set! test-source-hash
+                       (call-with-input-file file-name port-sha256))))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
+               (begin
+                 (mkdir "foo-1.0.0.dist-info")
+                 (with-output-to-file "foo-1.0.0.dist-info/metadata.json"
+                   (lambda ()
+                     (display test-metadata)))
+                 (let ((zip-file (string-append file-name ".zip")))
+		   ;; zip always adds a "zip" extension to the file it creates,
+                   ;; so we need to rename it.
+                   (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
+                   (rename-file zip-file file-name))
+                 (delete-file-recursively "foo-1.0.0.dist-info")))
              (_ (error "Unexpected URL: " url)))))
     (match (pypi->guix-package "foo")
       (('package
-- 
2.6.2

Re: [PATCH] import: pypi: read requirements from wheels.

[Ludovic Courtès]

Cyril Roelandt <tipecaml@gmail.com> skribis:

> * guix/import/pypi.scm (latest-wheel-release): New procedure.

Could you list the other changes (new procedures, new tests, changed
procedures, etc.) in the commit log?

Also, could you add a note in “Invoking guix import” to mention how
Wheels is used?  The ‘unzip’ requirement should also be mentioned in a
footnote or something.

> +(define (wheel-url->extracted-directory wheel-url)
> +  (string-append
> +   (string-join
> +    (list-head
> +     (string-split (last (string-split wheel-url #\/))  #\-) 2)
> +    "-")
> +   ".dist-info"))

I find it a bit hard to follow.  What about something along these lines
(untested):

     (match (string-split (basename wheel-url) #/-)
       ((name version _ ...)
        (string-append name "-" version ".dist-info")))

> +  (define (read-wheel-metadata wheel-archive)
> +    ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
> +    ;; requirements.
> +    (let* ((dirname (wheel-url->extracted-directory wheel-url))
> +           (json-file (string-append dirname "/metadata.json")))
> +      (and (system* "unzip" "-q" wheel-archive json-file)

Should be:

  (and (zero? (system* …)) …)

> +(test-assert "pypi->guix-package, wheels"
> +  ;; Replace network resources with sample data.
> +  (mock ((guix import utils) url-fetch
> +         (lambda (url file-name)
> +           (match url
> +             ("https://pypi.python.org/pypi/foo/json"
> +              (with-output-to-file file-name
> +                (lambda ()
> +                  (display test-json))))
> +             ("https://example.com/foo-1.0.0.tar.gz"
> +               (begin
> +                 (mkdir "foo-1.0.0")
> +                 (with-output-to-file "foo-1.0.0/requirements.txt"
> +                   (lambda ()
> +                     (display test-requirements)))
> +                 (system* "tar" "czvf" file-name "foo-1.0.0/")
> +                 (delete-file-recursively "foo-1.0.0")
> +                 (set! test-source-hash
> +                       (call-with-input-file file-name port-sha256))))
> +             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
> +               (begin
> +                 (mkdir "foo-1.0.0.dist-info")
> +                 (with-output-to-file "foo-1.0.0.dist-info/metadata.json"
> +                   (lambda ()
> +                     (display test-metadata)))
> +                 (let ((zip-file (string-append file-name ".zip")))
> +		   ;; zip always adds a "zip" extension to the file it creates,
   ^
Please remove tabs.

> +                   ;; so we need to rename it.
> +                   (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
> +                   (rename-file zip-file file-name))

This will fail if ‘zip’ is unavailable.  If this command is really
needed, the test should be skipped when it’s missing, for instance by
adding something like this above the test

  (test-skip (if (which "zip") 0 1))

… using ‘which’ from (guix build utils).

It seems that all these files are left behind no?  Could you make sure
they are removed?  That probably means putting them in a separate
directory, and then having:

  (dynamic-wind
    (const #t)
    (lambda ()
      ;; the body
      )
    (lambda ()
      (delete-file-recursively temp-dir)))

Could you send an updated patch?

Thank you!  It’s good to see the importer getting smarter.  :-)

Ludo’.

Re: [PATCH] import: pypi: read requirements from wheels.

[Cyril Roelandt]

On 03/02/2016 10:54 AM, Ludovic Courtès wrote:
> It seems that all these files are left behind no?  Could you make sure
> they are removed? 

I think they are deleted. I call delete-file-recursively to remove
everything, and I can't see the files anywhere, so I'm assuming they're
really gone.


Cyril.

[PATCH] import: pypi: read requirements from wheels.

[Cyril Roelandt]

* doc/guix.tex (Invoking guix import): Mention that the pypi importer
works better with "unzip".
* guix/import/pypi.scm (latest-wheel-release,
wheel-url->extracted-directory): New procedures.
* tests/pypi.scm (("pypi->guix-package, wheels"): New test.
---
 doc/guix.texi        |   4 +-
 guix/import/pypi.scm | 113 +++++++++++++++++++++++++++++++++++++++------------
 tests/pypi.scm       |  78 ++++++++++++++++++++++++++++++++++-
 3 files changed, 166 insertions(+), 29 deletions(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index 08d7a08..918a8f5 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -4478,7 +4478,9 @@ Import metadata from the @uref{https://pypi.python.org/, Python Package
 Index}@footnote{This functionality requires Guile-JSON to be installed.
 @xref{Requirements}.}.  Information is taken from the JSON-formatted
 description available at @code{pypi.python.org} and usually includes all
-the relevant information, including package dependencies.
+the relevant information, including package dependencies.  For maximum
+efficiency, it is recommended to install the @command{unzip} utility, so
+that the importer can unzip Python wheels and gather data from them.
 
 The command below imports metadata for the @code{itsdangerous} Python
 package:
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8ae4948..c7f8d33 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -72,6 +72,16 @@ or #f on failure."
         (raise (condition (&missing-source-error
                            (package pypi-package)))))))
 
+(define (latest-wheel-release pypi-package)
+  "Return the url of the wheel for the latest release of pypi-package,
+or #f if there isn't any."
+  (let ((releases (assoc-ref* pypi-package "releases"
+                              (assoc-ref* pypi-package "info" "version"))))
+    (or (find (lambda (release)
+                (string=? "bdist_wheel" (assoc-ref release "packagetype")))
+              releases)
+        #f)))
+
 (define (python->package-name name)
   "Given the NAME of a package on PyPI, return a Guix-compliant name for the
 package."
@@ -89,6 +99,11 @@ package on PyPI."
     ;; '/' + package name + '/' + ...
     (substring source-url 42 (string-rindex source-url #\/))))
 
+(define (wheel-url->extracted-directory wheel-url)
+  (match (string-split (basename wheel-url) #\-)
+    ((name version _ ...)
+     (string-append name "-" version ".dist-info"))))
+
 (define (maybe-inputs package-inputs)
   "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a
 package definition."
@@ -98,10 +113,10 @@ package definition."
     ((package-inputs ...)
      `((inputs (,'quasiquote ,package-inputs))))))
 
-(define (guess-requirements source-url tarball)
-  "Given SOURCE-URL and a TARBALL of the package, return a list of the required
-packages specified in the requirements.txt file. TARBALL will be extracted in
-the current directory, and will be deleted."
+(define (guess-requirements source-url wheel-url tarball)
+  "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of
+the required packages specified in the requirements.txt file. TARBALL will be
+extracted in the current directory, and will be deleted."
 
   (define (tarball-directory url)
     ;; Given the URL of the package's tarball, return the name of the directory
@@ -148,26 +163,69 @@ cannot determine package dependencies"))
                   (loop (cons (python->package-name (clean-requirement line))
                               result))))))))))
 
-  (let ((dirname (tarball-directory source-url)))
-    (if (string? dirname)
-        (let* ((req-file (string-append dirname "/requirements.txt"))
-               (exit-code (system* "tar" "xf" tarball req-file)))
-          ;; TODO: support more formats.
-          (if (zero? exit-code)
-              (dynamic-wind
-                (const #t)
-                (lambda ()
-                  (read-requirements req-file))
-                (lambda ()
-                  (delete-file req-file)
-                  (rmdir dirname)))
-              (begin
-                (warning (_ "'tar xf' failed with exit code ~a\n")
-                         exit-code)
-                '())))
-        '())))
+  (define (read-wheel-metadata wheel-archive)
+    ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
+    ;; requirements.
+    (let* ((dirname (wheel-url->extracted-directory wheel-url))
+           (json-file (string-append dirname "/metadata.json")))
+      (and (zero? (system* "unzip" "-q" wheel-archive json-file))
+           (dynamic-wind
+             (const #t)
+             (lambda ()
+               (call-with-input-file json-file
+                 (lambda (port)
+                   (let* ((metadata (json->scm port))
+                          (run_requires (hash-ref metadata "run_requires"))
+                          (requirements (hash-ref (list-ref run_requires 0)
+                                                  "requires")))
+                     (map (lambda (r)
+                            (python->package-name (clean-requirement r)))
+                          requirements)))))
+             (lambda ()
+               (delete-file json-file)
+               (rmdir dirname))))))
+
+  (define (guess-requirements-from-wheel)
+    ;; Return the package's requirements using the wheel, or #f if an error
+    ;; occurs.
+    (call-with-temporary-output-file
+     (lambda (temp port)
+       (if wheel-url
+         (and (url-fetch wheel-url temp)
+              (read-wheel-metadata temp))
+         #f))))
+
+
+  (define (guess-requirements-from-source)
+    ;; Return the package's requirements by guessing them from the source.
+    (let ((dirname (tarball-directory source-url)))
+      (if (string? dirname)
+          (let* ((req-file (string-append dirname "/requirements.txt"))
+                 (exit-code (system* "tar" "xf" tarball req-file)))
+            ;; TODO: support more formats.
+            (if (zero? exit-code)
+                (dynamic-wind
+                  (const #t)
+                  (lambda ()
+                    (read-requirements req-file))
+                  (lambda ()
+                    (delete-file req-file)
+                    (rmdir dirname)))
+                (begin
+                  (warning (_ "'tar xf' failed with exit code ~a\n")
+                           exit-code)
+                  '())))
+          '())))
+
+  ;; First, try to compute the requirements using the wheel, since that is the
+  ;; most reliable option. If a wheel is not provided for this package, try
+  ;; getting them by reading the "requirements.txt" file from the source. Note
+  ;; that "requirements.txt" is not mandatory, so this is likely to fail.
+  (or (guess-requirements-from-wheel)
+      (guess-requirements-from-source)))
+
 
-(define (compute-inputs source-url tarball)
+(define (compute-inputs source-url wheel-url tarball)
   "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
 name/variable pairs describing the required inputs of this package."
   (sort
@@ -176,13 +234,13 @@ name/variable pairs describing the required inputs of this package."
          (append '("python-setuptools")
                  ;; Argparse has been part of Python since 2.7.
                  (remove (cut string=? "python-argparse" <>)
-                         (guess-requirements source-url tarball))))
+                         (guess-requirements source-url wheel-url tarball))))
     (lambda args
       (match args
         (((a _ ...) (b _ ...))
          (string-ci<? a b))))))
 
-(define (make-pypi-sexp name version source-url home-page synopsis
+(define (make-pypi-sexp name version source-url wheel-url home-page synopsis
                         description license)
   "Return the `package' s-expression for a python package with the given NAME,
 VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
@@ -207,7 +265,7 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
                         (base32
                          ,(guix-hash-url temp)))))
              (build-system python-build-system)
-             ,@(maybe-inputs (compute-inputs source-url temp))
+             ,@(maybe-inputs (compute-inputs source-url wheel-url temp))
              (home-page ,home-page)
              (synopsis ,synopsis)
              (description ,description)
@@ -226,11 +284,12 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
            (let ((name (assoc-ref* package "info" "name"))
                  (version (assoc-ref* package "info" "version"))
                  (release (assoc-ref (latest-source-release package) "url"))
+                 (wheel (assoc-ref (latest-wheel-release package) "url"))
                  (synopsis (assoc-ref* package "info" "summary"))
                  (description (assoc-ref* package "info" "summary"))
                  (home-page (assoc-ref* package "info" "home_page"))
                  (license (string->license (assoc-ref* package "info" "license"))))
-             (make-pypi-sexp name version release home-page synopsis
+             (make-pypi-sexp name version release wheel home-page synopsis
                              description license))))))
 
 (define (pypi-package? package)
diff --git a/tests/pypi.scm b/tests/pypi.scm
index cf351a5..640f59f 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -21,7 +21,7 @@
   #:use-module (guix base32)
   #:use-module (guix hash)
   #:use-module (guix tests)
-  #:use-module ((guix build utils) #:select (delete-file-recursively))
+  #:use-module ((guix build utils) #:select (delete-file-recursively which))
   #:use-module (srfi srfi-64)
   #:use-module (ice-9 match))
 
@@ -42,6 +42,9 @@
       }, {
         \"url\": \"https://example.com/foo-1.0.0.tar.gz\",
         \"packagetype\": \"sdist\",
+      }, {
+        \"url\": \"https://example.com/foo-1.0.0-py2.py3-none-any.whl\",
+        \"packagetype\": \"bdist_wheel\",
       }
     ]
   }
@@ -56,6 +59,18 @@
 bar
 baz > 13.37")
 
+(define test-metadata
+  "{
+  \"run_requires\": [
+    {
+      \"requires\": [
+        \"bar\",
+        \"baz (>13.37)\"
+      ]
+    }
+  ]
+}")
+
 (test-begin "pypi")
 
 (test-assert "pypi->guix-package"
@@ -77,6 +92,67 @@ baz > 13.37")
                  (delete-file-recursively "foo-1.0.0")
                  (set! test-source-hash
                        (call-with-input-file file-name port-sha256))))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
+             (_ (error "Unexpected URL: " url)))))
+    (match (pypi->guix-package "foo")
+      (('package
+         ('name "python-foo")
+         ('version "1.0.0")
+         ('source ('origin
+                    ('method 'url-fetch)
+                    ('uri (string-append "https://example.com/foo-"
+                                         version ".tar.gz"))
+                    ('sha256
+                     ('base32
+                      (? string? hash)))))
+         ('build-system 'python-build-system)
+         ('inputs
+          ('quasiquote
+           (("python-bar" ('unquote 'python-bar))
+            ("python-baz" ('unquote 'python-baz))
+            ("python-setuptools" ('unquote 'python-setuptools)))))
+         ('home-page "http://example.com")
+         ('synopsis "summary")
+         ('description "summary")
+         ('license 'lgpl2.0))
+       (string=? (bytevector->nix-base32-string
+                  test-source-hash)
+                 hash))
+      (x
+       (pk 'fail x #f)))))
+
+(test-skip (if (which "zip") 0 1))
+(test-assert "pypi->guix-package, wheels"
+  ;; Replace network resources with sample data.
+  (mock ((guix import utils) url-fetch
+         (lambda (url file-name)
+           (match url
+             ("https://pypi.python.org/pypi/foo/json"
+              (with-output-to-file file-name
+                (lambda ()
+                  (display test-json))))
+             ("https://example.com/foo-1.0.0.tar.gz"
+               (begin
+                 (mkdir "foo-1.0.0")
+                 (with-output-to-file "foo-1.0.0/requirements.txt"
+                   (lambda ()
+                     (display test-requirements)))
+                 (system* "tar" "czvf" file-name "foo-1.0.0/")
+                 (delete-file-recursively "foo-1.0.0")
+                 (set! test-source-hash
+                       (call-with-input-file file-name port-sha256))))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
+               (begin
+                 (mkdir "foo-1.0.0.dist-info")
+                 (with-output-to-file "foo-1.0.0.dist-info/metadata.json"
+                   (lambda ()
+                     (display test-metadata)))
+                 (let ((zip-file (string-append file-name ".zip")))
+                   ;; zip always adds a "zip" extension to the file it creates,
+                   ;; so we need to rename it.
+                   (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
+                   (rename-file zip-file file-name))
+                 (delete-file-recursively "foo-1.0.0.dist-info")))
              (_ (error "Unexpected URL: " url)))))
     (match (pypi->guix-package "foo")
       (('package
-- 
2.6.2

Re: [PATCH] import: pypi: read requirements from wheels.

[Leo Famulari]

On Sat, Mar 26, 2016 at 02:45:50AM +0100, Cyril Roelandt wrote:
> * doc/guix.tex (Invoking guix import): Mention that the pypi importer
> works better with "unzip".
> * guix/import/pypi.scm (latest-wheel-release,
> wheel-url->extracted-directory): New procedures.
> * tests/pypi.scm (("pypi->guix-package, wheels"): New test.

Seems like a useful addition! What is the status of this patch?

Re: [PATCH] import: pypi: read requirements from wheels.

[Ludovic Courtès]

Leo Famulari <leo@famulari.name> skribis:

> On Sat, Mar 26, 2016 at 02:45:50AM +0100, Cyril Roelandt wrote:
>> * doc/guix.tex (Invoking guix import): Mention that the pypi importer
>> works better with "unzip".
>> * guix/import/pypi.scm (latest-wheel-release,
>> wheel-url->extracted-directory): New procedures.
>> * tests/pypi.scm (("pypi->guix-package, wheels"): New test.
>
> Seems like a useful addition! What is the status of this patch?

(A week passes…)  Well, it looks OK now!

Cyril, if there’s nothing from the previous review round that’s missing,
please push!

Thank you, and sorry for the delay!

Ludo’.

Re: [PATCH] import: pypi: read requirements from wheels.

[Cyril Roelandt]

On 05/15/2016 09:49 PM, Ludovic Courtès wrote:
> 
> Cyril, if there’s nothing from the previous review round that’s missing,
> please push!

Finally done!

Cyril.