bug#24450: [PATCHv3] Re: pypi importer outputs strange character series in optional dependency case.

[Maxim Cournoyer <maxim.cournoyer@gmail.com>]

[-- Attachment #1.1: Type: text/plain, Size: 105 bytes --]

Here's the current patch set, version 3, with the modifications
as discussed in the previous exchanges:


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: 0001-import-pypi-Do-not-consider-requirements.txt-files.patch --]
[-- Type: text/x-patch, Size: 6551 bytes --]

From 215d0bde103e7e8df0d1d1df81965e973f38783a Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 00:26:00 -0400
Subject: [PATCH 01/12] import: pypi: Do not consider requirements.txt files.

PyPI packages are mandated to have a setup.py file, which contains a listing
of the required dependencies.  The setuptools/distutils machinery embed
metadata in the archives they produce, which contains this information. There
is no need nor gain to collect the requirements from a "requirements.txt"
file, as it is not the true record of dependencies for PyPI packages and may
contain extraneous requirements or not exist at all.

* guix/import/pypi.scm (guess-requirements): Update comment.
[guess-requirements-from-source]: Do not attempt to parse the file
requirements.txt.  Streamline logic.
* tests/pypi.scm (test-requires.txt): Rename from test-requirements, to hint
at the file being tested.
("pypi->guix-package"): Adapt so that the fake package contains a requires.txt
file rather than a requirements.txt file.
("pypi->guix-package, wheels"): Likewise.
---
 guix/import/pypi.scm | 35 +++++++++++++----------------------
 tests/pypi.scm       | 23 ++++++++++++-----------
 2 files changed, 25 insertions(+), 33 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 3a20fc4b9b..8269aa61d7 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -206,35 +206,26 @@ cannot determine package dependencies"))
           (call-with-temporary-directory
            (lambda (dir)
              (let* ((pypi-name (string-take dirname (string-rindex dirname #\-)))
-                    (req-files (list (string-append dirname "/requirements.txt")
-                                     (string-append dirname "/" pypi-name ".egg-info"
-                                                    "/requires.txt")))
-                    (exit-codes (map (lambda (file-name)
-                                       (parameterize ((current-error-port (%make-void-port "rw+"))
-                                                      (current-output-port (%make-void-port "rw+")))
-                                         (system* "tar" "xf" tarball "-C" dir file-name)))
-                                     req-files)))
-               ;; Only one of these files needs to exist.
-               (if (any zero? exit-codes)
-                   (match (find-files dir)
-                     ((file . _)
-                      (read-requirements file))
-                     (()
-                      (warning (G_ "No requirements file found.\n"))))
+                    (requires.txt (string-append dirname "/" pypi-name
+                                                 ".egg-info" "/requires.txt"))
+                    (exit-code (parameterize ((current-error-port (%make-void-port "rw+"))
+                                              (current-output-port (%make-void-port "rw+")))
+                                 (system* "tar" "xf" tarball "-C" dir requires.txt))))
+               (if (zero? exit-code)
+                   (read-requirements (string-append dir "/" requires.txt))
                    (begin
-                     (warning (G_ "Failed to extract requirements files\n"))
+                     (warning
+                      (G_ "Failed to extract file: ~a from source.~%")
+                      requires.txt)
                      '())))))
           '())))
 
-  ;; First, try to compute the requirements using the wheel, since that is the
-  ;; most reliable option. If a wheel is not provided for this package, try
-  ;; getting them by reading either the "requirements.txt" file or the
-  ;; "requires.txt" from the egg-info directory from the source tarball. Note
-  ;; that "requirements.txt" is not mandatory, so this is likely to fail.
+  ;; First, try to compute the requirements using the wheel, else, fallback to
+  ;; reading the "requires.txt" from the egg-info directory from the source
+  ;; tarball.
   (or (guess-requirements-from-wheel)
       (guess-requirements-from-source)))
 
-
 (define (compute-inputs source-url wheel-url tarball)
   "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
 name/variable pairs describing the required inputs of this package.  Also
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 6daa44a6e7..a0271fffad 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -23,7 +23,7 @@
   #:use-module (gcrypt hash)
   #:use-module (guix tests)
   #:use-module (guix build-system python)
-  #:use-module ((guix build utils) #:select (delete-file-recursively which))
+  #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
   #:use-module (srfi srfi-64)
   #:use-module (ice-9 match))
 
@@ -55,11 +55,12 @@
 (define test-source-hash
   "")
 
-(define test-requirements
-"# A comment
+(define test-requires.txt "\
+# A comment
  # A comment after a space
 bar
-baz > 13.37")
+baz > 13.37
+")
 
 (define test-metadata
   "{
@@ -107,10 +108,10 @@ baz > 13.37")
              (match url
                ("https://example.com/foo-1.0.0.tar.gz"
                 (begin
-                  (mkdir "foo-1.0.0")
-                  (with-output-to-file "foo-1.0.0/requirements.txt"
+                  (mkdir-p "foo-1.0.0/foo.egg-info/")
+                  (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                     (lambda ()
-                      (display test-requirements)))
+                      (display test-requires.txt)))
                   (system* "tar" "czvf" file-name "foo-1.0.0/")
                   (delete-file-recursively "foo-1.0.0")
                   (set! test-source-hash
@@ -157,11 +158,11 @@ baz > 13.37")
          (lambda (url file-name)
            (match url
              ("https://example.com/foo-1.0.0.tar.gz"
-               (begin
-                 (mkdir "foo-1.0.0")
-                 (with-output-to-file "foo-1.0.0/requirements.txt"
+              (begin
+                (mkdir-p "foo-1.0.0/foo.egg-info/")
+                (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                    (lambda ()
-                     (display test-requirements)))
+                     (display test-requires.txt)))
                  (system* "tar" "czvf" file-name "foo-1.0.0/")
                  (delete-file-recursively "foo-1.0.0")
                  (set! test-source-hash
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.3: 0001-import-pypi-Do-not-parse-optional-requirements-from-.patch --]
[-- Type: text/x-patch, Size: 6516 bytes --]

From 22da422c3cea1f6d05a11123dd201c1ec54c9ff8 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 00:26:00 -0400
Subject: [PATCH] import: pypi: Do not parse optional requirements from source.

* guix/import/pypi.scm: Export PARSE-REQUIRES.TXT.
(clean-requirement): Move procedure to the top level.
(guess-requirements): Move the READ-REQUIREMENTS procedure to the top level,
and rename it to PARSE-REQUIRES.TXT.  Move the CLEAN-REQUIREMENT procedure to
the top level.  Move the COMMENT? functions inside the PARSE-REQUIRES.TXT
procedure.
(parse-requires.txt): Add a SECTION-HEADER? predicate, and use it to prevent
parsing optional requirements.

* tests/pypi.scm (test-requires-with-sections): New variable.
("parse-requires.txt, with sections"): New test.
---
 guix/import/pypi.scm | 74 ++++++++++++++++++++++++++------------------
 tests/pypi.scm       | 14 +++++++++
 2 files changed, 58 insertions(+), 30 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8269aa61d7..d9db876222 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -47,7 +47,8 @@
   #:use-module (guix upstream)
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
-  #:export (guix-package->pypi-name
+  #:export (parse-requires.txt
+            guix-package->pypi-name
             pypi-recursive-import
             pypi->guix-package
             %pypi-updater))
@@ -117,6 +118,47 @@ package definition."
     ((package-inputs ...)
      `((propagated-inputs (,'quasiquote ,package-inputs))))))
 
+(define (clean-requirement s)
+  ;; Given a requirement LINE, as can be found in a setuptools requires.txt
+  ;; file, remove everything other than the actual name of the required
+  ;; package, and return it.
+  (cond
+   ((string-index s (char-set #\space #\> #\= #\<)) => (cut string-take s <>))
+   (else s)))
+
+(define (parse-requires.txt requires.txt)
+  "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
+requirement names."
+  ;; This is a very incomplete parser, whose job is to select the non-optional
+  ;; dependencies and strip them out of any version information.
+  ;; Alternatively, we could implement a PEG parser with the (ice-9 peg)
+  ;; library and the requirements grammar defined by PEP-0508
+  ;; (https://www.python.org/dev/peps/pep-0508/).
+
+  (define (comment? line)
+    ;; Return #t if the given LINE is a comment, #f otherwise.
+    (string-prefix? "#" (string-trim line)))
+
+  (define (section-header? line)
+    ;; Return #t if the given LINE is a section header, #f otherwise.
+    (string-prefix? "[" (string-trim line)))
+
+  (call-with-input-file requires.txt
+    (lambda (port)
+      (let loop ((result '()))
+        (let ((line (read-line port)))
+          ;; Stop when a section is encountered, as sections contain optional
+          ;; (extra) requirements.  Non-optional requirements must appear
+          ;; before any section is defined.
+          (if (or (eof-object? line) (section-header? line))
+              (reverse result)
+              (cond
+               ((or (string-null? line) (comment? line))
+                (loop result))
+               (else
+                (loop (cons (clean-requirement line)
+                            result))))))))))
+
 (define (guess-requirements source-url wheel-url tarball)
   "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list
 of the required packages specified in the requirements.txt file.  TARBALL will
@@ -139,34 +181,6 @@ be extracted in a temporary directory."
 cannot determine package dependencies"))
           #f)))))
 
-  (define (clean-requirement s)
-    ;; Given a requirement LINE, as can be found in a Python requirements.txt
-    ;; file, remove everything other than the actual name of the required
-    ;; package, and return it.
-    (string-take s
-      (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<))))
-          (string-length s))))
-
-  (define (comment? line)
-    ;; Return #t if the given LINE is a comment, #f otherwise.
-    (eq? (string-ref (string-trim line) 0) #\#))
-
-  (define (read-requirements requirements-file)
-    ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list
-    ;; of name/variable pairs describing the requirements.
-    (call-with-input-file requirements-file
-      (lambda (port)
-        (let loop ((result '()))
-          (let ((line (read-line port)))
-            (if (eof-object? line)
-                result
-                (cond
-                 ((or (string-null? line) (comment? line))
-                  (loop result))
-                 (else
-                  (loop (cons (clean-requirement line)
-                              result))))))))))
-
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
     ;; requirements.
@@ -212,7 +226,7 @@ cannot determine package dependencies"))
                                               (current-output-port (%make-void-port "rw+")))
                                  (system* "tar" "xf" tarball "-C" dir requires.txt))))
                (if (zero? exit-code)
-                   (read-requirements (string-append dir "/" requires.txt))
+                   (parse-requires.txt (string-append dir "/" requires.txt))
                    (begin
                      (warning
                       (G_ "Failed to extract file: ~a from source.~%")
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 6df69073dc..03455ba6be 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -62,6 +62,14 @@ bar
 baz > 13.37
 ")
 
+(define test-requires-with-sections "\
+foo ~= 3
+bar != 2
+
+[test]
+pytest (>=2.5.0)
+")
+
 (define test-metadata
   "{
   \"run_requires\": [
@@ -101,6 +109,12 @@ baz > 13.37
                     (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
                                (pypi-uri "cram" "0.7"))))))))
 
+(test-equal "parse-requires.txt, with sections"
+  '("foo" "bar")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-requires.txt test-requires-with-sections)))
+
 (test-assert "pypi->guix-package"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.4: 0002-tests-pypi-Mute-the-output-of-tar.patch --]
[-- Type: text/x-patch, Size: 1827 bytes --]

From 17419a5f9572b7b886544531324b7b2ec431555e Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Mon, 10 Jun 2019 12:20:08 +0900
Subject: [PATCH 02/12] tests: pypi: Mute the output of tar.

The output of tar when creating archives for the purpose of tests is not
useful, so we mute it.

* tests/pypi.scm ("pypi->guix-package"): Mute the output of tar.
("pypi->guix-package, wheels"): Likewise.
---
 tests/pypi.scm | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/pypi.scm b/tests/pypi.scm
index a0271fffad..6df69073dc 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -112,7 +112,8 @@ baz > 13.37
                   (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                     (lambda ()
                       (display test-requires.txt)))
-                  (system* "tar" "czvf" file-name "foo-1.0.0/")
+                  (parameterize ((current-output-port (%make-void-port "rw+")))
+                    (system* "tar" "czvf" file-name "foo-1.0.0/"))
                   (delete-file-recursively "foo-1.0.0")
                   (set! test-source-hash
                     (call-with-input-file file-name port-sha256))))
@@ -163,7 +164,8 @@ baz > 13.37
                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                    (lambda ()
                      (display test-requires.txt)))
-                 (system* "tar" "czvf" file-name "foo-1.0.0/")
+                (parameterize ((current-output-port (%make-void-port "rw+")))
+                  (system* "tar" "czvf" file-name "foo-1.0.0/"))
                  (delete-file-recursively "foo-1.0.0")
                  (set! test-source-hash
                        (call-with-input-file file-name port-sha256))))
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.5: 0003-import-pypi-Do-not-parse-optional-requirements-from-.patch --]
[-- Type: text/x-patch, Size: 6524 bytes --]

From 22da422c3cea1f6d05a11123dd201c1ec54c9ff8 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 00:26:00 -0400
Subject: [PATCH 03/12] import: pypi: Do not parse optional requirements from
 source.

* guix/import/pypi.scm: Export PARSE-REQUIRES.TXT.
(clean-requirement): Move procedure to the top level.
(guess-requirements): Move the READ-REQUIREMENTS procedure to the top level,
and rename it to PARSE-REQUIRES.TXT.  Move the CLEAN-REQUIREMENT procedure to
the top level.  Move the COMMENT? functions inside the PARSE-REQUIRES.TXT
procedure.
(parse-requires.txt): Add a SECTION-HEADER? predicate, and use it to prevent
parsing optional requirements.

* tests/pypi.scm (test-requires-with-sections): New variable.
("parse-requires.txt, with sections"): New test.
---
 guix/import/pypi.scm | 74 ++++++++++++++++++++++++++------------------
 tests/pypi.scm       | 14 +++++++++
 2 files changed, 58 insertions(+), 30 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8269aa61d7..d9db876222 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -47,7 +47,8 @@
   #:use-module (guix upstream)
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
-  #:export (guix-package->pypi-name
+  #:export (parse-requires.txt
+            guix-package->pypi-name
             pypi-recursive-import
             pypi->guix-package
             %pypi-updater))
@@ -117,6 +118,47 @@ package definition."
     ((package-inputs ...)
      `((propagated-inputs (,'quasiquote ,package-inputs))))))
 
+(define (clean-requirement s)
+  ;; Given a requirement LINE, as can be found in a setuptools requires.txt
+  ;; file, remove everything other than the actual name of the required
+  ;; package, and return it.
+  (cond
+   ((string-index s (char-set #\space #\> #\= #\<)) => (cut string-take s <>))
+   (else s)))
+
+(define (parse-requires.txt requires.txt)
+  "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
+requirement names."
+  ;; This is a very incomplete parser, whose job is to select the non-optional
+  ;; dependencies and strip them out of any version information.
+  ;; Alternatively, we could implement a PEG parser with the (ice-9 peg)
+  ;; library and the requirements grammar defined by PEP-0508
+  ;; (https://www.python.org/dev/peps/pep-0508/).
+
+  (define (comment? line)
+    ;; Return #t if the given LINE is a comment, #f otherwise.
+    (string-prefix? "#" (string-trim line)))
+
+  (define (section-header? line)
+    ;; Return #t if the given LINE is a section header, #f otherwise.
+    (string-prefix? "[" (string-trim line)))
+
+  (call-with-input-file requires.txt
+    (lambda (port)
+      (let loop ((result '()))
+        (let ((line (read-line port)))
+          ;; Stop when a section is encountered, as sections contain optional
+          ;; (extra) requirements.  Non-optional requirements must appear
+          ;; before any section is defined.
+          (if (or (eof-object? line) (section-header? line))
+              (reverse result)
+              (cond
+               ((or (string-null? line) (comment? line))
+                (loop result))
+               (else
+                (loop (cons (clean-requirement line)
+                            result))))))))))
+
 (define (guess-requirements source-url wheel-url tarball)
   "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list
 of the required packages specified in the requirements.txt file.  TARBALL will
@@ -139,34 +181,6 @@ be extracted in a temporary directory."
 cannot determine package dependencies"))
           #f)))))
 
-  (define (clean-requirement s)
-    ;; Given a requirement LINE, as can be found in a Python requirements.txt
-    ;; file, remove everything other than the actual name of the required
-    ;; package, and return it.
-    (string-take s
-      (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<))))
-          (string-length s))))
-
-  (define (comment? line)
-    ;; Return #t if the given LINE is a comment, #f otherwise.
-    (eq? (string-ref (string-trim line) 0) #\#))
-
-  (define (read-requirements requirements-file)
-    ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list
-    ;; of name/variable pairs describing the requirements.
-    (call-with-input-file requirements-file
-      (lambda (port)
-        (let loop ((result '()))
-          (let ((line (read-line port)))
-            (if (eof-object? line)
-                result
-                (cond
-                 ((or (string-null? line) (comment? line))
-                  (loop result))
-                 (else
-                  (loop (cons (clean-requirement line)
-                              result))))))))))
-
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
     ;; requirements.
@@ -212,7 +226,7 @@ cannot determine package dependencies"))
                                               (current-output-port (%make-void-port "rw+")))
                                  (system* "tar" "xf" tarball "-C" dir requires.txt))))
                (if (zero? exit-code)
-                   (read-requirements (string-append dir "/" requires.txt))
+                   (parse-requires.txt (string-append dir "/" requires.txt))
                    (begin
                      (warning
                       (G_ "Failed to extract file: ~a from source.~%")
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 6df69073dc..03455ba6be 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -62,6 +62,14 @@ bar
 baz > 13.37
 ")
 
+(define test-requires-with-sections "\
+foo ~= 3
+bar != 2
+
+[test]
+pytest (>=2.5.0)
+")
+
 (define test-metadata
   "{
   \"run_requires\": [
@@ -101,6 +109,12 @@ baz > 13.37
                     (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
                                (pypi-uri "cram" "0.7"))))))))
 
+(test-equal "parse-requires.txt, with sections"
+  '("foo" "bar")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-requires.txt test-requires-with-sections)))
+
 (test-assert "pypi->guix-package"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.6: 0004-import-pypi-Improve-parsing-of-requirement-specifica.patch --]
[-- Type: text/x-patch, Size: 6147 bytes --]

From bbdfb0ff2cc0347df73bb21b1443d6ef1f138a43 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 00:26:01 -0400
Subject: [PATCH 04/12] import: pypi: Improve parsing of requirement
 specifications.

The previous solution was fragile and could leave unwanted characters in a
requirement name, such as '[' or ']'.

Partially fixes <https://bugs.gnu.org/33047>.

* guix/import/pypi.scm (use-modules): Export SPECIFICATION->REQUIREMENT-NAME
(%requirement-name-regexp): New variable.
(clean-requirement): Rename to...
(specification->requirement-name): this, which now uses
%requirement-name-regexp to select the requirement name from the requirement
specification.
(parse-requires.txt): Adapt.
---
 guix/import/pypi.scm | 54 ++++++++++++++++++++++++++++++++------------
 tests/pypi.scm       | 12 ++++++++++
 2 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index d9db876222..6a881bda12 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -48,6 +48,7 @@
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
   #:export (parse-requires.txt
+            specification->requirement-name
             guix-package->pypi-name
             pypi-recursive-import
             pypi->guix-package
@@ -118,22 +119,47 @@ package definition."
     ((package-inputs ...)
      `((propagated-inputs (,'quasiquote ,package-inputs))))))
 
-(define (clean-requirement s)
-  ;; Given a requirement LINE, as can be found in a setuptools requires.txt
-  ;; file, remove everything other than the actual name of the required
-  ;; package, and return it.
-  (cond
-   ((string-index s (char-set #\space #\> #\= #\<)) => (cut string-take s <>))
-   (else s)))
+(define %requirement-name-regexp
+  ;; Regexp to match the requirement name in a requirement specification.
+
+  ;; Some grammar, taken from PEP-0508 (see:
+  ;; https://www.python.org/dev/peps/pep-0508/).
+
+  ;; Using this grammar makes the PEP-0508 regexp easier to understand for
+  ;; humans.  The use of a regexp is preferred to more primitive string
+  ;; manipulations because we can more directly match what upstream uses
+  ;; (again, per PEP-0508).  The regexp approach is also easier to extend,
+  ;; should we want to implement more completely the grammar of PEP-0508.
+
+  ;; The unified rule can be expressed as:
+  ;; specification = wsp* ( url_req | name_req ) wsp*
+
+  ;; where url_req is:
+  ;; url_req = name wsp* extras? wsp* urlspec wsp+ quoted_marker?
+
+  ;; and where name_req is:
+  ;; name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker?
+
+  ;; Thus, we need only matching NAME, which is expressed as:
+  ;; identifer_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit)
+  ;; identifier    = letterOrDigit identifier_end*
+  ;; name          = identifier
+  (let* ((letter-or-digit "[A-Za-z0-9]")
+         (identifier-end (string-append "(" letter-or-digit "|"
+                                        "[-_.]*" letter-or-digit ")"))
+         (identifier (string-append "^" letter-or-digit identifier-end "*"))
+         (name identifier))
+    (make-regexp name)))
+
+(define (specification->requirement-name spec)
+  "Given a specification SPEC, return the requirement name."
+  (match:substring
+   (or (regexp-exec %requirement-name-regexp spec)
+       (error (G_ "Could not extract requirement name in spec:") spec))))
 
 (define (parse-requires.txt requires.txt)
   "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
 requirement names."
-  ;; This is a very incomplete parser, whose job is to select the non-optional
-  ;; dependencies and strip them out of any version information.
-  ;; Alternatively, we could implement a PEG parser with the (ice-9 peg)
-  ;; library and the requirements grammar defined by PEP-0508
-  ;; (https://www.python.org/dev/peps/pep-0508/).
 
   (define (comment? line)
     ;; Return #t if the given LINE is a comment, #f otherwise.
@@ -156,7 +182,7 @@ requirement names."
                ((or (string-null? line) (comment? line))
                 (loop result))
                (else
-                (loop (cons (clean-requirement line)
+                (loop (cons (specification->requirement-name line)
                             result))))))))))
 
 (define (guess-requirements source-url wheel-url tarball)
@@ -198,7 +224,7 @@ cannot determine package dependencies"))
                                             (hash-ref (list-ref run_requires 0)
                                                        "requires")
                                             '())))
-                     (map clean-requirement requirements)))))
+                     (map specification->requirement-name requirements)))))
              (lambda ()
                (delete-file json-file)
                (rmdir dirname))))))
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 03455ba6be..c40be6c21d 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -55,6 +55,14 @@
 (define test-source-hash
   "")
 
+(define test-specifications
+  '("Fizzy [foo, bar]"
+    "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1"
+    "SomethingWithMarker[foo]>1.0;python_version<\"2.7\""
+    "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\""
+    "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\
+sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
+
 (define test-requires.txt "\
 # A comment
  # A comment after a space
@@ -109,6 +117,10 @@ pytest (>=2.5.0)
                     (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
                                (pypi-uri "cram" "0.7"))))))))
 
+(test-equal "specification->requirement-name"
+  '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
+  (map specification->requirement-name test-specifications))
+
 (test-equal "parse-requires.txt, with sections"
   '("foo" "bar")
   (mock ((ice-9 ports) call-with-input-file
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.7: 0005-import-pypi-Deduplicate-requirements.patch --]
[-- Type: text/x-patch, Size: 1255 bytes --]

From 475d5e483e7bfdae8db271fc513172c4f05e7358 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 00:26:01 -0400
Subject: [PATCH 05/12] import: pypi: Deduplicate requirements.

* guix/import/pypi.scm (parse-requires.txt): Remove potential duplicates.
---
 guix/import/pypi.scm | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 6a881bda12..ad59a8b731 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -177,7 +177,11 @@ requirement names."
           ;; (extra) requirements.  Non-optional requirements must appear
           ;; before any section is defined.
           (if (or (eof-object? line) (section-header? line))
-              (reverse result)
+              ;; Duplicates can occur, since the same requirement can be
+              ;; listed multiple times with different conditional markers, e.g.
+              ;; pytest >= 3 ; python_version >= "3.3"
+              ;; pytest < 3 ; python_version < "3.3"
+              (reverse (delete-duplicates result))
               (cond
                ((or (string-null? line) (comment? line))
                 (loop result))
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.8: 0006-import-pypi-Support-more-types-of-archives.patch --]
[-- Type: text/x-patch, Size: 10481 bytes --]

From 49b2232b6a3421e3b7d3b59be5b965c5bb11392e Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 00:26:02 -0400
Subject: [PATCH 06/12] import: pypi: Support more types of archives.

This change enables the PyPI importer to look for requirements in a source
archive of a different type than "tar.gz" or "tar.bz2".  Also, scan the source
archive to find a requires.txt file.

* guix/import/pypi.scm: (guess-requirements)[tarball-directory]: Remove procedure.
[guess-requirements-from-source]: Use COMRESSED-FILE? to determine if an
archive type is supported, and some file extension logic that chooses either
"tar" or "unzip" as the extractor.  Search for the requires.txt file in the
archive instead of using a static, expected location.
(guess-requirements): Rename the TARBALL argument to ARCHIVE, to denote the
archive format is no longer bound specifically to the Tar format.
(compute-inputs): Likewise.
* tests/pypi.scm ("pypi->guix-package, no wheel"): Mock the requires.txt at a
non-standard location.
("pypi->guix-package, no usable requirement file."): New test.
---
 guix/import/pypi.scm | 77 +++++++++++++++++++-------------------------
 tests/pypi.scm       | 52 ++++++++++++++++++++++++++++--
 2 files changed, 83 insertions(+), 46 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index ad59a8b731..a6106ab4ec 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -39,7 +39,8 @@
   #:use-module ((guix build utils)
                 #:select ((package-name->name+version
                            . hyphen-package-name->name+version)
-                          find-files))
+                          find-files
+                          invoke))
   #:use-module (guix import utils)
   #:use-module ((guix download) #:prefix download:)
   #:use-module (guix import json)
@@ -189,28 +190,11 @@ requirement names."
                 (loop (cons (specification->requirement-name line)
                             result))))))))))
 
-(define (guess-requirements source-url wheel-url tarball)
-  "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list
-of the required packages specified in the requirements.txt file.  TARBALL will
+(define (guess-requirements source-url wheel-url archive)
+  "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
+of the required packages specified in the requirements.txt file.  ARCHIVE will
 be extracted in a temporary directory."
 
-  (define (tarball-directory url)
-    ;; Given the URL of the package's tarball, return the name of the directory
-    ;; that will be created upon decompressing it. If the filetype is not
-    ;; supported, return #f.
-    ;; TODO: Support more archive formats.
-    (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
-      (cond
-       ((string-suffix? ".tar.gz" basename)
-        (string-drop-right basename 7))
-       ((string-suffix? ".tar.bz2" basename)
-        (string-drop-right basename 8))
-       (else
-        (begin
-          (warning (G_ "Unsupported archive format: \
-cannot determine package dependencies"))
-          #f)))))
-
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
     ;; requirements.
@@ -239,29 +223,34 @@ cannot determine package dependencies"))
     (call-with-temporary-output-file
      (lambda (temp port)
        (if wheel-url
-         (and (url-fetch wheel-url temp)
-              (read-wheel-metadata temp))
-         #f))))
+           (and (url-fetch wheel-url temp)
+                (read-wheel-metadata temp))
+           #f))))
 
   (define (guess-requirements-from-source)
     ;; Return the package's requirements by guessing them from the source.
-    (let ((dirname (tarball-directory source-url)))
-      (if (string? dirname)
-          (call-with-temporary-directory
-           (lambda (dir)
-             (let* ((pypi-name (string-take dirname (string-rindex dirname #\-)))
-                    (requires.txt (string-append dirname "/" pypi-name
-                                                 ".egg-info" "/requires.txt"))
-                    (exit-code (parameterize ((current-error-port (%make-void-port "rw+"))
-                                              (current-output-port (%make-void-port "rw+")))
-                                 (system* "tar" "xf" tarball "-C" dir requires.txt))))
-               (if (zero? exit-code)
-                   (parse-requires.txt (string-append dir "/" requires.txt))
-                   (begin
-                     (warning
-                      (G_ "Failed to extract file: ~a from source.~%")
-                      requires.txt)
-                     '())))))
+    (if (compressed-file? source-url)
+        (call-with-temporary-directory
+         (lambda (dir)
+           (parameterize ((current-error-port (%make-void-port "rw+"))
+                          (current-output-port (%make-void-port "rw+")))
+             (if (string=? "zip" (file-extension source-url))
+                 (invoke "unzip" archive "-d" dir)
+                 (invoke "tar" "xf" archive "-C" dir)))
+           (let ((requires.txt-files
+                  (find-files dir (lambda (abs-file-name _)
+		                    (string-match "\\.egg-info/requires.txt$"
+                                                  abs-file-name)))))
+             (match requires.txt-files
+               (()
+                (warning (G_ "Cannot guess requirements from source archive:\
+ no requires.txt file found.~%"))
+                '())
+               (else (parse-requires.txt (first requires.txt-files)))))))
+        (begin
+          (warning (G_ "Unsupported archive format; \
+cannot determine package dependencies from source archive: ~a~%")
+                   (basename source-url))
           '())))
 
   ;; First, try to compute the requirements using the wheel, else, fallback to
@@ -270,13 +259,13 @@ cannot determine package dependencies"))
   (or (guess-requirements-from-wheel)
       (guess-requirements-from-source)))
 
-(define (compute-inputs source-url wheel-url tarball)
-  "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
+(define (compute-inputs source-url wheel-url archive)
+  "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of
 name/variable pairs describing the required inputs of this package.  Also
 return the unaltered list of upstream dependency names."
   (let ((dependencies
          (remove (cut string=? "argparse" <>)
-                 (guess-requirements source-url wheel-url tarball))))
+                 (guess-requirements source-url wheel-url archive))))
     (values (sort
              (map (lambda (input)
                     (let ((guix-name (python->package-name input)))
diff --git a/tests/pypi.scm b/tests/pypi.scm
index c40be6c21d..b45d2c9d2f 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -20,6 +20,7 @@
 (define-module (test-pypi)
   #:use-module (guix import pypi)
   #:use-module (guix base32)
+  #:use-module (guix memoization)
   #:use-module (gcrypt hash)
   #:use-module (guix tests)
   #:use-module (guix build-system python)
@@ -134,8 +135,9 @@ pytest (>=2.5.0)
              (match url
                ("https://example.com/foo-1.0.0.tar.gz"
                 (begin
-                  (mkdir-p "foo-1.0.0/foo.egg-info/")
-                  (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
+                  ;; Unusual requires.txt location should still be found.
+                  (mkdir-p "foo-1.0.0/src/bizarre.egg-info")
+                  (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt"
                     (lambda ()
                       (display test-requires.txt)))
                   (parameterize ((current-output-port (%make-void-port "rw+")))
@@ -241,4 +243,50 @@ pytest (>=2.5.0)
                 (x
                  (pk 'fail x #f))))))
 
+(test-assert "pypi->guix-package, no usable requirement file."
+  ;; Replace network resources with sample data.
+  (mock ((guix import utils) url-fetch
+         (lambda (url file-name)
+           (match url
+             ("https://example.com/foo-1.0.0.tar.gz"
+              (mkdir-p "foo-1.0.0/foo.egg-info/")
+              (parameterize ((current-output-port (%make-void-port "rw+")))
+                (system* "tar" "czvf" file-name "foo-1.0.0/"))
+              (delete-file-recursively "foo-1.0.0")
+              (set! test-source-hash
+                (call-with-input-file file-name port-sha256)))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
+             (_ (error "Unexpected URL: " url)))))
+        (mock ((guix http-client) http-fetch
+               (lambda (url . rest)
+                 (match url
+                   ("https://pypi.org/pypi/foo/json"
+                    (values (open-input-string test-json)
+                            (string-length test-json)))
+                   ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
+                   (_ (error "Unexpected URL: " url)))))
+              ;; Not clearing the memoization cache here would mean returning the value
+              ;; computed in the previous test.
+              (invalidate-memoization! pypi->guix-package)
+              (match (pypi->guix-package "foo")
+                (('package
+                   ('name "python-foo")
+                   ('version "1.0.0")
+                   ('source ('origin
+                              ('method 'url-fetch)
+                              ('uri ('pypi-uri "foo" 'version))
+                              ('sha256
+                               ('base32
+                                (? string? hash)))))
+                   ('build-system 'python-build-system)
+                   ('home-page "http://example.com")
+                   ('synopsis "summary")
+                   ('description "summary")
+                   ('license 'license:lgpl2.0))
+                 (string=? (bytevector->nix-base32-string
+                            test-source-hash)
+                           hash))
+                (x
+                 (pk 'fail x #f))))))
+
 (test-end "pypi")
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.9: 0007-import-pypi-Parse-wheel-METADATA-instead-of-metadata.patch --]
[-- Type: text/x-patch, Size: 10948 bytes --]

From 3a0082de1bed3eca6dedf71dec34efd0457bd58f Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 00:26:03 -0400
Subject: [PATCH 07/12] import: pypi: Parse wheel METADATA instead of
 metadata.json.

With newer Wheel releases, there is no more metadata.json file; the METADATA
file should be used instead (see: https://github.com/pypa/wheel/issues/195).

This change updates our PyPI importer so that it uses the latter.

* guix/import/pypi.scm (define-module): Remove unnecessary modules and export
the PARSE-WHEEL-METADATA procedure.
(parse-wheel-metadata): Add procedure.
(guess-requirements): Use it.
* tests/pypi.scm (test-metadata): Test it.
---
 guix/import/pypi.scm | 90 +++++++++++++++++++++++++++-----------------
 tests/pypi.scm       | 60 ++++++++++++++++++++++-------
 2 files changed, 101 insertions(+), 49 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index a6106ab4ec..7cf1e92101 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,9 +21,7 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (guix import pypi)
-  #:use-module (ice-9 binary-ports)
   #:use-module (ice-9 match)
-  #:use-module (ice-9 pretty-print)
   #:use-module (ice-9 regex)
   #:use-module (ice-9 receive)
   #:use-module ((ice-9 rdelim) #:select (read-line))
@@ -31,9 +29,6 @@
   #:use-module (srfi srfi-26)
   #:use-module (srfi srfi-34)
   #:use-module (srfi srfi-35)
-  #:use-module (rnrs bytevectors)
-  #:use-module (json)
-  #:use-module (web uri)
   #:use-module (guix ui)
   #:use-module (guix utils)
   #:use-module ((guix build utils)
@@ -49,6 +44,7 @@
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
   #:export (parse-requires.txt
+            parse-wheel-metadata
             specification->requirement-name
             guix-package->pypi-name
             pypi-recursive-import
@@ -177,18 +173,49 @@ requirement names."
           ;; Stop when a section is encountered, as sections contain optional
           ;; (extra) requirements.  Non-optional requirements must appear
           ;; before any section is defined.
-          (if (or (eof-object? line) (section-header? line))
-              ;; Duplicates can occur, since the same requirement can be
-              ;; listed multiple times with different conditional markers, e.g.
-              ;; pytest >= 3 ; python_version >= "3.3"
-              ;; pytest < 3 ; python_version < "3.3"
-              (reverse (delete-duplicates result))
-              (cond
-               ((or (string-null? line) (comment? line))
-                (loop result))
-               (else
-                (loop (cons (specification->requirement-name line)
-                            result))))))))))
+          (cond
+           ((or (eof-object? line) (section-header? line))
+            ;; Duplicates can occur, since the same requirement can be
+            ;; listed multiple times with different conditional markers, e.g.
+            ;; pytest >= 3 ; python_version >= "3.3"
+            ;; pytest < 3 ; python_version < "3.3"
+            (reverse (delete-duplicates result)))
+           ((or (string-null? line) (comment? line))
+            (loop result))
+           (else
+            (loop (cons (specification->requirement-name line)
+                        result)))))))))
+
+(define (parse-wheel-metadata metadata)
+  "Given METADATA, a Wheel metadata file, return a list of requirement names."
+  ;; METADATA is a RFC-2822-like, header based file.
+
+  (define (requires-dist-header? line)
+    ;; Return #t if the given LINE is a Requires-Dist header.
+    (string-match "^Requires-Dist: " line))
+
+  (define (requires-dist-value line)
+    (string-drop line (string-length "Requires-Dist: ")))
+
+  (define (extra? line)
+    ;; Return #t if the given LINE is an "extra" requirement.
+    (string-match "extra == '(.*)'" line))
+
+  (call-with-input-file metadata
+    (lambda (port)
+      (let loop ((requirements '()))
+        (let ((line (read-line port)))
+          ;; Stop at the first 'Provides-Extra' section: the non-optional
+          ;; requirements appear before the optional ones.
+          (cond
+           ((eof-object? line)
+            (reverse (delete-duplicates requirements)))
+           ((and (requires-dist-header? line) (not (extra? line)))
+            (loop (cons (specification->requirement-name
+                         (requires-dist-value line))
+                        requirements)))
+           (else
+            (loop requirements))))))))
 
 (define (guess-requirements source-url wheel-url archive)
   "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
@@ -197,25 +224,18 @@ be extracted in a temporary directory."
 
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
-    ;; requirements.
+    ;; requirements, or #f if the metadata file contained therein couldn't be
+    ;; extracted.
     (let* ((dirname (wheel-url->extracted-directory wheel-url))
-           (json-file (string-append dirname "/metadata.json")))
-      (and (zero? (system* "unzip" "-q" wheel-archive json-file))
-           (dynamic-wind
-             (const #t)
-             (lambda ()
-               (call-with-input-file json-file
-                 (lambda (port)
-                   (let* ((metadata (json->scm port))
-                          (run_requires (hash-ref metadata "run_requires"))
-                          (requirements (if run_requires
-                                            (hash-ref (list-ref run_requires 0)
-                                                       "requires")
-                                            '())))
-                     (map specification->requirement-name requirements)))))
-             (lambda ()
-               (delete-file json-file)
-               (rmdir dirname))))))
+           (metadata (string-append dirname "/METADATA")))
+      (call-with-temporary-directory
+       (lambda (dir)
+         (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata))
+             (parse-wheel-metadata (string-append dir "/" metadata))
+             (begin
+               (warning
+                (G_ "Failed to extract file: ~a from wheel.~%") metadata)
+               #f))))))
 
   (define (guess-requirements-from-wheel)
     ;; Return the package's requirements using the wheel, or #f if an error
diff --git a/tests/pypi.scm b/tests/pypi.scm
index b45d2c9d2f..8b42c2f071 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -22,6 +22,7 @@
   #:use-module (guix base32)
   #:use-module (guix memoization)
   #:use-module (gcrypt hash)
+  #:use-module (guix memoization)
   #:use-module (guix tests)
   #:use-module (guix build-system python)
   #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
@@ -79,17 +80,33 @@ bar != 2
 pytest (>=2.5.0)
 ")
 
-(define test-metadata
-  "{
-  \"run_requires\": [
-    {
-      \"requires\": [
-        \"bar\",
-        \"baz (>13.37)\"
-      ]
-    }
-  ]
-}")
+(define test-metadata "\
+Classifier: Programming Language :: Python :: 3.7
+Requires-Dist: baz ~= 3
+Requires-Dist: bar != 2
+Provides-Extra: test
+pytest (>=2.5.0)
+")
+
+(define test-metadata-with-extras "
+Classifier: Programming Language :: Python :: 3.7
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: wrapt (<2,>=1)
+Requires-Dist: bar
+
+Provides-Extra: dev
+Requires-Dist: tox ; extra == 'dev'
+Requires-Dist: bumpversion (<1) ; extra == 'dev'
+")
+
+;;; Provides-Extra can appear before Requires-Dist.
+(define test-metadata-with-extras-jedi "\
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Provides-Extra: testing
+Requires-Dist: parso (>=0.3.0)
+Provides-Extra: testing
+Requires-Dist: pytest (>=3.1.0); extra == 'testing'
+")
 
 (test-begin "pypi")
 
@@ -128,6 +145,18 @@ pytest (>=2.5.0)
          call-with-input-string)
         (parse-requires.txt test-requires-with-sections)))
 
+(test-equal "parse-wheel-metadata, with extras"
+  '("wrapt" "bar")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras)))
+
+(test-equal "parse-wheel-metadata, with extras - Jedi"
+  '("parso")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras-jedi)))
+
 (test-assert "pypi->guix-package"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
@@ -191,7 +220,7 @@ pytest (>=2.5.0)
                 (mkdir-p "foo-1.0.0/foo.egg-info/")
                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                    (lambda ()
-                     (display test-requires.txt)))
+                     (display "wrong data to make sure we're testing wheels ")))
                 (parameterize ((current-output-port (%make-void-port "rw+")))
                   (system* "tar" "czvf" file-name "foo-1.0.0/"))
                  (delete-file-recursively "foo-1.0.0")
@@ -200,13 +229,13 @@ pytest (>=2.5.0)
              ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
                (begin
                  (mkdir "foo-1.0.0.dist-info")
-                 (with-output-to-file "foo-1.0.0.dist-info/metadata.json"
+                 (with-output-to-file "foo-1.0.0.dist-info/METADATA"
                    (lambda ()
                      (display test-metadata)))
                  (let ((zip-file (string-append file-name ".zip")))
                    ;; zip always adds a "zip" extension to the file it creates,
                    ;; so we need to rename it.
-                   (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
+                   (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
                    (rename-file zip-file file-name))
                  (delete-file-recursively "foo-1.0.0.dist-info")))
              (_ (error "Unexpected URL: " url)))))
@@ -218,6 +247,9 @@ pytest (>=2.5.0)
                             (string-length test-json)))
                    ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
                    (_ (error "Unexpected URL: " url)))))
+              ;; Not clearing the memoization cache here would mean returning the value
+              ;; computed in the previous test.
+              (invalidate-memoization! pypi->guix-package)
               (match (pypi->guix-package "foo")
                 (('package
                    ('name "python-foo")
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.10: 0008-import-pypi-Fix-typo-in-docstring.patch --]
[-- Type: text/x-patch, Size: 958 bytes --]

From 04e119d50d8622bf5bf8f1a6c6c80cd51e3e1658 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Wed, 12 Jun 2019 11:34:23 +0900
Subject: [PATCH 08/12] import: pypi: Fix typo in docstring.

* guix/import/pypi.scm (guess-requirements): Fix typo.
---
 guix/import/pypi.scm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 7cf1e92101..d861dd960d 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -218,7 +218,7 @@ requirement names."
             (loop requirements))))))))
 
 (define (guess-requirements source-url wheel-url archive)
-  "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
+  "Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list
 of the required packages specified in the requirements.txt file.  ARCHIVE will
 be extracted in a temporary directory."
 
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.11: 0009-import-pypi-Completely-mute-the-output-of-the-unzip-.patch --]
[-- Type: text/x-patch, Size: 1236 bytes --]

From 141551b4dae1f7d9920495eddfd85d49e3a45262 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Wed, 12 Jun 2019 11:36:39 +0900
Subject: [PATCH 09/12] import: pypi: Completely mute the output of the "unzip"
 command.

* guix/import/pypi.scm (guess-requirements): Completely mute the output of the
"unzip" command.
---
 guix/import/pypi.scm | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index d861dd960d..23a1e69061 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -230,7 +230,10 @@ be extracted in a temporary directory."
            (metadata (string-append dirname "/METADATA")))
       (call-with-temporary-directory
        (lambda (dir)
-         (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata))
+         (if (zero?
+              (parameterize ((current-error-port (%make-void-port "rw+"))
+                             (current-output-port (%make-void-port "rw+")))
+                (system* "unzip" wheel-archive "-d" dir metadata)))
              (parse-wheel-metadata (string-append dir "/" metadata))
              (begin
                (warning
-- 
2.21.0


[-- Attachment #1.12: 0010-import-pypi-Include-optional-test-inputs-as-native-i.patch --]
[-- Type: text/x-patch, Size: 19751 bytes --]

From b435bb15a1905fee13fcee5423a6d2dc2300fa9a Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 28 Mar 2019 23:12:26 -0400
Subject: [PATCH 10/12] import: pypi: Include optional test inputs as
 native-inputs.

* guix/import/pypi.scm (maybe-inputs): Add INPUT-TYPE argument, and use it.
(test-section?): New predicate.
(parse-requires.txt): Collect the optional test inputs, and return them as the
second element of the returned list.
(parse-wheel-metadata): Likewise.
(guess-requirements): Adapt.
(make-pypi-sexp): Likewise, and include the test inputs requirements as native
inputs in the returned package expression.

* tests/pypi.scm (test-requires.txt): Include a test section in the
test-requires.txt data.
(test-requires.txt-beaker): New variable.
("parse-requires.txt"): Adapt.
("parse-requires.txt - Beaker"): New test.
("parse-wheel-metadata, with extras"): Adapt.
("parse-wheel-metadata, with extras - Jedi"): Adapt.
("pypi->guix-package, no wheel"): Re-indent, and add the expected
native-inputs.
("pypi->guix-package, wheels"): Likewise.
---
 guix/import/pypi.scm | 177 ++++++++++++++++++++++++++++---------------
 tests/pypi.scm       |  79 ++++++++++++-------
 2 files changed, 166 insertions(+), 90 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 23a1e69061..537431dd69 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -4,6 +4,7 @@
 ;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
 ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com>
 ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -26,6 +27,7 @@
   #:use-module (ice-9 receive)
   #:use-module ((ice-9 rdelim) #:select (read-line))
   #:use-module (srfi srfi-1)
+  #:use-module (srfi srfi-11)
   #:use-module (srfi srfi-26)
   #:use-module (srfi srfi-34)
   #:use-module (srfi srfi-35)
@@ -107,14 +109,15 @@ package on PyPI."
     ((name version _ ...)
      (string-append name "-" version ".dist-info"))))
 
-(define (maybe-inputs package-inputs)
+(define (maybe-inputs package-inputs input-type)
   "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a
-package definition."
+package definition.  INPUT-TYPE, a symbol, is used to populate the name of
+the input field."
   (match package-inputs
     (()
      '())
     ((package-inputs ...)
-     `((propagated-inputs (,'quasiquote ,package-inputs))))))
+     `((,input-type (,'quasiquote ,package-inputs))))))
 
 (define %requirement-name-regexp
   ;; Regexp to match the requirement name in a requirement specification.
@@ -154,9 +157,19 @@ package definition."
    (or (regexp-exec %requirement-name-regexp spec)
        (error (G_ "Could not extract requirement name in spec:") spec))))
 
+(define (test-section? name)
+  "Return #t if the section name contains 'test' or 'dev'."
+  (any (cut string-contains-ci name <>)
+       '("test" "dev")))
+
 (define (parse-requires.txt requires.txt)
-  "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
-requirement names."
+  "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of lists
+of requirements.
+
+The first list contains the required dependencies while the second the
+optional test dependencies.  Note that currently, optional, non-test
+dependencies are omitted since these can be difficult or expensive to
+satisfy."
 
   (define (comment? line)
     ;; Return #t if the given LINE is a comment, #f otherwise.
@@ -168,26 +181,49 @@ requirement names."
 
   (call-with-input-file requires.txt
     (lambda (port)
-      (let loop ((result '()))
+      (let loop ((required-deps '())
+                 (test-deps '())
+                 (inside-test-section? #f)
+                 (optional? #f))
         (let ((line (read-line port)))
-          ;; Stop when a section is encountered, as sections contain optional
-          ;; (extra) requirements.  Non-optional requirements must appear
-          ;; before any section is defined.
           (cond
-           ((or (eof-object? line) (section-header? line))
+           ((eof-object? line)
             ;; Duplicates can occur, since the same requirement can be
             ;; listed multiple times with different conditional markers, e.g.
             ;; pytest >= 3 ; python_version >= "3.3"
             ;; pytest < 3 ; python_version < "3.3"
-            (reverse (delete-duplicates result)))
+            (map (compose reverse delete-duplicates)
+                 (list required-deps test-deps)))
            ((or (string-null? line) (comment? line))
-            (loop result))
-           (else
+            (loop required-deps test-deps inside-test-section? optional?))
+           ((section-header? line)
+            ;; Encountering a section means that all the requirements
+            ;; listed below are optional. Since we want to pick only the
+            ;; test dependencies from the optional dependencies, we must
+            ;; track those separately.
+            (loop required-deps test-deps (test-section? line) #t))
+           (inside-test-section?
+            (loop required-deps
+                  (cons (specification->requirement-name line)
+                        test-deps)
+                  inside-test-section? optional?))
+           ((not optional?)
             (loop (cons (specification->requirement-name line)
-                        result)))))))))
+                        required-deps)
+                  test-deps inside-test-section? optional?))
+           (optional?
+            ;; Skip optional items.
+            (loop required-deps test-deps inside-test-section? optional?))
+           (else
+            (warning (G_ "parse-requires.txt reached an unexpected \
+condition on line ~a~%") line))))))))
 
 (define (parse-wheel-metadata metadata)
-  "Given METADATA, a Wheel metadata file, return a list of requirement names."
+  "Given METADATA, a Wheel metadata file, return a list of lists of
+requirements.
+
+Refer to the documentation of PARSE-REQUIRES.TXT for a description of the
+returned value."
   ;; METADATA is a RFC-2822-like, header based file.
 
   (define (requires-dist-header? line)
@@ -201,21 +237,29 @@ requirement names."
     ;; Return #t if the given LINE is an "extra" requirement.
     (string-match "extra == '(.*)'" line))
 
+  (define (test-requirement? line)
+    (and=> (match:substring (extra? line) 1) test-section?))
+
   (call-with-input-file metadata
     (lambda (port)
-      (let loop ((requirements '()))
+      (let loop ((required-deps '())
+                 (test-deps '()))
         (let ((line (read-line port)))
-          ;; Stop at the first 'Provides-Extra' section: the non-optional
-          ;; requirements appear before the optional ones.
           (cond
            ((eof-object? line)
-            (reverse (delete-duplicates requirements)))
+            (map (compose reverse delete-duplicates)
+                 (list required-deps test-deps)))
            ((and (requires-dist-header? line) (not (extra? line)))
             (loop (cons (specification->requirement-name
                          (requires-dist-value line))
-                        requirements)))
+                        required-deps)
+                  test-deps))
+           ((and (requires-dist-header? line) (test-requirement? line))
+            (loop required-deps
+                  (cons (specification->requirement-name (requires-dist-value line))
+                        test-deps)))
            (else
-            (loop requirements))))))))
+            (loop required-deps test-deps)))))))) ;skip line
 
 (define (guess-requirements source-url wheel-url archive)
   "Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list
@@ -268,37 +312,46 @@ be extracted in a temporary directory."
                (()
                 (warning (G_ "Cannot guess requirements from source archive:\
  no requires.txt file found.~%"))
-                '())
+                (list '() '()))
                (else (parse-requires.txt (first requires.txt-files)))))))
         (begin
           (warning (G_ "Unsupported archive format; \
 cannot determine package dependencies from source archive: ~a~%")
                    (basename source-url))
-          '())))
+          (list '() '()))))
 
   ;; First, try to compute the requirements using the wheel, else, fallback to
   ;; reading the "requires.txt" from the egg-info directory from the source
-  ;; tarball.
+  ;; archive.
   (or (guess-requirements-from-wheel)
       (guess-requirements-from-source)))
 
 (define (compute-inputs source-url wheel-url archive)
-  "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of
-name/variable pairs describing the required inputs of this package.  Also
+  "Given the SOURCE-URL and WHEEL-URL of an already downloaded ARCHIVE, return
+a pair of lists, each consisting of a list of name/variable pairs, for the
+propagated inputs and the native inputs, respectively.  Also
 return the unaltered list of upstream dependency names."
-  (let ((dependencies
-         (remove (cut string=? "argparse" <>)
-                 (guess-requirements source-url wheel-url archive))))
-    (values (sort
-             (map (lambda (input)
-                    (let ((guix-name (python->package-name input)))
-                      (list guix-name (list 'unquote (string->symbol guix-name)))))
-                  dependencies)
-             (lambda args
-               (match args
-                 (((a _ ...) (b _ ...))
-                  (string-ci<? a b)))))
-            dependencies)))
+
+  (define (strip-argparse deps)
+    (remove (cut string=? "argparse" <>) deps))
+
+  (define (requirement->package-name/sort deps)
+    (sort
+     (map (lambda (input)
+            (let ((guix-name (python->package-name input)))
+              (list guix-name (list 'unquote (string->symbol guix-name)))))
+          deps)
+     (lambda args
+       (match args
+         (((a _ ...) (b _ ...))
+          (string-ci<? a b))))))
+
+  (define process-requirements
+    (compose requirement->package-name/sort strip-argparse))
+
+  (let ((dependencies (guess-requirements source-url wheel-url archive)))
+    (values (map process-requirements dependencies)
+            (concatenate dependencies))))
 
 (define (make-pypi-sexp name version source-url wheel-url home-page synopsis
                         description license)
@@ -307,29 +360,31 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
   (call-with-temporary-output-file
    (lambda (temp port)
      (and (url-fetch source-url temp)
-          (receive (input-package-names upstream-dependency-names)
+          (receive (guix-dependencies upstream-dependencies)
               (compute-inputs source-url wheel-url temp)
-            (values
-             `(package
-                (name ,(python->package-name name))
-                (version ,version)
-                (source (origin
-                          (method url-fetch)
-
-                          ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
-                          ;; cases in NAME, for instance, as is the case with
-                          ;; "uwsgi".  In that case, fall back to a full URL.
-                          (uri (pypi-uri ,(string-downcase name) version))
-                          (sha256
-                           (base32
-                            ,(guix-hash-url temp)))))
-                (build-system python-build-system)
-                ,@(maybe-inputs input-package-names)
-                (home-page ,home-page)
-                (synopsis ,synopsis)
-                (description ,description)
-                (license ,(license->symbol license)))
-             upstream-dependency-names))))))
+            (match guix-dependencies
+              ((required-inputs test-inputs)
+               (values
+                `(package
+                   (name ,(python->package-name name))
+                   (version ,version)
+                   (source (origin
+                             (method url-fetch)
+                             ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
+                             ;; cases in NAME, for instance, as is the case with
+                             ;; "uwsgi".  In that case, fall back to a full URL.
+                             (uri (pypi-uri ,(string-downcase name) version))
+                             (sha256
+                              (base32
+                               ,(guix-hash-url temp)))))
+                   (build-system python-build-system)
+                   ,@(maybe-inputs required-inputs 'propagated-inputs)
+                   ,@(maybe-inputs test-inputs 'native-inputs)
+                   (home-page ,home-page)
+                   (synopsis ,synopsis)
+                   (description ,description)
+                   (license ,(license->symbol license)))
+                upstream-dependencies))))))))
 
 (define pypi->guix-package
   (memoize
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 8b42c2f071..43d45f1dd8 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -1,6 +1,7 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2014 David Thompson <davet@gnu.org>
 ;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -68,11 +69,6 @@ sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
 (define test-requires.txt "\
 # A comment
  # A comment after a space
-bar
-baz > 13.37
-")
-
-(define test-requires-with-sections "\
 foo ~= 3
 bar != 2
 
@@ -80,12 +76,25 @@ bar != 2
 pytest (>=2.5.0)
 ")
 
+;; Beaker contains only optional dependencies.
+(define test-requires.txt-beaker "\
+[crypto]
+pycryptopp>=0.5.12
+
+[cryptography]
+cryptography
+
+[testsuite]
+Mock
+coverage
+")
+
 (define test-metadata "\
 Classifier: Programming Language :: Python :: 3.7
 Requires-Dist: baz ~= 3
 Requires-Dist: bar != 2
 Provides-Extra: test
-pytest (>=2.5.0)
+Requires-Dist: pytest (>=2.5.0) ; extra == 'test'
 ")
 
 (define test-metadata-with-extras "
@@ -139,25 +148,31 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
   '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
   (map specification->requirement-name test-specifications))
 
-(test-equal "parse-requires.txt, with sections"
-  '("foo" "bar")
+(test-equal "parse-requires.txt"
+  (list '("foo" "bar") '("pytest"))
   (mock ((ice-9 ports) call-with-input-file
          call-with-input-string)
-        (parse-requires.txt test-requires-with-sections)))
+        (parse-requires.txt test-requires.txt)))
+
+(test-equal "parse-requires.txt - Beaker"
+  (list '() '("Mock" "coverage"))
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-requires.txt test-requires.txt-beaker)))
 
 (test-equal "parse-wheel-metadata, with extras"
-  '("wrapt" "bar")
+  (list '("wrapt" "bar") '("tox" "bumpversion"))
   (mock ((ice-9 ports) call-with-input-file
          call-with-input-string)
         (parse-wheel-metadata test-metadata-with-extras)))
 
 (test-equal "parse-wheel-metadata, with extras - Jedi"
-  '("parso")
+  (list '("parso") '("pytest"))
   (mock ((ice-9 ports) call-with-input-file
          call-with-input-string)
         (parse-wheel-metadata test-metadata-with-extras-jedi)))
 
-(test-assert "pypi->guix-package"
+(test-assert "pypi->guix-package, no wheel"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
            (lambda (url file-name)
@@ -198,7 +213,10 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                      ('propagated-inputs
                       ('quasiquote
                        (("python-bar" ('unquote 'python-bar))
-                        ("python-baz" ('unquote 'python-baz)))))
+                        ("python-foo" ('unquote 'python-foo)))))
+                     ('native-inputs
+                      ('quasiquote
+                       (("python-pytest" ('unquote 'python-pytest)))))
                      ('home-page "http://example.com")
                      ('synopsis "summary")
                      ('description "summary")
@@ -219,25 +237,25 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
               (begin
                 (mkdir-p "foo-1.0.0/foo.egg-info/")
                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
-                   (lambda ()
-                     (display "wrong data to make sure we're testing wheels ")))
+                  (lambda ()
+                    (display "wrong data to make sure we're testing wheels ")))
                 (parameterize ((current-output-port (%make-void-port "rw+")))
                   (system* "tar" "czvf" file-name "foo-1.0.0/"))
-                 (delete-file-recursively "foo-1.0.0")
-                 (set! test-source-hash
-                       (call-with-input-file file-name port-sha256))))
+                (delete-file-recursively "foo-1.0.0")
+                (set! test-source-hash
+                  (call-with-input-file file-name port-sha256))))
              ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
-               (begin
-                 (mkdir "foo-1.0.0.dist-info")
-                 (with-output-to-file "foo-1.0.0.dist-info/METADATA"
-                   (lambda ()
-                     (display test-metadata)))
-                 (let ((zip-file (string-append file-name ".zip")))
-                   ;; zip always adds a "zip" extension to the file it creates,
-                   ;; so we need to rename it.
-                   (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
-                   (rename-file zip-file file-name))
-                 (delete-file-recursively "foo-1.0.0.dist-info")))
+              (begin
+                (mkdir "foo-1.0.0.dist-info")
+                (with-output-to-file "foo-1.0.0.dist-info/METADATA"
+                  (lambda ()
+                    (display test-metadata)))
+                (let ((zip-file (string-append file-name ".zip")))
+                  ;; zip always adds a "zip" extension to the file it creates,
+                  ;; so we need to rename it.
+                  (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA")
+                  (rename-file zip-file file-name))
+                (delete-file-recursively "foo-1.0.0.dist-info")))
              (_ (error "Unexpected URL: " url)))))
         (mock ((guix http-client) http-fetch
                (lambda (url . rest)
@@ -265,6 +283,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                     ('quasiquote
                      (("python-bar" ('unquote 'python-bar))
                       ("python-baz" ('unquote 'python-baz)))))
+                   ('native-inputs
+                    ('quasiquote
+                     (("python-pytest" ('unquote 'python-pytest)))))
                    ('home-page "http://example.com")
                    ('synopsis "summary")
                    ('description "summary")
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.13: 0011-import-pypi-Update-the-host-URI.patch --]
[-- Type: text/x-patch, Size: 1043 bytes --]

From 736e91e7c1d9d7e7c6f93342766fdcd4b78cbfa1 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Sun, 16 Jun 2019 14:52:25 +0900
Subject: [PATCH 11/12] import: pypi: Update the host URI.

* guix/build-system/python.scm (pypi-uri): Update the host URI to
"files.pythonhosted.org".
---
 guix/build-system/python.scm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm
index b753940bad..e39c06528e 100644
--- a/guix/build-system/python.scm
+++ b/guix/build-system/python.scm
@@ -50,7 +50,7 @@
   "Return a URI string for the Python package hosted on the Python Package
 Index (PyPI) corresponding to NAME and VERSION.  EXTENSION is the file name
 extension, such as '.tar.gz'."
-  (string-append "https://pypi.org/packages/source/"
+  (string-append "https://files.pythonhosted.org/packages/source/"
                  (string-take name 1) "/" name "/"
                  name "-" version extension))
 
-- 
2.21.0


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.14: 0012-import-pypi-Preserve-package-name-case-when-forming-.patch --]
[-- Type: text/x-patch, Size: 2535 bytes --]

From 5373f50088cd0d37e1dbc64e6ca505224a452d8e Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Sat, 30 Mar 2019 20:27:35 -0400
Subject: [PATCH 12/12] import: pypi: Preserve package name case when forming
 pypi-uri.

Fixes <https://bugs.gnu.org/33046>.

* guix/build-system/python.scm (pypi-uri): Update the host URI to
"files.pythonhosted.org".
* guix/import/pypi.scm (make-pypi-sexp): Preserve the package name case when
the source URL calls for it.
---
 guix/import/pypi.scm | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 537431dd69..ab7a024ee0 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -368,15 +368,20 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
                 `(package
                    (name ,(python->package-name name))
                    (version ,version)
-                   (source (origin
-                             (method url-fetch)
-                             ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
-                             ;; cases in NAME, for instance, as is the case with
-                             ;; "uwsgi".  In that case, fall back to a full URL.
-                             (uri (pypi-uri ,(string-downcase name) version))
-                             (sha256
-                              (base32
-                               ,(guix-hash-url temp)))))
+                   (source
+                    (origin
+                      (method url-fetch)
+                      ;; PyPI URL are case sensitive, but sometimes a project
+                      ;; named using mixed case has a URL using lower case, so
+                      ;; we must work around this inconsistency.  For actual
+                      ;; examples, compare the URLs of the "Deprecated" and
+                      ;; "uWSGI" PyPI packages.
+                      (uri ,(if (string-contains source-url name)
+                                `(pypi-uri ,name version)
+                                `(pypi-uri ,(string-downcase name) version)))
+                      (sha256
+                       (base32
+                        ,(guix-hash-url temp)))))
                    (build-system python-build-system)
                    ,@(maybe-inputs required-inputs 'propagated-inputs)
                    ,@(maybe-inputs test-inputs 'native-inputs)
-- 
2.21.0


[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 832 bytes --]