bug#24450: [PATCH] bug#24450: pypi importer outputs strange character series in optional dependency case.

[Maxim Cournoyer <maxim.cournoyer@gmail.com>]

[-- Attachment #1.1: Type: text/plain, Size: 442 bytes --]

Hello!

I've yet another commit to add on the top of the current stack of
patches already sent. This one makes it so that the source archive is
searched for the '[...].egg-info/requires.txt' file rather than check
the more common, fixed location.

This makes the importer more useful, as some packages such as
robotframework-sshlibrary use a "src" directory in their hierarchy which
would cause the previous scheme to not find requires.txt.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: 0008-import-pypi-Scan-source-archive-to-find-requires.txt.patch --]
[-- Type: text/x-patch, Size: 7538 bytes --]

From cfde6e09f8f8c692fe252d76ed27e8c50a9e5377 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Sat, 30 Mar 2019 23:13:26 -0400
Subject: [PATCH] import: pypi: Scan source archive to find requires.txt file.

* guix/import/pypi.scm (use-modules): Use invoke from (guix build utils).
(guess-requirements)[archive-root-directory]: Remove procedure.
[guess-requirements-from-wheel]: Re-ident.
[guess-requirements-from-source]: Search for the requires.txt file in the
archive instead of using a static, expected location.
* tests/pypi.scm ("pypi->guix-package, no wheel"): Mock the requires.txt at a
non-standard location to test the new feature.
("pypi->guix-package, no usable requirement file."): Adapt.
---
 guix/import/pypi.scm | 65 ++++++++++++++++++--------------------------
 tests/pypi.scm       | 17 +++++++-----
 2 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 099768f0c8..a2ce14b192 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -36,7 +36,8 @@
   #:use-module ((guix build utils)
                 #:select ((package-name->name+version
                            . hyphen-package-name->name+version)
-                          find-files))
+                          find-files
+                          invoke))
   #:use-module (guix import utils)
   #:use-module ((guix download) #:prefix download:)
   #:use-module (guix import json)
@@ -267,19 +268,6 @@ omitted since these can be difficult or expensive to satisfy."
 of the required packages specified in the requirements.txt file.  ARCHIVE will
 be extracted in a temporary directory."
 
-  (define (archive-root-directory url)
-    ;; Given the URL of the package's archive, return the name of the directory
-    ;; that will be created upon decompressing it. If the filetype is not
-    ;; supported, return #f.
-    (if (compressed-file? url)
-        (let ((root-directory (file-sans-extension (basename url))))
-          (if (string=? "tar" (file-extension root-directory))
-              (file-sans-extension root-directory)
-              root-directory))
-        (begin
-          (warning (G_ "Unsupported archive format (~a): \
-cannot determine package dependencies") (file-extension url))
-          #f)))
 
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
@@ -305,33 +293,34 @@ cannot determine package dependencies") (file-extension url))
     (call-with-temporary-output-file
      (lambda (temp port)
        (if wheel-url
-         (and (url-fetch wheel-url temp)
-              (read-wheel-metadata temp))
-         #f))))
+           (and (url-fetch wheel-url temp)
+                (read-wheel-metadata temp))
+           #f))))
 
   (define (guess-requirements-from-source)
     ;; Return the package's requirements by guessing them from the source.
-    (let ((dirname (archive-root-directory source-url))
-          (extension (file-extension source-url)))
-      (if (string? dirname)
-          (call-with-temporary-directory
-           (lambda (dir)
-             (let* ((pypi-name (string-take dirname (string-rindex dirname #\-)))
-                    (requires.txt (string-append dirname "/" pypi-name
-                                                 ".egg-info" "/requires.txt"))
-                    (exit-code
-                     (parameterize ((current-error-port (%make-void-port "rw+"))
-                                    (current-output-port (%make-void-port "rw+")))
-                       (if (string=? "zip" extension)
-                           (system* "unzip" archive "-d" dir requires.txt)
-                           (system* "tar" "xf" archive "-C" dir requires.txt)))))
-               (if (zero? exit-code)
-                   (parse-requires.txt (string-append dir "/" requires.txt))
-                   (begin
-                     (warning
-                      (G_ "Failed to extract file: ~a from source.~%")
-                      requires.txt)
-                     (list '() '()))))))
+    (if (compressed-file? source-url)
+        (call-with-temporary-directory
+         (lambda (dir)
+           (parameterize ((current-error-port (%make-void-port "rw+"))
+                          (current-output-port (%make-void-port "rw+")))
+             (if (string=? "zip" (file-extension source-url))
+                 (invoke "unzip" archive "-d" dir)
+                 (invoke "tar" "xf" archive "-C" dir)))
+           (let ((requires.txt-files
+                  (find-files dir (lambda (abs-file-name _)
+		                    (string-match "\\.egg-info/requires.txt$"
+                                                  abs-file-name)))))
+             (if (> (length requires.txt-files) 0)
+                 (begin
+                   (parse-requires.txt (first requires.txt-files)))
+                 (begin (warning (G_ "Cannot guess requirements from source archive:\
+ no requires.txt file found.~%"))
+                        (list '() '()))))))
+        (begin
+          (warning (G_ "Unsupported archive format; \
+cannot determine package dependencies from source archive: ~a~%")
+                   (basename source-url))
           (list '() '()))))
 
   ;; First, try to compute the requirements using the wheel, else, fallback to
diff --git a/tests/pypi.scm b/tests/pypi.scm
index aa08e2cb54..ad188df16c 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -177,8 +177,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
              (match url
                ("https://example.com/foo-1.0.0.tar.gz"
                 (begin
-                  (mkdir-p "foo-1.0.0/foo.egg-info/")
-                  (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
+                  ;; Unusual requires.txt location should still be found.
+                  (mkdir-p "foo-1.0.0/src/bizarre.egg-info")
+                  (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt"
                     (lambda ()
                       (display test-requires.txt)))
                   (parameterize ((current-output-port (%make-void-port "rw+")))
@@ -299,10 +300,13 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
          (lambda (url file-name)
            (match url
              ("https://example.com/foo-1.0.0.tar.gz"
+              (mkdir-p "foo-1.0.0/foo.egg-info/")
+              (parameterize ((current-output-port (%make-void-port "rw+")))
+                (system* "tar" "czvf" file-name "foo-1.0.0/"))
+              (delete-file-recursively "foo-1.0.0")
               (set! test-source-hash
-                (call-with-input-file file-name port-sha256))
-              #t)
-             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #t)
+                (call-with-input-file file-name port-sha256)))
+             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
              (_ (error "Unexpected URL: " url)))))
         (mock ((guix http-client) http-fetch
                (lambda (url . rest)
@@ -334,7 +338,6 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
                             test-source-hash)
                            hash))
                 (x
-                 (pk 'fail x #f)))
-              )))
+                 (pk 'fail x #f))))))
 
 (test-end "pypi")
-- 
2.20.1


[-- Attachment #1.3: Type: text/plain, Size: 16 bytes --]


Thanks,

Maxim

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 832 bytes --]