From mboxrd@z Thu Jan 1 00:00:00 1970 From: Cyril Roelandt Subject: [PATCH] import: pypi: Detect inputs. Date: Fri, 5 Jun 2015 00:56:54 +0200 Message-ID: <1433458614-4189-1-git-send-email-tipecaml@gmail.com> References: <874mp6iplj.fsf@fsf.org> Return-path: Received: from eggs.gnu.org ([2001:4830:134:3::10]:50016) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1Z0e4Q-00080N-Tw for guix-devel@gnu.org; Thu, 04 Jun 2015 18:57:08 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1Z0e4M-0002A8-QN for guix-devel@gnu.org; Thu, 04 Jun 2015 18:57:06 -0400 Received: from mail-wi0-x231.google.com ([2a00:1450:400c:c05::231]:36310) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1Z0e4M-0002A0-FQ for guix-devel@gnu.org; Thu, 04 Jun 2015 18:57:02 -0400 Received: by wiam3 with SMTP id m3so1554764wia.1 for ; Thu, 04 Jun 2015 15:57:01 -0700 (PDT) In-Reply-To: <874mp6iplj.fsf@fsf.org> List-Id: "Development of GNU Guix and the GNU System distribution." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-devel-bounces+gcggd-guix-devel=m.gmane.org@gnu.org Sender: guix-devel-bounces+gcggd-guix-devel=m.gmane.org@gnu.org To: guix-devel@gnu.org * guix/import/pypi.scm (compute-inputs, guess-requirements): New procedures. --- guix/import/pypi.scm | 149 ++++++++++++++++++++++++++++++++++++++++----------- tests/pypi.scm | 41 +++++++++----- 2 files changed, 146 insertions(+), 44 deletions(-) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 8567cad..cc2651a 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -21,7 +21,9 @@ #:use-module (ice-9 match) #:use-module (ice-9 pretty-print) #:use-module (ice-9 regex) + #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) #:use-module (rnrs bytevectors) #:use-module (json) #:use-module (web uri) @@ -77,42 +79,127 @@ or #f on failure." with dashes." (string-join (string-split (string-downcase str) #\_) "-")) -(define (guix-hash-url url) - "Download the resource at URL and return the hash in nix-base32 format." - (call-with-temporary-output-file - (lambda (temp port) - (and (url-fetch url temp) - (bytevector->nix-base32-string - (call-with-input-file temp port-sha256)))))) +(define (guix-hash-url filename) + "Return the hash of FILENAME in nix-base32 format." + (bytevector->nix-base32-string (file-sha256 filename))) + +(define (python->package-name name) + "Given the NAME of a package on PyPI, return a Guix-compliant name for the +package." + (if (string-prefix? "python-" name) + (snake-case name) + (string-append "python-" (snake-case name)))) + +(define (maybe-inputs python->package-name inputs) + (match inputs + (() + '()) + ((inputs ...) + `((,python->package-name (,'quasiquote ,inputs)))))) + +(define (guess-requirements source-url tarball) + "Given SOURCE-URL and a TARBALL of the package, return a list of the required +packages specified in the requirements.txt file." + + (define (tarball-directory url) + "Given the URL of the package's tarball, return the name of the directory +that will be created upon decompressing it." + ;; TODO: Support more archive formats. + (let ((basename (substring url (+ 1 (string-rindex url #\/))))) + (cond + ((string-suffix? ".tar.gz" basename) + (string-drop-right basename 7)) + ((string-suffix? ".tar.bz2" basename) + (string-drop-right basename 8)) + (else #f)))) + + (define (clean-requirement s) + "Given a requirement LINE, as can be found in a Python requirements.txt +file, remove everything other than the actual name of the required package, and +return it." + (string-take s + (or (string-index s #\space) + (string-length s)))) + + (define (comment? line) + "Return #t if the given LINE is a comment, #f otherwise." + (eq? (string-ref (string-trim line) 0) #\#)) + + (define (read-requirements requirements-file) + "Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list of +name/variable pairs describing the requirements." + (call-with-input-file requirements-file + (lambda (port) + (let loop ((result '())) + (let ((line (read-line port))) + (if (eof-object? line) + result + (cond + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (python->package-name (clean-requirement line)) + result)))))))))) + + (let ((dirname (tarball-directory source-url))) + (if (string? dirname) + (let ((req-file (string-append dirname "/requirements.txt"))) + ;; TODO: support more formats. + (if (zero? (system* "tar" "xf" tarball req-file)) + (dynamic-wind + (const #t) + (lambda () + (read-requirements req-file)) + (lambda () + (delete-file req-file) + (rmdir dirname))) + '())) + '()))) + +(define (compute-inputs source-url tarball) + "Given the SOURCE-URL of an already downloaded TARBALL, return a list of +name/variable pairs describing the required inputs of this package." + (sort + (map (lambda (input) + (list input (list 'unquote (string->symbol input)))) + (append '("python-setuptools") + ;; Argparse has been part of Python since 2.7. + (remove (cut string=? "python-argparse" <>) + (guess-requirements source-url tarball)))) + (lambda args + (match args + (((a _ ...) (b _ ...)) + (string-cipackage-name name)) + (version ,version) + (source (origin + (method url-fetch) + (uri (string-append ,@(factorize-uri source-url version))) + (sha256 + (base32 + ,(guix-hash-url temp))))) + (build-system python-build-system) + ,@(maybe-inputs 'inputs + (compute-inputs source-url temp)) + (home-page ,home-page) + (synopsis ,synopsis) + (description ,description) + (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0) + (,gpl3 . gpl3) + (,bsd-3 . bsd-3) + (,expat . expat) + (,public-domain . public-domain) + (,asl2.0 . asl2.0)) + license))))))) (define (pypi->guix-package package-name) "Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the diff --git a/tests/pypi.scm b/tests/pypi.scm index 45cf7ca..12cc10d 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -46,8 +46,14 @@ } }") -(define test-source - "foobar") +(define test-source-hash + "") + +(define test-requirements +"# A comment + # A comment after a space +bar +baz > 13.37") (test-begin "pypi") @@ -55,15 +61,22 @@ ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch (lambda (url file-name) - (with-output-to-file file-name - (lambda () - (display - (match url - ("https://pypi.python.org/pypi/foo/json" - test-json) - ("https://example.com/foo-1.0.0.tar.gz" - test-source) - (_ (error "Unexpected URL: " url)))))))) + (match url + ("https://pypi.python.org/pypi/foo/json" + (with-output-to-file file-name + (lambda () + (display test-json)))) + ("https://example.com/foo-1.0.0.tar.gz" + (begin + (mkdir "foo-1.0.0") + (with-output-to-file "foo-1.0.0/requirements.txt" + (lambda () + (display test-requirements))) + (system* "tar" "czvf" file-name "foo-1.0.0/") + (system* "rm" "-rf" "foo-1.0.0") + (set! test-source-hash + (call-with-input-file file-name port-sha256)))) + (_ (error "Unexpected URL: " url))))) (match (pypi->guix-package "foo") (('package ('name "python-foo") @@ -78,13 +91,15 @@ ('build-system 'python-build-system) ('inputs ('quasiquote - (("python-setuptools" ('unquote 'python-setuptools))))) + (("python-bar" ('unquote 'python-bar)) + ("python-baz" ('unquote 'python-baz)) + ("python-setuptools" ('unquote 'python-setuptools))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") ('license 'lgpl2.0)) (string=? (bytevector->nix-base32-string - (call-with-input-string test-source port-sha256)) + test-source-hash) hash)) (x (pk 'fail x #f))))) -- 1.8.4.rc3