* [PATCH] import: pypi: detect requirements from requirements.txt.
@ 2015-03-07 23:51 Cyril Roelandt
2015-03-08 0:41 ` David Thompson
0 siblings, 1 reply; 18+ messages in thread
From: Cyril Roelandt @ 2015-03-07 23:51 UTC (permalink / raw)
To: guix-devel
* guix/import/pypi.scm (compute-inputs, guess-requirements): New functions.
---
guix/import/pypi.scm | 146 ++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 115 insertions(+), 31 deletions(-)
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8567cad..9ac3342 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,6 +21,7 @@
#:use-module (ice-9 match)
#:use-module (ice-9 pretty-print)
#:use-module (ice-9 regex)
+ #:use-module ((ice-9 rdelim) #:select (read-line))
#:use-module (srfi srfi-1)
#:use-module (rnrs bytevectors)
#:use-module (json)
@@ -77,42 +78,125 @@ or #f on failure."
with dashes."
(string-join (string-split (string-downcase str) #\_) "-"))
-(define (guix-hash-url url)
- "Download the resource at URL and return the hash in nix-base32 format."
- (call-with-temporary-output-file
- (lambda (temp port)
- (and (url-fetch url temp)
- (bytevector->nix-base32-string
- (call-with-input-file temp port-sha256))))))
+(define (guix-hash-url filename)
+ "Return the hash of FILENAME in nix-base32 format."
+ (bytevector->nix-base32-string
+ (call-with-input-file filename port-sha256)))
+
+(define (guix-name name)
+ (if (string-prefix? "python-" name)
+ (snake-case name)
+ (string-append "python-" (snake-case name))))
+
+(define (maybe-inputs guix-name inputs)
+ (match inputs
+ (()
+ '())
+ ((inputs ...)
+ (list (list guix-name
+ (list 'quasiquote inputs))))))
+
+(define (guess-requirements source-url tarball)
+ "Given SOURCE-URL and a TARBALL of the package, return a list of the required
+packages specified in the requirements.txt file."
+
+ (define (tarball-directory url)
+ "Given the URL of the package's tarball, return the name of the directory
+that will be created upon decompressing it."
+ ;; TODO: Support more archive formats.
+ (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
+ (cond
+ ((string-suffix? ".tar.gz" basename)
+ (string-drop-right basename 7))
+ ((string-suffix? ".tar.bz2" basename)
+ (string-drop-right basename 8))
+ (else #f))))
+
+ (define (read-requirements requirements-file)
+ "Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list of
+name/variable pairs describing the requirements."
+ (call-with-input-file requirements-file
+ (lambda (port)
+ (let loop ((result '()))
+ (let ((line (read-line port)))
+ (if (eof-object? line)
+ result
+ (loop (if (or (string-null? line)
+ (eq? (string-ref line 0) #\#))
+ result
+ (let ((input (guix-name (clean-requirement line))))
+ ;; Argparse has been part of Python since 2.7.
+ (if (string=? input "python-argparse")
+ result
+ (cons input result)))))))))))
+
+ (define (clean-requirement s)
+ "Given a requirement LINE, as can be found in a Python requirements.txt
+file, remove everything other than the actual name of the required package, and
+return it."
+ (string-take s
+ (or (string-index s (lambda (c)
+ (member c '(#\< #\> #\= #\#))))
+ (string-length s))))
+
+ (let* ((dirname (tarball-directory source-url))
+ (req-file (if (string? dirname)
+ (string-append dirname "/requirements.txt")
+ #f)))
+ (if (and (string? req-file)
+ ;; TODO: support more formats.
+ (zero? (system* "tar" "xf" tarball req-file)))
+ (dynamic-wind
+ (const #t)
+ (lambda ()
+ (read-requirements req-file))
+ (lambda ()
+ (delete-file req-file)
+ (rmdir dirname)))
+ '())))
+
+(define (compute-inputs source-url tarball)
+ "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
+name/variable pairs describing the required inputs of this package."
+ (sort
+ (map (lambda (input)
+ (list input (list 'unquote (string->symbol input))))
+ (append '("python-setuptools")
+ (guess-requirements source-url tarball)))
+ (lambda args
+ (match args
+ (((a _ ...) (b _ ...))
+ (string-ci<? a b))))))
(define (make-pypi-sexp name version source-url home-page synopsis
description license)
"Return the `package' s-expression for a python package with the given NAME,
VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
- `(package
- (name ,(if (string-prefix? "python-" name)
- (snake-case name)
- (string-append "python-" (snake-case name))))
- (version ,version)
- (source (origin
- (method url-fetch)
- (uri (string-append ,@(factorize-uri source-url version)))
- (sha256
- (base32
- ,(guix-hash-url source-url)))))
- (build-system python-build-system)
- (inputs
- `(("python-setuptools" ,python-setuptools)))
- (home-page ,home-page)
- (synopsis ,synopsis)
- (description ,description)
- (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
- (,gpl3 . gpl3)
- (,bsd-3 . bsd-3)
- (,expat . expat)
- (,public-domain . public-domain)
- (,asl2.0 . asl2.0))
- license))))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (and (url-fetch source-url temp)
+ `(package
+ (name ,(guix-name name))
+ (version ,version)
+ (source (origin
+ (method url-fetch)
+ (uri (string-append ,@(factorize-uri source-url version)))
+ (sha256
+ (base32
+ ,(guix-hash-url temp)))))
+ (build-system python-build-system)
+ ,@(maybe-inputs 'inputs
+ (compute-inputs source-url temp))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
+ (,gpl3 . gpl3)
+ (,bsd-3 . bsd-3)
+ (,expat . expat)
+ (,public-domain . public-domain)
+ (,asl2.0 . asl2.0))
+ license)))))))
(define (pypi->guix-package package-name)
"Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the
--
1.8.4.rc3
^ permalink raw reply related [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: detect requirements from requirements.txt.
2015-03-07 23:51 [PATCH] import: pypi: detect requirements from requirements.txt Cyril Roelandt
@ 2015-03-08 0:41 ` David Thompson
2015-03-22 21:05 ` [PATCH] import: pypi: Detect inputs Cyril Roelandt
2015-03-22 21:05 ` [PATCH] import: pypi: detect requirements from requirements.txt Cyril Roelandt
0 siblings, 2 replies; 18+ messages in thread
From: David Thompson @ 2015-03-08 0:41 UTC (permalink / raw)
To: Cyril Roelandt, guix-devel
Awesome! :)
How about this for the commit log:
"import: pypi: Detect inputs."
?
Cyril Roelandt <tipecaml@gmail.com> writes:
> * guix/import/pypi.scm (compute-inputs, guess-requirements): New functions.
"New procedures."
> ---
> guix/import/pypi.scm | 146 ++++++++++++++++++++++++++++++++++++++++-----------
> 1 file changed, 115 insertions(+), 31 deletions(-)
>
> diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
> index 8567cad..9ac3342 100644
> --- a/guix/import/pypi.scm
> +++ b/guix/import/pypi.scm
> @@ -21,6 +21,7 @@
> #:use-module (ice-9 match)
> #:use-module (ice-9 pretty-print)
> #:use-module (ice-9 regex)
> + #:use-module ((ice-9 rdelim) #:select (read-line))
> #:use-module (srfi srfi-1)
> #:use-module (rnrs bytevectors)
> #:use-module (json)
> @@ -77,42 +78,125 @@ or #f on failure."
> with dashes."
> (string-join (string-split (string-downcase str) #\_) "-"))
>
> -(define (guix-hash-url url)
> - "Download the resource at URL and return the hash in nix-base32 format."
> - (call-with-temporary-output-file
> - (lambda (temp port)
> - (and (url-fetch url temp)
> - (bytevector->nix-base32-string
> - (call-with-input-file temp port-sha256))))))
> +(define (guix-hash-url filename)
> + "Return the hash of FILENAME in nix-base32 format."
> + (bytevector->nix-base32-string
> + (call-with-input-file filename port-sha256)))
> +
> +(define (guix-name name)
> + (if (string-prefix? "python-" name)
> + (snake-case name)
> + (string-append "python-" (snake-case name))))
> +
> +(define (maybe-inputs guix-name inputs)
> + (match inputs
> + (()
> + '())
> + ((inputs ...)
> + (list (list guix-name
> + (list 'quasiquote inputs))))))
How about:
`((,guix-name ,(list 'quasiquote inputs)))
Or:
`((,guix-name (,'quasiquote ,inputs)))
> +
> +(define (guess-requirements source-url tarball)
> + "Given SOURCE-URL and a TARBALL of the package, return a list of the required
> +packages specified in the requirements.txt file."
> +
> + (define (tarball-directory url)
> + "Given the URL of the package's tarball, return the name of the directory
> +that will be created upon decompressing it."
> + ;; TODO: Support more archive formats.
> + (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
> + (cond
> + ((string-suffix? ".tar.gz" basename)
> + (string-drop-right basename 7))
> + ((string-suffix? ".tar.bz2" basename)
> + (string-drop-right basename 8))
> + (else #f))))
> +
> + (define (read-requirements requirements-file)
> + "Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list of
> +name/variable pairs describing the requirements."
> + (call-with-input-file requirements-file
> + (lambda (port)
> + (let loop ((result '()))
> + (let ((line (read-line port)))
> + (if (eof-object? line)
> + result
> + (loop (if (or (string-null? line)
> + (eq? (string-ref line 0) #\#))
For readability, consider extracting `(eq? (string-ref line 0) #\#)` to
an inner procedure named 'comment?'
> + result
> + (let ((input (guix-name (clean-requirement line))))
> + ;; Argparse has been part of Python since 2.7.
> + (if (string=? input "python-argparse")
> + result
> + (cons input result)))))))))))
There's a lot of nested 'if' forms here. I think it could be
restructured into a more readable 'cond' by moving these conditionals
outside of the call to 'loop' and filtering out "python-argpase" as a
final step rather than doing it while you are still reading the file.
> +
> + (define (clean-requirement s)
> + "Given a requirement LINE, as can be found in a Python requirements.txt
> +file, remove everything other than the actual name of the required package, and
> +return it."
> + (string-take s
> + (or (string-index s (lambda (c)
> + (member c '(#\< #\> #\= #\#))))
> + (string-length s))))
> +
> + (let* ((dirname (tarball-directory source-url))
> + (req-file (if (string? dirname)
> + (string-append dirname "/requirements.txt")
> + #f)))
(and (string? dirname) (string-append dirname "/requirements.txt"))
> + (if (and (string? req-file)
If 'dirname' was a string, then 'req-file' is certainly a string, so I
think you can just test dirname above and never go down this code path
if it isn't a string.
> + ;; TODO: support more formats.
> + (zero? (system* "tar" "xf" tarball req-file)))
> + (dynamic-wind
> + (const #t)
> + (lambda ()
> + (read-requirements req-file))
> + (lambda ()
> + (delete-file req-file)
> + (rmdir dirname)))
> + '())))
> +
> +(define (compute-inputs source-url tarball)
> + "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
> +name/variable pairs describing the required inputs of this package."
> + (sort
> + (map (lambda (input)
> + (list input (list 'unquote (string->symbol input))))
> + (append '("python-setuptools")
> + (guess-requirements source-url tarball)))
> + (lambda args
> + (match args
> + (((a _ ...) (b _ ...))
> + (string-ci<? a b))))))
>
> (define (make-pypi-sexp name version source-url home-page synopsis
> description license)
> "Return the `package' s-expression for a python package with the given NAME,
> VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
> - `(package
> - (name ,(if (string-prefix? "python-" name)
> - (snake-case name)
> - (string-append "python-" (snake-case name))))
> - (version ,version)
> - (source (origin
> - (method url-fetch)
> - (uri (string-append ,@(factorize-uri source-url version)))
> - (sha256
> - (base32
> - ,(guix-hash-url source-url)))))
> - (build-system python-build-system)
> - (inputs
> - `(("python-setuptools" ,python-setuptools)))
> - (home-page ,home-page)
> - (synopsis ,synopsis)
> - (description ,description)
> - (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
> - (,gpl3 . gpl3)
> - (,bsd-3 . bsd-3)
> - (,expat . expat)
> - (,public-domain . public-domain)
> - (,asl2.0 . asl2.0))
> - license))))
> + (call-with-temporary-output-file
> + (lambda (temp port)
> + (and (url-fetch source-url temp)
> + `(package
> + (name ,(guix-name name))
> + (version ,version)
> + (source (origin
> + (method url-fetch)
> + (uri (string-append ,@(factorize-uri source-url version)))
> + (sha256
> + (base32
> + ,(guix-hash-url temp)))))
> + (build-system python-build-system)
> + ,@(maybe-inputs 'inputs
> + (compute-inputs source-url temp))
Fix identation: |------------^
> + (home-page ,home-page)
> + (synopsis ,synopsis)
> + (description ,description)
> + (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
> + (,gpl3 . gpl3)
> + (,bsd-3 . bsd-3)
> + (,expat . expat)
> + (,public-domain . public-domain)
> + (,asl2.0 . asl2.0))
> + license)))))))
>
> (define (pypi->guix-package package-name)
> "Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the
> --
> 1.8.4.rc3
>
>
Hope the above wasn't overwhelming. Could you send an updated patch?
Thanks for working on this!
--
David Thompson
Web Developer - Free Software Foundation - http://fsf.org
GPG Key: 0FF1D807
Support the FSF: https://fsf.org/donate
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH] import: pypi: Detect inputs.
2015-03-08 0:41 ` David Thompson
@ 2015-03-22 21:05 ` Cyril Roelandt
2015-03-26 13:15 ` Ludovic Courtès
2015-03-27 12:36 ` David Thompson
2015-03-22 21:05 ` [PATCH] import: pypi: detect requirements from requirements.txt Cyril Roelandt
1 sibling, 2 replies; 18+ messages in thread
From: Cyril Roelandt @ 2015-03-22 21:05 UTC (permalink / raw)
To: guix-devel
* guix/import/pypi.scm (compute-inputs, guess-requirements): New procedures.
---
guix/import/pypi.scm | 149 ++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 118 insertions(+), 31 deletions(-)
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8567cad..1091deb 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,6 +21,7 @@
#:use-module (ice-9 match)
#:use-module (ice-9 pretty-print)
#:use-module (ice-9 regex)
+ #:use-module ((ice-9 rdelim) #:select (read-line))
#:use-module (srfi srfi-1)
#:use-module (rnrs bytevectors)
#:use-module (json)
@@ -77,42 +78,128 @@ or #f on failure."
with dashes."
(string-join (string-split (string-downcase str) #\_) "-"))
-(define (guix-hash-url url)
- "Download the resource at URL and return the hash in nix-base32 format."
- (call-with-temporary-output-file
- (lambda (temp port)
- (and (url-fetch url temp)
- (bytevector->nix-base32-string
- (call-with-input-file temp port-sha256))))))
+(define (guix-hash-url filename)
+ "Return the hash of FILENAME in nix-base32 format."
+ (bytevector->nix-base32-string
+ (call-with-input-file filename port-sha256)))
+
+(define (guix-name name)
+ (if (string-prefix? "python-" name)
+ (snake-case name)
+ (string-append "python-" (snake-case name))))
+
+(define (maybe-inputs guix-name inputs)
+ (match inputs
+ (()
+ '())
+ ((inputs ...)
+ `((,guix-name (,'quasiquote ,inputs))))))
+
+(define (guess-requirements source-url tarball)
+ "Given SOURCE-URL and a TARBALL of the package, return a list of the required
+packages specified in the requirements.txt file."
+
+ (define (tarball-directory url)
+ "Given the URL of the package's tarball, return the name of the directory
+that will be created upon decompressing it."
+ ;; TODO: Support more archive formats.
+ (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
+ (cond
+ ((string-suffix? ".tar.gz" basename)
+ (string-drop-right basename 7))
+ ((string-suffix? ".tar.bz2" basename)
+ (string-drop-right basename 8))
+ (else #f))))
+
+ (define (clean-requirement s)
+ "Given a requirement LINE, as can be found in a Python requirements.txt
+file, remove everything other than the actual name of the required package, and
+return it."
+ (string-take s
+ (or (string-index s (lambda (c)
+ (member c '(#\< #\> #\= #\#))))
+ (string-length s))))
+
+ (define (comment? line)
+ "Return #t if the given LINE is a comment, #f otherwise."
+ (eq? (string-ref line 0) #\#))
+
+ (define (read-requirements requirements-file)
+ "Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list of
+name/variable pairs describing the requirements."
+ (call-with-input-file requirements-file
+ (lambda (port)
+ (let loop ((result '()))
+ (let ((line (read-line port)))
+ (if (eof-object? line)
+ result
+ (cond
+ ((or (string-null? line) (comment? line))
+ (loop result))
+ (else
+ (loop (cons (guix-name (clean-requirement line))
+ result))))))))))
+
+ (let ((dirname (tarball-directory source-url)))
+ (if (string? dirname)
+ (let ((req-file (string-append dirname "/requirements.txt")))
+ ;; TODO: support more formats.
+ (if (zero? (system* "tar" "xf" tarball req-file))
+ (dynamic-wind
+ (const #t)
+ (lambda ()
+ (read-requirements req-file))
+ (lambda ()
+ (delete-file req-file)
+ (rmdir dirname)))
+ '()))
+ '())))
+
+(define (compute-inputs source-url tarball)
+ "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
+name/variable pairs describing the required inputs of this package."
+ (sort
+ (map (lambda (input)
+ (list input (list 'unquote (string->symbol input))))
+ (append '("python-setuptools")
+ ;; Argparse has been part of Python since 2.7.
+ (filter (lambda (input)
+ (not (string=? "python-argparse" input)))
+ (guess-requirements source-url tarball))))
+ (lambda args
+ (match args
+ (((a _ ...) (b _ ...))
+ (string-ci<? a b))))))
(define (make-pypi-sexp name version source-url home-page synopsis
description license)
"Return the `package' s-expression for a python package with the given NAME,
VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
- `(package
- (name ,(if (string-prefix? "python-" name)
- (snake-case name)
- (string-append "python-" (snake-case name))))
- (version ,version)
- (source (origin
- (method url-fetch)
- (uri (string-append ,@(factorize-uri source-url version)))
- (sha256
- (base32
- ,(guix-hash-url source-url)))))
- (build-system python-build-system)
- (inputs
- `(("python-setuptools" ,python-setuptools)))
- (home-page ,home-page)
- (synopsis ,synopsis)
- (description ,description)
- (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
- (,gpl3 . gpl3)
- (,bsd-3 . bsd-3)
- (,expat . expat)
- (,public-domain . public-domain)
- (,asl2.0 . asl2.0))
- license))))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (and (url-fetch source-url temp)
+ `(package
+ (name ,(guix-name name))
+ (version ,version)
+ (source (origin
+ (method url-fetch)
+ (uri (string-append ,@(factorize-uri source-url version)))
+ (sha256
+ (base32
+ ,(guix-hash-url temp)))))
+ (build-system python-build-system)
+ ,@(maybe-inputs 'inputs
+ (compute-inputs source-url temp))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
+ (,gpl3 . gpl3)
+ (,bsd-3 . bsd-3)
+ (,expat . expat)
+ (,public-domain . public-domain)
+ (,asl2.0 . asl2.0))
+ license)))))))
(define (pypi->guix-package package-name)
"Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the
--
1.8.4.rc3
^ permalink raw reply related [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: detect requirements from requirements.txt.
2015-03-08 0:41 ` David Thompson
2015-03-22 21:05 ` [PATCH] import: pypi: Detect inputs Cyril Roelandt
@ 2015-03-22 21:05 ` Cyril Roelandt
1 sibling, 0 replies; 18+ messages in thread
From: Cyril Roelandt @ 2015-03-22 21:05 UTC (permalink / raw)
To: David Thompson, guix-devel
On 03/08/2015 01:41 AM, David Thompson wrote:
>
> Hope the above wasn't overwhelming. Could you send an updated patch?
> Thanks for working on this!
>
Thanks for your feedback! I sent an updated patch.
Cyril Roelandt.
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-03-22 21:05 ` [PATCH] import: pypi: Detect inputs Cyril Roelandt
@ 2015-03-26 13:15 ` Ludovic Courtès
2015-03-27 12:36 ` David Thompson
1 sibling, 0 replies; 18+ messages in thread
From: Ludovic Courtès @ 2015-03-26 13:15 UTC (permalink / raw)
To: Cyril Roelandt; +Cc: guix-devel
Cyril Roelandt <tipecaml@gmail.com> skribis:
> * guix/import/pypi.scm (compute-inputs, guess-requirements): New procedures.
Just some superficial comments. David, anything else?
> +(define (guix-hash-url filename)
> + "Return the hash of FILENAME in nix-base32 format."
> + (bytevector->nix-base32-string
> + (call-with-input-file filename port-sha256)))
Please remove and use:
(bytevector->nix-base32-string (file-sha256 file))
> +(define (guix-name name)
‘python->package-name’? + docstring
Could you augment tests/pypi.scm with a test where dependencies are
parsed?
Thanks!
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-03-22 21:05 ` [PATCH] import: pypi: Detect inputs Cyril Roelandt
2015-03-26 13:15 ` Ludovic Courtès
@ 2015-03-27 12:36 ` David Thompson
2015-03-29 13:46 ` Ludovic Courtès
2015-06-04 22:56 ` Cyril Roelandt
1 sibling, 2 replies; 18+ messages in thread
From: David Thompson @ 2015-03-27 12:36 UTC (permalink / raw)
To: Cyril Roelandt, guix-devel
Cyril Roelandt <tipecaml@gmail.com> writes:
> * guix/import/pypi.scm (compute-inputs, guess-requirements): New procedures.
> ---
> guix/import/pypi.scm | 149 ++++++++++++++++++++++++++++++++++++++++-----------
> 1 file changed, 118 insertions(+), 31 deletions(-)
>
> diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
> index 8567cad..1091deb 100644
> --- a/guix/import/pypi.scm
> +++ b/guix/import/pypi.scm
> @@ -21,6 +21,7 @@
> #:use-module (ice-9 match)
> #:use-module (ice-9 pretty-print)
> #:use-module (ice-9 regex)
> + #:use-module ((ice-9 rdelim) #:select (read-line))
> #:use-module (srfi srfi-1)
> #:use-module (rnrs bytevectors)
> #:use-module (json)
> @@ -77,42 +78,128 @@ or #f on failure."
> with dashes."
> (string-join (string-split (string-downcase str) #\_) "-"))
>
> -(define (guix-hash-url url)
> - "Download the resource at URL and return the hash in nix-base32 format."
> - (call-with-temporary-output-file
> - (lambda (temp port)
> - (and (url-fetch url temp)
> - (bytevector->nix-base32-string
> - (call-with-input-file temp port-sha256))))))
> +(define (guix-hash-url filename)
> + "Return the hash of FILENAME in nix-base32 format."
> + (bytevector->nix-base32-string
> + (call-with-input-file filename port-sha256)))
> +
> +(define (guix-name name)
> + (if (string-prefix? "python-" name)
> + (snake-case name)
> + (string-append "python-" (snake-case name))))
> +
> +(define (maybe-inputs guix-name inputs)
> + (match inputs
> + (()
> + '())
> + ((inputs ...)
> + `((,guix-name (,'quasiquote ,inputs))))))
> +
> +(define (guess-requirements source-url tarball)
> + "Given SOURCE-URL and a TARBALL of the package, return a list of the required
> +packages specified in the requirements.txt file."
> +
> + (define (tarball-directory url)
> + "Given the URL of the package's tarball, return the name of the directory
> +that will be created upon decompressing it."
> + ;; TODO: Support more archive formats.
> + (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
> + (cond
> + ((string-suffix? ".tar.gz" basename)
> + (string-drop-right basename 7))
> + ((string-suffix? ".tar.bz2" basename)
> + (string-drop-right basename 8))
> + (else #f))))
> +
> + (define (clean-requirement s)
> + "Given a requirement LINE, as can be found in a Python requirements.txt
> +file, remove everything other than the actual name of the required package, and
> +return it."
> + (string-take s
> + (or (string-index s (lambda (c)
> + (member c '(#\< #\> #\= #\#))))
> + (string-length s))))
> +
> + (define (comment? line)
> + "Return #t if the given LINE is a comment, #f otherwise."
> + (eq? (string-ref line 0) #\#))
Couldn't there be any amount of whitespace before the "#"?
> +
> + (define (read-requirements requirements-file)
> + "Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list of
> +name/variable pairs describing the requirements."
> + (call-with-input-file requirements-file
> + (lambda (port)
> + (let loop ((result '()))
> + (let ((line (read-line port)))
> + (if (eof-object? line)
> + result
> + (cond
> + ((or (string-null? line) (comment? line))
> + (loop result))
> + (else
> + (loop (cons (guix-name (clean-requirement line))
> + result))))))))))
> +
> + (let ((dirname (tarball-directory source-url)))
> + (if (string? dirname)
> + (let ((req-file (string-append dirname "/requirements.txt")))
> + ;; TODO: support more formats.
> + (if (zero? (system* "tar" "xf" tarball req-file))
> + (dynamic-wind
> + (const #t)
> + (lambda ()
> + (read-requirements req-file))
> + (lambda ()
> + (delete-file req-file)
> + (rmdir dirname)))
> + '()))
> + '())))
> +
> +(define (compute-inputs source-url tarball)
> + "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
> +name/variable pairs describing the required inputs of this package."
> + (sort
> + (map (lambda (input)
> + (list input (list 'unquote (string->symbol input))))
Could be rephrased as:
`(,input (,'unqoute ,(string->symbol input)))
Not sure if it's a good idea, though. Ludo? ;)
> + (append '("python-setuptools")
> + ;; Argparse has been part of Python since 2.7.
> + (filter (lambda (input)
> + (not (string=? "python-argparse" input)))
> + (guess-requirements source-url tarball))))
Rather than using 'filter' with a predicate that uses 'not', use
'remove' instead.
(remove (cut string=? "python-argparse" <>)
(guess-requirements source-url tarball))
> + (lambda args
> + (match args
> + (((a _ ...) (b _ ...))
> + (string-ci<? a b))))))
>
> (define (make-pypi-sexp name version source-url home-page synopsis
> description license)
> "Return the `package' s-expression for a python package with the given NAME,
> VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
> - `(package
> - (name ,(if (string-prefix? "python-" name)
> - (snake-case name)
> - (string-append "python-" (snake-case name))))
> - (version ,version)
> - (source (origin
> - (method url-fetch)
> - (uri (string-append ,@(factorize-uri source-url version)))
> - (sha256
> - (base32
> - ,(guix-hash-url source-url)))))
> - (build-system python-build-system)
> - (inputs
> - `(("python-setuptools" ,python-setuptools)))
> - (home-page ,home-page)
> - (synopsis ,synopsis)
> - (description ,description)
> - (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
> - (,gpl3 . gpl3)
> - (,bsd-3 . bsd-3)
> - (,expat . expat)
> - (,public-domain . public-domain)
> - (,asl2.0 . asl2.0))
> - license))))
> + (call-with-temporary-output-file
> + (lambda (temp port)
> + (and (url-fetch source-url temp)
> + `(package
> + (name ,(guix-name name))
> + (version ,version)
> + (source (origin
> + (method url-fetch)
> + (uri (string-append ,@(factorize-uri source-url version)))
> + (sha256
> + (base32
> + ,(guix-hash-url temp)))))
> + (build-system python-build-system)
> + ,@(maybe-inputs 'inputs
> + (compute-inputs source-url temp))
> + (home-page ,home-page)
> + (synopsis ,synopsis)
> + (description ,description)
> + (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
> + (,gpl3 . gpl3)
> + (,bsd-3 . bsd-3)
> + (,expat . expat)
> + (,public-domain . public-domain)
> + (,asl2.0 . asl2.0))
> + license)))))))
>
> (define (pypi->guix-package package-name)
> "Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the
> --
> 1.8.4.rc3
>
>
And as Ludo requested, a test case would be good.
Thanks!
--
David Thompson
Web Developer - Free Software Foundation - http://fsf.org
GPG Key: 0FF1D807
Support the FSF: https://fsf.org/donate
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-03-27 12:36 ` David Thompson
@ 2015-03-29 13:46 ` Ludovic Courtès
2015-06-04 22:56 ` Cyril Roelandt
1 sibling, 0 replies; 18+ messages in thread
From: Ludovic Courtès @ 2015-03-29 13:46 UTC (permalink / raw)
To: David Thompson; +Cc: guix-devel
David Thompson <dthompson2@worcester.edu> skribis:
>> + (map (lambda (input)
>> + (list input (list 'unquote (string->symbol input))))
>
> Could be rephrased as:
>
> `(,input (,'unqoute ,(string->symbol input)))
>
> Not sure if it's a good idea, though. Ludo? ;)
Not sure either. :-) The original may be better.
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH] import: pypi: Detect inputs.
2015-03-27 12:36 ` David Thompson
2015-03-29 13:46 ` Ludovic Courtès
@ 2015-06-04 22:56 ` Cyril Roelandt
2015-06-07 20:03 ` Ludovic Courtès
1 sibling, 1 reply; 18+ messages in thread
From: Cyril Roelandt @ 2015-06-04 22:56 UTC (permalink / raw)
To: guix-devel
* guix/import/pypi.scm (compute-inputs, guess-requirements): New procedures.
---
guix/import/pypi.scm | 149 ++++++++++++++++++++++++++++++++++++++++-----------
tests/pypi.scm | 41 +++++++++-----
2 files changed, 146 insertions(+), 44 deletions(-)
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8567cad..cc2651a 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,7 +21,9 @@
#:use-module (ice-9 match)
#:use-module (ice-9 pretty-print)
#:use-module (ice-9 regex)
+ #:use-module ((ice-9 rdelim) #:select (read-line))
#:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-26)
#:use-module (rnrs bytevectors)
#:use-module (json)
#:use-module (web uri)
@@ -77,42 +79,127 @@ or #f on failure."
with dashes."
(string-join (string-split (string-downcase str) #\_) "-"))
-(define (guix-hash-url url)
- "Download the resource at URL and return the hash in nix-base32 format."
- (call-with-temporary-output-file
- (lambda (temp port)
- (and (url-fetch url temp)
- (bytevector->nix-base32-string
- (call-with-input-file temp port-sha256))))))
+(define (guix-hash-url filename)
+ "Return the hash of FILENAME in nix-base32 format."
+ (bytevector->nix-base32-string (file-sha256 filename)))
+
+(define (python->package-name name)
+ "Given the NAME of a package on PyPI, return a Guix-compliant name for the
+package."
+ (if (string-prefix? "python-" name)
+ (snake-case name)
+ (string-append "python-" (snake-case name))))
+
+(define (maybe-inputs python->package-name inputs)
+ (match inputs
+ (()
+ '())
+ ((inputs ...)
+ `((,python->package-name (,'quasiquote ,inputs))))))
+
+(define (guess-requirements source-url tarball)
+ "Given SOURCE-URL and a TARBALL of the package, return a list of the required
+packages specified in the requirements.txt file."
+
+ (define (tarball-directory url)
+ "Given the URL of the package's tarball, return the name of the directory
+that will be created upon decompressing it."
+ ;; TODO: Support more archive formats.
+ (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
+ (cond
+ ((string-suffix? ".tar.gz" basename)
+ (string-drop-right basename 7))
+ ((string-suffix? ".tar.bz2" basename)
+ (string-drop-right basename 8))
+ (else #f))))
+
+ (define (clean-requirement s)
+ "Given a requirement LINE, as can be found in a Python requirements.txt
+file, remove everything other than the actual name of the required package, and
+return it."
+ (string-take s
+ (or (string-index s #\space)
+ (string-length s))))
+
+ (define (comment? line)
+ "Return #t if the given LINE is a comment, #f otherwise."
+ (eq? (string-ref (string-trim line) 0) #\#))
+
+ (define (read-requirements requirements-file)
+ "Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list of
+name/variable pairs describing the requirements."
+ (call-with-input-file requirements-file
+ (lambda (port)
+ (let loop ((result '()))
+ (let ((line (read-line port)))
+ (if (eof-object? line)
+ result
+ (cond
+ ((or (string-null? line) (comment? line))
+ (loop result))
+ (else
+ (loop (cons (python->package-name (clean-requirement line))
+ result))))))))))
+
+ (let ((dirname (tarball-directory source-url)))
+ (if (string? dirname)
+ (let ((req-file (string-append dirname "/requirements.txt")))
+ ;; TODO: support more formats.
+ (if (zero? (system* "tar" "xf" tarball req-file))
+ (dynamic-wind
+ (const #t)
+ (lambda ()
+ (read-requirements req-file))
+ (lambda ()
+ (delete-file req-file)
+ (rmdir dirname)))
+ '()))
+ '())))
+
+(define (compute-inputs source-url tarball)
+ "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
+name/variable pairs describing the required inputs of this package."
+ (sort
+ (map (lambda (input)
+ (list input (list 'unquote (string->symbol input))))
+ (append '("python-setuptools")
+ ;; Argparse has been part of Python since 2.7.
+ (remove (cut string=? "python-argparse" <>)
+ (guess-requirements source-url tarball))))
+ (lambda args
+ (match args
+ (((a _ ...) (b _ ...))
+ (string-ci<? a b))))))
(define (make-pypi-sexp name version source-url home-page synopsis
description license)
"Return the `package' s-expression for a python package with the given NAME,
VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
- `(package
- (name ,(if (string-prefix? "python-" name)
- (snake-case name)
- (string-append "python-" (snake-case name))))
- (version ,version)
- (source (origin
- (method url-fetch)
- (uri (string-append ,@(factorize-uri source-url version)))
- (sha256
- (base32
- ,(guix-hash-url source-url)))))
- (build-system python-build-system)
- (inputs
- `(("python-setuptools" ,python-setuptools)))
- (home-page ,home-page)
- (synopsis ,synopsis)
- (description ,description)
- (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
- (,gpl3 . gpl3)
- (,bsd-3 . bsd-3)
- (,expat . expat)
- (,public-domain . public-domain)
- (,asl2.0 . asl2.0))
- license))))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (and (url-fetch source-url temp)
+ `(package
+ (name ,(python->package-name name))
+ (version ,version)
+ (source (origin
+ (method url-fetch)
+ (uri (string-append ,@(factorize-uri source-url version)))
+ (sha256
+ (base32
+ ,(guix-hash-url temp)))))
+ (build-system python-build-system)
+ ,@(maybe-inputs 'inputs
+ (compute-inputs source-url temp))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
+ (,gpl3 . gpl3)
+ (,bsd-3 . bsd-3)
+ (,expat . expat)
+ (,public-domain . public-domain)
+ (,asl2.0 . asl2.0))
+ license)))))))
(define (pypi->guix-package package-name)
"Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 45cf7ca..12cc10d 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -46,8 +46,14 @@
}
}")
-(define test-source
- "foobar")
+(define test-source-hash
+ "")
+
+(define test-requirements
+"# A comment
+ # A comment after a space
+bar
+baz > 13.37")
(test-begin "pypi")
@@ -55,15 +61,22 @@
;; Replace network resources with sample data.
(mock ((guix import utils) url-fetch
(lambda (url file-name)
- (with-output-to-file file-name
- (lambda ()
- (display
- (match url
- ("https://pypi.python.org/pypi/foo/json"
- test-json)
- ("https://example.com/foo-1.0.0.tar.gz"
- test-source)
- (_ (error "Unexpected URL: " url))))))))
+ (match url
+ ("https://pypi.python.org/pypi/foo/json"
+ (with-output-to-file file-name
+ (lambda ()
+ (display test-json))))
+ ("https://example.com/foo-1.0.0.tar.gz"
+ (begin
+ (mkdir "foo-1.0.0")
+ (with-output-to-file "foo-1.0.0/requirements.txt"
+ (lambda ()
+ (display test-requirements)))
+ (system* "tar" "czvf" file-name "foo-1.0.0/")
+ (system* "rm" "-rf" "foo-1.0.0")
+ (set! test-source-hash
+ (call-with-input-file file-name port-sha256))))
+ (_ (error "Unexpected URL: " url)))))
(match (pypi->guix-package "foo")
(('package
('name "python-foo")
@@ -78,13 +91,15 @@
('build-system 'python-build-system)
('inputs
('quasiquote
- (("python-setuptools" ('unquote 'python-setuptools)))))
+ (("python-bar" ('unquote 'python-bar))
+ ("python-baz" ('unquote 'python-baz))
+ ("python-setuptools" ('unquote 'python-setuptools)))))
('home-page "http://example.com")
('synopsis "summary")
('description "summary")
('license 'lgpl2.0))
(string=? (bytevector->nix-base32-string
- (call-with-input-string test-source port-sha256))
+ test-source-hash)
hash))
(x
(pk 'fail x #f)))))
--
1.8.4.rc3
^ permalink raw reply related [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-04 22:56 ` Cyril Roelandt
@ 2015-06-07 20:03 ` Ludovic Courtès
2015-06-15 1:25 ` Cyril Roelandt
0 siblings, 1 reply; 18+ messages in thread
From: Ludovic Courtès @ 2015-06-07 20:03 UTC (permalink / raw)
To: Cyril Roelandt; +Cc: guix-devel
Cyril Roelandt <tipecaml@gmail.com> skribis:
> * guix/import/pypi.scm (compute-inputs, guess-requirements): New procedures.
Nice!
Please mention the ‘guix-hash-url’, ‘make-pypi-sexp’,
‘python->package-name’, etc. changes and the tests.
> +(define (maybe-inputs python->package-name inputs)
> + (match inputs
> + (()
> + '())
> + ((inputs ...)
> + `((,python->package-name (,'quasiquote ,inputs))))))
Please add a docstring. The first argument is misnamed: There’s only on
call site and its value is 'inputs, so I think you should remove this
argument.
> +(define (guess-requirements source-url tarball)
> + "Given SOURCE-URL and a TARBALL of the package, return a list of the required
> +packages specified in the requirements.txt file."
Perhaps this should mention that TARBALL is extracted in the current
directory.
> + (define (tarball-directory url)
> + "Given the URL of the package's tarball, return the name of the directory
> +that will be created upon decompressing it."
Use comments instead of docstrings for internal procedures. Also
mention the #f return value.
> + (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
> + (cond
The opening paren should be under the ‘e’ of ‘let’.
> + ((string-suffix? ".tar.gz" basename)
> + (string-drop-right basename 7))
> + ((string-suffix? ".tar.bz2" basename)
> + (string-drop-right basename 8))
> + (else #f))))
Would be nice to emit a warning like “unsupported archive format; cannot
determine package dependencies” in the #f case.
> + (if (zero? (system* "tar" "xf" tarball req-file))
> + (dynamic-wind
> + (const #t)
> + (lambda ()
> + (read-requirements req-file))
> + (lambda ()
> + (delete-file req-file)
> + (rmdir dirname)))
> + '()))
When ‘tar’ exits with non-zero, it would be nice to report the failure
and exit value using ‘warning’ from (guix ui).
> + ,@(maybe-inputs 'inputs
> + (compute-inputs source-url temp))
Second line should be aligned with 'inputs.
> + (match url
> + ("https://pypi.python.org/pypi/foo/json"
> + (with-output-to-file file-name
> + (lambda ()
> + (display test-json))))
> + ("https://example.com/foo-1.0.0.tar.gz"
> + (begin
> + (mkdir "foo-1.0.0")
> + (with-output-to-file "foo-1.0.0/requirements.txt"
> + (lambda ()
> + (display test-requirements)))
> + (system* "tar" "czvf" file-name "foo-1.0.0/")
> + (system* "rm" "-rf" "foo-1.0.0")
Use ‘delete-file-recursively’ here.
Could you send an updated patch?
Thank you!
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread
* [PATCH] import: pypi: Detect inputs.
2015-06-07 20:03 ` Ludovic Courtès
@ 2015-06-15 1:25 ` Cyril Roelandt
2015-06-18 10:45 ` Amirouche Boubekki
0 siblings, 1 reply; 18+ messages in thread
From: Cyril Roelandt @ 2015-06-15 1:25 UTC (permalink / raw)
To: guix-devel
* guix/import/pypi.scm (python->package-name, maybe-inputs, compute-inputs,
guess-requirements): New procedures.
* guix/import/pypi.scm (guix-hash-url): Now takes a filename instead of an
URL as input.
* guix/import/pypi.scm (make-pypi-sexp): Now tries to generate the inputs
automagically.
* tests/pypi.scm: Update the test.
---
guix/import/pypi.scm | 160 +++++++++++++++++++++++++++++++++++++++++----------
tests/pypi.scm | 42 +++++++++-----
2 files changed, 158 insertions(+), 44 deletions(-)
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 8567cad..cf0a7bb 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,10 +21,13 @@
#:use-module (ice-9 match)
#:use-module (ice-9 pretty-print)
#:use-module (ice-9 regex)
+ #:use-module ((ice-9 rdelim) #:select (read-line))
#:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-26)
#:use-module (rnrs bytevectors)
#:use-module (json)
#:use-module (web uri)
+ #:use-module (guix ui)
#:use-module (guix utils)
#:use-module (guix import utils)
#:use-module (guix import json)
@@ -77,42 +80,137 @@ or #f on failure."
with dashes."
(string-join (string-split (string-downcase str) #\_) "-"))
-(define (guix-hash-url url)
- "Download the resource at URL and return the hash in nix-base32 format."
- (call-with-temporary-output-file
- (lambda (temp port)
- (and (url-fetch url temp)
- (bytevector->nix-base32-string
- (call-with-input-file temp port-sha256))))))
+(define (guix-hash-url filename)
+ "Return the hash of FILENAME in nix-base32 format."
+ (bytevector->nix-base32-string (file-sha256 filename)))
+
+(define (python->package-name name)
+ "Given the NAME of a package on PyPI, return a Guix-compliant name for the
+package."
+ (if (string-prefix? "python-" name)
+ (snake-case name)
+ (string-append "python-" (snake-case name))))
+
+(define (maybe-inputs package-inputs)
+ "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a
+package definition."
+ (match package-inputs
+ (()
+ '())
+ ((package-inputs ...)
+ `((inputs (,'quasiquote ,package-inputs))))))
+
+(define (guess-requirements source-url tarball)
+ "Given SOURCE-URL and a TARBALL of the package, return a list of the required
+packages specified in the requirements.txt file. TARBALL will be extracted in
+the current directory, and will be deleted."
+
+ (define (tarball-directory url)
+ ;; Given the URL of the package's tarball, return the name of the directory
+ ;; that will be created upon decompressing it. If the filetype is not
+ ;; supported, return #f.
+ ;; TODO: Support more archive formats.
+ (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
+ (cond
+ ((string-suffix? ".tar.gz" basename)
+ (string-drop-right basename 7))
+ ((string-suffix? ".tar.bz2" basename)
+ (string-drop-right basename 8))
+ (else
+ (begin
+ (warning (_ "Unsupported archive format: \
+cannot determine package dependencies"))
+ #f)))))
+
+ (define (clean-requirement s)
+ ;; Given a requirement LINE, as can be found in a Python requirements.txt
+ ;; file, remove everything other than the actual name of the required
+ ;; package, and return it.
+ (string-take s
+ (or (string-index s #\space)
+ (string-length s))))
+
+ (define (comment? line)
+ ;; Return #t if the given LINE is a comment, #f otherwise.
+ (eq? (string-ref (string-trim line) 0) #\#))
+
+ (define (read-requirements requirements-file)
+ ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list
+ ;; of name/variable pairs describing the requirements.
+ (call-with-input-file requirements-file
+ (lambda (port)
+ (let loop ((result '()))
+ (let ((line (read-line port)))
+ (if (eof-object? line)
+ result
+ (cond
+ ((or (string-null? line) (comment? line))
+ (loop result))
+ (else
+ (loop (cons (python->package-name (clean-requirement line))
+ result))))))))))
+
+ (let ((dirname (tarball-directory source-url)))
+ (if (string? dirname)
+ (let* ((req-file (string-append dirname "/requirements.txt"))
+ (exit-code (system* "tar" "xf" tarball req-file)))
+ ;; TODO: support more formats.
+ (if (zero? exit-code)
+ (dynamic-wind
+ (const #t)
+ (lambda ()
+ (read-requirements req-file))
+ (lambda ()
+ (delete-file req-file)
+ (rmdir dirname)))
+ (begin
+ (warning (_ "tar xf failed with exit code ~a") exit-code)
+ '())))
+ '())))
+
+(define (compute-inputs source-url tarball)
+ "Given the SOURCE-URL of an already downloaded TARBALL, return a list of
+name/variable pairs describing the required inputs of this package."
+ (sort
+ (map (lambda (input)
+ (list input (list 'unquote (string->symbol input))))
+ (append '("python-setuptools")
+ ;; Argparse has been part of Python since 2.7.
+ (remove (cut string=? "python-argparse" <>)
+ (guess-requirements source-url tarball))))
+ (lambda args
+ (match args
+ (((a _ ...) (b _ ...))
+ (string-ci<? a b))))))
(define (make-pypi-sexp name version source-url home-page synopsis
description license)
"Return the `package' s-expression for a python package with the given NAME,
VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
- `(package
- (name ,(if (string-prefix? "python-" name)
- (snake-case name)
- (string-append "python-" (snake-case name))))
- (version ,version)
- (source (origin
- (method url-fetch)
- (uri (string-append ,@(factorize-uri source-url version)))
- (sha256
- (base32
- ,(guix-hash-url source-url)))))
- (build-system python-build-system)
- (inputs
- `(("python-setuptools" ,python-setuptools)))
- (home-page ,home-page)
- (synopsis ,synopsis)
- (description ,description)
- (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
- (,gpl3 . gpl3)
- (,bsd-3 . bsd-3)
- (,expat . expat)
- (,public-domain . public-domain)
- (,asl2.0 . asl2.0))
- license))))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (and (url-fetch source-url temp)
+ `(package
+ (name ,(python->package-name name))
+ (version ,version)
+ (source (origin
+ (method url-fetch)
+ (uri (string-append ,@(factorize-uri source-url version)))
+ (sha256
+ (base32
+ ,(guix-hash-url temp)))))
+ (build-system python-build-system)
+ ,@(maybe-inputs (compute-inputs source-url temp))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
+ (,gpl3 . gpl3)
+ (,bsd-3 . bsd-3)
+ (,expat . expat)
+ (,public-domain . public-domain)
+ (,asl2.0 . asl2.0))
+ license)))))))
(define (pypi->guix-package package-name)
"Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 45cf7ca..c772474 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -21,6 +21,7 @@
#:use-module (guix base32)
#:use-module (guix hash)
#:use-module (guix tests)
+ #:use-module ((guix build utils) #:select (delete-file-recursively))
#:use-module (srfi srfi-64)
#:use-module (ice-9 match))
@@ -46,8 +47,14 @@
}
}")
-(define test-source
- "foobar")
+(define test-source-hash
+ "")
+
+(define test-requirements
+"# A comment
+ # A comment after a space
+bar
+baz > 13.37")
(test-begin "pypi")
@@ -55,15 +62,22 @@
;; Replace network resources with sample data.
(mock ((guix import utils) url-fetch
(lambda (url file-name)
- (with-output-to-file file-name
- (lambda ()
- (display
- (match url
- ("https://pypi.python.org/pypi/foo/json"
- test-json)
- ("https://example.com/foo-1.0.0.tar.gz"
- test-source)
- (_ (error "Unexpected URL: " url))))))))
+ (match url
+ ("https://pypi.python.org/pypi/foo/json"
+ (with-output-to-file file-name
+ (lambda ()
+ (display test-json))))
+ ("https://example.com/foo-1.0.0.tar.gz"
+ (begin
+ (mkdir "foo-1.0.0")
+ (with-output-to-file "foo-1.0.0/requirements.txt"
+ (lambda ()
+ (display test-requirements)))
+ (system* "tar" "czvf" file-name "foo-1.0.0/")
+ (delete-file-recursively "foo-1.0.0")
+ (set! test-source-hash
+ (call-with-input-file file-name port-sha256))))
+ (_ (error "Unexpected URL: " url)))))
(match (pypi->guix-package "foo")
(('package
('name "python-foo")
@@ -78,13 +92,15 @@
('build-system 'python-build-system)
('inputs
('quasiquote
- (("python-setuptools" ('unquote 'python-setuptools)))))
+ (("python-bar" ('unquote 'python-bar))
+ ("python-baz" ('unquote 'python-baz))
+ ("python-setuptools" ('unquote 'python-setuptools)))))
('home-page "http://example.com")
('synopsis "summary")
('description "summary")
('license 'lgpl2.0))
(string=? (bytevector->nix-base32-string
- (call-with-input-string test-source port-sha256))
+ test-source-hash)
hash))
(x
(pk 'fail x #f)))))
--
1.8.4.rc3
^ permalink raw reply related [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-15 1:25 ` Cyril Roelandt
@ 2015-06-18 10:45 ` Amirouche Boubekki
2015-06-19 15:32 ` Christopher Allan Webber
` (2 more replies)
0 siblings, 3 replies; 18+ messages in thread
From: Amirouche Boubekki @ 2015-06-18 10:45 UTC (permalink / raw)
To: Cyril Roelandt; +Cc: guix-devel, guix-devel-bounces+amirouche=hypermove.net
[-- Attachment #1: Type: text/plain, Size: 12655 bytes --]
Héllo,
If I'm not mistaken this patch relies only on the presence of
requirements.txt. This is not a required file in python packaging.
otherwise said, we miss a lot using this method. I think the best way to
do that would be to:
- download the package and extract it
- create an environment (#)
- create a virtual env with access to system site package of the
environment (#)
- enter the venv and install the package
- use `pip freeze -l` to retrieve the full set of dependencies
If it fails (because of missing system dependencies) fallback to parse
setup.py (with guile-log?) and plain requirements.txt. It would be nice
to allow to drop to guix environment (#) when the first option fails to
inspect and install missing system dependencies manually.
Maybe [1] can be helpful, I attached both data and a script to extract.
the dataset is missing and needs cleanup. It helped me to see that *a
lot* of django packages miss django dependency on pypi.
WDYT?
[1]
https://ogirardot.wordpress.com/2013/01/31/sharing-pypimaven-dependency-data/
On 2015-06-15 03:25, Cyril Roelandt wrote:
> * guix/import/pypi.scm (python->package-name, maybe-inputs,
> compute-inputs,
> guess-requirements): New procedures.
> * guix/import/pypi.scm (guix-hash-url): Now takes a filename instead of
> an
> URL as input.
> * guix/import/pypi.scm (make-pypi-sexp): Now tries to generate the
> inputs
> automagically.
> * tests/pypi.scm: Update the test.
> ---
> guix/import/pypi.scm | 160
> +++++++++++++++++++++++++++++++++++++++++----------
> tests/pypi.scm | 42 +++++++++-----
> 2 files changed, 158 insertions(+), 44 deletions(-)
>
> diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
> index 8567cad..cf0a7bb 100644
> --- a/guix/import/pypi.scm
> +++ b/guix/import/pypi.scm
> @@ -21,10 +21,13 @@
> #:use-module (ice-9 match)
> #:use-module (ice-9 pretty-print)
> #:use-module (ice-9 regex)
> + #:use-module ((ice-9 rdelim) #:select (read-line))
> #:use-module (srfi srfi-1)
> + #:use-module (srfi srfi-26)
> #:use-module (rnrs bytevectors)
> #:use-module (json)
> #:use-module (web uri)
> + #:use-module (guix ui)
> #:use-module (guix utils)
> #:use-module (guix import utils)
> #:use-module (guix import json)
> @@ -77,42 +80,137 @@ or #f on failure."
> with dashes."
> (string-join (string-split (string-downcase str) #\_) "-"))
>
> -(define (guix-hash-url url)
> - "Download the resource at URL and return the hash in nix-base32
> format."
> - (call-with-temporary-output-file
> - (lambda (temp port)
> - (and (url-fetch url temp)
> - (bytevector->nix-base32-string
> - (call-with-input-file temp port-sha256))))))
> +(define (guix-hash-url filename)
> + "Return the hash of FILENAME in nix-base32 format."
> + (bytevector->nix-base32-string (file-sha256 filename)))
> +
> +(define (python->package-name name)
> + "Given the NAME of a package on PyPI, return a Guix-compliant name
> for the
> +package."
> + (if (string-prefix? "python-" name)
> + (snake-case name)
> + (string-append "python-" (snake-case name))))
> +
> +(define (maybe-inputs package-inputs)
> + "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs'
> field of a
> +package definition."
> + (match package-inputs
> + (()
> + '())
> + ((package-inputs ...)
> + `((inputs (,'quasiquote ,package-inputs))))))
> +
> +(define (guess-requirements source-url tarball)
> + "Given SOURCE-URL and a TARBALL of the package, return a list of the
> required
> +packages specified in the requirements.txt file. TARBALL will be
> extracted in
> +the current directory, and will be deleted."
> +
> + (define (tarball-directory url)
> + ;; Given the URL of the package's tarball, return the name of the
> directory
> + ;; that will be created upon decompressing it. If the filetype is
> not
> + ;; supported, return #f.
> + ;; TODO: Support more archive formats.
> + (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
> + (cond
> + ((string-suffix? ".tar.gz" basename)
> + (string-drop-right basename 7))
> + ((string-suffix? ".tar.bz2" basename)
> + (string-drop-right basename 8))
> + (else
> + (begin
> + (warning (_ "Unsupported archive format: \
> +cannot determine package dependencies"))
> + #f)))))
> +
> + (define (clean-requirement s)
> + ;; Given a requirement LINE, as can be found in a Python
> requirements.txt
> + ;; file, remove everything other than the actual name of the
> required
> + ;; package, and return it.
> + (string-take s
> + (or (string-index s #\space)
> + (string-length s))))
> +
> + (define (comment? line)
> + ;; Return #t if the given LINE is a comment, #f otherwise.
> + (eq? (string-ref (string-trim line) 0) #\#))
> +
> + (define (read-requirements requirements-file)
> + ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return
> a list
> + ;; of name/variable pairs describing the requirements.
> + (call-with-input-file requirements-file
> + (lambda (port)
> + (let loop ((result '()))
> + (let ((line (read-line port)))
> + (if (eof-object? line)
> + result
> + (cond
> + ((or (string-null? line) (comment? line))
> + (loop result))
> + (else
> + (loop (cons (python->package-name (clean-requirement
> line))
> + result))))))))))
> +
> + (let ((dirname (tarball-directory source-url)))
> + (if (string? dirname)
> + (let* ((req-file (string-append dirname "/requirements.txt"))
> + (exit-code (system* "tar" "xf" tarball req-file)))
> + ;; TODO: support more formats.
> + (if (zero? exit-code)
> + (dynamic-wind
> + (const #t)
> + (lambda ()
> + (read-requirements req-file))
> + (lambda ()
> + (delete-file req-file)
> + (rmdir dirname)))
> + (begin
> + (warning (_ "tar xf failed with exit code ~a")
> exit-code)
> + '())))
> + '())))
> +
> +(define (compute-inputs source-url tarball)
> + "Given the SOURCE-URL of an already downloaded TARBALL, return a
> list of
> +name/variable pairs describing the required inputs of this package."
> + (sort
> + (map (lambda (input)
> + (list input (list 'unquote (string->symbol input))))
> + (append '("python-setuptools")
> + ;; Argparse has been part of Python since 2.7.
> + (remove (cut string=? "python-argparse" <>)
> + (guess-requirements source-url tarball))))
> + (lambda args
> + (match args
> + (((a _ ...) (b _ ...))
> + (string-ci<? a b))))))
>
> (define (make-pypi-sexp name version source-url home-page synopsis
> description license)
> "Return the `package' s-expression for a python package with the
> given NAME,
> VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
> - `(package
> - (name ,(if (string-prefix? "python-" name)
> - (snake-case name)
> - (string-append "python-" (snake-case name))))
> - (version ,version)
> - (source (origin
> - (method url-fetch)
> - (uri (string-append ,@(factorize-uri source-url
> version)))
> - (sha256
> - (base32
> - ,(guix-hash-url source-url)))))
> - (build-system python-build-system)
> - (inputs
> - `(("python-setuptools" ,python-setuptools)))
> - (home-page ,home-page)
> - (synopsis ,synopsis)
> - (description ,description)
> - (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
> - (,gpl3 . gpl3)
> - (,bsd-3 . bsd-3)
> - (,expat . expat)
> - (,public-domain . public-domain)
> - (,asl2.0 . asl2.0))
> - license))))
> + (call-with-temporary-output-file
> + (lambda (temp port)
> + (and (url-fetch source-url temp)
> + `(package
> + (name ,(python->package-name name))
> + (version ,version)
> + (source (origin
> + (method url-fetch)
> + (uri (string-append ,@(factorize-uri
> source-url version)))
> + (sha256
> + (base32
> + ,(guix-hash-url temp)))))
> + (build-system python-build-system)
> + ,@(maybe-inputs (compute-inputs source-url temp))
> + (home-page ,home-page)
> + (synopsis ,synopsis)
> + (description ,description)
> + (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
> + (,gpl3 . gpl3)
> + (,bsd-3 . bsd-3)
> + (,expat . expat)
> + (,public-domain . public-domain)
> + (,asl2.0 . asl2.0))
> + license)))))))
>
> (define (pypi->guix-package package-name)
> "Fetch the metadata for PACKAGE-NAME from pypi.python.org, and
> return the
> diff --git a/tests/pypi.scm b/tests/pypi.scm
> index 45cf7ca..c772474 100644
> --- a/tests/pypi.scm
> +++ b/tests/pypi.scm
> @@ -21,6 +21,7 @@
> #:use-module (guix base32)
> #:use-module (guix hash)
> #:use-module (guix tests)
> + #:use-module ((guix build utils) #:select (delete-file-recursively))
> #:use-module (srfi srfi-64)
> #:use-module (ice-9 match))
>
> @@ -46,8 +47,14 @@
> }
> }")
>
> -(define test-source
> - "foobar")
> +(define test-source-hash
> + "")
> +
> +(define test-requirements
> +"# A comment
> + # A comment after a space
> +bar
> +baz > 13.37")
>
> (test-begin "pypi")
>
> @@ -55,15 +62,22 @@
> ;; Replace network resources with sample data.
> (mock ((guix import utils) url-fetch
> (lambda (url file-name)
> - (with-output-to-file file-name
> - (lambda ()
> - (display
> - (match url
> - ("https://pypi.python.org/pypi/foo/json"
> - test-json)
> - ("https://example.com/foo-1.0.0.tar.gz"
> - test-source)
> - (_ (error "Unexpected URL: " url))))))))
> + (match url
> + ("https://pypi.python.org/pypi/foo/json"
> + (with-output-to-file file-name
> + (lambda ()
> + (display test-json))))
> + ("https://example.com/foo-1.0.0.tar.gz"
> + (begin
> + (mkdir "foo-1.0.0")
> + (with-output-to-file "foo-1.0.0/requirements.txt"
> + (lambda ()
> + (display test-requirements)))
> + (system* "tar" "czvf" file-name "foo-1.0.0/")
> + (delete-file-recursively "foo-1.0.0")
> + (set! test-source-hash
> + (call-with-input-file file-name port-sha256))))
> + (_ (error "Unexpected URL: " url)))))
> (match (pypi->guix-package "foo")
> (('package
> ('name "python-foo")
> @@ -78,13 +92,15 @@
> ('build-system 'python-build-system)
> ('inputs
> ('quasiquote
> - (("python-setuptools" ('unquote 'python-setuptools)))))
> + (("python-bar" ('unquote 'python-bar))
> + ("python-baz" ('unquote 'python-baz))
> + ("python-setuptools" ('unquote 'python-setuptools)))))
> ('home-page "http://example.com")
> ('synopsis "summary")
> ('description "summary")
> ('license 'lgpl2.0))
> (string=? (bytevector->nix-base32-string
> - (call-with-input-string test-source port-sha256))
> + test-source-hash)
> hash))
> (x
> (pk 'fail x #f)))))
--
Amirouche ~ amz3 ~ http://www.hyperdev.fr
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: extract.py --]
[-- Type: text/x-python; name=extract.py, Size: 755 bytes --]
from re import compile
from json import loads
from pathlib import Path
from base64 import b64decode
NAME = compile('^([\w\.]+)')
# the pypi dataset was built by O. Girardot
# https://ogirardot.wordpress.com/2013/01/31/sharing-pypimaven-dependency-data/
with Path('./import.log').open('w') as log:
with Path('./pypi-deps.csv').resolve().open() as f:
lines = f.read().split('\n')
count = len(lines)
for num, line in enumerate(lines):
name, version, dependencies = line.split('\t')
dependencies = loads(b64decode(dependencies).decode('utf-8'))
dependencies = map(lambda x: x.strip(), dependencies)
dependencies = list(dependencies)
print(name, version, dependencies)
[-- Attachment #3: pypi-deps.csv.gz --]
[-- Type: application/x-gzip, Size: 223837 bytes --]
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-18 10:45 ` Amirouche Boubekki
@ 2015-06-19 15:32 ` Christopher Allan Webber
2015-06-20 18:01 ` Amirouche Boubekki
2015-06-21 20:56 ` Ludovic Courtès
2015-06-23 21:04 ` Cyril Roelandt
2 siblings, 1 reply; 18+ messages in thread
From: Christopher Allan Webber @ 2015-06-19 15:32 UTC (permalink / raw)
To: Amirouche Boubekki; +Cc: guix-devel, guix-devel-bounces+amirouche=hypermove.net
Amirouche Boubekki writes:
> Héllo,
>
>
> If I'm not mistaken this patch relies only on the presence of
> requirements.txt. This is not a required file in python packaging.
> otherwise said, we miss a lot using this method. I think the best way to
> do that would be to:
>
> - download the package and extract it
> - create an environment (#)
> - create a virtual env with access to system site package of the
> environment (#)
> - enter the venv and install the package
> - use `pip freeze -l` to retrieve the full set of dependencies
Using pip freeze is an interesting idea.
Setting up a virtualenv... that's interesting. Would it be written to a
temporary directory?
> If it fails (because of missing system dependencies) fallback to parse
> setup.py (with guile-log?) and plain requirements.txt. It would be nice
> to allow to drop to guix environment (#) when the first option fails to
> inspect and install missing system dependencies manually.
>
> Maybe [1] can be helpful, I attached both data and a script to extract.
> the dataset is missing and needs cleanup. It helped me to see that *a
> lot* of django packages miss django dependency on pypi.
>
> WDYT?
>
> [1]
> https://ogirardot.wordpress.com/2013/01/31/sharing-pypimaven-dependency-data/
>
>
> On 2015-06-15 03:25, Cyril Roelandt wrote:
>> * guix/import/pypi.scm (python->package-name, maybe-inputs,
>> compute-inputs,
>> guess-requirements): New procedures.
>> * guix/import/pypi.scm (guix-hash-url): Now takes a filename instead of
>> an
>> URL as input.
>> * guix/import/pypi.scm (make-pypi-sexp): Now tries to generate the
>> inputs
>> automagically.
>> * tests/pypi.scm: Update the test.
>> ---
>> guix/import/pypi.scm | 160
>> +++++++++++++++++++++++++++++++++++++++++----------
>> tests/pypi.scm | 42 +++++++++-----
>> 2 files changed, 158 insertions(+), 44 deletions(-)
>>
>> diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
>> index 8567cad..cf0a7bb 100644
>> --- a/guix/import/pypi.scm
>> +++ b/guix/import/pypi.scm
>> @@ -21,10 +21,13 @@
>> #:use-module (ice-9 match)
>> #:use-module (ice-9 pretty-print)
>> #:use-module (ice-9 regex)
>> + #:use-module ((ice-9 rdelim) #:select (read-line))
>> #:use-module (srfi srfi-1)
>> + #:use-module (srfi srfi-26)
>> #:use-module (rnrs bytevectors)
>> #:use-module (json)
>> #:use-module (web uri)
>> + #:use-module (guix ui)
>> #:use-module (guix utils)
>> #:use-module (guix import utils)
>> #:use-module (guix import json)
>> @@ -77,42 +80,137 @@ or #f on failure."
>> with dashes."
>> (string-join (string-split (string-downcase str) #\_) "-"))
>>
>> -(define (guix-hash-url url)
>> - "Download the resource at URL and return the hash in nix-base32
>> format."
>> - (call-with-temporary-output-file
>> - (lambda (temp port)
>> - (and (url-fetch url temp)
>> - (bytevector->nix-base32-string
>> - (call-with-input-file temp port-sha256))))))
>> +(define (guix-hash-url filename)
>> + "Return the hash of FILENAME in nix-base32 format."
>> + (bytevector->nix-base32-string (file-sha256 filename)))
>> +
>> +(define (python->package-name name)
>> + "Given the NAME of a package on PyPI, return a Guix-compliant name
>> for the
>> +package."
>> + (if (string-prefix? "python-" name)
>> + (snake-case name)
>> + (string-append "python-" (snake-case name))))
>> +
>> +(define (maybe-inputs package-inputs)
>> + "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs'
>> field of a
>> +package definition."
>> + (match package-inputs
>> + (()
>> + '())
>> + ((package-inputs ...)
>> + `((inputs (,'quasiquote ,package-inputs))))))
>> +
>> +(define (guess-requirements source-url tarball)
>> + "Given SOURCE-URL and a TARBALL of the package, return a list of the
>> required
>> +packages specified in the requirements.txt file. TARBALL will be
>> extracted in
>> +the current directory, and will be deleted."
>> +
>> + (define (tarball-directory url)
>> + ;; Given the URL of the package's tarball, return the name of the
>> directory
>> + ;; that will be created upon decompressing it. If the filetype is
>> not
>> + ;; supported, return #f.
>> + ;; TODO: Support more archive formats.
>> + (let ((basename (substring url (+ 1 (string-rindex url #\/)))))
>> + (cond
>> + ((string-suffix? ".tar.gz" basename)
>> + (string-drop-right basename 7))
>> + ((string-suffix? ".tar.bz2" basename)
>> + (string-drop-right basename 8))
>> + (else
>> + (begin
>> + (warning (_ "Unsupported archive format: \
>> +cannot determine package dependencies"))
>> + #f)))))
>> +
>> + (define (clean-requirement s)
>> + ;; Given a requirement LINE, as can be found in a Python
>> requirements.txt
>> + ;; file, remove everything other than the actual name of the
>> required
>> + ;; package, and return it.
>> + (string-take s
>> + (or (string-index s #\space)
>> + (string-length s))))
>> +
>> + (define (comment? line)
>> + ;; Return #t if the given LINE is a comment, #f otherwise.
>> + (eq? (string-ref (string-trim line) 0) #\#))
>> +
>> + (define (read-requirements requirements-file)
>> + ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return
>> a list
>> + ;; of name/variable pairs describing the requirements.
>> + (call-with-input-file requirements-file
>> + (lambda (port)
>> + (let loop ((result '()))
>> + (let ((line (read-line port)))
>> + (if (eof-object? line)
>> + result
>> + (cond
>> + ((or (string-null? line) (comment? line))
>> + (loop result))
>> + (else
>> + (loop (cons (python->package-name (clean-requirement
>> line))
>> + result))))))))))
>> +
>> + (let ((dirname (tarball-directory source-url)))
>> + (if (string? dirname)
>> + (let* ((req-file (string-append dirname "/requirements.txt"))
>> + (exit-code (system* "tar" "xf" tarball req-file)))
>> + ;; TODO: support more formats.
>> + (if (zero? exit-code)
>> + (dynamic-wind
>> + (const #t)
>> + (lambda ()
>> + (read-requirements req-file))
>> + (lambda ()
>> + (delete-file req-file)
>> + (rmdir dirname)))
>> + (begin
>> + (warning (_ "tar xf failed with exit code ~a")
>> exit-code)
>> + '())))
>> + '())))
>> +
>> +(define (compute-inputs source-url tarball)
>> + "Given the SOURCE-URL of an already downloaded TARBALL, return a
>> list of
>> +name/variable pairs describing the required inputs of this package."
>> + (sort
>> + (map (lambda (input)
>> + (list input (list 'unquote (string->symbol input))))
>> + (append '("python-setuptools")
>> + ;; Argparse has been part of Python since 2.7.
>> + (remove (cut string=? "python-argparse" <>)
>> + (guess-requirements source-url tarball))))
>> + (lambda args
>> + (match args
>> + (((a _ ...) (b _ ...))
>> + (string-ci<? a b))))))
>>
>> (define (make-pypi-sexp name version source-url home-page synopsis
>> description license)
>> "Return the `package' s-expression for a python package with the
>> given NAME,
>> VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
>> - `(package
>> - (name ,(if (string-prefix? "python-" name)
>> - (snake-case name)
>> - (string-append "python-" (snake-case name))))
>> - (version ,version)
>> - (source (origin
>> - (method url-fetch)
>> - (uri (string-append ,@(factorize-uri source-url
>> version)))
>> - (sha256
>> - (base32
>> - ,(guix-hash-url source-url)))))
>> - (build-system python-build-system)
>> - (inputs
>> - `(("python-setuptools" ,python-setuptools)))
>> - (home-page ,home-page)
>> - (synopsis ,synopsis)
>> - (description ,description)
>> - (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
>> - (,gpl3 . gpl3)
>> - (,bsd-3 . bsd-3)
>> - (,expat . expat)
>> - (,public-domain . public-domain)
>> - (,asl2.0 . asl2.0))
>> - license))))
>> + (call-with-temporary-output-file
>> + (lambda (temp port)
>> + (and (url-fetch source-url temp)
>> + `(package
>> + (name ,(python->package-name name))
>> + (version ,version)
>> + (source (origin
>> + (method url-fetch)
>> + (uri (string-append ,@(factorize-uri
>> source-url version)))
>> + (sha256
>> + (base32
>> + ,(guix-hash-url temp)))))
>> + (build-system python-build-system)
>> + ,@(maybe-inputs (compute-inputs source-url temp))
>> + (home-page ,home-page)
>> + (synopsis ,synopsis)
>> + (description ,description)
>> + (license ,(assoc-ref `((,lgpl2.0 . lgpl2.0)
>> + (,gpl3 . gpl3)
>> + (,bsd-3 . bsd-3)
>> + (,expat . expat)
>> + (,public-domain . public-domain)
>> + (,asl2.0 . asl2.0))
>> + license)))))))
>>
>> (define (pypi->guix-package package-name)
>> "Fetch the metadata for PACKAGE-NAME from pypi.python.org, and
>> return the
>> diff --git a/tests/pypi.scm b/tests/pypi.scm
>> index 45cf7ca..c772474 100644
>> --- a/tests/pypi.scm
>> +++ b/tests/pypi.scm
>> @@ -21,6 +21,7 @@
>> #:use-module (guix base32)
>> #:use-module (guix hash)
>> #:use-module (guix tests)
>> + #:use-module ((guix build utils) #:select (delete-file-recursively))
>> #:use-module (srfi srfi-64)
>> #:use-module (ice-9 match))
>>
>> @@ -46,8 +47,14 @@
>> }
>> }")
>>
>> -(define test-source
>> - "foobar")
>> +(define test-source-hash
>> + "")
>> +
>> +(define test-requirements
>> +"# A comment
>> + # A comment after a space
>> +bar
>> +baz > 13.37")
>>
>> (test-begin "pypi")
>>
>> @@ -55,15 +62,22 @@
>> ;; Replace network resources with sample data.
>> (mock ((guix import utils) url-fetch
>> (lambda (url file-name)
>> - (with-output-to-file file-name
>> - (lambda ()
>> - (display
>> - (match url
>> - ("https://pypi.python.org/pypi/foo/json"
>> - test-json)
>> - ("https://example.com/foo-1.0.0.tar.gz"
>> - test-source)
>> - (_ (error "Unexpected URL: " url))))))))
>> + (match url
>> + ("https://pypi.python.org/pypi/foo/json"
>> + (with-output-to-file file-name
>> + (lambda ()
>> + (display test-json))))
>> + ("https://example.com/foo-1.0.0.tar.gz"
>> + (begin
>> + (mkdir "foo-1.0.0")
>> + (with-output-to-file "foo-1.0.0/requirements.txt"
>> + (lambda ()
>> + (display test-requirements)))
>> + (system* "tar" "czvf" file-name "foo-1.0.0/")
>> + (delete-file-recursively "foo-1.0.0")
>> + (set! test-source-hash
>> + (call-with-input-file file-name port-sha256))))
>> + (_ (error "Unexpected URL: " url)))))
>> (match (pypi->guix-package "foo")
>> (('package
>> ('name "python-foo")
>> @@ -78,13 +92,15 @@
>> ('build-system 'python-build-system)
>> ('inputs
>> ('quasiquote
>> - (("python-setuptools" ('unquote 'python-setuptools)))))
>> + (("python-bar" ('unquote 'python-bar))
>> + ("python-baz" ('unquote 'python-baz))
>> + ("python-setuptools" ('unquote 'python-setuptools)))))
>> ('home-page "http://example.com")
>> ('synopsis "summary")
>> ('description "summary")
>> ('license 'lgpl2.0))
>> (string=? (bytevector->nix-base32-string
>> - (call-with-input-string test-source port-sha256))
>> + test-source-hash)
>> hash))
>> (x
>> (pk 'fail x #f)))))
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-19 15:32 ` Christopher Allan Webber
@ 2015-06-20 18:01 ` Amirouche Boubekki
0 siblings, 0 replies; 18+ messages in thread
From: Amirouche Boubekki @ 2015-06-20 18:01 UTC (permalink / raw)
To: Christopher Allan Webber
Cc: guix-devel, guix-devel-bounces+amirouche=hypermove.net
[-- Attachment #1: Type: text/plain, Size: 1207 bytes --]
On 2015-06-19 17:32, Christopher Allan Webber wrote:
> Amirouche Boubekki writes:
>
>> Héllo,
>>
>>
>> If I'm not mistaken this patch relies only on the presence of
>> requirements.txt. This is not a required file in python packaging.
>> otherwise said, we miss a lot using this method. I think the best way
>> to
>> do that would be to:
>>
>> - download the package and extract it
>> - create an environment (#)
>> - create a virtual env with access to system site package of the
>> environment (#)
>> - enter the venv and install the package
>> - use `pip freeze -l` to retrieve the full set of dependencies
>
> Using pip freeze is an interesting idea.
>
> Setting up a virtualenv... that's interesting. Would it be written to
> a
> temporary directory?
My bad, it's probably not a good idea to have that without containers,
as it execute some code that we don't know what it does - the setup.py.
The best way to go is to parse the setup.py *and* requirements.txt.
I attached a script that does what I described without the `guix
environment`.
Using `guix environment` might be a good idea to prepapre a recipe
without polluting its own profile.
HTH.
[-- Attachment #2: pypi-guess-deps.sh --]
[-- Type: text/plain, Size: 362 bytes --]
if [ $1 ]
then
wget -q $1 -O - | tar xz > .pypi-guess-deps.log 2>&1
virtualenv .venv >> .pypi-guess-deps.log 2>&1
source .venv/bin/activate >> .pypi-guess-deps.log 2>&1
cd * && python setup.py install >> .pypi-guess-deps.log 2>&1
pip freeze -l
else
echo "Usage: pypi-guess-deps.sh PACKAGE-URL
log is written to `pypi-guess-deps.log`
"
fi
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-18 10:45 ` Amirouche Boubekki
2015-06-19 15:32 ` Christopher Allan Webber
@ 2015-06-21 20:56 ` Ludovic Courtès
2015-06-21 21:32 ` Amirouche Boubekki
2015-06-23 21:04 ` Cyril Roelandt
2 siblings, 1 reply; 18+ messages in thread
From: Ludovic Courtès @ 2015-06-21 20:56 UTC (permalink / raw)
To: Amirouche Boubekki; +Cc: guix-devel, guix-devel-bounces+amirouche=hypermove.net
Amirouche Boubekki <amirouche@hypermove.net> skribis:
> If I'm not mistaken this patch relies only on the presence of
> requirements.txt. This is not a required file in python
> packaging. otherwise said, we miss a lot using this method. I think
> the best way to do that would be to:
>
> - download the package and extract it
> - create an environment (#)
> - create a virtual env with access to system site package of the
> environment (#)
> - enter the venv and install the package
> - use `pip freeze -l` to retrieve the full set of dependencies
Cyril, WDYT?
I’m not familiar with the details, but my feeling is that reading
requirements.txt is more lightweight, simpler, and more robust than what
you propose.
So I’d be inclined to apply Cyril’s patch ASAP.
Thanks,
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-21 20:56 ` Ludovic Courtès
@ 2015-06-21 21:32 ` Amirouche Boubekki
0 siblings, 0 replies; 18+ messages in thread
From: Amirouche Boubekki @ 2015-06-21 21:32 UTC (permalink / raw)
To: ludo; +Cc: guix-devel, guix-devel-bounces+amirouche=hypermove.net
On 2015-06-21 22:56, ludo@gnu.org wrote:
> Amirouche Boubekki <amirouche@hypermove.net> skribis:
>
>> If I'm not mistaken this patch relies only on the presence of
>> requirements.txt. This is not a required file in python
>> packaging. otherwise said, we miss a lot using this method. I think
>> the best way to do that would be to:
>>
>> - download the package and extract it
>> - create an environment (#)
>> - create a virtual env with access to system site package of the
>> environment (#)
>> - enter the venv and install the package
>> - use `pip freeze -l` to retrieve the full set of dependencies
>
> Cyril, WDYT?
>
> I’m not familiar with the details, but my feeling is that reading
> requirements.txt is more lightweight, simpler, and more robust than
> what
> you propose.
>
> So I’d be inclined to apply Cyril’s patch ASAP.
Yes of course, it super helpful. It's not comprehensive, that's all I
liked to say, that is all.
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-18 10:45 ` Amirouche Boubekki
2015-06-19 15:32 ` Christopher Allan Webber
2015-06-21 20:56 ` Ludovic Courtès
@ 2015-06-23 21:04 ` Cyril Roelandt
2015-06-24 19:49 ` Ludovic Courtès
2 siblings, 1 reply; 18+ messages in thread
From: Cyril Roelandt @ 2015-06-23 21:04 UTC (permalink / raw)
To: Amirouche Boubekki; +Cc: guix-devel
On 06/18/2015 12:45 PM, Amirouche Boubekki wrote:
> Héllo,
>
Hey,
>
> If I'm not mistaken this patch relies only on the presence of
> requirements.txt. This is not a required file in python packaging.
> otherwise said, we miss a lot using this method. I think the best way to
> do that would be to:
>
Indeed, requirements.txt is not required. Note that if this file does
not exist, "guix import pypi" will not fail, so this patch should not be
breaking anything.
> - download the package and extract it
> - create an environment (#)
> - create a virtual env with access to system site package of the
> environment (#)
> - enter the venv and install the package
> - use `pip freeze -l` to retrieve the full set of dependencies
>
That seems nice, but:
1) I would like to be able to determine whether we have the "right"
version of all the dependencies in the current list of packages (if
requirements.txt specifies "foobar>=2.0" and we only have python-foo-1.0
in Guix, the user should be told that python-foo should be upgraded),
and pip freeze will just give you "pinned" versions (foo==2.0);
2) "pip freeze" will list everything that is installed in the virtual
environment, including the dependencies of the actual dependencies, and
we do not want that.
> If it fails (because of missing system dependencies) fallback to parse
> setup.py (with guile-log?) and plain requirements.txt. It would be nice
> to allow to drop to guix environment (#) when the first option fails to
> inspect and install missing system dependencies manually.
>
I really do not want to parse setup.py, because you can put any Python
code you want in there, so parsing it properly would require a Python
parser, and an AI that can determine what the code is meant to do. That
might be a bit too complex.
WDYT?
Cyril.
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-23 21:04 ` Cyril Roelandt
@ 2015-06-24 19:49 ` Ludovic Courtès
2015-06-24 21:42 ` Cyril Roelandt
0 siblings, 1 reply; 18+ messages in thread
From: Ludovic Courtès @ 2015-06-24 19:49 UTC (permalink / raw)
To: Cyril Roelandt; +Cc: guix-devel
Cyril Roelandt <tipecaml@gmail.com> skribis:
> WDYT?
Makes sense to me.
The latest version of the patch (posted on Monday 15th) LGTM;
please push.
Thank you!
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread
* Re: [PATCH] import: pypi: Detect inputs.
2015-06-24 19:49 ` Ludovic Courtès
@ 2015-06-24 21:42 ` Cyril Roelandt
0 siblings, 0 replies; 18+ messages in thread
From: Cyril Roelandt @ 2015-06-24 21:42 UTC (permalink / raw)
To: Ludovic Courtès; +Cc: guix-devel
On 06/24/2015 09:49 PM, Ludovic Courtès wrote:
> Cyril Roelandt <tipecaml@gmail.com> skribis:
>
>> WDYT?
>
> Makes sense to me.
>
> The latest version of the patch (posted on Monday 15th) LGTM;
> please push.
>
Pushed in ff986890ece06b0623a7c8b3048dea1206f076ea .
Cyril.
^ permalink raw reply [flat|nested] 18+ messages in thread
end of thread, other threads:[~2015-06-24 21:42 UTC | newest]
Thread overview: 18+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2015-03-07 23:51 [PATCH] import: pypi: detect requirements from requirements.txt Cyril Roelandt
2015-03-08 0:41 ` David Thompson
2015-03-22 21:05 ` [PATCH] import: pypi: Detect inputs Cyril Roelandt
2015-03-26 13:15 ` Ludovic Courtès
2015-03-27 12:36 ` David Thompson
2015-03-29 13:46 ` Ludovic Courtès
2015-06-04 22:56 ` Cyril Roelandt
2015-06-07 20:03 ` Ludovic Courtès
2015-06-15 1:25 ` Cyril Roelandt
2015-06-18 10:45 ` Amirouche Boubekki
2015-06-19 15:32 ` Christopher Allan Webber
2015-06-20 18:01 ` Amirouche Boubekki
2015-06-21 20:56 ` Ludovic Courtès
2015-06-21 21:32 ` Amirouche Boubekki
2015-06-23 21:04 ` Cyril Roelandt
2015-06-24 19:49 ` Ludovic Courtès
2015-06-24 21:42 ` Cyril Roelandt
2015-03-22 21:05 ` [PATCH] import: pypi: detect requirements from requirements.txt Cyril Roelandt
Code repositories for project(s) associated with this public inbox
https://git.savannah.gnu.org/cgit/guix.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).