From 10a2c699f28daa29f434fbd4fa0e67fbbac4224b Mon Sep 17 00:00:00 2001 From: Martin Becze Date: Tue, 10 Nov 2020 22:39:30 +0100 Subject: [PATCH 3/7] import: crate: Use guile-semver to resolve module versions. * guix/import/crate.scm: Add guile-semver as a soft dependency. (make-crate-sexp): Don't allow other keys. Add '#:skip-build?' to build system args. Pass a VERSION argument to 'cargo-inputs'. Move 'package-definition' from scripts/import/crate.scm to here. (crate->guix-package): Use guile-semver to resolve the correct module versions. Treat "build" dependencies as normal dependencies. Add key "repo" and make "name" a key. (crate-name->package-name): Reuse the procedure 'guix-name' instead of duplicating its logic. (crate-recursive-import): Use (crate->guix-package) directly since it now takes the correct arguments. * guix/import/utils.scm (package-names->package-inputs): Implement handling of (name version) pairs. * guix/scripts/import/crate.scm (guix-import-crate): Remove the public definitions since the are now returned by (crate-recursive-import). * tests/crate.scm: (recursive-import) Add version data to the test. --- guix/import/crate.scm | 89 +++++---- guix/import/utils.scm | 21 ++- guix/scripts/import/crate.scm | 11 +- tests/crate.scm | 330 +++++++++++++++++++--------------- 4 files changed, 261 insertions(+), 190 deletions(-) diff --git a/guix/import/crate.scm b/guix/import/crate.scm index 8c2b76cab4..9c93a80fbd 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016 David Craven ;;; Copyright © 2019, 2020 Ludovic Courtès -;;; Copyright © 2019 Martin Becze +;;; Copyright © 2019, 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -37,6 +37,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-2) #:use-module (srfi srfi-26) + #:use-module (srfi srfi-71) #:export (crate->guix-package guix-package->crate-name string->license @@ -85,10 +86,15 @@ crate-dependency? json->crate-dependency (id crate-dependency-id "crate_id") ;string - (kind crate-dependency-kind "kind" ;'normal | 'dev + (kind crate-dependency-kind "kind" ;'normal | 'dev | 'build string->symbol) (requirement crate-dependency-requirement "req")) ;string +(module-autoload! (current-module) + '(semver) '(string->semver)) +(module-autoload! (current-module) + '(semver ranges) '(string->semver-range semver-range-contains?)) + (define (lookup-crate name) "Look up NAME on https://crates.io and return the corresopnding record or #f if it was not found." @@ -142,16 +148,22 @@ record or #f if it was not found." `((arguments (,'quasiquote ,args)))))) (define* (make-crate-sexp #:key name version cargo-inputs cargo-development-inputs - home-page synopsis description license - #:allow-other-keys) + home-page synopsis description license) "Return the `package' s-expression for a rust package with the given NAME, VERSION, CARGO-INPUTS, CARGO-DEVELOPMENT-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." + (define (format-inputs inputs) + (map + (match-lambda + ((name version) + (list (crate-name->package-name name) + (version-major+minor version)))) + inputs)) + (let* ((port (http-fetch (crate-uri name version))) (guix-name (crate-name->package-name name)) - (cargo-inputs (map crate-name->package-name cargo-inputs)) - (cargo-development-inputs (map crate-name->package-name - cargo-development-inputs)) + (cargo-inputs (format-inputs cargo-inputs)) + (cargo-development-inputs (format-inputs cargo-development-inputs)) (pkg `(package (name ,guix-name) (version ,version) @@ -163,7 +175,8 @@ and LICENSE." (base32 ,(bytevector->nix-base32-string (port-sha256 port)))))) (build-system cargo-build-system) - ,@(maybe-arguments (append (maybe-cargo-inputs cargo-inputs) + ,@(maybe-arguments (append '(#:skip-build? #t) + (maybe-cargo-inputs cargo-inputs) (maybe-cargo-development-inputs cargo-development-inputs))) (home-page ,(match home-page @@ -176,7 +189,7 @@ and LICENSE." ((license) license) (_ `(list ,@license))))))) (close-port port) - pkg)) + (package->definition pkg #t))) (define (string->license string) (filter-map (lambda (license) @@ -187,14 +200,19 @@ and LICENSE." 'unknown-license!))) (string-split string (string->char-set " /")))) -(define* (crate->guix-package crate-name #:optional version) +(define* (crate->guix-package crate-name #:key version repo) "Fetch the metadata for CRATE-NAME from crates.io, and return the `package' s-expression corresponding to that package, or #f on failure. When VERSION is specified, attempt to fetch that version; otherwise fetch the latest version of CRATE-NAME." + (define (semver-range-contains-string? range version) + (semver-range-contains? (string->semver-range range) + (string->semver version))) + (define (normal-dependency? dependency) - (eq? (crate-dependency-kind dependency) 'normal)) + (or (eq? (crate-dependency-kind dependency) 'build) + (eq? (crate-dependency-kind dependency) 'normal))) (define crate (lookup-crate crate-name)) @@ -204,22 +222,34 @@ latest version of CRATE-NAME." (or version (crate-latest-version crate)))) + ;; finds the a version of a crate that fulfills the semver + (define (find-version crate range) + (find (lambda (version) + (semver-range-contains-string? + range + (crate-version-number version))) + (crate-versions crate))) + (define version* (and crate - (find (lambda (version) - (string=? (crate-version-number version) - version-number)) - (crate-versions crate)))) + (find-version crate version-number))) + + ;; maps the dependencies to a list containing pairs of (name version) + (define (map-dependencies deps) + (map (lambda (dep) + (let* ((name (crate-dependency-id dep)) + (crate (lookup-crate* name)) + (req (crate-dependency-requirement dep)) + (ver (find-version crate req))) + (list name + (crate-version-number ver)))) + deps)) (and crate version* - (let* ((dependencies (crate-version-dependencies version*)) - (dep-crates (filter normal-dependency? dependencies)) - (dev-dep-crates (remove normal-dependency? dependencies)) - (cargo-inputs (sort (map crate-dependency-id dep-crates) - string-cilicense)) (append cargo-inputs cargo-development-inputs))))) -(define* (crate-recursive-import crate-name #:optional version) - (recursive-import crate-name #f - #:repo->guix-package - (lambda (name repo) - (let ((version (and (string=? name crate-name) - version))) - (crate->guix-package name version))) +(define* (crate-recursive-import crate-name #:key version) + (recursive-import crate-name + #:repo->guix-package crate->guix-package + #:version version #:guix-name crate-name->package-name)) (define (guix-package->crate-name package) @@ -254,7 +281,7 @@ latest version of CRATE-NAME." ((name _ ...) name)))) (define (crate-name->package-name name) - (string-append "rust-" (string-join (string-split name #\_) "-"))) + (guix-name "rust-" name)) ;;; diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 9213eca8a0..66445f006f 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -229,13 +229,20 @@ into a proper sentence and by using two spaces between sentences." cleaned 'pre ". " 'post))) (define* (package-names->package-inputs names #:optional (output #f)) - "Given a list of PACKAGE-NAMES, and an optional OUTPUT, tries to generate a -quoted list of inputs, as suitable to use in an 'inputs' field of a package -definition." - (map (lambda (input) - (cons* input (list 'unquote (string->symbol input)) - (or (and output (list output)) - '()))) + "Given a list of PACKAGE-NAMES or (PACKAGE-NAME VERSION) pairs, and an +optional OUTPUT, tries to generate a quoted list of inputs, as suitable to +use in an 'inputs' field of a package definition." + (define (make-input input version) + (cons* input (list 'unquote (string->symbol + (if version + (string-append input "-" version) + input))) + (or (and output (list output)) + '()))) + + (map (match-lambda + ((input version) (make-input input version)) + (input (make-input input #f))) names)) (define* (maybe-inputs package-names #:optional (output #f)) diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm index d834518c18..552628cfc7 100644 --- a/guix/scripts/import/crate.scm +++ b/guix/scripts/import/crate.scm @@ -2,7 +2,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson ;;; Copyright © 2016 David Craven -;;; Copyright © 2019 Martin Becze +;;; Copyright © 2019, 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -95,13 +95,8 @@ Import and convert the crate.io package for PACKAGE-NAME.\n")) (package-name->name+version spec)) (if (assoc-ref opts 'recursive) - (map (match-lambda - ((and ('package ('name name) . rest) pkg) - `(define-public ,(string->symbol name) - ,pkg)) - (_ #f)) - (crate-recursive-import name version)) - (let ((sexp (crate->guix-package name version))) + (crate-recursive-import name #:version version) + (let ((sexp (crate->guix-package name #:version version))) (unless sexp (leave (G_ "failed to download meta-data for package '~a'~%") (if version diff --git a/tests/crate.scm b/tests/crate.scm index 61a04f986b..beaa696be0 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2014 David Thompson ;;; Copyright © 2016 David Craven ;;; Copyright © 2019, 2020 Ludovic Courtès +;;; Copyright © 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -36,8 +37,8 @@ \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ { \"id\": \"foo\", \"num\": \"1.0.0\", @@ -54,8 +55,9 @@ "{ \"dependencies\": [ { - \"crate_id\": \"bar\", - \"kind\": \"normal\" + \"crate_id\": \"leaf-alice\", + \"kind\": \"normal\", + \"req\": \"1.0.0\" } ] }") @@ -68,8 +70,8 @@ \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ { \"id\": \"foo\", \"num\": \"1.0.0\", @@ -87,19 +89,23 @@ \"dependencies\": [ { \"crate_id\": \"intermediate-1\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" }, { \"crate_id\": \"intermediate-2\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" } { \"crate_id\": \"leaf-alice\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" }, { \"crate_id\": \"leaf-bob\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" } ] }") @@ -112,8 +118,8 @@ \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ { \"id\": \"intermediate-1\", \"num\": \"1.0.0\", @@ -131,15 +137,18 @@ \"dependencies\": [ { \"crate_id\": \"intermediate-2\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" }, { \"crate_id\": \"leaf-alice\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" }, { \"crate_id\": \"leaf-bob\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" } ] }") @@ -152,8 +161,8 @@ \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ { \"id\": \"intermediate-2\", \"num\": \"1.0.0\", @@ -171,7 +180,8 @@ \"dependencies\": [ { \"crate_id\": \"leaf-bob\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"1.0.0\" } ] }") @@ -184,8 +194,8 @@ \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ { \"id\": \"leaf-alice\", \"num\": \"1.0.0\", @@ -211,7 +221,7 @@ \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], + \"keywords\": [\"dummy\", \"test\"], \"categories\": [\"test\"] \"actual_versions\": [ { \"id\": \"leaf-bob\", @@ -253,34 +263,48 @@ (open-input-string test-foo-crate)) ("https://crates.io/api/v1/crates/foo/1.0.0/download" (set! test-source-hash - (bytevector->nix-base32-string - (sha256 (string->bytevector "empty file\n" "utf-8")))) + (bytevector->nix-base32-string + (sha256 (string->bytevector "empty file\n" "utf-8")))) (open-input-string "empty file\n")) ("https://crates.io/api/v1/crates/foo/1.0.0/dependencies" (open-input-string test-foo-dependencies)) + ("https://crates.io/api/v1/crates/leaf-alice" + (open-input-string test-leaf-alice-crate)) + ("https://crates.io/api/v1/crates/leaf-alice/1.0.0/download" + (set! test-source-hash + (bytevector->nix-base32-string + (sha256 (string->bytevector "empty file\n" "utf-8")))) + (open-input-string "empty file\n")) + ("https://crates.io/api/v1/crates/leaf-alice/1.0.0/dependencies" + (open-input-string test-leaf-alice-dependencies)) (_ (error "Unexpected URL: " url))))) - (match (crate->guix-package "foo") - (('package - ('name "rust-foo") - ('version "1.0.0") - ('source ('origin - ('method 'url-fetch) - ('uri ('crate-uri "foo" 'version)) - ('file-name ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-bar" ('unquote rust-bar)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - (string=? test-source-hash hash)) - (x - (pk 'fail x #f))))) + + (match (crate->guix-package "foo") + ((define-public rust-foo-1.0.0 + (package (name "rust-foo") + (version "1.0.0") + (source + (origin + (method url-fetch) + (uri (crate-uri "foo" 'version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system 'cargo-build-system) + (arguments + ('quasiquote + (#:skip-build? #t + #:cargo-inputs + (("rust-leaf-alice-1.0.0" ('unquote rust-leaf-alice-1.0.0)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + + (string=? test-source-hash hash)) + (x + (pk 'fail x #f))))) (test-assert "cargo-recursive-import" ;; Replace network resources with sample data. @@ -335,105 +359,123 @@ (_ (error "Unexpected URL: " url))))) (match (crate-recursive-import "root") ;; rust-intermediate-2 has no dependency on the rust-leaf-alice package, so this is a valid ordering - ((('package - ('name "rust-leaf-alice") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "leaf-alice" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-leaf-bob") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "leaf-bob" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-intermediate-2") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "intermediate-2" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-leaf-bob" ('unquote rust-leaf-bob)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-intermediate-1") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "intermediate-1" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-intermediate-2" ('unquote rust-intermediate-2)) - ("rust-leaf-alice" ('unquote rust-leaf-alice)) - ("rust-leaf-bob" ('unquote rust-leaf-bob)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-root") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "root" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-intermediate-1" ('unquote rust-intermediate-1)) - ("rust-intermediate-2" ('unquote rust-intermediate-2)) - ("rust-leaf-alice" ('unquote rust-leaf-alice)) - ("rust-leaf-bob" ('unquote rust-leaf-bob)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0)))) + (((define-public rust-leaf-alice-1.0.0 + (package + (name "rust-leaf-alice") + (version (? string? ver)) + (source + (origin + (method url-fetch) + (uri (crate-uri "leaf-alice" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments ('quasiquote (#:skip-build? #t))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public rust-leaf-bob-1.0.0 + (package + (name "rust-leaf-bob") + (version (? string? ver)) + (source + (origin + (method url-fetch) + (uri (crate-uri "leaf-bob" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments ('quasiquote (#:skip-build? #t))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public rust-intermediate-2-1.0.0 + (package + (name "rust-intermediate-2") + (version (? string? ver)) + (source + (origin + (method url-fetch) + (uri (crate-uri "intermediate-2" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments + ('quasiquote (#:skip-build? #t + #:cargo-inputs + (("rust-leaf-bob-1.0.0" + ('unquote rust-leaf-bob-1.0.0)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public rust-intermediate-1-1.0.0 + (package + (name "rust-intermediate-1") + (version (? string? ver)) + (source + (origin + (method url-fetch) + (uri (crate-uri "intermediate-1" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments + ('quasiquote (#:skip-build? #t + #:cargo-inputs + (("rust-intermediate-2-1.0.0" + ,rust-intermediate-2-1.0.0) + ("rust-leaf-alice-1.0.0" + ('unquote rust-leaf-alice-1.0.0)) + ("rust-leaf-bob-1.0.0" + ('unquote rust-leaf-bob-1.0.0)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public rust-root-1.0.0 + (package + (name "rust-root") + (version (? string? ver)) + (source + (origin + (method url-fetch) + (uri (crate-uri "root" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments + ('quasiquote (#:skip-build? + #t #:cargo-inputs + (("rust-intermediate-1-1.0.0" + ('unquote rust-intermediate-1-1.0.0)) + ("rust-intermediate-2-1.0.0" + ('unquote rust-intermediate-2-1.0.0)) + ("rust-leaf-alice-1.0.0" + ('unquote rust-leaf-alice-1.0.0)) + ("rust-leaf-bob-1.0.0" + ('unquote rust-leaf-bob-1.0.0)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0))))) #t) (x (pk 'fail x #f))))) -- 2.29.2