From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:470:142:3::10]:58001) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1ib5k5-0006bG-4g for guix-patches@gnu.org; Sat, 30 Nov 2019 11:37:21 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1ib5jy-0005fs-Mh for guix-patches@gnu.org; Sat, 30 Nov 2019 11:37:09 -0500 Received: from debbugs.gnu.org ([209.51.188.43]:57375) by eggs.gnu.org with esmtps (TLS1.0:RSA_AES_128_CBC_SHA1:16) (Exim 4.71) (envelope-from ) id 1ib5jy-0005fL-BB for guix-patches@gnu.org; Sat, 30 Nov 2019 11:37:02 -0500 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1ib5jy-0005VJ-88 for guix-patches@gnu.org; Sat, 30 Nov 2019 11:37:02 -0500 Subject: [bug#38408] [PATCH 3/3] Rewrote some of guix/import/crate.scm to use recursive-import-semver and updated script and test. Resent-Message-ID: MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="=_ad807c347f5e3a60f456e53b0edbeabf" Date: Sat, 30 Nov 2019 08:36:20 -0800 From: Martin Becze In-Reply-To: <052524339786cd4c0db5fda81547239c8bee6003.1574897905.git.mjbecze@riseup.net> References: <052524339786cd4c0db5fda81547239c8bee6003.1574897905.git.mjbecze@riseup.net> Message-ID: <42cb010759c8355943b9e2cb71a66b93@riseup.net> List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-patches-bounces+kyle=kyleam.com@gnu.org Sender: "Guix-patches" To: 38408@debbugs.gnu.org Cc: efraim@flashner.co.il --=_ad807c347f5e3a60f456e53b0edbeabf Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset=US-ASCII On 2019-11-28 00:16, Martin Becze wrote: > * guix/import/crate.scm (make-crate-sexp): Use as args > * guix/import/crate.scm (crate->crate-version): New Procedure > * guix/import/crate.scm (crate->versions): New Procedure > * guix/import/crate.scm (crate-recursive-import): Updated to user > recursive-import-semver > * guix/scripts/import/crate.scm (guix-import-crate): Remove > `define-public` generation from UI > * guix/tests/crate.scm: Updated tests > --- > guix/import/crate.scm | 165 ++++++++++++++++++---------------- > guix/scripts/import/crate.scm | 9 +- > tests/crate.scm | 2 +- > 3 files changed, 91 insertions(+), 85 deletions(-) > > diff --git a/guix/import/crate.scm b/guix/import/crate.scm > index 8dc014d232..da92c43b8c 100644 > --- a/guix/import/crate.scm > +++ b/guix/import/crate.scm > @@ -38,6 +38,7 @@ > #:use-module (srfi srfi-1) > #:use-module (srfi srfi-2) > #:use-module (srfi srfi-26) > + #:use-module (srfi srfi-71) > #:export (crate->guix-package > guix-package->crate-name > crate-recursive-import > @@ -85,7 +86,7 @@ > crate-dependency? > json->crate-dependency > (id crate-dependency-id "crate_id") ;string > - (kind crate-dependency-kind "kind" ;'normal | 'dev > + (kind crate-dependency-kind "kind" ;'normal | 'dev | 'build > string->symbol) > (requirement crate-dependency-requirement "req")) ;string > > @@ -111,7 +112,9 @@ record or #f if it was not found." > (url (string-append (%crate-base-url) path))) > (match (assoc-ref (or (json-fetch url) '()) "dependencies") > ((? vector? vector) > - (map json->crate-dependency (vector->list vector))) > + (filter (lambda (dep) > + (not (eq? (crate-dependency-kind dep) 'dev))) > + (map json->crate-dependency (vector->list vector)))) > (_ > '())))) > > @@ -141,62 +144,84 @@ record or #f if it was not found." > ((args ...) > `((arguments (,'quasiquote ,args)))))) > > -(define* (make-crate-sexp #:key name version cargo-inputs > cargo-development-inputs > - home-page synopsis description license > - #:allow-other-keys) > - "Return the `package' s-expression for a rust package with the given NAME, > -VERSION, CARGO-INPUTS, CARGO-DEVELOPMENT-INPUTS, HOME-PAGE, SYNOPSIS, > DESCRIPTION, > -and LICENSE." > - (let* ((port (http-fetch (crate-uri name version))) > +(define (make-crate-sexp crate version* dependencies) > + "Return the `package' s-expression for a rust package given , > + and a list of " > + (define normal-dependency? > + (match-lambda ((_ dep) (not (eq? (crate-dependency-kind dep) 'dev))))) > + > + (define (string->license string) > + (match (regexp-exec %dual-license-rx string) > + (#f (list (spdx-string->license string))) > + (m (list (spdx-string->license (match:substring m 1)) > + (spdx-string->license (match:substring m 2)))))) > + > + (let* ((dep-crates dev-dep-crates (partition normal-dependency? > dependencies)) > + (cargo-inputs (sort (unzip1 dep-crates) > + string-ci + (cargo-development-inputs > + (sort (unzip1 dev-dep-crates) > + string-ci + (name (crate-name crate)) > + (version (crate-version-number version*)) > + (home-page (or (crate-home-page crate) > + (crate-repository crate))) > + (synopsis (crate-description crate)) > + (description (crate-description crate)) > + (license (and=> (crate-version-license version*) > + string->license)) > + (port (http-fetch (crate-uri name version)) ) > (guix-name (crate-name->package-name name)) > - (cargo-inputs (map crate-name->package-name cargo-inputs)) > - (cargo-development-inputs (map crate-name->package-name > - cargo-development-inputs)) > (pkg `(package > - (name ,guix-name) > - (version ,version) > - (source (origin > - (method url-fetch) > - (uri (crate-uri ,name version)) > - (file-name (string-append name "-" > version ".tar.gz")) > - (sha256 > - (base32 > - ,(bytevector->nix-base32-string > (port-sha256 port)))))) > - (build-system cargo-build-system) > - ,@(maybe-arguments (append (maybe-cargo-inputs cargo-inputs) > - (maybe-cargo-development-inputs > - cargo-development-inputs))) > - (home-page ,(match home-page > - (() "") > - (_ home-page))) > - (synopsis ,synopsis) > - (description ,(beautify-description description)) > - (license ,(match license > - (() #f) > - ((license) license) > - (_ `(list ,@license))))))) > - (close-port port) > - pkg)) > + (name ,guix-name) > + (version ,version) > + (source (origin > + (method url-fetch) > + (uri (crate-uri ,name version)) > + (file-name (string-append name "-" version > ".crate")) > + (sha256 > + (base32 > + ,(bytevector->nix-base32-string > (port-sha256 port)))))) > + (build-system cargo-build-system) > + ,@(maybe-arguments (append (maybe-cargo-inputs cargo-inputs) > + (maybe-cargo-development-inputs > + cargo-development-inputs))) > + (home-page ,(match home-page > + (() "") > + (_ home-page))) > + (synopsis ,synopsis) > + (description ,(beautify-description description)) > + (license ,(match license > + (() #f) > + ((license) license) > + (_ `(list ,@license))))))) > + > + (close-port port) > + pkg)) > > (define %dual-license-rx > ;; Dual licensing is represented by a string such as "MIT OR Apache-2.0". > ;; This regexp matches that. > (make-regexp "^(.*) OR (.*)$")) > > +(define (crate->crate-version crate version-number) > + "returns the for a given CRATE and VERSION-NUMBER" > + (find (lambda (version) > + (string=? (crate-version-number version) > + version-number)) > + (crate-versions crate))) > + > +(define (crate->versions crate) > + "Returns a list of versions for a given CRATE" > + (map (lambda (version) > + (crate-version-number version)) > + (crate-versions crate))) > + > (define* (crate->guix-package crate-name #:optional version) > "Fetch the metadata for CRATE-NAME from crates.io, and return the > `package' s-expression corresponding to that package, or #f on failure. > When VERSION is specified, attempt to fetch that version; otherwise fetch the > latest version of CRATE-NAME." > - (define (string->license string) > - (match (regexp-exec %dual-license-rx string) > - (#f (list (spdx-string->license string))) > - (m (list (spdx-string->license (match:substring m 1)) > - (spdx-string->license (match:substring m 2)))))) > - > - (define (normal-dependency? dependency) > - (eq? (crate-dependency-kind dependency) 'normal)) > - > (define crate > (lookup-crate crate-name)) > > @@ -205,38 +230,27 @@ latest version of CRATE-NAME." > (crate-latest-version crate))) > > (define version* > - (find (lambda (version) > - (string=? (crate-version-number version) > - version-number)) > - (crate-versions crate))) > + (crate->crate-version crate version-number)) > > - (and crate version* > - (let* ((dependencies (crate-version-dependencies version*)) > - (dep-crates (filter normal-dependency? dependencies)) > - (dev-dep-crates (remove normal-dependency? dependencies)) > - (cargo-inputs (sort (map crate-dependency-id dep-crates) > - string-ci - (cargo-development-inputs > - (sort (map crate-dependency-id dev-dep-crates) > - string-ci - (values > - (make-crate-sexp #:name crate-name > - #:version (crate-version-number version*) > - #:cargo-inputs cargo-inputs > - #:cargo-development-inputs cargo-development-inputs > - #:home-page (or (crate-home-page crate) > - (crate-repository crate)) > - #:synopsis (crate-description crate) > - #:description (crate-description crate) > - #:license (and=> (crate-version-license version*) > - string->license)) > - (append cargo-inputs cargo-development-inputs))))) > + (define dependencies (map > + (lambda (dep) > + (list (crate-name->package-name > + (crate-dependency-id dep)) dep)) > + (crate-version-dependencies version*))) > + (make-crate-sexp crate version* dependencies)) > > -(define (crate-recursive-import crate-name) > - (recursive-import crate-name #f > - #:repo->guix-package (lambda (name repo) > - (crate->guix-package name)) > - #:guix-name crate-name->package-name)) > +(define* (crate-recursive-import name #:optional version) > + (recursive-import-semver > + #:name name > + #:version version > + #:name->metadata lookup-crate > + #:metadata->package crate->crate-version > + #:metadata-versions crate->versions > + #:package-dependencies crate-version-dependencies > + #:dependency-name crate-dependency-id > + #:dependency-range crate-dependency-requirement > + #:guix-name crate-name->package-name > + #:make-sexp make-crate-sexp)) > > (define (guix-package->crate-name package) > "Return the crate name of PACKAGE." > @@ -285,4 +299,3 @@ latest version of CRATE-NAME." > (description "Updater for crates.io packages") > (pred crate-package?) > (latest latest-release))) > - > diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm > index 4690cceb4d..85ae6fbe59 100644 > --- a/guix/scripts/import/crate.scm > +++ b/guix/scripts/import/crate.scm > @@ -96,14 +96,7 @@ Import and convert the crate.io package for > PACKAGE-NAME.\n")) > (package-name->name+version spec)) > > (if (assoc-ref opts 'recursive) > - (map (match-lambda > - ((and ('package ('name name) . rest) pkg) > - `(define-public ,(string->symbol name) > - ,pkg)) > - (_ #f)) > - (reverse > - (stream->list > - (crate-recursive-import name)))) > + (stream->list (crate-recursive-import name version)) > (let ((sexp (crate->guix-package name version))) > (unless sexp > (leave (G_ "failed to download meta-data for package '~a'~%") > diff --git a/tests/crate.scm b/tests/crate.scm > index c14862ad9f..b77cbb08c6 100644 > --- a/tests/crate.scm > +++ b/tests/crate.scm > @@ -95,7 +95,7 @@ > ('source ('origin > ('method 'url-fetch) > ('uri ('crate-uri "foo" 'version)) > - ('file-name ('string-append 'name "-" 'version ".tar.gz")) > + ('file-name ('string-append 'name "-" 'version ".crate")) > ('sha256 > ('base32 > (? string? hash))))) I'm added a patch that will skips the building of libraries which I would assume most of the packages being imported are. This could be parametrized in the future. --=_ad807c347f5e3a60f456e53b0edbeabf Content-Transfer-Encoding: base64 Content-Type: text/x-diff; name=0001-added-skip-build-t-to-the-output-of-make-crate-sexp-.patch Content-Disposition: attachment; filename=0001-added-skip-build-t-to-the-output-of-make-crate-sexp-.patch; size=1876 RnJvbSAzZjJmZjNiNGRjNGNkZjhiMDI4MjMxNmI5YzI0MjYyOTFkYThhNmM3IE1vbiBTZXAgMTcg MDA6MDA6MDAgMjAwMQpGcm9tOiBNYXJ0aW4gQmVjemUgPG1qYmVjemVAcmlzZXVwLm5ldD4KRGF0 ZTogU2F0LCAzMCBOb3YgMjAxOSAxMToyNzowNSAtMDUwMApTdWJqZWN0OiBbUEFUQ0hdIGFkZGVk ICIjOnNraXAtYnVpbGQ/ICN0IiB0byB0aGUgb3V0cHV0IG9mIChtYWtlLWNyYXRlLXNleHApLgog TW9zdCB0aGUgdGhlIHBhY2thZ2VzIGltcG9ydGVkIHdpbGwgYmUgbGliYXJpZXMgYW5kIHdvbid0 IG5lZWQgdG8gYnVpbGQuIFRoZQogdG9wIGxldmVsIHBhY2thZ2Ugd2lsbCBidWlsZCB0aGVtIHRo b3VnaC4KCiogZ3VpeC9pbXBvcnQvY3JhdGUuc2NtIChtYWtlLWNyYXRlLXNleHApOiBhZGRlZCAi Izpza2lwLWJ1aWxkPyAjdCIgdG8gdGhlIG91dHB1dAotLS0KIGd1aXgvaW1wb3J0L2NyYXRlLnNj bSB8IDMgKystCiB0ZXN0cy9jcmF0ZS5zY20gICAgICAgfCAzICsrLQogMiBmaWxlcyBjaGFuZ2Vk LCA0IGluc2VydGlvbnMoKyksIDIgZGVsZXRpb25zKC0pCgpkaWZmIC0tZ2l0IGEvZ3VpeC9pbXBv cnQvY3JhdGUuc2NtIGIvZ3VpeC9pbXBvcnQvY3JhdGUuc2NtCmluZGV4IGRhOTJjNDNiOGMuLjU2 ODMzNjliN2EgMTAwNjQ0Ci0tLSBhL2d1aXgvaW1wb3J0L2NyYXRlLnNjbQorKysgYi9ndWl4L2lt cG9ydC9jcmF0ZS5zY20KQEAgLTE4Myw3ICsxODMsOCBAQCByZWNvcmQgb3IgI2YgaWYgaXQgd2Fz IG5vdCBmb3VuZC4iCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKGJhc2UzMgogICAgICAg ICAgICAgICAgICAgICAgICAgICAgICAsKGJ5dGV2ZWN0b3ItPm5peC1iYXNlMzItc3RyaW5nIChw b3J0LXNoYTI1NiBwb3J0KSkpKSkpCiAgICAgICAgICAgICAgICAgIChidWlsZC1zeXN0ZW0gY2Fy Z28tYnVpbGQtc3lzdGVtKQotICAgICAgICAgICAgICAgICAsQChtYXliZS1hcmd1bWVudHMgKGFw cGVuZCAobWF5YmUtY2FyZ28taW5wdXRzIGNhcmdvLWlucHV0cykKKyAgICAgICAgICAgICAgICAg LEAobWF5YmUtYXJndW1lbnRzIChhcHBlbmQgYCgjOnNraXAtYnVpbGQ/ICN0KQorICAgICAgICAg ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAobWF5YmUtY2FyZ28taW5wdXRzIGNh cmdvLWlucHV0cykKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg KG1heWJlLWNhcmdvLWRldmVsb3BtZW50LWlucHV0cwogICAgICAgICAgICAgICAgICAgICAgICAg ICAgICAgICAgICAgICAgICAgICAgY2FyZ28tZGV2ZWxvcG1lbnQtaW5wdXRzKSkpCiAgICAgICAg ICAgICAgICAgIChob21lLXBhZ2UgLChtYXRjaCBob21lLXBhZ2UKZGlmZiAtLWdpdCBhL3Rlc3Rz L2NyYXRlLnNjbSBiL3Rlc3RzL2NyYXRlLnNjbQppbmRleCBiNzdjYmIwOGM2Li42NGU1YjY5MzJl IDEwMDY0NAotLS0gYS90ZXN0cy9jcmF0ZS5zY20KKysrIGIvdGVzdHMvY3JhdGUuc2NtCkBAIC0x MDIsNyArMTAyLDggQEAKICAgICAgICAgICgnYnVpbGQtc3lzdGVtICdjYXJnby1idWlsZC1zeXN0 ZW0pCiAgICAgICAgICAoJ2FyZ3VtZW50cwogICAgICAgICAgICgncXVhc2lxdW90ZQotICAgICAg ICAgICAoJyM6Y2FyZ28taW5wdXRzICgoInJ1c3QtYmFyIiAoJ3VucXVvdGUgcnVzdC1iYXIpKSkp KSkKKyAgICAgICAgICAgKCcjOnNraXAtYnVpbGQ/ICN0CisgICAgICAgICAgICAjOmNhcmdvLWlu cHV0cyAoKCJydXN0LWJhciIgKCd1bnF1b3RlIHJ1c3QtYmFyKSkpKSkpCiAgICAgICAgICAoJ2hv bWUtcGFnZSAiaHR0cDovL2V4YW1wbGUuY29tIikKICAgICAgICAgICgnc3lub3BzaXMgInN1bW1h cnkiKQogICAgICAgICAgKCdkZXNjcmlwdGlvbiAic3VtbWFyeSIpCi0tIAoyLjI0LjAKCg== --=_ad807c347f5e3a60f456e53b0edbeabf--