* [bug#62375] [PATCH] import: Add binary npm importer.
2023-03-22 11:25 [bug#62375] [PATCH 0/1] npm binary importer jlicht
@ 2023-03-22 11:27 ` jlicht
2023-03-28 15:49 ` Ludovic Courtès
2024-02-08 0:59 ` Nicolas Graves via Guix-patches via
` (5 subsequent siblings)
6 siblings, 1 reply; 18+ messages in thread
From: jlicht @ 2023-03-22 11:27 UTC (permalink / raw)
To: 62375
Cc: Timothy Sample, Josselin Poiret, Tobias Geerinckx-Rice,
Simon Tournier, Mathieu Othacehe, Ludovic Courtès,
Christopher Baines, Lars-Dominik Braun, Ricardo Wurmus,
Jelle Licht
From: Jelle Licht <jlicht@fsfe.org>
* guix/scripts/import.scm: (importers): Add "npm-binary".
* guix/import/npm-binary.scm: New file.
* guix/scripts/import/npm-binary.scm: New file.
* Makefile.am: Add them.
Co-authored-by: Timothy Sample <samplet@ngyro.com>
Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
---
Makefile.am | 2 +
guix/import/npm-binary.scm | 269 +++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/npm-binary.scm | 113 ++++++++++++
4 files changed, 385 insertions(+), 1 deletion(-)
create mode 100644 guix/import/npm-binary.scm
create mode 100644 guix/scripts/import/npm-binary.scm
diff --git a/Makefile.am b/Makefile.am
index 23b939b674..52def58ae2 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -288,6 +288,7 @@ MODULES = \
guix/import/kde.scm \
guix/import/launchpad.scm \
guix/import/minetest.scm \
+ guix/import/npm-binary.scm \
guix/import/opam.scm \
guix/import/print.scm \
guix/import/pypi.scm \
@@ -339,6 +340,7 @@ MODULES = \
guix/scripts/import/hexpm.scm \
guix/scripts/import/json.scm \
guix/scripts/import/minetest.scm \
+ guix/scripts/import/npm-binary.scm \
guix/scripts/import/opam.scm \
guix/scripts/import/pypi.scm \
guix/scripts/import/stackage.scm \
diff --git a/guix/import/npm-binary.scm b/guix/import/npm-binary.scm
new file mode 100644
index 0000000000..f9b54263e4
--- /dev/null
+++ b/guix/import/npm-binary.scm
@@ -0,0 +1,269 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2019, 2020 Timothy Sample <samplet@ngyro.com>
+;;; Copyright © 2020, 2023 Jelle Licht <jlicht@fsfe.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix import npm-binary)
+ #:use-module (guix import json)
+ #:use-module (guix import utils)
+ #:use-module (guix memoization)
+ #:use-module ((gnu services configuration) #:select (alist?))
+ #:use-module (guix utils)
+ #:use-module (gnu packages)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 regex)
+ #:use-module (ice-9 receive)
+ #:use-module (json)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-41)
+ #:use-module (web client)
+ #:use-module (web response)
+ #:use-module (web uri)
+ #:export (npm-binary-recursive-import
+ npm-binary->guix-package
+ package-json->guix-package
+ make-versioned-package
+ name+version->symbol))
+
+;; Autoload Guile-Semver so we only have a soft dependency.
+(module-autoload! (current-module)
+ '(semver) '(string->semver semver? semver->string semver=? semver>?))
+(module-autoload! (current-module)
+ '(semver ranges) '(*semver-range-any* string->semver-range semver-range-contains?))
+
+;; Dist-tags
+(define-json-mapping <dist-tags> make-dist-tags dist-tags?
+ json->dist-tags
+ (latest dist-tags-latest "latest" string->semver))
+
+(define-record-type <versioned-package>
+ (make-versioned-package name version)
+ versioned-package?
+ (name versioned-package-name) ;string
+ (version versioned-package-version)) ;string
+
+(define (dependencies->versioned-packages entries)
+ (match entries
+ (((names . versions) ...)
+ (map make-versioned-package names versions))
+ (_ '())))
+
+(define (extract-license license-string)
+ (if (unspecified? license-string)
+ 'unspecified!
+ (spdx-string->license license-string)))
+
+(define-json-mapping <dist> make-dist dist?
+ json->dist
+ (tarball dist-tarball))
+
+(define (empty-or-string s)
+ (if (string? s) s ""))
+
+(define-json-mapping <package-revision> make-package-revision package-revision?
+ json->package-revision
+ (name package-revision-name)
+ (version package-revision-version "version" string->semver) ;semver
+ (home-page package-revision-home-page "homepage") ;string
+ (dependencies package-revision-dependencies "dependencies" ;list of versioned-package
+ dependencies->versioned-packages)
+ (dev-dependencies package-revision-dev-dependencies ;list of versioned-package
+ "devDependencies" dependencies->versioned-packages)
+ (peer-dependencies package-revision-peer-dependencies ;list of versioned-package
+ "peerDependencies" dependencies->versioned-packages)
+ (license package-revision-license "license" ;license | #f
+ (match-lambda
+ ((? unspecified?) #f)
+ ((? string? str) (spdx-string->license str))
+ ((? alist? alist)
+ (match (assoc "type" alist)
+ ((_ . (? string? type))
+ (spdx-string->license type))
+ (_ #f)))))
+ (description package-revision-description ; string
+ "description" empty-or-string)
+ (dist package-revision-dist "dist" json->dist)) ;dist
+
+(define (versions->package-revisions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map json->package-revision package-spec))
+ (_ '())))
+
+(define (versions->package-versions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map string->semver versions))
+ (_ '())))
+
+(define-json-mapping <meta-package> make-meta-package meta-package?
+ json->meta-package
+ (name meta-package-name) ;string
+ (description meta-package-description) ;string
+ (dist-tags meta-package-dist-tags "dist-tags" json->dist-tags) ;dist-tags
+ (revisions meta-package-revisions "versions" versions->package-revisions))
+
+;; TODO: Support other registries
+(define *registry* "https://registry.npmjs.org")
+(define *default-page* "https://www.npmjs.com/package")
+
+(define (lookup-meta-package name)
+ (let ((json (json-fetch (string-append *registry* "/" (uri-encode name)))))
+ (and=> json json->meta-package)))
+
+(define lookup-meta-package* (memoize lookup-meta-package))
+
+(define (http-error-code arglist)
+ (match arglist
+ (('http-error _ _ _ (code)) code)
+ (_ #f)))
+
+(define (meta-package-versions meta)
+ (map package-revision-version
+ (meta-package-revisions meta)))
+
+(define (meta-package-latest meta)
+ (and=> (meta-package-dist-tags meta) dist-tags-latest))
+
+(define* (meta-package-package meta #:optional
+ (version (meta-package-latest meta)))
+ (match version
+ ((? semver?) (find (lambda (revision)
+ (semver=? version (package-revision-version revision)))
+ (meta-package-revisions meta)))
+ ((? string?) (meta-package-package meta (string->semver version)))
+ (_ #f)))
+
+(define* (semver-latest svs #:optional (svr *semver-range-any*))
+ (find (cut semver-range-contains? svr <>)
+ (sort svs semver>?)))
+
+(define* (resolve-package name #:optional (svr *semver-range-any*))
+ (let ((meta (lookup-meta-package* name)))
+ (and meta
+ (let* ((version (semver-latest (or (meta-package-versions meta) '()) svr))
+ (pkg (meta-package-package meta version)))
+ pkg))))
+
+\f
+;;;
+;;; Converting packages
+;;;
+
+(define (hash-url url)
+ "Downloads the resource at URL and computes the base32 hash for it."
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (begin ((@ (guix import utils) url-fetch) url temp)
+ (guix-hash-url temp)))))
+
+(define (npm-name->name npm-name)
+ "Return a Guix package name for the npm package with name NPM-NAME."
+ (define (clean name)
+ (string-map (lambda (chr) (if (char=? chr #\/) #\- chr))
+ (string-filter (negate (cut char=? <> #\@)) name)))
+ (guix-name "node-" (clean npm-name)))
+
+(define (name+version->symbol name version)
+ (string->symbol (string-append name "-" version)))
+
+(define (package-revision->symbol package)
+ (let* ((npm-name (package-revision-name package))
+ (version (semver->string (package-revision-version package)))
+ (name (npm-name->name npm-name)))
+ (name+version->symbol name version)))
+
+(define (package-revision->input package)
+ "Return the `inputs' entry for PACKAGE."
+ (let* ((npm-name (package-revision-name package))
+ (name (npm-name->name npm-name)))
+ `(,name
+ (,'unquote ,(package-revision->symbol package)))))
+
+(define (npm-package->package-sexp npm-package)
+ "Return the `package' s-expression for an NPM-PACKAGE."
+ (define (new-or-existing-inputs resolved-deps)
+ (map package-revision->input resolved-deps))
+
+ (match npm-package
+ (($ <package-revision> name version home-page dependencies dev-dependencies peer-dependencies license description dist)
+ (let* ((name (npm-name->name name))
+ (url (dist-tarball dist))
+ (home-page (if (string? home-page)
+ home-page
+ (string-append *default-page* "/" (uri-encode name))))
+ (synopsis description)
+ (resolved-deps (map (match-lambda (($ <versioned-package> name version)
+ (resolve-package name (string->semver-range version)))) (append dependencies peer-dependencies)))
+ (peer-names (map versioned-package-name peer-dependencies))
+ ;; lset-difference for treating peer-dependencies as dependencies, which leads to dependency cycles.
+ ;; lset-union for treating them as (ignored) dev-dependencies, which leads to broken packages.
+ (dev-names (lset-union string= (map versioned-package-name dev-dependencies) peer-names))
+ (extra-phases (match dev-names
+ (() '())
+ ((dev-names ...)
+ `((add-after 'patch-dependencies 'delete-dev-dependencies
+ (lambda _
+ (delete-dependencies '(,@(reverse dev-names))))))))))
+ (values
+ `(package
+ (name ,name)
+ (version ,(semver->string (package-revision-version npm-package)))
+ (source (origin
+ (method url-fetch)
+ (uri ,url)
+ (sha256 (base32 ,(hash-url url)))))
+ (build-system node-build-system)
+ (arguments
+ '(#:tests? #f
+ #:phases (modify-phases %standard-phases
+ (delete 'build)
+ ,@extra-phases)))
+ ,@(match dependencies
+ (() '())
+ ((dependencies ...)
+ `((inputs
+ (,'quasiquote ,(map package-revision->input resolved-deps))))))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,license))
+ (map (match-lambda (($ <package-revision> name version)
+ (list name (semver->string version))))
+ resolved-deps))))
+ (_ #f)))
+
+\f
+;;;
+;;; Interface
+;;;
+
+(define npm-binary->guix-package
+ (lambda* (name #:key (version *semver-range-any*) #:allow-other-keys)
+ (let* ((svr (match version
+ ((? string?) (string->semver-range version))
+ (_ version)))
+ (pkg (resolve-package name svr)))
+ (and=> pkg npm-package->package-sexp))))
+
+(define* (npm-binary-recursive-import package-name #:key version)
+ (recursive-import package-name
+ #:repo->guix-package (memoize npm-binary->guix-package)
+ #:version version
+ #:guix-name npm-name->name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index f84a964a53..dccf6488b2 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -47,7 +47,7 @@ (define %standard-import-options '())
(define importers '("gnu" "pypi" "cpan" "hackage" "stackage" "egg" "elpa"
"gem" "go" "cran" "crate" "texlive" "json" "opam"
- "minetest" "elm" "hexpm"))
+ "minetest" "elm" "hexpm" "npm-binary"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/npm-binary.scm b/guix/scripts/import/npm-binary.scm
new file mode 100644
index 0000000000..825c43bbc3
--- /dev/null
+++ b/guix/scripts/import/npm-binary.scm
@@ -0,0 +1,113 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2014 David Thompson <davet@gnu.org>
+;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Timothy Sample <samplet@ngyro.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import npm-binary)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import npm-binary)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (srfi srfi-41)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-npm-binary))
+
+\f
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import npm-binary PACKAGE-NAME [VERSION]
+Import and convert the NPM package PACKAGE-NAME using the
+`npm-build-system' (but without building the package from source)."))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -r, --recursive import packages recursively"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import npm-binary")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ %standard-import-options))
+
+\f
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-npm-binary . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (let loop ((args args))
+ (match args
+ ((package-name version)
+ (if (assoc-ref opts 'recursive)
+ ;; Recursive import
+ (map (match-lambda
+ ((and ('package ('name name) ('version version) . rest) pkg)
+ `(define-public ,(name+version->symbol name version)
+ ,pkg))
+ (_ #f))
+ (npm-binary-recursive-import package-name #:version version))
+ ;; Single import
+ (let ((sexp (npm-binary->guix-package package-name #:version version)))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for package '~a@~a'~%")
+ package-name version))
+ sexp)))
+ ((package-name)
+ (loop (list package-name "*")))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%")))))))
--
2.39.2
^ permalink raw reply related [flat|nested] 18+ messages in thread* [bug#62375] [PATCH] import: Add binary npm importer.
2023-03-22 11:27 ` [bug#62375] [PATCH] import: Add binary npm importer jlicht
@ 2023-03-28 15:49 ` Ludovic Courtès
2023-04-08 18:29 ` Jelle Licht
0 siblings, 1 reply; 18+ messages in thread
From: Ludovic Courtès @ 2023-03-28 15:49 UTC (permalink / raw)
To: jlicht
Cc: Timothy Sample, Josselin Poiret, Tobias Geerinckx-Rice,
Simon Tournier, Mathieu Othacehe, Christopher Baines,
Lars-Dominik Braun, 62375, Ricardo Wurmus
Hello Jelle!
jlicht@fsfe.org skribis:
> From: Jelle Licht <jlicht@fsfe.org>
>
> * guix/scripts/import.scm: (importers): Add "npm-binary".
> * guix/import/npm-binary.scm: New file.
> * guix/scripts/import/npm-binary.scm: New file.
> * Makefile.am: Add them.
>
> Co-authored-by: Timothy Sample <samplet@ngyro.com>
> Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
Woohoo! I think it’ll be useful to many, even if we know it’s far from
“ideal” by our standards.
We’ll need doc under “Invoking guix import” and tests in
‘tests/npm-binary.scm’, similar to what’s done for the other importers.
The doc should clarify why it’s called that way and what the limitations
are.
For tests, I’d recommend mocking the npmjs.org HTTP servers using
‘with-http-server’ as in ‘tests/cpan.scm’.
Also, please add docstrings to top-level procedures.
> +;; TODO: Support other registries
> +(define *registry* "https://registry.npmjs.org")
> +(define *default-page* "https://www.npmjs.com/package")
The convention currently is more like ‘%registry’.
> +(define (http-error-code arglist)
> + (match arglist
> + (('http-error _ _ _ (code)) code)
> + (_ #f)))
Unused. :-)
> +(define (hash-url url)
> + "Downloads the resource at URL and computes the base32 hash for it."
> + (call-with-temporary-output-file
> + (lambda (temp port)
> + (begin ((@ (guix import utils) url-fetch) url temp)
> + (guix-hash-url temp)))))
Maybe something more like: (port-sha256 (http-fetch …)) ?
> +(define (npm-package->package-sexp npm-package)
> + "Return the `package' s-expression for an NPM-PACKAGE."
> + (define (new-or-existing-inputs resolved-deps)
> + (map package-revision->input resolved-deps))
> +
> + (match npm-package
> + (($ <package-revision> name version home-page dependencies dev-dependencies peer-dependencies license description dist)
Please use ‘match-record’ instead and keep lines below 80 chars. :-)
> + (let* ((name (npm-name->name name))
> + (url (dist-tarball dist))
> + (home-page (if (string? home-page)
> + home-page
> + (string-append *default-page* "/" (uri-encode name))))
> + (synopsis description)
> + (resolved-deps (map (match-lambda (($ <versioned-package> name version)
> + (resolve-package name (string->semver-range version)))) (append dependencies peer-dependencies)))
Likewise.
That’s it! Could you send an updated patch?
Thanks,
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread* [bug#62375] [PATCH] import: Add binary npm importer.
2023-03-28 15:49 ` Ludovic Courtès
@ 2023-04-08 18:29 ` Jelle Licht
2023-04-17 21:14 ` [bug#62375] [PATCH 0/1] npm binary importer Ludovic Courtès
0 siblings, 1 reply; 18+ messages in thread
From: Jelle Licht @ 2023-04-08 18:29 UTC (permalink / raw)
To: Ludovic Courtès
Cc: Timothy Sample, Josselin Poiret, Tobias Geerinckx-Rice,
Simon Tournier, Mathieu Othacehe, Christopher Baines,
Lars-Dominik Braun, 62375, Ricardo Wurmus
Ludovic Courtès <ludo@gnu.org> writes:
> Hello Jelle!
>
> jlicht@fsfe.org skribis:
>
>> From: Jelle Licht <jlicht@fsfe.org>
>>
>> * guix/scripts/import.scm: (importers): Add "npm-binary".
>> * guix/import/npm-binary.scm: New file.
>> * guix/scripts/import/npm-binary.scm: New file.
>> * Makefile.am: Add them.
>>
>> Co-authored-by: Timothy Sample <samplet@ngyro.com>
>> Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
>
> Woohoo! I think it’ll be useful to many, even if we know it’s far from
> “ideal” by our standards.
>
> We’ll need doc under “Invoking guix import” and tests in
> ‘tests/npm-binary.scm’, similar to what’s done for the other importers.
> The doc should clarify why it’s called that way and what the limitations
> are.
>
> For tests, I’d recommend mocking the npmjs.org HTTP servers using
> ‘with-http-server’ as in ‘tests/cpan.scm’.
>
> Also, please add docstrings to top-level procedures.
>
>> +;; TODO: Support other registries
>> +(define *registry* "https://registry.npmjs.org")
>> +(define *default-page* "https://www.npmjs.com/package")
>
> The convention currently is more like ‘%registry’.
>
>> +(define (http-error-code arglist)
>> + (match arglist
>> + (('http-error _ _ _ (code)) code)
>> + (_ #f)))
>
> Unused. :-)
>
>> +(define (hash-url url)
>> + "Downloads the resource at URL and computes the base32 hash for it."
>> + (call-with-temporary-output-file
>> + (lambda (temp port)
>> + (begin ((@ (guix import utils) url-fetch) url temp)
>> + (guix-hash-url temp)))))
>
> Maybe something more like: (port-sha256 (http-fetch …)) ?
>
>> +(define (npm-package->package-sexp npm-package)
>> + "Return the `package' s-expression for an NPM-PACKAGE."
>> + (define (new-or-existing-inputs resolved-deps)
>> + (map package-revision->input resolved-deps))
>> +
>> + (match npm-package
>> + (($ <package-revision> name version home-page dependencies dev-dependencies peer-dependencies license description dist)
>
> Please use ‘match-record’ instead and keep lines below 80 chars. :-)
The records generated by `define-json-mapping' through
`define-record-type' seem to not work with `match-record'.
I could define our very own `define-json-mapping*' that works /w
`define-record-type*' instead (and through that, `match-record'), but I
thought I'd ask if you had something else in mind first :).
>> + (let* ((name (npm-name->name name))
>> + (url (dist-tarball dist))
>> + (home-page (if (string? home-page)
>> + home-page
>> + (string-append *default-page* "/" (uri-encode name))))
>> + (synopsis description)
>> + (resolved-deps (map (match-lambda (($ <versioned-package> name version)
>> + (resolve-package name (string->semver-range version)))) (append dependencies peer-dependencies)))
>
> Likewise.
>
> That’s it! Could you send an updated patch?
Will do, once I got some proper test cases set up.
Thanks again for the review!
- Jelle
^ permalink raw reply [flat|nested] 18+ messages in thread* [bug#62375] [PATCH 0/1] npm binary importer
2023-04-08 18:29 ` Jelle Licht
@ 2023-04-17 21:14 ` Ludovic Courtès
2023-06-18 21:03 ` Ludovic Courtès
0 siblings, 1 reply; 18+ messages in thread
From: Ludovic Courtès @ 2023-04-17 21:14 UTC (permalink / raw)
To: Jelle Licht
Cc: Timothy Sample, Josselin Poiret, Christopher Baines,
Simon Tournier, Mathieu Othacehe, Tobias Geerinckx-Rice,
Lars-Dominik Braun, 62375, Ricardo Wurmus
Hi,
Jelle Licht <jlicht@fsfe.org> skribis:
> Ludovic Courtès <ludo@gnu.org> writes:
[...]
>>> + (match npm-package
>>> + (($ <package-revision> name version home-page dependencies dev-dependencies peer-dependencies license description dist)
>>
>> Please use ‘match-record’ instead and keep lines below 80 chars. :-)
>
> The records generated by `define-json-mapping' through
> `define-record-type' seem to not work with `match-record'.
Oh right, my bad.
The other option is to call record accessors; it would avoid the risk
associated with index-based matches but may be more verbose. Your call!
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread
* [bug#62375] [PATCH 0/1] npm binary importer
2023-04-17 21:14 ` [bug#62375] [PATCH 0/1] npm binary importer Ludovic Courtès
@ 2023-06-18 21:03 ` Ludovic Courtès
2023-06-22 9:39 ` Jelle Licht
0 siblings, 1 reply; 18+ messages in thread
From: Ludovic Courtès @ 2023-06-18 21:03 UTC (permalink / raw)
To: Jelle Licht
Cc: Timothy Sample, Josselin Poiret, Christopher Baines,
Simon Tournier, Mathieu Othacehe, Tobias Geerinckx-Rice,
Lars-Dominik Braun, 62375, Ricardo Wurmus
Hey Jelle,
How far are we from merging? :-)
TIA,
Ludo’.
Ludovic Courtès <ludo@gnu.org> skribis:
> Hi,
>
> Jelle Licht <jlicht@fsfe.org> skribis:
>
>> Ludovic Courtès <ludo@gnu.org> writes:
>
> [...]
>
>>>> + (match npm-package
>>>> + (($ <package-revision> name version home-page dependencies dev-dependencies peer-dependencies license description dist)
>>>
>>> Please use ‘match-record’ instead and keep lines below 80 chars. :-)
>>
>> The records generated by `define-json-mapping' through
>> `define-record-type' seem to not work with `match-record'.
>
> Oh right, my bad.
>
> The other option is to call record accessors; it would avoid the risk
> associated with index-based matches but may be more verbose. Your call!
>
> Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread
* [bug#62375] [PATCH 0/1] npm binary importer
2023-06-18 21:03 ` Ludovic Courtès
@ 2023-06-22 9:39 ` Jelle Licht
0 siblings, 0 replies; 18+ messages in thread
From: Jelle Licht @ 2023-06-22 9:39 UTC (permalink / raw)
To: Ludovic Courtès
Cc: Timothy Sample, Josselin Poiret, Christopher Baines,
Simon Tournier, Mathieu Othacehe, Tobias Geerinckx-Rice,
Lars-Dominik Braun, 62375, Ricardo Wurmus
Ludovic Courtès <ludo@gnu.org> writes:
> Hey Jelle,
>
> How far are we from merging? :-)
Hey Ludo,
Thanks for checking in! I haven't had a whole lot of time recently due
to a bunch of (positive) personal things, but I'll reach out soon~ish
with the next iteration.
Cheers,
- Jelle
^ permalink raw reply [flat|nested] 18+ messages in thread
* [bug#62375] [PATCH 0/1] npm binary importer
2023-03-22 11:25 [bug#62375] [PATCH 0/1] npm binary importer jlicht
2023-03-22 11:27 ` [bug#62375] [PATCH] import: Add binary npm importer jlicht
@ 2024-02-08 0:59 ` Nicolas Graves via Guix-patches via
2024-03-24 14:54 ` [bug#62375] Continue the npm-binary importer Pablo Zamora
` (4 subsequent siblings)
6 siblings, 0 replies; 18+ messages in thread
From: Nicolas Graves via Guix-patches via @ 2024-02-08 0:59 UTC (permalink / raw)
To: jlicht, 62375
Cc: dev, mail, zimon.toutoune, othacehe, ludo, me, rekado,
Jelle Licht
On 2023-03-22 12:25, jlicht@fsfe.org wrote:
> From: Jelle Licht <jlicht@fsfe.org>
>
> Folks,
>
> Here a revised patch to add the npm binary importer. To give some context,
> 'binary' here refers to the fact that this downloads archives straight from
> the npm registry at https://registry.npmjs.org. Some of these downloaded
> archives may not contain the original sources, so unless properly vetted the
> output of this importer does not allow one to easily generate package
> expressions for inclusion in guix's collection of packages.
>
> It should work as-is for most simple NPM packages. As noted in an inline
> comment somewhere, the way both npm and our very own node-build-system treats
> peer dependencies may require some manual intervention from time to time to
> hook in the right dependency at the right spot. The upside here is that when
> this happens, it's either trivial for a human to spot and fix, or wholly
> incompatible with our current approach so not fixable without writing a custom
> build system to deal with dependency cycles :-).
>
> Please test and review, let us get this merged as it might be useful to many
> people for building out their personal channels and/or package expressions.
>
> Special thanks to Timothy Sample and Lars-Dominik Braun for involvement in
> realising this, way too long ago.
>
>
> Jelle Licht (1):
> import: Add binary npm importer.
>
> Makefile.am | 2 +
> guix/import/npm-binary.scm | 269 +++++++++++++++++++++++++++++
> guix/scripts/import.scm | 2 +-
> guix/scripts/import/npm-binary.scm | 113 ++++++++++++
> 4 files changed, 385 insertions(+), 1 deletion(-)
> create mode 100644 guix/import/npm-binary.scm
> create mode 100644 guix/scripts/import/npm-binary.scm
Hey Jelle,
Just to understand, is there a reason why you didn't went further with
the sucrase approach?
--
Best regards,
Nicolas Graves
^ permalink raw reply [flat|nested] 18+ messages in thread* [bug#62375] Continue the npm-binary importer
2023-03-22 11:25 [bug#62375] [PATCH 0/1] npm binary importer jlicht
2023-03-22 11:27 ` [bug#62375] [PATCH] import: Add binary npm importer jlicht
2024-02-08 0:59 ` Nicolas Graves via Guix-patches via
@ 2024-03-24 14:54 ` Pablo Zamora
2024-03-31 19:57 ` Jelle Licht
2024-03-31 19:46 ` [bug#62375] [PATCH v2] import: Add binary npm importer jlicht
` (3 subsequent siblings)
6 siblings, 1 reply; 18+ messages in thread
From: Pablo Zamora @ 2024-03-24 14:54 UTC (permalink / raw)
To: 62375@debbugs.gnu.org
[-- Attachment #1: Type: text/plain, Size: 512 bytes --]
Dear Guix,
I am stuck while trying to define the 'bitwarden' package definition on my personal channel here https://github.com/Pablo12345678901/guix-custom-channel/blob/master/dev/my-bitwarden.scm . I am facing issues due to the npm dependencies.
One of the potential solution would be this importer. I would like to finish writing it and perform the tests.
On which branch should I send git pull request to continue what has been done so far ? This would be my first commit with guix.
Pablo Zamora
[-- Attachment #2: Type: text/html, Size: 2125 bytes --]
^ permalink raw reply [flat|nested] 18+ messages in thread* [bug#62375] Continue the npm-binary importer
2024-03-24 14:54 ` [bug#62375] Continue the npm-binary importer Pablo Zamora
@ 2024-03-31 19:57 ` Jelle Licht
2024-03-31 22:03 ` Pablo Zamora
0 siblings, 1 reply; 18+ messages in thread
From: Jelle Licht @ 2024-03-31 19:57 UTC (permalink / raw)
To: Pablo Zamora, 62375@debbugs.gnu.org
Hi Pablo,
Pablo Zamora <pablo.zamora@outlook.fr> writes:
> Dear Guix,
>
> I am stuck while trying to define the 'bitwarden' package definition on my personal channel here https://github.com/Pablo12345678901/guix-custom-channel/blob/master/dev/my-bitwarden.scm . I am facing issues due to the npm dependencies.
>
> One of the potential solution would be this importer. I would like to finish writing it and perform the tests.
That is nice to hear! I just sent v2 of this patch addressing most of
the shortcomings that were identified earlier.
> On which branch should I send git pull request to continue what has been done so far ? This would be my first commit with guix.
I see two ways forward, depending on how much work you foresee and/or
are willing to do:
1) This patch gets through another round of review, and afterwards you
can send patches based on the 'master' branch.
2) You work on improving this patch while it is still under review, and
send a V{3,4,5} patch as a reply to this very thread.
Kind regards,
Jelle Licht
^ permalink raw reply [flat|nested] 18+ messages in thread* [bug#62375] Continue the npm-binary importer
2024-03-31 19:57 ` Jelle Licht
@ 2024-03-31 22:03 ` Pablo Zamora
0 siblings, 0 replies; 18+ messages in thread
From: Pablo Zamora @ 2024-03-31 22:03 UTC (permalink / raw)
To: 62375@debbugs.gnu.org; +Cc: Jelle Licht
[-- Attachment #1: Type: text/plain, Size: 1588 bytes --]
Dear Jelle,
Thank you very much for your help.
I will shortly (this week) take a thorough look on your patches so I think of choosing '2)', even temporarily in order to become accustomed on how to develop a Guix importer.
Kind regards,
Pablo Zamora
________________________________
De : Jelle Licht <jlicht@fsfe.org>
Envoyé : dimanche 31 mars 2024 21:57
À : Pablo Zamora <pablo.zamora@outlook.fr>; 62375@debbugs.gnu.org <62375@debbugs.gnu.org>
Objet : Re: [bug#62375] Continue the npm-binary importer
Hi Pablo,
Pablo Zamora <pablo.zamora@outlook.fr> writes:
> Dear Guix,
>
> I am stuck while trying to define the 'bitwarden' package definition on my personal channel here https://github.com/Pablo12345678901/guix-custom-channel/blob/master/dev/my-bitwarden.scm . I am facing issues due to the npm dependencies.
>
> One of the potential solution would be this importer. I would like to finish writing it and perform the tests.
That is nice to hear! I just sent v2 of this patch addressing most of
the shortcomings that were identified earlier.
> On which branch should I send git pull request to continue what has been done so far ? This would be my first commit with guix.
I see two ways forward, depending on how much work you foresee and/or
are willing to do:
1) This patch gets through another round of review, and afterwards you
can send patches based on the 'master' branch.
2) You work on improving this patch while it is still under review, and
send a V{3,4,5} patch as a reply to this very thread.
Kind regards,
Jelle Licht
[-- Attachment #2: Type: text/html, Size: 3711 bytes --]
^ permalink raw reply [flat|nested] 18+ messages in thread
* [bug#62375] [PATCH v2] import: Add binary npm importer.
2023-03-22 11:25 [bug#62375] [PATCH 0/1] npm binary importer jlicht
` (2 preceding siblings ...)
2024-03-24 14:54 ` [bug#62375] Continue the npm-binary importer Pablo Zamora
@ 2024-03-31 19:46 ` jlicht
2024-03-31 20:37 ` [bug#62375] [PATCH v3] " jlicht
` (2 subsequent siblings)
6 siblings, 0 replies; 18+ messages in thread
From: jlicht @ 2024-03-31 19:46 UTC (permalink / raw)
To: 62375
Cc: Timothy Sample, Jelle Licht, Lars-Dominik Braun, pablo.zamora,
Christopher Baines, Josselin Poiret, Ludovic Courtès,
Mathieu Othacehe, Ricardo Wurmus, Simon Tournier,
Tobias Geerinckx-Rice
From: Jelle Licht <jlicht@fsfe.org>
* guix/scripts/import.scm: (importers): Add "npm-binary".
* guix/import/npm-binary.scm: New file.
* guix/scripts/import/npm-binary.scm: New file.
* Makefile.am: Add them.
Co-authored-by: Timothy Sample <samplet@ngyro.com>
Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
Change-Id: I98a45068cf5b9c42790664cc743feaa7ac76f807
---
Changes in v2:
- Change *SOME-VAR* to %SOME-VAR
- Removed unused http-error-code
- Rebase on master
- Refactor hash-url to use port-sha256 helper
- use explicit record accessors instead of order-sensitive destructuring
- address line-width styling issues
- added basic documentation
- added some basic tests (using simple mocks instead of with-http-server)
- simplify import script entrypoint
Makefile.am | 3 +
doc/guix.texi | 33 ++++
guix/import/npm-binary.scm | 277 +++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/npm-binary.scm | 108 +++++++++++
tests/npm-binary.scm | 146 +++++++++++++++
6 files changed, 568 insertions(+), 1 deletion(-)
create mode 100644 guix/import/npm-binary.scm
create mode 100644 guix/scripts/import/npm-binary.scm
create mode 100755 tests/npm-binary.scm
diff --git a/Makefile.am b/Makefile.am
index 1c5688ac13..459c47a954 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -305,6 +305,7 @@ MODULES = \
guix/import/kde.scm \
guix/import/launchpad.scm \
guix/import/minetest.scm \
+ guix/import/npm-binary.scm \
guix/import/opam.scm \
guix/import/print.scm \
guix/import/pypi.scm \
@@ -359,6 +360,7 @@ MODULES = \
guix/scripts/import/hexpm.scm \
guix/scripts/import/json.scm \
guix/scripts/import/minetest.scm \
+ guix/scripts/import/npm-binary.scm \
guix/scripts/import/opam.scm \
guix/scripts/import/pypi.scm \
guix/scripts/import/stackage.scm \
@@ -557,6 +559,7 @@ SCM_TESTS = \
tests/modules.scm \
tests/monads.scm \
tests/nar.scm \
+ tests/npm-binary.scm \
tests/networking.scm \
tests/opam.scm \
tests/openpgp.scm \
diff --git a/doc/guix.texi b/doc/guix.texi
index 69a904473c..566af6e849 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -14641,6 +14641,39 @@ Invoking guix import
in Guix.
@end table
+@item npm-binary
+@cindex npm
+@cindex Node.js
+Import metadata from the @uref{https://registry.npmjs.org, npm
+Registry}, as in this example:
+
+@example
+guix import npm-binary buffer-crc32
+@end example
+
+The npm-binary importer also allows you to specify a version string:
+
+@example
+guix import npm-binary buffer-crc32 1.0.0
+@end example
+
+@quotation Note
+Generated package expressions skip the build step of the
+@code{node-build-system}. As such, generated package expressions often
+refer to transpiled or generated files, instead of being built from
+source.
+@end quotation
+
+Additional options include:
+
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
+
@item opam
@cindex OPAM
@cindex OCaml
diff --git a/guix/import/npm-binary.scm b/guix/import/npm-binary.scm
new file mode 100644
index 0000000000..57c985baf2
--- /dev/null
+++ b/guix/import/npm-binary.scm
@@ -0,0 +1,277 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2019, 2020 Timothy Sample <samplet@ngyro.com>
+;;; Copyright © 2020, 2023, 2024 Jelle Licht <jlicht@fsfe.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix import npm-binary)
+ #:use-module ((gnu services configuration) #:select (alist?))
+ #:use-module (gcrypt hash)
+ #:use-module (gnu packages)
+ #:use-module (guix base32)
+ #:use-module (guix http-client)
+ #:use-module (guix import json)
+ #:use-module (guix import utils)
+ #:use-module (guix memoization)
+ #:use-module (guix utils)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (json)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-41)
+ #:use-module (srfi srfi-9)
+ #:use-module (web client)
+ #:use-module (web response)
+ #:use-module (web uri)
+ #:export (npm-binary-recursive-import
+ npm-binary->guix-package
+ make-versioned-package
+ name+version->symbol))
+
+;; Autoload Guile-Semver so we only have a soft dependency.
+(module-autoload! (current-module)
+ '(semver)
+ '(string->semver semver? semver->string semver=? semver>?))
+(module-autoload! (current-module)
+ '(semver ranges)
+ '(*semver-range-any* string->semver-range semver-range-contains?))
+
+;; Dist-tags
+(define-json-mapping <dist-tags> make-dist-tags dist-tags?
+ json->dist-tags
+ (latest dist-tags-latest "latest" string->semver))
+
+(define-record-type <versioned-package>
+ (make-versioned-package name version)
+ versioned-package?
+ (name versioned-package-name) ;string
+ (version versioned-package-version)) ;string
+
+(define (dependencies->versioned-packages entries)
+ (match entries
+ (((names . versions) ...)
+ (map make-versioned-package names versions))
+ (_ '())))
+
+(define (extract-license license-string)
+ (if (unspecified? license-string)
+ 'unspecified!
+ (spdx-string->license license-string)))
+
+(define-json-mapping <dist> make-dist dist?
+ json->dist
+ (tarball dist-tarball))
+
+(define (empty-or-string s)
+ (if (string? s) s ""))
+
+(define-json-mapping <package-revision> make-package-revision package-revision?
+ json->package-revision
+ (name package-revision-name)
+ (version package-revision-version "version" ;semver
+ string->semver)
+ (home-page package-revision-home-page "homepage") ;string
+ (dependencies package-revision-dependencies ;list of versioned-package
+ "dependencies"
+ dependencies->versioned-packages)
+ (dev-dependencies package-revision-dev-dependencies ;list of versioned-package
+ "devDependencies" dependencies->versioned-packages)
+ (peer-dependencies package-revision-peer-dependencies ;list of versioned-package
+ "peerDependencies" dependencies->versioned-packages)
+ (license package-revision-license "license" ;license | #f
+ (match-lambda
+ ((? unspecified?) #f)
+ ((? string? str) (spdx-string->license str))
+ ((? alist? alist)
+ (match (assoc "type" alist)
+ ((_ . (? string? type))
+ (spdx-string->license type))
+ (_ #f)))))
+ (description package-revision-description ;string
+ "description" empty-or-string)
+ (dist package-revision-dist "dist" json->dist)) ;dist
+
+(define (versions->package-revisions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map json->package-revision package-spec))
+ (_ '())))
+
+(define (versions->package-versions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map string->semver versions))
+ (_ '())))
+
+(define-json-mapping <meta-package> make-meta-package meta-package?
+ json->meta-package
+ (name meta-package-name) ;string
+ (description meta-package-description) ;string
+ (dist-tags meta-package-dist-tags "dist-tags" json->dist-tags) ;dist-tags
+ (revisions meta-package-revisions "versions" versions->package-revisions))
+
+;; TODO: Support other registries
+(define %registry "https://registry.npmjs.org")
+(define %default-page "https://www.npmjs.com/package")
+
+(define (lookup-meta-package name)
+ (let ((json (json-fetch (string-append %registry "/" (uri-encode name)))))
+ (and=> json json->meta-package)))
+
+(define lookup-meta-package* (memoize lookup-meta-package))
+
+(define (meta-package-versions meta)
+ (map package-revision-version
+ (meta-package-revisions meta)))
+
+(define (meta-package-latest meta)
+ (and=> (meta-package-dist-tags meta) dist-tags-latest))
+
+(define* (meta-package-package meta #:optional
+ (version (meta-package-latest meta)))
+ (match version
+ ((? semver?) (find (lambda (revision)
+ (semver=? version (package-revision-version revision)))
+ (meta-package-revisions meta)))
+ ((? string?) (meta-package-package meta (string->semver version)))
+ (_ #f)))
+
+(define* (semver-latest svs #:optional (svr *semver-range-any*))
+ (find (cut semver-range-contains? svr <>)
+ (sort svs semver>?)))
+
+(define* (resolve-package name #:optional (svr *semver-range-any*))
+ (let ((meta (lookup-meta-package* name)))
+ (and meta
+ (let* ((version (semver-latest (or (meta-package-versions meta) '()) svr))
+ (pkg (meta-package-package meta version)))
+ pkg))))
+
+\f
+;;;
+;;; Converting packages
+;;;
+
+(define (hash-url url)
+ "Downloads the resource at URL and computes the base32 hash for it."
+ (bytevector->nix-base32-string (port-sha256 (http-fetch url))))
+
+(define (npm-name->name npm-name)
+ "Return a Guix package name for the npm package with name NPM-NAME."
+ (define (clean name)
+ (string-map (lambda (chr) (if (char=? chr #\/) #\- chr))
+ (string-filter (negate (cut char=? <> #\@)) name)))
+ (guix-name "node-" (clean npm-name)))
+
+(define (name+version->symbol name version)
+ (string->symbol (string-append name "-" version)))
+
+(define (package-revision->symbol package)
+ (let* ((npm-name (package-revision-name package))
+ (version (semver->string (package-revision-version package)))
+ (name (npm-name->name npm-name)))
+ (name+version->symbol name version)))
+
+(define (npm-package->package-sexp npm-package)
+ "Return the `package' s-expression for an NPM-PACKAGE."
+ (define resolve-spec
+ (match-lambda
+ (($ <versioned-package> name version)
+ (resolve-package name (string->semver-range version)))))
+
+ (if (package-revision? npm-package)
+ (let ((name (package-revision-name npm-package))
+ (version (package-revision-version npm-package))
+ (home-page (package-revision-home-page npm-package))
+ (dependencies (package-revision-dependencies npm-package))
+ (dev-dependencies (package-revision-dev-dependencies npm-package))
+ (peer-dependencies (package-revision-peer-dependencies npm-package))
+ (license (package-revision-license npm-package))
+ (description (package-revision-description npm-package))
+ (dist (package-revision-dist npm-package)))
+ (let* ((name (npm-name->name name))
+ (url (dist-tarball dist))
+ (home-page (if (string? home-page)
+ home-page
+ (string-append %default-page "/" (uri-encode name))))
+ (synopsis description)
+ (resolved-deps (map resolve-spec
+ (append dependencies peer-dependencies)))
+ (peer-names (map versioned-package-name peer-dependencies))
+ ;; lset-difference for treating peer-dependencies as dependencies,
+ ;; which leads to dependency cycles. lset-union for treating them as
+ ;; (ignored) dev-dependencies, which leads to broken packages.
+ (dev-names
+ (lset-union string=
+ (map versioned-package-name dev-dependencies)
+ peer-names))
+ (extra-phases
+ (match dev-names
+ (() '())
+ ((dev-names ...)
+ `((add-after 'patch-dependencies 'delete-dev-dependencies
+ (lambda _
+ (delete-dependencies '(,@(reverse dev-names))))))))))
+ (values
+ `(package
+ (name ,name)
+ (version ,(semver->string (package-revision-version npm-package)))
+ (source (origin
+ (method url-fetch)
+ (uri ,url)
+ (sha256 (base32 ,(hash-url url)))))
+ (build-system node-build-system)
+ (arguments
+ (list
+ #:tests? #f
+ #:phases
+ #~(modify-phases %standard-phases
+ (delete 'build)
+ ,@extra-phases)))
+ ,@(match dependencies
+ (() '())
+ ((dependencies ...)
+ `((inputs
+ (list ,@(map package-revision->symbol resolved-deps))))))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,license))
+ (map (match-lambda (($ <package-revision> name version)
+ (list name (semver->string version))))
+ resolved-deps))))
+ (values #f '())))
+
+\f
+;;;
+;;; Interface
+;;;
+
+(define npm-binary->guix-package
+ (lambda* (name #:key (version *semver-range-any*) #:allow-other-keys)
+ (let* ((svr (match version
+ ((? string?) (string->semver-range version))
+ (_ version)))
+ (pkg (resolve-package name svr)))
+ (npm-package->package-sexp pkg))))
+
+(define* (npm-binary-recursive-import package-name #:key version)
+ (recursive-import package-name
+ #:repo->guix-package (memoize npm-binary->guix-package)
+ #:version version
+ #:guix-name npm-name->name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 1f34cab088..d724f2bca3 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -49,7 +49,7 @@ (define %standard-import-options '())
(define importers '("gnu" "pypi" "cpan" "hackage" "stackage" "egg" "elpa"
"gem" "go" "cran" "crate" "texlive" "json" "opam"
- "minetest" "elm" "hexpm" "composer"))
+ "minetest" "elm" "hexpm" "composer" "npm-binary"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/npm-binary.scm b/guix/scripts/import/npm-binary.scm
new file mode 100644
index 0000000000..d16b0f15b0
--- /dev/null
+++ b/guix/scripts/import/npm-binary.scm
@@ -0,0 +1,108 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2014 David Thompson <davet@gnu.org>
+;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Timothy Sample <samplet@ngyro.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import npm-binary)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import npm-binary)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (srfi srfi-41)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-npm-binary))
+
+\f
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import npm-binary PACKAGE-NAME [VERSION]
+Import and convert the npm package PACKAGE-NAME using the
+`node-build-system' (but without building the package from source)."))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -r, --recursive import packages recursively"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import npm-binary")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ %standard-import-options))
+
+\f
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-npm-binary . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (let loop ((args args))
+ (match args
+ ((package-name version)
+ (match (if (assoc-ref opts 'recursive)
+ ;; Recursive import
+ (npm-binary-recursive-import package-name #:version version)
+ ;; Single import
+ (npm-binary->guix-package package-name #:version version))
+ ((or #f '())
+ (leave (G_ "failed to download meta-data for package '~a@~a'~%")
+ package-name version))
+ ((? list? sexps) sexps)
+ (sexp (list sexp))))
+ ((package-name)
+ (loop (list package-name "*")))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%")))))))
diff --git a/tests/npm-binary.scm b/tests/npm-binary.scm
new file mode 100755
index 0000000000..cf85e572b3
--- /dev/null
+++ b/tests/npm-binary.scm
@@ -0,0 +1,146 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2024 Jelle Licht <jlicht@fsfe.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+(define-module (test-npm-binary)
+ #:use-module ((gcrypt hash)
+ #:select ((sha256 . gcrypt-sha256)))
+ #:use-module (guix import npm-binary)
+ #:use-module (guix base32)
+ #:use-module (guix tests)
+ #:use-module (srfi srfi-64)
+ #:use-module (ice-9 iconv)
+ #:use-module (ice-9 match)
+ #:export (run-test))
+
+(define foo-json
+ "{
+ \"name\": \"foo\",
+ \"dist-tags\": {
+ \"latest\": \"1.2.3\",
+ \"next\": \"2.0.1-beta4\"
+ },
+ \"description\": \"General purpose utilities to foo your bars\",
+ \"homepage\": \"https://github.com/quartz/foo\",
+ \"repository\": \"quartz/foo\",
+ \"versions\": {
+ \"1.2.3\": {
+ \"name\": \"foo\",
+ \"description\": \"General purpose utilities to foo your bars\",
+ \"version\": \"1.2.3\",
+ \"author\": \"Jelle Licht <jlicht@fsfe.org>\",
+ \"devDependencies\": {
+ \"node-megabuilder\": \"^0.0.2\"
+ },
+ \"dependencies\": {
+ \"bar\": \"^0.1.0\"
+ },
+ \"repository\": {
+ \"url\": \"quartz/foo\"
+ },
+ \"homepage\": \"https://github.com/quartz/foo\",
+ \"license\": \"MIT\",
+ \"dist\": {
+ \"tarball\": \"https://registry.npmjs.org/foo/-/foo-1.2.3.tgz\"
+ }
+ }
+ }
+}")
+
+(define bar-json
+ "{
+ \"name\": \"bar\",
+ \"dist-tags\": {
+ \"latest\": \"0.1.2\"
+ },
+ \"description\": \"Core module in FooBar\",
+ \"homepage\": \"https://github.com/quartz/bar\",
+ \"repository\": \"quartz/bar\",
+ \"versions\": {
+ \"0.1.2\": {
+ \"name\": \"bar\",
+ \"description\": \"Core module in FooBar\",
+ \"version\": \"0.1.2\",
+ \"author\": \"Jelle Licht <jlicht@fsfe.org>\",
+ \"repository\": {
+ \"url\": \"quartz/bar\"
+ },
+ \"homepage\": \"https://github.com/quartz/bar\",
+ \"license\": \"MIT\",
+ \"dist\": {
+ \"tarball\": \"https://registry.npmjs.org/bar/-/bar-0.1.2.tgz\"
+ }
+ }
+ }
+}")
+
+(define test-source-hash
+ "")
+
+(define test-source
+ "Empty file\n")
+
+(define have-guile-semver?
+ (false-if-exception (resolve-interface '(semver))))
+
+(test-begin "npm")
+
+(unless have-guile-semver? (test-skip 1))
+(test-assert "npm-binary->guix-package"
+ (mock ((guix http-client) http-fetch
+ (lambda* (url #:rest _)
+ (match url
+ ("https://registry.npmjs.org/foo"
+ (values (open-input-string foo-json)
+ (string-length foo-json)))
+ ("https://registry.npmjs.org/bar"
+ (values (open-input-string bar-json)
+ (string-length bar-json)))
+ ("https://registry.npmjs.org/foo/-/foo-1.2.3.tgz"
+ (set! test-source-hash
+ (bytevector->nix-base32-string
+ (gcrypt-sha256 (string->bytevector test-source "utf-8"))))
+ (values (open-input-string test-source)
+ (string-length test-source))))))
+ (match (npm-binary->guix-package "foo")
+ (`(package
+ (name "node-foo")
+ (version "1.2.3")
+ (source (origin
+ (method url-fetch)
+ (uri "https://registry.npmjs.org/foo/-/foo-1.2.3.tgz")
+ (sha256
+ (base32
+ ,test-source-hash))))
+ (build-system node-build-system)
+ (arguments
+ (list #:tests? #f
+ #:phases
+ (gexp (modify-phases %standard-phases
+ (delete 'build)
+ (add-after 'patch-dependencies 'delete-dev-dependencies
+ (lambda _
+ (delete-dependencies '("node-megabuilder"))))))))
+ (inputs (list node-bar-0.1.2))
+ (home-page "https://github.com/quartz/foo")
+ (synopsis "General purpose utilities to foo your bars")
+ (description "General purpose utilities to foo your bars")
+ (license license:expat))
+ #t)
+ (x
+ (pk 'fail x #f)))))
+
+(test-end "npm")
base-commit: 4d79a9cd6b5f0d8c5afbab0c6b70ae42740d5470
--
2.41.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [bug#62375] [PATCH v3] import: Add binary npm importer.
2023-03-22 11:25 [bug#62375] [PATCH 0/1] npm binary importer jlicht
` (3 preceding siblings ...)
2024-03-31 19:46 ` [bug#62375] [PATCH v2] import: Add binary npm importer jlicht
@ 2024-03-31 20:37 ` jlicht
2024-04-01 20:41 ` Ludovic Courtès
2024-04-01 22:01 ` [bug#62375] [PATCH 0/1] npm binary importer Jonathan Brielmaier via Guix-patches via
2024-04-02 14:13 ` [bug#62375] [PATCH v4] import: Add binary npm importer jlicht
6 siblings, 1 reply; 18+ messages in thread
From: jlicht @ 2024-03-31 20:37 UTC (permalink / raw)
To: 62375
Cc: Timothy Sample, Jelle Licht, Lars-Dominik Braun,
Christopher Baines, Josselin Poiret, Ludovic Courtès,
Mathieu Othacehe, Ricardo Wurmus, Simon Tournier,
Tobias Geerinckx-Rice
From: Jelle Licht <jlicht@fsfe.org>
* guix/scripts/import.scm: (importers): Add "npm-binary".
* guix/import/npm-binary.scm: New file.
* guix/scripts/import/npm-binary.scm: New file.
* Makefile.am: Add them.
Co-authored-by: Timothy Sample <samplet@ngyro.com>
Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
Change-Id: I98a45068cf5b9c42790664cc743feaa7ac76f807
---
Changes in v3:
- Ensure that package bindings generated during recursive import match the
package bindings used in the list of inputs, as this was a regression introduced in the V2 series.
Changes in v2:
- Change *SOME-VAR* to %SOME-VAR
- Removed unused http-error-code
- Rebase on master
- Refactor hash-url to use port-sha256 helper
- use explicit record accessors instead of order-sensitive destructuring
- address line-width styling issues
- added basic documentation
- added some basic tests (using simple mocks instead of with-http-server)
- simplify import script entrypoint
Makefile.am | 3 +
doc/guix.texi | 33 ++++
guix/import/npm-binary.scm | 277 +++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/npm-binary.scm | 114 ++++++++++++
tests/npm-binary.scm | 146 +++++++++++++++
6 files changed, 574 insertions(+), 1 deletion(-)
create mode 100644 guix/import/npm-binary.scm
create mode 100644 guix/scripts/import/npm-binary.scm
create mode 100755 tests/npm-binary.scm
diff --git a/Makefile.am b/Makefile.am
index 1c5688ac13..459c47a954 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -305,6 +305,7 @@ MODULES = \
guix/import/kde.scm \
guix/import/launchpad.scm \
guix/import/minetest.scm \
+ guix/import/npm-binary.scm \
guix/import/opam.scm \
guix/import/print.scm \
guix/import/pypi.scm \
@@ -359,6 +360,7 @@ MODULES = \
guix/scripts/import/hexpm.scm \
guix/scripts/import/json.scm \
guix/scripts/import/minetest.scm \
+ guix/scripts/import/npm-binary.scm \
guix/scripts/import/opam.scm \
guix/scripts/import/pypi.scm \
guix/scripts/import/stackage.scm \
@@ -557,6 +559,7 @@ SCM_TESTS = \
tests/modules.scm \
tests/monads.scm \
tests/nar.scm \
+ tests/npm-binary.scm \
tests/networking.scm \
tests/opam.scm \
tests/openpgp.scm \
diff --git a/doc/guix.texi b/doc/guix.texi
index 69a904473c..566af6e849 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -14641,6 +14641,39 @@ Invoking guix import
in Guix.
@end table
+@item npm-binary
+@cindex npm
+@cindex Node.js
+Import metadata from the @uref{https://registry.npmjs.org, npm
+Registry}, as in this example:
+
+@example
+guix import npm-binary buffer-crc32
+@end example
+
+The npm-binary importer also allows you to specify a version string:
+
+@example
+guix import npm-binary buffer-crc32 1.0.0
+@end example
+
+@quotation Note
+Generated package expressions skip the build step of the
+@code{node-build-system}. As such, generated package expressions often
+refer to transpiled or generated files, instead of being built from
+source.
+@end quotation
+
+Additional options include:
+
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
+
@item opam
@cindex OPAM
@cindex OCaml
diff --git a/guix/import/npm-binary.scm b/guix/import/npm-binary.scm
new file mode 100644
index 0000000000..57c985baf2
--- /dev/null
+++ b/guix/import/npm-binary.scm
@@ -0,0 +1,277 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2019, 2020 Timothy Sample <samplet@ngyro.com>
+;;; Copyright © 2020, 2023, 2024 Jelle Licht <jlicht@fsfe.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix import npm-binary)
+ #:use-module ((gnu services configuration) #:select (alist?))
+ #:use-module (gcrypt hash)
+ #:use-module (gnu packages)
+ #:use-module (guix base32)
+ #:use-module (guix http-client)
+ #:use-module (guix import json)
+ #:use-module (guix import utils)
+ #:use-module (guix memoization)
+ #:use-module (guix utils)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (json)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-41)
+ #:use-module (srfi srfi-9)
+ #:use-module (web client)
+ #:use-module (web response)
+ #:use-module (web uri)
+ #:export (npm-binary-recursive-import
+ npm-binary->guix-package
+ make-versioned-package
+ name+version->symbol))
+
+;; Autoload Guile-Semver so we only have a soft dependency.
+(module-autoload! (current-module)
+ '(semver)
+ '(string->semver semver? semver->string semver=? semver>?))
+(module-autoload! (current-module)
+ '(semver ranges)
+ '(*semver-range-any* string->semver-range semver-range-contains?))
+
+;; Dist-tags
+(define-json-mapping <dist-tags> make-dist-tags dist-tags?
+ json->dist-tags
+ (latest dist-tags-latest "latest" string->semver))
+
+(define-record-type <versioned-package>
+ (make-versioned-package name version)
+ versioned-package?
+ (name versioned-package-name) ;string
+ (version versioned-package-version)) ;string
+
+(define (dependencies->versioned-packages entries)
+ (match entries
+ (((names . versions) ...)
+ (map make-versioned-package names versions))
+ (_ '())))
+
+(define (extract-license license-string)
+ (if (unspecified? license-string)
+ 'unspecified!
+ (spdx-string->license license-string)))
+
+(define-json-mapping <dist> make-dist dist?
+ json->dist
+ (tarball dist-tarball))
+
+(define (empty-or-string s)
+ (if (string? s) s ""))
+
+(define-json-mapping <package-revision> make-package-revision package-revision?
+ json->package-revision
+ (name package-revision-name)
+ (version package-revision-version "version" ;semver
+ string->semver)
+ (home-page package-revision-home-page "homepage") ;string
+ (dependencies package-revision-dependencies ;list of versioned-package
+ "dependencies"
+ dependencies->versioned-packages)
+ (dev-dependencies package-revision-dev-dependencies ;list of versioned-package
+ "devDependencies" dependencies->versioned-packages)
+ (peer-dependencies package-revision-peer-dependencies ;list of versioned-package
+ "peerDependencies" dependencies->versioned-packages)
+ (license package-revision-license "license" ;license | #f
+ (match-lambda
+ ((? unspecified?) #f)
+ ((? string? str) (spdx-string->license str))
+ ((? alist? alist)
+ (match (assoc "type" alist)
+ ((_ . (? string? type))
+ (spdx-string->license type))
+ (_ #f)))))
+ (description package-revision-description ;string
+ "description" empty-or-string)
+ (dist package-revision-dist "dist" json->dist)) ;dist
+
+(define (versions->package-revisions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map json->package-revision package-spec))
+ (_ '())))
+
+(define (versions->package-versions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map string->semver versions))
+ (_ '())))
+
+(define-json-mapping <meta-package> make-meta-package meta-package?
+ json->meta-package
+ (name meta-package-name) ;string
+ (description meta-package-description) ;string
+ (dist-tags meta-package-dist-tags "dist-tags" json->dist-tags) ;dist-tags
+ (revisions meta-package-revisions "versions" versions->package-revisions))
+
+;; TODO: Support other registries
+(define %registry "https://registry.npmjs.org")
+(define %default-page "https://www.npmjs.com/package")
+
+(define (lookup-meta-package name)
+ (let ((json (json-fetch (string-append %registry "/" (uri-encode name)))))
+ (and=> json json->meta-package)))
+
+(define lookup-meta-package* (memoize lookup-meta-package))
+
+(define (meta-package-versions meta)
+ (map package-revision-version
+ (meta-package-revisions meta)))
+
+(define (meta-package-latest meta)
+ (and=> (meta-package-dist-tags meta) dist-tags-latest))
+
+(define* (meta-package-package meta #:optional
+ (version (meta-package-latest meta)))
+ (match version
+ ((? semver?) (find (lambda (revision)
+ (semver=? version (package-revision-version revision)))
+ (meta-package-revisions meta)))
+ ((? string?) (meta-package-package meta (string->semver version)))
+ (_ #f)))
+
+(define* (semver-latest svs #:optional (svr *semver-range-any*))
+ (find (cut semver-range-contains? svr <>)
+ (sort svs semver>?)))
+
+(define* (resolve-package name #:optional (svr *semver-range-any*))
+ (let ((meta (lookup-meta-package* name)))
+ (and meta
+ (let* ((version (semver-latest (or (meta-package-versions meta) '()) svr))
+ (pkg (meta-package-package meta version)))
+ pkg))))
+
+\f
+;;;
+;;; Converting packages
+;;;
+
+(define (hash-url url)
+ "Downloads the resource at URL and computes the base32 hash for it."
+ (bytevector->nix-base32-string (port-sha256 (http-fetch url))))
+
+(define (npm-name->name npm-name)
+ "Return a Guix package name for the npm package with name NPM-NAME."
+ (define (clean name)
+ (string-map (lambda (chr) (if (char=? chr #\/) #\- chr))
+ (string-filter (negate (cut char=? <> #\@)) name)))
+ (guix-name "node-" (clean npm-name)))
+
+(define (name+version->symbol name version)
+ (string->symbol (string-append name "-" version)))
+
+(define (package-revision->symbol package)
+ (let* ((npm-name (package-revision-name package))
+ (version (semver->string (package-revision-version package)))
+ (name (npm-name->name npm-name)))
+ (name+version->symbol name version)))
+
+(define (npm-package->package-sexp npm-package)
+ "Return the `package' s-expression for an NPM-PACKAGE."
+ (define resolve-spec
+ (match-lambda
+ (($ <versioned-package> name version)
+ (resolve-package name (string->semver-range version)))))
+
+ (if (package-revision? npm-package)
+ (let ((name (package-revision-name npm-package))
+ (version (package-revision-version npm-package))
+ (home-page (package-revision-home-page npm-package))
+ (dependencies (package-revision-dependencies npm-package))
+ (dev-dependencies (package-revision-dev-dependencies npm-package))
+ (peer-dependencies (package-revision-peer-dependencies npm-package))
+ (license (package-revision-license npm-package))
+ (description (package-revision-description npm-package))
+ (dist (package-revision-dist npm-package)))
+ (let* ((name (npm-name->name name))
+ (url (dist-tarball dist))
+ (home-page (if (string? home-page)
+ home-page
+ (string-append %default-page "/" (uri-encode name))))
+ (synopsis description)
+ (resolved-deps (map resolve-spec
+ (append dependencies peer-dependencies)))
+ (peer-names (map versioned-package-name peer-dependencies))
+ ;; lset-difference for treating peer-dependencies as dependencies,
+ ;; which leads to dependency cycles. lset-union for treating them as
+ ;; (ignored) dev-dependencies, which leads to broken packages.
+ (dev-names
+ (lset-union string=
+ (map versioned-package-name dev-dependencies)
+ peer-names))
+ (extra-phases
+ (match dev-names
+ (() '())
+ ((dev-names ...)
+ `((add-after 'patch-dependencies 'delete-dev-dependencies
+ (lambda _
+ (delete-dependencies '(,@(reverse dev-names))))))))))
+ (values
+ `(package
+ (name ,name)
+ (version ,(semver->string (package-revision-version npm-package)))
+ (source (origin
+ (method url-fetch)
+ (uri ,url)
+ (sha256 (base32 ,(hash-url url)))))
+ (build-system node-build-system)
+ (arguments
+ (list
+ #:tests? #f
+ #:phases
+ #~(modify-phases %standard-phases
+ (delete 'build)
+ ,@extra-phases)))
+ ,@(match dependencies
+ (() '())
+ ((dependencies ...)
+ `((inputs
+ (list ,@(map package-revision->symbol resolved-deps))))))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,license))
+ (map (match-lambda (($ <package-revision> name version)
+ (list name (semver->string version))))
+ resolved-deps))))
+ (values #f '())))
+
+\f
+;;;
+;;; Interface
+;;;
+
+(define npm-binary->guix-package
+ (lambda* (name #:key (version *semver-range-any*) #:allow-other-keys)
+ (let* ((svr (match version
+ ((? string?) (string->semver-range version))
+ (_ version)))
+ (pkg (resolve-package name svr)))
+ (npm-package->package-sexp pkg))))
+
+(define* (npm-binary-recursive-import package-name #:key version)
+ (recursive-import package-name
+ #:repo->guix-package (memoize npm-binary->guix-package)
+ #:version version
+ #:guix-name npm-name->name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 1f34cab088..d724f2bca3 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -49,7 +49,7 @@ (define %standard-import-options '())
(define importers '("gnu" "pypi" "cpan" "hackage" "stackage" "egg" "elpa"
"gem" "go" "cran" "crate" "texlive" "json" "opam"
- "minetest" "elm" "hexpm" "composer"))
+ "minetest" "elm" "hexpm" "composer" "npm-binary"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/npm-binary.scm b/guix/scripts/import/npm-binary.scm
new file mode 100644
index 0000000000..4a4daa7945
--- /dev/null
+++ b/guix/scripts/import/npm-binary.scm
@@ -0,0 +1,114 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2014 David Thompson <davet@gnu.org>
+;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Timothy Sample <samplet@ngyro.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import npm-binary)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import npm-binary)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (srfi srfi-41)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-npm-binary))
+
+\f
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import npm-binary PACKAGE-NAME [VERSION]
+Import and convert the npm package PACKAGE-NAME using the
+`node-build-system' (but without building the package from source)."))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -r, --recursive import packages recursively"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import npm-binary")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ %standard-import-options))
+
+\f
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-npm-binary . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (let loop ((args args))
+ (match args
+ ((package-name version)
+ (match (if (assoc-ref opts 'recursive)
+ ;; Recursive import
+ (npm-binary-recursive-import package-name #:version version)
+ ;; Single import
+ (npm-binary->guix-package package-name #:version version))
+ ((or #f '())
+ (leave (G_ "failed to download meta-data for package '~a@~a'~%")
+ package-name version))
+ ((? list? sexps)
+ (map (match-lambda
+ ((and ('package ('name name) ('version version) . rest) pkg)
+ `(define-public ,(name+version->symbol name version)
+ ,pkg))
+ (_ #f))
+ sexps))
+ (sexp (list sexp))))
+ ((package-name)
+ (loop (list package-name "*")))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%")))))))
diff --git a/tests/npm-binary.scm b/tests/npm-binary.scm
new file mode 100755
index 0000000000..cf85e572b3
--- /dev/null
+++ b/tests/npm-binary.scm
@@ -0,0 +1,146 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2024 Jelle Licht <jlicht@fsfe.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+(define-module (test-npm-binary)
+ #:use-module ((gcrypt hash)
+ #:select ((sha256 . gcrypt-sha256)))
+ #:use-module (guix import npm-binary)
+ #:use-module (guix base32)
+ #:use-module (guix tests)
+ #:use-module (srfi srfi-64)
+ #:use-module (ice-9 iconv)
+ #:use-module (ice-9 match)
+ #:export (run-test))
+
+(define foo-json
+ "{
+ \"name\": \"foo\",
+ \"dist-tags\": {
+ \"latest\": \"1.2.3\",
+ \"next\": \"2.0.1-beta4\"
+ },
+ \"description\": \"General purpose utilities to foo your bars\",
+ \"homepage\": \"https://github.com/quartz/foo\",
+ \"repository\": \"quartz/foo\",
+ \"versions\": {
+ \"1.2.3\": {
+ \"name\": \"foo\",
+ \"description\": \"General purpose utilities to foo your bars\",
+ \"version\": \"1.2.3\",
+ \"author\": \"Jelle Licht <jlicht@fsfe.org>\",
+ \"devDependencies\": {
+ \"node-megabuilder\": \"^0.0.2\"
+ },
+ \"dependencies\": {
+ \"bar\": \"^0.1.0\"
+ },
+ \"repository\": {
+ \"url\": \"quartz/foo\"
+ },
+ \"homepage\": \"https://github.com/quartz/foo\",
+ \"license\": \"MIT\",
+ \"dist\": {
+ \"tarball\": \"https://registry.npmjs.org/foo/-/foo-1.2.3.tgz\"
+ }
+ }
+ }
+}")
+
+(define bar-json
+ "{
+ \"name\": \"bar\",
+ \"dist-tags\": {
+ \"latest\": \"0.1.2\"
+ },
+ \"description\": \"Core module in FooBar\",
+ \"homepage\": \"https://github.com/quartz/bar\",
+ \"repository\": \"quartz/bar\",
+ \"versions\": {
+ \"0.1.2\": {
+ \"name\": \"bar\",
+ \"description\": \"Core module in FooBar\",
+ \"version\": \"0.1.2\",
+ \"author\": \"Jelle Licht <jlicht@fsfe.org>\",
+ \"repository\": {
+ \"url\": \"quartz/bar\"
+ },
+ \"homepage\": \"https://github.com/quartz/bar\",
+ \"license\": \"MIT\",
+ \"dist\": {
+ \"tarball\": \"https://registry.npmjs.org/bar/-/bar-0.1.2.tgz\"
+ }
+ }
+ }
+}")
+
+(define test-source-hash
+ "")
+
+(define test-source
+ "Empty file\n")
+
+(define have-guile-semver?
+ (false-if-exception (resolve-interface '(semver))))
+
+(test-begin "npm")
+
+(unless have-guile-semver? (test-skip 1))
+(test-assert "npm-binary->guix-package"
+ (mock ((guix http-client) http-fetch
+ (lambda* (url #:rest _)
+ (match url
+ ("https://registry.npmjs.org/foo"
+ (values (open-input-string foo-json)
+ (string-length foo-json)))
+ ("https://registry.npmjs.org/bar"
+ (values (open-input-string bar-json)
+ (string-length bar-json)))
+ ("https://registry.npmjs.org/foo/-/foo-1.2.3.tgz"
+ (set! test-source-hash
+ (bytevector->nix-base32-string
+ (gcrypt-sha256 (string->bytevector test-source "utf-8"))))
+ (values (open-input-string test-source)
+ (string-length test-source))))))
+ (match (npm-binary->guix-package "foo")
+ (`(package
+ (name "node-foo")
+ (version "1.2.3")
+ (source (origin
+ (method url-fetch)
+ (uri "https://registry.npmjs.org/foo/-/foo-1.2.3.tgz")
+ (sha256
+ (base32
+ ,test-source-hash))))
+ (build-system node-build-system)
+ (arguments
+ (list #:tests? #f
+ #:phases
+ (gexp (modify-phases %standard-phases
+ (delete 'build)
+ (add-after 'patch-dependencies 'delete-dev-dependencies
+ (lambda _
+ (delete-dependencies '("node-megabuilder"))))))))
+ (inputs (list node-bar-0.1.2))
+ (home-page "https://github.com/quartz/foo")
+ (synopsis "General purpose utilities to foo your bars")
+ (description "General purpose utilities to foo your bars")
+ (license license:expat))
+ #t)
+ (x
+ (pk 'fail x #f)))))
+
+(test-end "npm")
base-commit: 4d79a9cd6b5f0d8c5afbab0c6b70ae42740d5470
--
2.41.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [bug#62375] [PATCH v3] import: Add binary npm importer.
2024-03-31 20:37 ` [bug#62375] [PATCH v3] " jlicht
@ 2024-04-01 20:41 ` Ludovic Courtès
2024-04-02 14:12 ` Jelle Licht
0 siblings, 1 reply; 18+ messages in thread
From: Ludovic Courtès @ 2024-04-01 20:41 UTC (permalink / raw)
To: jlicht
Cc: Timothy Sample, Josselin Poiret, Simon Tournier, Mathieu Othacehe,
Tobias Geerinckx-Rice, Lars-Dominik Braun, 62375,
Christopher Baines, Ricardo Wurmus
Hi!
jlicht@fsfe.org skribis:
> From: Jelle Licht <jlicht@fsfe.org>
>
> * guix/scripts/import.scm: (importers): Add "npm-binary".
> * guix/import/npm-binary.scm: New file.
> * guix/scripts/import/npm-binary.scm: New file.
> * Makefile.am: Add them.
>
> Co-authored-by: Timothy Sample <samplet@ngyro.com>
> Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
>
> Change-Id: I98a45068cf5b9c42790664cc743feaa7ac76f807
Yay!
> +The npm-binary importer also allows you to specify a version string:
> +
> +@example
> +guix import npm-binary buffer-crc32 1.0.0
> +@end example
For consistency with other importers (pypi, gem, cran), could you change
the syntax to:
guix import npm-binary buffer-crc32@1.0.0
?
That’s the last remaining issue for me.
> +;; TODO: Support other registries
> +(define %registry "https://registry.npmjs.org")
For the purposes of tests, you could make it:
(define %npm-registry
(make-parameter "https://registry.npmjs.org"))
That would allow you to write tests using ‘with-http-server’ and
‘parameterize’ as done in ‘tests/pypi.scm’ and others, which I find
nicer and more robust than ‘mock’.
Not a blocker though.
Thanks!
Ludo’.
^ permalink raw reply [flat|nested] 18+ messages in thread* [bug#62375] [PATCH v3] import: Add binary npm importer.
2024-04-01 20:41 ` Ludovic Courtès
@ 2024-04-02 14:12 ` Jelle Licht
0 siblings, 0 replies; 18+ messages in thread
From: Jelle Licht @ 2024-04-02 14:12 UTC (permalink / raw)
To: Ludovic Courtès
Cc: Timothy Sample, Josselin Poiret, Simon Tournier, Mathieu Othacehe,
Tobias Geerinckx-Rice, Lars-Dominik Braun, 62375,
Christopher Baines, Ricardo Wurmus
Hi Ludo',
Thanks for the speedy review!
Ludovic Courtès <ludo@gnu.org> writes:
> Hi!
>
> jlicht@fsfe.org skribis:
>
>> From: Jelle Licht <jlicht@fsfe.org>
>>
>> * guix/scripts/import.scm: (importers): Add "npm-binary".
>> * guix/import/npm-binary.scm: New file.
>> * guix/scripts/import/npm-binary.scm: New file.
>> * Makefile.am: Add them.
>>
>> Co-authored-by: Timothy Sample <samplet@ngyro.com>
>> Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
>>
>> Change-Id: I98a45068cf5b9c42790664cc743feaa7ac76f807
>
> Yay!
>
>> +The npm-binary importer also allows you to specify a version string:
>> +
>> +@example
>> +guix import npm-binary buffer-crc32 1.0.0
>> +@end example
>
> For consistency with other importers (pypi, gem, cran), could you change
> the syntax to:
>
> guix import npm-binary buffer-crc32@1.0.0
>
> ?
>
> That’s the last remaining issue for me.
>
I needed some custom logic to support the npm scoped packages
("@linthtml/linthtml@1.2.3"), but it should now work.
>> +;; TODO: Support other registries
>> +(define %registry "https://registry.npmjs.org")
>
> For the purposes of tests, you could make it:
>
> (define %npm-registry
> (make-parameter "https://registry.npmjs.org"))
Easy!
> That would allow you to write tests using ‘with-http-server’ and
> ‘parameterize’ as done in ‘tests/pypi.scm’ and others, which I find
> nicer and more robust than ‘mock’.
The `with-http-server' construct does not play well with an interactive
REPL workflow due to not cleaning up the used port in all situations,
making that port unavailable for running the test again[1]. `mock',
brittle as it may be, does not to suffer from this drawback, so I'd
vastly prefer using that for now.
> Not a blocker though.
>
> Thanks!
>
> Ludo’.
I'll send a v4, let it simmer for some days and then merge it if no big
blockers or requested changes show up.
- Jelle
^ permalink raw reply [flat|nested] 18+ messages in thread
* [bug#62375] [PATCH 0/1] npm binary importer
2023-03-22 11:25 [bug#62375] [PATCH 0/1] npm binary importer jlicht
` (4 preceding siblings ...)
2024-03-31 20:37 ` [bug#62375] [PATCH v3] " jlicht
@ 2024-04-01 22:01 ` Jonathan Brielmaier via Guix-patches via
2024-04-02 14:16 ` Jelle Licht
2024-04-02 14:13 ` [bug#62375] [PATCH v4] import: Add binary npm importer jlicht
6 siblings, 1 reply; 18+ messages in thread
From: Jonathan Brielmaier via Guix-patches via @ 2024-04-01 22:01 UTC (permalink / raw)
To: 62375
Hi Jelle,
one rather unimportant remark. The commit message mentions Lars-Dominik
as co-author:
https://issues.guix.gnu.org/62375#12-lineno8
But there is no copyright header for him in any of the added files...
~Jonathan
^ permalink raw reply [flat|nested] 18+ messages in thread* [bug#62375] [PATCH v4] import: Add binary npm importer.
2023-03-22 11:25 [bug#62375] [PATCH 0/1] npm binary importer jlicht
` (5 preceding siblings ...)
2024-04-01 22:01 ` [bug#62375] [PATCH 0/1] npm binary importer Jonathan Brielmaier via Guix-patches via
@ 2024-04-02 14:13 ` jlicht
6 siblings, 0 replies; 18+ messages in thread
From: jlicht @ 2024-04-02 14:13 UTC (permalink / raw)
To: 62375
Cc: Timothy Sample, Jelle Licht, Lars-Dominik Braun,
Christopher Baines, Josselin Poiret, Ludovic Courtès,
Mathieu Othacehe, Ricardo Wurmus, Simon Tournier,
Tobias Geerinckx-Rice
From: Jelle Licht <jlicht@fsfe.org>
* guix/scripts/import.scm: (importers): Add "npm-binary".
* guix/import/npm-binary.scm: New file.
* guix/scripts/import/npm-binary.scm: New file.
* Makefile.am: Add them.
Co-authored-by: Timothy Sample <samplet@ngyro.com>
Co-authored-by: Lars-Dominik Braun <lars@6xq.net>
Change-Id: I98a45068cf5b9c42790664cc743feaa7ac76f807
---
Changes in v4:
- Add copyright line for LDB
- Use package-name@version-spec notation on CLI
- Simplify CLI argument handling
- Turn %registry into a parameter named %npm-registry
Changes in v3:
- Ensure that package bindings generated during recursive import match the
package bindings used in the list of inputs
Changes in v2:
- Change *SOME-VAR* to %SOME-VAR
- Removed unused http-error-code
- Rebase on master
- Refactor hash-url to use port-sha256 helper
- use explicit record accessors instead of order-sensitive destructuring
- address line-width styling issues
- added basic documentation
- added some basic tests (using simple mocks instead of with-http-server)
- simplify import script entrypoint
Makefile.am | 3 +
doc/guix.texi | 33 ++++
guix/import/npm-binary.scm | 279 +++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/npm-binary.scm | 121 +++++++++++++
tests/npm-binary.scm | 146 +++++++++++++++
6 files changed, 583 insertions(+), 1 deletion(-)
create mode 100644 guix/import/npm-binary.scm
create mode 100644 guix/scripts/import/npm-binary.scm
create mode 100755 tests/npm-binary.scm
diff --git a/Makefile.am b/Makefile.am
index 1c5688ac13..459c47a954 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -305,6 +305,7 @@ MODULES = \
guix/import/kde.scm \
guix/import/launchpad.scm \
guix/import/minetest.scm \
+ guix/import/npm-binary.scm \
guix/import/opam.scm \
guix/import/print.scm \
guix/import/pypi.scm \
@@ -359,6 +360,7 @@ MODULES = \
guix/scripts/import/hexpm.scm \
guix/scripts/import/json.scm \
guix/scripts/import/minetest.scm \
+ guix/scripts/import/npm-binary.scm \
guix/scripts/import/opam.scm \
guix/scripts/import/pypi.scm \
guix/scripts/import/stackage.scm \
@@ -557,6 +559,7 @@ SCM_TESTS = \
tests/modules.scm \
tests/monads.scm \
tests/nar.scm \
+ tests/npm-binary.scm \
tests/networking.scm \
tests/opam.scm \
tests/openpgp.scm \
diff --git a/doc/guix.texi b/doc/guix.texi
index 69a904473c..f8f8f7bf3a 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -14641,6 +14641,39 @@ Invoking guix import
in Guix.
@end table
+@item npm-binary
+@cindex npm
+@cindex Node.js
+Import metadata from the @uref{https://registry.npmjs.org, npm
+Registry}, as in this example:
+
+@example
+guix import npm-binary buffer-crc32
+@end example
+
+The npm-binary importer also allows you to specify a version string:
+
+@example
+guix import npm-binary buffer-crc32@@1.0.0
+@end example
+
+@quotation Note
+Generated package expressions skip the build step of the
+@code{node-build-system}. As such, generated package expressions often
+refer to transpiled or generated files, instead of being built from
+source.
+@end quotation
+
+Additional options include:
+
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
+
@item opam
@cindex OPAM
@cindex OCaml
diff --git a/guix/import/npm-binary.scm b/guix/import/npm-binary.scm
new file mode 100644
index 0000000000..6dfedc4910
--- /dev/null
+++ b/guix/import/npm-binary.scm
@@ -0,0 +1,279 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2019, 2020 Timothy Sample <samplet@ngyro.com>
+;;; Copyright © 2021 Lars-Dominik Braun <lars@6xq.net>
+;;; Copyright © 2020, 2023, 2024 Jelle Licht <jlicht@fsfe.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix import npm-binary)
+ #:use-module ((gnu services configuration) #:select (alist?))
+ #:use-module (gcrypt hash)
+ #:use-module (gnu packages)
+ #:use-module (guix base32)
+ #:use-module (guix http-client)
+ #:use-module (guix import json)
+ #:use-module (guix import utils)
+ #:use-module (guix memoization)
+ #:use-module (guix utils)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (json)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-41)
+ #:use-module (srfi srfi-9)
+ #:use-module (web client)
+ #:use-module (web response)
+ #:use-module (web uri)
+ #:export (npm-binary-recursive-import
+ npm-binary->guix-package
+ %npm-registry
+ make-versioned-package
+ name+version->symbol))
+
+;; Autoload Guile-Semver so we only have a soft dependency.
+(module-autoload! (current-module)
+ '(semver)
+ '(string->semver semver? semver->string semver=? semver>?))
+(module-autoload! (current-module)
+ '(semver ranges)
+ '(*semver-range-any* string->semver-range semver-range-contains?))
+
+;; Dist-tags
+(define-json-mapping <dist-tags> make-dist-tags dist-tags?
+ json->dist-tags
+ (latest dist-tags-latest "latest" string->semver))
+
+(define-record-type <versioned-package>
+ (make-versioned-package name version)
+ versioned-package?
+ (name versioned-package-name) ;string
+ (version versioned-package-version)) ;string
+
+(define (dependencies->versioned-packages entries)
+ (match entries
+ (((names . versions) ...)
+ (map make-versioned-package names versions))
+ (_ '())))
+
+(define (extract-license license-string)
+ (if (unspecified? license-string)
+ 'unspecified!
+ (spdx-string->license license-string)))
+
+(define-json-mapping <dist> make-dist dist?
+ json->dist
+ (tarball dist-tarball))
+
+(define (empty-or-string s)
+ (if (string? s) s ""))
+
+(define-json-mapping <package-revision> make-package-revision package-revision?
+ json->package-revision
+ (name package-revision-name)
+ (version package-revision-version "version" ;semver
+ string->semver)
+ (home-page package-revision-home-page "homepage") ;string
+ (dependencies package-revision-dependencies ;list of versioned-package
+ "dependencies"
+ dependencies->versioned-packages)
+ (dev-dependencies package-revision-dev-dependencies ;list of versioned-package
+ "devDependencies" dependencies->versioned-packages)
+ (peer-dependencies package-revision-peer-dependencies ;list of versioned-package
+ "peerDependencies" dependencies->versioned-packages)
+ (license package-revision-license "license" ;license | #f
+ (match-lambda
+ ((? unspecified?) #f)
+ ((? string? str) (spdx-string->license str))
+ ((? alist? alist)
+ (match (assoc "type" alist)
+ ((_ . (? string? type))
+ (spdx-string->license type))
+ (_ #f)))))
+ (description package-revision-description ;string
+ "description" empty-or-string)
+ (dist package-revision-dist "dist" json->dist)) ;dist
+
+(define (versions->package-revisions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map json->package-revision package-spec))
+ (_ '())))
+
+(define (versions->package-versions versions)
+ (match versions
+ (((version . package-spec) ...)
+ (map string->semver versions))
+ (_ '())))
+
+(define-json-mapping <meta-package> make-meta-package meta-package?
+ json->meta-package
+ (name meta-package-name) ;string
+ (description meta-package-description) ;string
+ (dist-tags meta-package-dist-tags "dist-tags" json->dist-tags) ;dist-tags
+ (revisions meta-package-revisions "versions" versions->package-revisions))
+
+(define %npm-registry
+ (make-parameter "https://registry.npmjs.org"))
+(define %default-page "https://www.npmjs.com/package")
+
+(define (lookup-meta-package name)
+ (let ((json (json-fetch (string-append (%npm-registry) "/" (uri-encode name)))))
+ (and=> json json->meta-package)))
+
+(define lookup-meta-package* (memoize lookup-meta-package))
+
+(define (meta-package-versions meta)
+ (map package-revision-version
+ (meta-package-revisions meta)))
+
+(define (meta-package-latest meta)
+ (and=> (meta-package-dist-tags meta) dist-tags-latest))
+
+(define* (meta-package-package meta #:optional
+ (version (meta-package-latest meta)))
+ (match version
+ ((? semver?) (find (lambda (revision)
+ (semver=? version (package-revision-version revision)))
+ (meta-package-revisions meta)))
+ ((? string?) (meta-package-package meta (string->semver version)))
+ (_ #f)))
+
+(define* (semver-latest svs #:optional (svr *semver-range-any*))
+ (find (cut semver-range-contains? svr <>)
+ (sort svs semver>?)))
+
+(define* (resolve-package name #:optional (svr *semver-range-any*))
+ (let ((meta (lookup-meta-package* name)))
+ (and meta
+ (let* ((version (semver-latest (or (meta-package-versions meta) '()) svr))
+ (pkg (meta-package-package meta version)))
+ pkg))))
+
+\f
+;;;
+;;; Converting packages
+;;;
+
+(define (hash-url url)
+ "Downloads the resource at URL and computes the base32 hash for it."
+ (bytevector->nix-base32-string (port-sha256 (http-fetch url))))
+
+(define (npm-name->name npm-name)
+ "Return a Guix package name for the npm package with name NPM-NAME."
+ (define (clean name)
+ (string-map (lambda (chr) (if (char=? chr #\/) #\- chr))
+ (string-filter (negate (cut char=? <> #\@)) name)))
+ (guix-name "node-" (clean npm-name)))
+
+(define (name+version->symbol name version)
+ (string->symbol (string-append name "-" version)))
+
+(define (package-revision->symbol package)
+ (let* ((npm-name (package-revision-name package))
+ (version (semver->string (package-revision-version package)))
+ (name (npm-name->name npm-name)))
+ (name+version->symbol name version)))
+
+(define (npm-package->package-sexp npm-package)
+ "Return the `package' s-expression for an NPM-PACKAGE."
+ (define resolve-spec
+ (match-lambda
+ (($ <versioned-package> name version)
+ (resolve-package name (string->semver-range version)))))
+
+ (if (package-revision? npm-package)
+ (let ((name (package-revision-name npm-package))
+ (version (package-revision-version npm-package))
+ (home-page (package-revision-home-page npm-package))
+ (dependencies (package-revision-dependencies npm-package))
+ (dev-dependencies (package-revision-dev-dependencies npm-package))
+ (peer-dependencies (package-revision-peer-dependencies npm-package))
+ (license (package-revision-license npm-package))
+ (description (package-revision-description npm-package))
+ (dist (package-revision-dist npm-package)))
+ (let* ((name (npm-name->name name))
+ (url (dist-tarball dist))
+ (home-page (if (string? home-page)
+ home-page
+ (string-append %default-page "/" (uri-encode name))))
+ (synopsis description)
+ (resolved-deps (map resolve-spec
+ (append dependencies peer-dependencies)))
+ (peer-names (map versioned-package-name peer-dependencies))
+ ;; lset-difference for treating peer-dependencies as dependencies,
+ ;; which leads to dependency cycles. lset-union for treating them as
+ ;; (ignored) dev-dependencies, which leads to broken packages.
+ (dev-names
+ (lset-union string=
+ (map versioned-package-name dev-dependencies)
+ peer-names))
+ (extra-phases
+ (match dev-names
+ (() '())
+ ((dev-names ...)
+ `((add-after 'patch-dependencies 'delete-dev-dependencies
+ (lambda _
+ (delete-dependencies '(,@(reverse dev-names))))))))))
+ (values
+ `(package
+ (name ,name)
+ (version ,(semver->string (package-revision-version npm-package)))
+ (source (origin
+ (method url-fetch)
+ (uri ,url)
+ (sha256 (base32 ,(hash-url url)))))
+ (build-system node-build-system)
+ (arguments
+ (list
+ #:tests? #f
+ #:phases
+ #~(modify-phases %standard-phases
+ (delete 'build)
+ ,@extra-phases)))
+ ,@(match dependencies
+ (() '())
+ ((dependencies ...)
+ `((inputs
+ (list ,@(map package-revision->symbol resolved-deps))))))
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,license))
+ (map (match-lambda (($ <package-revision> name version)
+ (list name (semver->string version))))
+ resolved-deps))))
+ (values #f '())))
+
+\f
+;;;
+;;; Interface
+;;;
+
+(define npm-binary->guix-package
+ (lambda* (name #:key (version *semver-range-any*) #:allow-other-keys)
+ (let* ((svr (match version
+ ((? string?) (string->semver-range version))
+ (_ version)))
+ (pkg (resolve-package name svr)))
+ (npm-package->package-sexp pkg))))
+
+(define* (npm-binary-recursive-import package-name #:key version)
+ (recursive-import package-name
+ #:repo->guix-package (memoize npm-binary->guix-package)
+ #:version version
+ #:guix-name npm-name->name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 1f34cab088..d724f2bca3 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -49,7 +49,7 @@ (define %standard-import-options '())
(define importers '("gnu" "pypi" "cpan" "hackage" "stackage" "egg" "elpa"
"gem" "go" "cran" "crate" "texlive" "json" "opam"
- "minetest" "elm" "hexpm" "composer"))
+ "minetest" "elm" "hexpm" "composer" "npm-binary"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/npm-binary.scm b/guix/scripts/import/npm-binary.scm
new file mode 100644
index 0000000000..3403a69bcc
--- /dev/null
+++ b/guix/scripts/import/npm-binary.scm
@@ -0,0 +1,121 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2014 David Thompson <davet@gnu.org>
+;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Timothy Sample <samplet@ngyro.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import npm-binary)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import npm-binary)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (srfi srfi-41)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-npm-binary))
+
+\f
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import npm-binary PACKAGE-NAME [VERSION]
+Import and convert the npm package PACKAGE-NAME using the
+`node-build-system' (but without building the package from source)."))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -r, --recursive import packages recursively"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import npm-binary")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ %standard-import-options))
+
+(define* (package-name->name+version* spec)
+ "Given SPEC, a package name like \"@scope/pac@^0.9.1\", return two values:
+\"@scope/pac\" and \"^0.9.1\". When the version part is unavailable, SPEC and \"*\"
+are returned. The first part may start with '@', the latter part must not contain
+contain '@'."
+ (match (string-rindex spec delimiter)
+ (#f (values spec "*"))
+ (0 (values spec "*"))
+ (idx (values (substring spec 0 idx)
+ (substring spec (1+ idx))))))
+
+\f
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-npm-binary . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (parse-command-line args %options (list %default-options)
+ #:build-options? #f))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (match args
+ ((spec)
+ (define-values (package-name version)
+ (package-name->name+version* spec))
+ (match (if (assoc-ref opts 'recursive)
+ ;; Recursive import
+ (npm-binary-recursive-import package-name #:version version)
+ ;; Single import
+ (npm-binary->guix-package package-name #:version version))
+ ((or #f '())
+ (leave (G_ "failed to download meta-data for package '~a@~a'~%")
+ package-name version))
+ (('package etc ...) `(package ,@etc))
+ ((? list? sexps)
+ (map (match-lambda
+ ((and ('package ('name name) ('version version) . rest) pkg)
+ `(define-public ,(name+version->symbol name version)
+ ,pkg))
+ (_ #f))
+ sexps))))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%"))))))
diff --git a/tests/npm-binary.scm b/tests/npm-binary.scm
new file mode 100755
index 0000000000..cf85e572b3
--- /dev/null
+++ b/tests/npm-binary.scm
@@ -0,0 +1,146 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2024 Jelle Licht <jlicht@fsfe.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+(define-module (test-npm-binary)
+ #:use-module ((gcrypt hash)
+ #:select ((sha256 . gcrypt-sha256)))
+ #:use-module (guix import npm-binary)
+ #:use-module (guix base32)
+ #:use-module (guix tests)
+ #:use-module (srfi srfi-64)
+ #:use-module (ice-9 iconv)
+ #:use-module (ice-9 match)
+ #:export (run-test))
+
+(define foo-json
+ "{
+ \"name\": \"foo\",
+ \"dist-tags\": {
+ \"latest\": \"1.2.3\",
+ \"next\": \"2.0.1-beta4\"
+ },
+ \"description\": \"General purpose utilities to foo your bars\",
+ \"homepage\": \"https://github.com/quartz/foo\",
+ \"repository\": \"quartz/foo\",
+ \"versions\": {
+ \"1.2.3\": {
+ \"name\": \"foo\",
+ \"description\": \"General purpose utilities to foo your bars\",
+ \"version\": \"1.2.3\",
+ \"author\": \"Jelle Licht <jlicht@fsfe.org>\",
+ \"devDependencies\": {
+ \"node-megabuilder\": \"^0.0.2\"
+ },
+ \"dependencies\": {
+ \"bar\": \"^0.1.0\"
+ },
+ \"repository\": {
+ \"url\": \"quartz/foo\"
+ },
+ \"homepage\": \"https://github.com/quartz/foo\",
+ \"license\": \"MIT\",
+ \"dist\": {
+ \"tarball\": \"https://registry.npmjs.org/foo/-/foo-1.2.3.tgz\"
+ }
+ }
+ }
+}")
+
+(define bar-json
+ "{
+ \"name\": \"bar\",
+ \"dist-tags\": {
+ \"latest\": \"0.1.2\"
+ },
+ \"description\": \"Core module in FooBar\",
+ \"homepage\": \"https://github.com/quartz/bar\",
+ \"repository\": \"quartz/bar\",
+ \"versions\": {
+ \"0.1.2\": {
+ \"name\": \"bar\",
+ \"description\": \"Core module in FooBar\",
+ \"version\": \"0.1.2\",
+ \"author\": \"Jelle Licht <jlicht@fsfe.org>\",
+ \"repository\": {
+ \"url\": \"quartz/bar\"
+ },
+ \"homepage\": \"https://github.com/quartz/bar\",
+ \"license\": \"MIT\",
+ \"dist\": {
+ \"tarball\": \"https://registry.npmjs.org/bar/-/bar-0.1.2.tgz\"
+ }
+ }
+ }
+}")
+
+(define test-source-hash
+ "")
+
+(define test-source
+ "Empty file\n")
+
+(define have-guile-semver?
+ (false-if-exception (resolve-interface '(semver))))
+
+(test-begin "npm")
+
+(unless have-guile-semver? (test-skip 1))
+(test-assert "npm-binary->guix-package"
+ (mock ((guix http-client) http-fetch
+ (lambda* (url #:rest _)
+ (match url
+ ("https://registry.npmjs.org/foo"
+ (values (open-input-string foo-json)
+ (string-length foo-json)))
+ ("https://registry.npmjs.org/bar"
+ (values (open-input-string bar-json)
+ (string-length bar-json)))
+ ("https://registry.npmjs.org/foo/-/foo-1.2.3.tgz"
+ (set! test-source-hash
+ (bytevector->nix-base32-string
+ (gcrypt-sha256 (string->bytevector test-source "utf-8"))))
+ (values (open-input-string test-source)
+ (string-length test-source))))))
+ (match (npm-binary->guix-package "foo")
+ (`(package
+ (name "node-foo")
+ (version "1.2.3")
+ (source (origin
+ (method url-fetch)
+ (uri "https://registry.npmjs.org/foo/-/foo-1.2.3.tgz")
+ (sha256
+ (base32
+ ,test-source-hash))))
+ (build-system node-build-system)
+ (arguments
+ (list #:tests? #f
+ #:phases
+ (gexp (modify-phases %standard-phases
+ (delete 'build)
+ (add-after 'patch-dependencies 'delete-dev-dependencies
+ (lambda _
+ (delete-dependencies '("node-megabuilder"))))))))
+ (inputs (list node-bar-0.1.2))
+ (home-page "https://github.com/quartz/foo")
+ (synopsis "General purpose utilities to foo your bars")
+ (description "General purpose utilities to foo your bars")
+ (license license:expat))
+ #t)
+ (x
+ (pk 'fail x #f)))))
+
+(test-end "npm")
base-commit: 4d79a9cd6b5f0d8c5afbab0c6b70ae42740d5470
--
2.41.0
^ permalink raw reply related [flat|nested] 18+ messages in thread