* [bug#47310] [PATCH 2/4] import: go: Improve synopsis and description parsing.
2021-03-22 4:20 ` [bug#47310] [PATCH 1/4] import: utils: Refactor maybe-inputs and add maybe-propagated-inputs Maxim Cournoyer
@ 2021-03-22 4:20 ` Maxim Cournoyer
2021-03-22 4:20 ` [bug#47310] [PATCH 3/4] import: go: Add an option to use pinned versions Maxim Cournoyer
2021-03-22 4:20 ` [bug#47310] [PATCH 4/4] import: go: Append version to symbol name in the pinned version mode Maxim Cournoyer
2 siblings, 0 replies; 12+ messages in thread
From: Maxim Cournoyer @ 2021-03-22 4:20 UTC (permalink / raw)
To: 47310; +Cc: Maxim Cournoyer
* guix/import/go.scm (%strict-tokenizer?): Set parameter to #t.
(go-path-escape): Redefine to prevent inlining.
(http-get*): Replace by ...
(http-fetch*): this ...
(json-fetch*): New procedure.
(go.pkg.dev-info): Use http-fetch*.
(go-package-licenses): Rewrite in terms of go.pkg.dev-info.
(go-package-description): Likewise.
(go-package-synopsis): Likewise.
(fetch-go.mod): Use the memoized http-fetch*.
(parse-go.mod): Adjust to receive content as a string.
(fetch-module-meta-data): Adjust to use http-fetch*.
(go-module->guix-package): Adjust to the modified fetch-go.mod return value.
[inputs]: Use propagated inputs, which is the most common situations for Go
libraries.
[description]: Beautify description.
[licenses]: Do no check for #f. The result of the license parsing is always a
list.
* tests/go.scm: Adjust following above changes.
---
guix/import/go.scm | 213 ++++++++++++++++++++++++++----------------
guix/import/utils.scm | 4 +-
tests/go.scm | 75 +++++++--------
3 files changed, 170 insertions(+), 122 deletions(-)
diff --git a/guix/import/go.scm b/guix/import/go.scm
index 7452b4c903..2ef2d872e8 100644
--- a/guix/import/go.scm
+++ b/guix/import/go.scm
@@ -32,7 +32,7 @@
#:use-module (guix http-client)
#:use-module ((guix licenses) #:prefix license:)
#:use-module (guix memoization)
- #:autoload (htmlprag) (html->sxml) ;from Guile-Lib
+ #:use-module (htmlprag) ;from Guile-Lib
#:autoload (guix git) (update-cached-checkout)
#:autoload (gcrypt hash) (open-hash-port hash-algorithm sha256)
#:autoload (guix serialization) (write-file)
@@ -42,20 +42,28 @@
#:use-module (ice-9 rdelim)
#:use-module (ice-9 receive)
#:use-module (ice-9 regex)
+ #:use-module (ice-9 textual-ports)
#:use-module ((rnrs io ports) #:select (call-with-port))
#:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-2)
#:use-module (srfi srfi-9)
#:use-module (srfi srfi-11)
#:use-module (srfi srfi-26)
- #:use-module (sxml xpath)
+ #:use-module (sxml match)
+ #:use-module ((sxml xpath) #:renamer (lambda (s)
+ (if (eq? 'filter s)
+ 'xfilter
+ s)))
#:use-module (web client)
#:use-module (web response)
#:use-module (web uri)
- #:export (go-path-escape
- go-module->guix-package
+ #:export (go-module->guix-package
go-module-recursive-import))
+;;; Parameterize htmlprag to parse valid HTML more reliably.
+(%strict-tokenizer? #t)
+
;;; Commentary:
;;;
;;; (guix import go) attempts to make it easier to create Guix package
@@ -89,6 +97,14 @@
;;; Code:
+(define http-fetch*
+ ;; Like http-fetch, but memoized and returning the body as a string.
+ (memoize (lambda args
+ (call-with-port (apply http-fetch args) get-string-all))))
+
+(define json-fetch*
+ (memoize json-fetch))
+
(define (go-path-escape path)
"Escape a module path by replacing every uppercase letter with an
exclamation mark followed with its lowercase equivalent, as per the module
@@ -98,54 +114,73 @@ https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths)."
(string-append "!" (string-downcase (match:substring occurrence))))
(regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
+;; Prevent inlining of this procedure, which is accessed by unit tests.
+(set! go-path-escape go-path-escape)
+
+(define (go.pkg.dev-info name)
+ (http-fetch* (string-append "https://pkg.go.dev/" name)))
+
(define (go-module-latest-version goproxy-url module-path)
"Fetch the version number of the latest version for MODULE-PATH from the
given GOPROXY-URL server."
- (assoc-ref (json-fetch (format #f "~a/~a/@latest" goproxy-url
- (go-path-escape module-path)))
+ (assoc-ref (json-fetch* (format #f "~a/~a/@latest" goproxy-url
+ (go-path-escape module-path)))
"Version"))
-
(define (go-package-licenses name)
"Retrieve the list of licenses that apply to NAME, a Go package or module
-name (e.g. \"github.com/golang/protobuf/proto\"). The data is scraped from
-the https://pkg.go.dev/ web site."
- (let*-values (((url) (string-append "https://pkg.go.dev/" name
- "?tab=licenses"))
- ((response body) (http-get url))
- ;; Extract the text contained in a h2 child node of any
- ;; element marked with a "License" class attribute.
- ((select) (sxpath `(// (* (@ (equal? (class "License"))))
- h2 // *text*))))
- (and (eq? (response-code response) 200)
- (match (select (html->sxml body))
- (() #f) ;nothing selected
- (licenses licenses)))))
-
-(define (go.pkg.dev-info name)
- (http-get (string-append "https://pkg.go.dev/" name)))
-(define go.pkg.dev-info*
- (memoize go.pkg.dev-info))
+name (e.g. \"github.com/golang/protobuf/proto\")."
+ (let* ((body (go.pkg.dev-info (string-append name "?tab=licenses")))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ (select (sxpath `(// (* (@ (equal? (class "License"))))
+ h2 // *text*))))
+ (select (html->sxml body))))
+
+(define (sxml->texi sxml-node)
+ "A very basic SXML to Texinfo converter which attempts to preserve HTML
+formatting and links as text."
+ (sxml-match sxml-node
+ ((strong ,text)
+ (format #f "@strong{~a}" text))
+ ((a (@ (href ,url)) ,text)
+ (format #f "@url{~a,~a}" url text))
+ ((code ,text)
+ (format #f "@code{~a}" text))
+ (,something-else something-else)))
(define (go-package-description name)
"Retrieve a short description for NAME, a Go package name,
-e.g. \"google.golang.org/protobuf/proto\". The data is scraped from the
-https://pkg.go.dev/ web site."
- (let*-values (((response body) (go.pkg.dev-info* name))
- ;; Extract the text contained in a h2 child node of any
- ;; element marked with a "License" class attribute.
- ((select) (sxpath
- `(// (section
- (@ (equal? (class "Documentation-overview"))))
- (p 1)))))
- (and (eq? (response-code response) 200)
- (match (select (html->sxml body))
- (() #f) ;nothing selected
- (((p . strings))
- ;; The paragraph text is returned as a list of strings embedding
- ;; newline characters. Join them and strip the newline
- ;; characters.
- (string-delete #\newline (string-join strings)))))))
+e.g. \"google.golang.org/protobuf/proto\"."
+ (let* ((body (go.pkg.dev-info name))
+ (sxml (html->sxml body))
+ (overview ((sxpath
+ `(//
+ (* (@ (equal? (class "Documentation-overview"))))
+ (p 1))) sxml))
+ ;; Sometimes, the first paragraph just contains images/links that
+ ;; has only "\n" for text. The following filter is designed to
+ ;; omit it.
+ (contains-text? (lambda (node)
+ (remove string-null?
+ (map string-trim-both
+ (filter (node-typeof? '*text*)
+ (cdr node))))))
+ (select-content (sxpath
+ `(//
+ (* (@ (equal? (class "UnitReadme-content"))))
+ div // p ,(xfilter contains-text?))))
+ ;; Fall-back to use content; this is less desirable as it is more
+ ;; verbose, but not every page has an overview.
+ (description (if (not (null? overview))
+ overview
+ (select-content sxml)))
+ (description* (and (not (null? description))
+ (first description))))
+ (match description*
+ (() #f) ;nothing selected
+ ((p elements ...)
+ (apply string-append (filter string? (map sxml->texi elements)))))))
(define (go-package-synopsis module-name)
"Retrieve a short synopsis for a Go module named MODULE-NAME,
@@ -153,17 +188,17 @@ e.g. \"google.golang.org/protobuf\". The data is scraped from
the https://pkg.go.dev/ web site."
;; Note: Only the *module* (rather than package) page has the README title
;; used as a synopsis on the https://pkg.go.dev web site.
- (let*-values (((response body) (go.pkg.dev-info* module-name))
- ;; Extract the text contained in a h2 child node of any
- ;; element marked with a "License" class attribute.
- ((select) (sxpath
- `(// (div (@ (equal? (class "UnitReadme-content"))))
- // h3 *text*))))
- (and (eq? (response-code response) 200)
- (match (select (html->sxml body))
- (() #f) ;nothing selected
- ((title more ...) ;title is the first string of the list
- (string-trim-both title))))))
+ (let* ((url (string-append "https://pkg.go.dev/" module-name))
+ (body (http-fetch* url))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ (select-title (sxpath
+ `(// (div (@ (equal? (class "UnitReadme-content"))))
+ // h3 *text*))))
+ (match (select-title (html->sxml body))
+ (() #f) ;nothing selected
+ ((title more ...) ;title is the first string of the list
+ (string-trim-both title)))))
(define (list->licenses licenses)
"Given a list of LICENSES mostly following the SPDX conventions, return the
@@ -188,13 +223,13 @@ corresponding Guix license or 'unknown-license!"
'unknown-license!)))
licenses))
-(define (fetch-go.mod goproxy-url module-path version)
- "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
-and VERSION."
- (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
+(define (fetch-go.mod goproxy module-path version)
+ "Fetch go.mod from the given GOPROXY server for the given MODULE-PATH
+and VERSION and return an input port."
+ (let ((url (format #f "~a/~a/@v/~a.mod" goproxy
(go-path-escape module-path)
(go-path-escape version))))
- (http-fetch url)))
+ (http-fetch* url)))
(define %go.mod-require-directive-rx
;; A line in a require directive is composed of a module path and
@@ -215,9 +250,8 @@ and VERSION."
"[[:blank:]]+" "=>" "[[:blank:]]+"
"([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
-(define (parse-go.mod port)
- "Parse the go.mod file accessible via the input PORT, returning a list of
-requirements."
+(define (parse-go.mod content)
+ "Parse the go.mod file CONTENT, returning a list of requirements."
(define-record-type <results>
(make-results requirements replacements)
results?
@@ -228,7 +262,7 @@ requirements."
(define (toplevel results)
"Main parser, RESULTS is a pair of alist serving as accumulator for
all encountered requirements and replacements."
- (let ((line (read-line port)))
+ (let ((line (read-line)))
(cond
((eof-object? line)
;; parsing ended, give back the result
@@ -254,7 +288,7 @@ requirements."
(toplevel results)))))
(define (in-require results)
- (let ((line (read-line port)))
+ (let ((line (read-line)))
(cond
((eof-object? line)
;; this should never happen here but we ignore silently
@@ -266,7 +300,7 @@ requirements."
(in-require (require-directive results line))))))
(define (in-replace results)
- (let ((line (read-line port)))
+ (let ((line (read-line)))
(cond
((eof-object? line)
;; this should never happen here but we ignore silently
@@ -305,7 +339,9 @@ requirements."
(($ <results> requirements replaced)
(make-results (alist-cons module-path version requirements) replaced)))))
- (let ((results (toplevel (make-results '() '()))))
+ (let ((results (with-input-from-string content
+ (lambda _
+ (toplevel (make-results '() '()))))))
(match results
(($ <results> requirements replaced)
;; At last we remove replaced modules from the requirements list
@@ -324,8 +360,10 @@ requirements."
(url-prefix vcs-url-prefix)
(root-regex vcs-root-regex)
(type vcs-type))
+
(define (make-vcs prefix regexp type)
- (%make-vcs prefix (make-regexp regexp) type))
+ (%make-vcs prefix (make-regexp regexp) type))
+
(define known-vcs
;; See the following URL for the official Go equivalent:
;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
@@ -384,6 +422,14 @@ hence the need to derive this information."
"." "-")
"/" "-"))))
+(define (strip-.git-suffix/maybe repo-url)
+ "Strip a repository URL '.git' suffix from REPO-URL if hosted at GitHub."
+ (match repo-url
+ ((and (? (cut string-prefix? "https://github.com" <>))
+ (? (cut string-suffix? ".git" <>)))
+ (string-drop-right repo-url 4))
+ (_ repo-url)))
+
(define-record-type <module-meta>
(make-module-meta import-prefix vcs repo-root)
module-meta?
@@ -396,21 +442,22 @@ hence the need to derive this information."
because goproxy servers don't currently provide all the information needed to
build a package."
;; <meta name="go-import" content="import-prefix vcs repo-root">
- (let* ((port (http-fetch (format #f "https://~a?go-get=1" module-path)))
+ (let* ((meta-data (http-fetch* (format #f "https://~a?go-get=1" module-path)))
(select (sxpath `(// head (meta (@ (equal? (name "go-import"))))
// content))))
- (match (select (call-with-port port html->sxml))
- (() #f) ;nothing selected
+ (match (select (html->sxml meta-data))
+ (() #f) ;nothing selected
(((content content-text))
(match (string-split content-text #\space)
((root-path vcs repo-url)
- (make-module-meta root-path (string->symbol vcs) repo-url)))))))
+ (make-module-meta root-path (string->symbol vcs)
+ (strip-.git-suffix/maybe repo-url))))))))
-(define (module-meta-data-repo-url meta-data goproxy-url)
+(define (module-meta-data-repo-url meta-data goproxy)
"Return the URL where the fetcher which will be used can download the
source."
(if (member (module-meta-vcs meta-data) '(fossil mod))
- goproxy-url
+ goproxy
(module-meta-repo-root meta-data)))
;; XXX: Copied from (guix scripts hash).
@@ -463,6 +510,9 @@ control system is being used."
(method git-fetch)
(uri (git-reference
(url ,vcs-repo-url)
+ ;; This is done because the version field of the package,
+ ;; which the generated quoted expression refers to, has been
+ ;; stripped of any 'v' prefixed.
(commit ,(if (and plain-version? v-prefixed?)
'(string-append "v" version)
'(go-version->git-ref version)))))
@@ -502,8 +552,8 @@ control system is being used."
(define* (go-module->guix-package module-path #:key
(goproxy-url "https://proxy.golang.org"))
(let* ((latest-version (go-module-latest-version goproxy-url module-path))
- (port (fetch-go.mod goproxy-url module-path latest-version))
- (dependencies (map car (call-with-port port parse-go.mod)))
+ (content (fetch-go.mod goproxy-url module-path latest-version))
+ (dependencies (map car (parse-go.mod content)))
(guix-name (go-module->guix-package-name module-path))
(root-module-path (module-path->repository-root module-path))
;; The VCS type and URL are not included in goproxy information. For
@@ -524,14 +574,17 @@ control system is being used."
(build-system go-build-system)
(arguments
'(#:import-path ,root-module-path))
- ,@(maybe-inputs (map go-module->guix-package-name dependencies))
+ ,@(maybe-propagated-inputs
+ (map go-module->guix-package-name dependencies))
(home-page ,(format #f "https://~a" root-module-path))
(synopsis ,synopsis)
- (description ,description)
- (license ,(match (and=> licenses list->licenses)
- ((license) license)
- ((licenses ...) `(list ,@licenses))
- (x x))))
+ (description ,(and=> description beautify-description))
+ (license ,(match (list->licenses licenses)
+ (() #f) ;unknown license
+ ((license) ;a single license
+ license)
+ ((license ...) ;a list of licenses
+ `(list ,@license)))))
dependencies)))
(define go-module->guix-package* (memoize go-module->guix-package))
diff --git a/guix/import/utils.scm b/guix/import/utils.scm
index e55258e306..21fafad388 100644
--- a/guix/import/utils.scm
+++ b/guix/import/utils.scm
@@ -445,8 +445,8 @@ obtain a node's uniquely identifying \"key\"."
"Return a list of package expressions for PACKAGE-NAME and all its
dependencies, sorted in topological order. For each package,
call (REPO->GUIX-PACKAGE NAME :KEYS version repo), which should return a
-package expression and a list of dependencies; call (GUIX-NAME NAME) to
-obtain the Guix package name corresponding to the upstream name."
+package expression and a list of dependencies; call (GUIX-NAME PACKAGE-NAME)
+to obtain the Guix package name corresponding to the upstream name."
(define-record-type <node>
(make-node name version package dependencies)
node?
diff --git a/tests/go.scm b/tests/go.scm
index 6ab99f508a..fa8fa7a2a6 100644
--- a/tests/go.scm
+++ b/tests/go.scm
@@ -180,13 +180,9 @@ require github.com/kr/pretty v0.2.1
(define (testing-parse-mod name expected input)
(define (inf? p1 p2)
(string<? (car p1) (car p2)))
- (let ((input-port (open-input-string input)))
- (test-equal name
- (sort expected inf?)
- (sort
- ( (@@ (guix import go) parse-go.mod)
- input-port)
- inf?))))
+ (test-equal name
+ (sort expected inf?)
+ (sort ((@@ (guix import go) parse-go.mod) input) inf?)))
(testing-parse-mod "parse-go.mod-simple"
'(("good/thing" . "v1.4.5")
@@ -249,44 +245,43 @@ require github.com/kr/pretty v0.2.1
(test-equal "go-module->guix-package"
'(package
- (name "go-github-com-go-check-check")
- (version "0.0.0-20201130134442-10cb98267c6c")
- (source
- (origin
- (method git-fetch)
- (uri (git-reference
- (url "https://github.com/go-check/check.git")
- (commit (go-version->git-ref version))))
- (file-name (git-file-name name version))
- (sha256
- (base32
- "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"))))
- (build-system go-build-system)
- (arguments
- (quote (#:import-path "github.com/go-check/check")))
- (inputs
- (quasiquote (("go-github-com-kr-pretty"
- (unquote go-github-com-kr-pretty)))))
- (home-page "https://github.com/go-check/check")
- (synopsis "Instructions")
- (description #f)
- (license license:bsd-2))
+ (name "go-github-com-go-check-check")
+ (version "0.0.0-20201130134442-10cb98267c6c")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/go-check/check")
+ (commit (go-version->git-ref version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"))))
+ (build-system go-build-system)
+ (arguments
+ '(#:import-path "github.com/go-check/check"))
+ (propagated-inputs
+ `(("go-github-com-kr-pretty" ,go-github-com-kr-pretty)))
+ (home-page "https://github.com/go-check/check")
+ (synopsis "Instructions")
+ (description "Package check is a rich testing extension for Go's testing \
+package.")
+ (license license:bsd-2))
;; Replace network resources with sample data.
(call-with-temporary-directory
(lambda (checkout)
(mock ((web client) http-get
(mock-http-get fixtures-go-check-test))
- (mock ((guix http-client) http-fetch
- (mock-http-fetch fixtures-go-check-test))
- (mock ((guix git) update-cached-checkout
- (lambda* (url #:key ref)
- ;; Return an empty directory and its hash.
- (values checkout
- (nix-base32-string->bytevector
- "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5")
- #f)))
- (go-module->guix-package "github.com/go-check/check")))))))
+ (mock ((guix http-client) http-fetch
+ (mock-http-fetch fixtures-go-check-test))
+ (mock ((guix git) update-cached-checkout
+ (lambda* (url #:key ref)
+ ;; Return an empty directory and its hash.
+ (values checkout
+ (nix-base32-string->bytevector
+ "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5")
+ #f)))
+ (go-module->guix-package "github.com/go-check/check")))))))
(test-end "go")
-
--
2.30.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [bug#47310] [PATCH 3/4] import: go: Add an option to use pinned versions.
2021-03-22 4:20 ` [bug#47310] [PATCH 1/4] import: utils: Refactor maybe-inputs and add maybe-propagated-inputs Maxim Cournoyer
2021-03-22 4:20 ` [bug#47310] [PATCH 2/4] import: go: Improve synopsis and description parsing Maxim Cournoyer
@ 2021-03-22 4:20 ` Maxim Cournoyer
2021-04-10 3:13 ` bug#47310: " Maxim Cournoyer
2021-03-22 4:20 ` [bug#47310] [PATCH 4/4] import: go: Append version to symbol name in the pinned version mode Maxim Cournoyer
2 siblings, 1 reply; 12+ messages in thread
From: Maxim Cournoyer @ 2021-03-22 4:20 UTC (permalink / raw)
To: 47310; +Cc: Maxim Cournoyer
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=yes, Size: 29859 bytes --]
The ability to pin versions is handy when having to deal to packages that
bootstrap themselves through a chain of former versions. Not using pinned
versions in these case could introduce dependency cycles.
* guix/build-system/go.scm (guix)
(%go-version-rx): Rename to...
(%go-pseudo-version-rx): ... this. Simplify the regular expression, which in
turns makes it more robust.
* guix/build-system/go.scm (go-version->git-ref): Adjust following the above
rename.
(go-pseudo-version?): New predicate.
(go-module-latest-version): Rename to ...
(go-module-version-string): ... this. Rename goproxy-url argument to just
goproxy. Add a VERSION keyword argument, update docstring and adjust to have
it used.
(go-module-available-versions): New procedure.
(%go.mod-require-directive-rx): Document regexp.
(parse-go.mod): Harmonize the way dependencies are recorded to a list of lists
rather than a list of pairs, as done for other importers. Rewrite to directly pass
multiple values rather than a record object. Filter the replaced modules in a
functional style.
(go-module->guix-package): Add docstring.
[version, pin-versions?]: New arguments. Rename the GOPROXY-URL argument to
GOPROXY. Adjust to the new returned value of fetch-go.mod, which is a string.
Fail when the provided version doesn't exist. Return a list dependencies and
their versions when in pinned versions mode, else just the dependencies.
(go-module-recursive-import)[version, pin-versions?]: New arguments.
Honor the new arguments and guard against network errors.
* guix/scripts/import/go.scm (%default-options): Register a default value for
the goproxy argument.
(show-help): Document that a version can be specified. Remove the --version
argument and add a --pin-versions argument.
(%options)[version]: Remove option.
[pin-versions]: Add option.
(guix-import-go): Adjust so the version provided from the module name is
honored, along the new pin-versions? argument.
* tests/go.scm: Adjust and add new tests.
---
guix/build-system/go.scm | 24 ++--
guix/import/go.scm | 239 ++++++++++++++++++++++---------------
guix/scripts/import/go.scm | 70 ++++++-----
tests/go.scm | 64 +++++-----
4 files changed, 234 insertions(+), 163 deletions(-)
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm
index 0e2c1cd2ee..8f55796e86 100644
--- a/guix/build-system/go.scm
+++ b/guix/build-system/go.scm
@@ -31,6 +31,7 @@
go-build
go-build-system
+ go-pseudo-version?
go-version->git-ref))
;; Commentary:
@@ -40,17 +41,19 @@
;;
;; Code:
-(define %go-version-rx
+(define %go-pseudo-version-rx
+ ;; Match only the end of the version string; this is so that matching the
+ ;; more complex leading semantic version pattern is not required.
(make-regexp (string-append
- "(v?[0-9]\\.[0-9]\\.[0-9])" ;"v" prefix can be omitted in version prefix
- "(-|-pre\\.0\\.|-0\\.)" ;separator
- "([0-9]{14})-" ;timestamp
- "([0-9A-Fa-f]{12})"))) ;commit hash
+ "([0-9]{14}-)" ;timestamp
+ "([0-9A-Fa-f]{12})" ;commit hash
+ "(\\+incompatible)?$"))) ;optional +incompatible tag
(define (go-version->git-ref version)
"Parse VERSION, a \"pseudo-version\" as defined at
<https://golang.org/ref/mod#pseudo-versions>, and extract the commit hash from
-it, defaulting to full VERSION if a pseudo-version pattern is not recognized."
+it, defaulting to full VERSION (stripped from the \"+incompatible\" suffix if
+present) if a pseudo-version pattern is not recognized."
;; A module version like v1.2.3 is introduced by tagging a revision in the
;; underlying source repository. Untagged revisions can be referred to
;; using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef, where
@@ -65,11 +68,16 @@ it, defaulting to full VERSION if a pseudo-version pattern is not recognized."
(if (string-suffix? "+incompatible" version)
(string-drop-right version 13)
version))
- (match (regexp-exec %go-version-rx version)))
+ (match (regexp-exec %go-pseudo-version-rx version)))
(if match
- (match:substring match 4)
+ (match:substring match 2)
version)))
+(define (go-pseudo-version? version)
+ "True if VERSION is a Go pseudo-version, i.e., a version string made of a
+commit hash and its date rather than a proper release tag."
+ (regexp-exec %go-pseudo-version-rx version))
+
(define %go-build-system-modules
;; Build-side modules imported and used by default.
`((guix build go-build-system)
diff --git a/guix/import/go.scm b/guix/import/go.scm
index 2ef2d872e8..2376336265 100644
--- a/guix/import/go.scm
+++ b/guix/import/go.scm
@@ -49,6 +49,7 @@
#:use-module (srfi srfi-9)
#:use-module (srfi srfi-11)
#:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-34)
#:use-module (sxml match)
#:use-module ((sxml xpath) #:renamer (lambda (s)
(if (eq? 'filter s)
@@ -91,9 +92,7 @@
;;; assumption that there will be no collision.
;;; TODO list
-;;; - get correct hash in vcs->origin
-;;; - print partial result during recursive imports (need to catch
-;;; exceptions)
+;;; - get correct hash in vcs->origin for Mercurial and Subversion
;;; Code:
@@ -120,12 +119,26 @@ https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths)."
(define (go.pkg.dev-info name)
(http-fetch* (string-append "https://pkg.go.dev/" name)))
-(define (go-module-latest-version goproxy-url module-path)
- "Fetch the version number of the latest version for MODULE-PATH from the
-given GOPROXY-URL server."
- (assoc-ref (json-fetch* (format #f "~a/~a/@latest" goproxy-url
- (go-path-escape module-path)))
- "Version"))
+(define* (go-module-version-string goproxy name #:key version)
+ "Fetch the version string of the latest version for NAME from the given
+GOPROXY server, or for VERSION when specified."
+ (let ((file (if version
+ (string-append "@v/" version ".info")
+ "@latest")))
+ (assoc-ref (json-fetch* (format #f "~a/~a/~a"
+ goproxy (go-path-escape name) file))
+ "Version")))
+
+(define* (go-module-available-versions goproxy name)
+ "Retrieve the available versions for a given module from the module proxy.
+Versions are being returned **unordered** and may contain different versioning
+styles for the same package."
+ (let* ((url (string-append goproxy "/" (go-path-escape name) "/@v/list"))
+ (body (http-fetch* url))
+ (versions (remove string-null? (string-split body #\newline))))
+ (if (null? versions)
+ (list (go-module-version-string goproxy name)) ;latest version
+ versions)))
(define (go-package-licenses name)
"Retrieve the list of licenses that apply to NAME, a Go package or module
@@ -237,119 +250,119 @@ and VERSION and return an input port."
;; the end.
(make-regexp
(string-append
- "^[[:blank:]]*"
- "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
- "([[:blank:]]+//.*)?")))
+ "^[[:blank:]]*([^[:blank:]]+)[[:blank:]]+" ;the module path
+ "([^[:blank:]]+)" ;the version
+ "([[:blank:]]+//.*)?"))) ;an optional comment
(define %go.mod-replace-directive-rx
;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
;; | ModulePath [ Version ] "=>" ModulePath Version newline .
(make-regexp
(string-append
- "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
- "[[:blank:]]+" "=>" "[[:blank:]]+"
- "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
+ "([^[:blank:]]+)" ;the module path
+ "([[:blank:]]+([^[:blank:]]+))?" ;optional version
+ "[[:blank:]]+=>[[:blank:]]+"
+ "([^[:blank:]]+)" ;the file or module path
+ "([[:blank:]]+([^[:blank:]]+))?"))) ;the version (if a module path)
(define (parse-go.mod content)
"Parse the go.mod file CONTENT, returning a list of requirements."
- (define-record-type <results>
- (make-results requirements replacements)
- results?
- (requirements results-requirements)
- (replacements results-replacements))
;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
;; which we think necessary for our use case.
- (define (toplevel results)
- "Main parser, RESULTS is a pair of alist serving as accumulator for
- all encountered requirements and replacements."
+ (define (toplevel requirements replaced)
+ "This is the main parser. The results are accumulated in THE REQUIREMENTS
+and REPLACED lists."
(let ((line (read-line)))
(cond
((eof-object? line)
;; parsing ended, give back the result
- results)
+ (values requirements replaced))
((string=? line "require (")
;; a require block begins, delegate parsing to IN-REQUIRE
- (in-require results))
+ (in-require requirements replaced))
((string=? line "replace (")
;; a replace block begins, delegate parsing to IN-REPLACE
- (in-replace results))
+ (in-replace requirements replaced))
((string-prefix? "require " line)
- ;; a standalone require directive
- (let* ((stripped-line (string-drop line 8))
- (new-results (require-directive results stripped-line)))
- (toplevel new-results)))
+ ;; a require directive by itself
+ (let* ((stripped-line (string-drop line 8)))
+ (call-with-values
+ (lambda ()
+ (require-directive requirements replaced stripped-line))
+ toplevel)))
((string-prefix? "replace " line)
- ;; a standalone replace directive
- (let* ((stripped-line (string-drop line 8))
- (new-results (replace-directive results stripped-line)))
- (toplevel new-results)))
+ ;; a replace directive by itself
+ (let* ((stripped-line (string-drop line 8)))
+ (call-with-values
+ (lambda ()
+ (replace-directive requirements replaced stripped-line))
+ toplevel)))
(#t
;; unrecognised line, ignore silently
- (toplevel results)))))
+ (toplevel requirements replaced)))))
- (define (in-require results)
+ (define (in-require requirements replaced)
(let ((line (read-line)))
(cond
((eof-object? line)
;; this should never happen here but we ignore silently
- results)
+ (values requirements replaced))
((string=? line ")")
;; end of block, coming back to toplevel
- (toplevel results))
+ (toplevel requirements replaced))
(#t
- (in-require (require-directive results line))))))
+ (call-with-values (lambda ()
+ (require-directive requirements replaced line))
+ in-require)))))
- (define (in-replace results)
+ (define (in-replace requirements replaced)
(let ((line (read-line)))
(cond
((eof-object? line)
;; this should never happen here but we ignore silently
- results)
+ (values requirements replaced))
((string=? line ")")
;; end of block, coming back to toplevel
- (toplevel results))
+ (toplevel requirements replaced))
(#t
- (in-replace (replace-directive results line))))))
-
- (define (replace-directive results line)
- "Extract replaced modules and new requirements from replace directive
- in LINE and add to RESULTS."
- (match results
- (($ <results> requirements replaced)
- (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line))
- (module-path (match:substring rx-match 1))
- (version (match:substring rx-match 3))
- (new-module-path (match:substring rx-match 4))
- (new-version (match:substring rx-match 6))
- (new-replaced (alist-cons module-path version replaced))
- (new-requirements
- (if (string-match "^\\.?\\./" new-module-path)
- requirements
- (alist-cons new-module-path new-version requirements))))
- (make-results new-requirements new-replaced)))))
- (define (require-directive results line)
- "Extract requirement from LINE and add it to RESULTS."
+ (call-with-values (lambda ()
+ (replace-directive requirements replaced line))
+ in-replace)))))
+
+ (define (replace-directive requirements replaced line)
+ "Extract replaced modules and new requirements from the replace directive
+in LINE and add them to the REQUIREMENTS and REPLACED lists."
+ (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line))
+ (module-path (match:substring rx-match 1))
+ (version (match:substring rx-match 3))
+ (new-module-path (match:substring rx-match 4))
+ (new-version (match:substring rx-match 6))
+ (new-replaced (cons (list module-path version) replaced))
+ (new-requirements
+ (if (string-match "^\\.?\\./" new-module-path)
+ requirements
+ (cons (list new-module-path new-version) requirements))))
+ (values new-requirements new-replaced)))
+
+ (define (require-directive requirements replaced line)
+ "Extract requirement from LINE and augment the REQUIREMENTS and REPLACED
+lists."
(let* ((rx-match (regexp-exec %go.mod-require-directive-rx line))
(module-path (match:substring rx-match 1))
- ;; we saw double-quoted string in the wild without escape
- ;; sequences so we just trim the quotes
+ ;; Double-quoted strings were seen in the wild without escape
+ ;; sequences; trim the quotes to be on the safe side.
(module-path (string-trim-both module-path #\"))
(version (match:substring rx-match 2)))
- (match results
- (($ <results> requirements replaced)
- (make-results (alist-cons module-path version requirements) replaced)))))
-
- (let ((results (with-input-from-string content
- (lambda _
- (toplevel (make-results '() '()))))))
- (match results
- (($ <results> requirements replaced)
- ;; At last we remove replaced modules from the requirements list
- (fold
- (lambda (replacedelem requirements)
- (alist-delete! (car replacedelem) requirements))
- requirements
- replaced)))))
+ (values (cons (list module-path version) requirements) replaced)))
+
+ (with-input-from-string content
+ (lambda ()
+ (receive (requirements replaced)
+ (toplevel '() '())
+ ;; At last remove the replaced modules from the requirements list.
+ (remove (lambda (r)
+ (assoc (car r) replaced))
+ requirements)))))
;; Prevent inlining of this procedure, which is accessed by unit tests.
(set! parse-go.mod parse-go.mod)
@@ -550,17 +563,32 @@ control system is being used."
vcs-type vcs-repo-url)))))
(define* (go-module->guix-package module-path #:key
- (goproxy-url "https://proxy.golang.org"))
- (let* ((latest-version (go-module-latest-version goproxy-url module-path))
- (content (fetch-go.mod goproxy-url module-path latest-version))
- (dependencies (map car (parse-go.mod content)))
+ (goproxy "https://proxy.golang.org")
+ version
+ pin-versions?)
+ "Return the package S-expression corresponding to MODULE-PATH at VERSION, a Go package.
+The meta-data is fetched from the GOPROXY server and https://pkg.go.dev/.
+When VERSION is unspecified, the latest version available is used."
+ (let* ((available-versions (go-module-available-versions goproxy module-path))
+ (version* (or version
+ (go-module-version-string goproxy module-path))) ;latest
+ ;; Pseudo-versions do not appear in the versions list; skip the
+ ;; following check.
+ (_ (unless (or (go-pseudo-version? version*)
+ (member version* available-versions))
+ (error (format #f "error: version ~s is not available
+hint: use one of the following available versions ~a\n"
+ version* available-versions))))
+ (content (fetch-go.mod goproxy module-path version*))
+ (dependencies+versions (parse-go.mod content))
+ (dependencies (map car dependencies+versions))
(guix-name (go-module->guix-package-name module-path))
(root-module-path (module-path->repository-root module-path))
;; The VCS type and URL are not included in goproxy information. For
;; this we need to fetch it from the official module page.
(meta-data (fetch-module-meta-data root-module-path))
(vcs-type (module-meta-vcs meta-data))
- (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url))
+ (vcs-repo-url (module-meta-data-repo-url meta-data goproxy))
(synopsis (go-package-synopsis root-module-path))
(description (go-package-description module-path))
(licenses (go-package-licenses module-path)))
@@ -568,14 +596,14 @@ control system is being used."
`(package
(name ,guix-name)
;; Elide the "v" prefix Go uses
- (version ,(string-trim latest-version #\v))
+ (version ,(string-trim version* #\v))
(source
- ,(vcs->origin vcs-type vcs-repo-url latest-version))
+ ,(vcs->origin vcs-type vcs-repo-url version*))
(build-system go-build-system)
(arguments
'(#:import-path ,root-module-path))
- ,@(maybe-propagated-inputs
- (map go-module->guix-package-name dependencies))
+ ,@(maybe-propagated-inputs (map go-module->guix-package-name
+ dependencies))
(home-page ,(format #f "https://~a" root-module-path))
(synopsis ,synopsis)
(description ,(and=> description beautify-description))
@@ -585,16 +613,37 @@ control system is being used."
license)
((license ...) ;a list of licenses
`(list ,@license)))))
- dependencies)))
+ (if pin-versions?
+ dependencies+versions
+ dependencies))))
(define go-module->guix-package* (memoize go-module->guix-package))
(define* (go-module-recursive-import package-name
- #:key (goproxy-url "https://proxy.golang.org"))
+ #:key (goproxy "https://proxy.golang.org")
+ version
+ pin-versions?)
+
(recursive-import
package-name
- #:repo->guix-package (lambda* (name . _)
- (go-module->guix-package*
- name
- #:goproxy-url goproxy-url))
- #:guix-name go-module->guix-package-name))
+ #:repo->guix-package
+ (lambda* (name #:key version repo)
+ ;; Disable output buffering so that the following warning gets printed
+ ;; consistently.
+ (setvbuf (current-error-port) 'none)
+ (guard (c ((http-get-error? c)
+ (warning (G_ "Failed to import package ~s.
+reason: ~s could not be fetched: HTTP error ~a (~s).
+This package and its dependencies won't be imported.~%")
+ name
+ (uri->string (http-get-error-uri c))
+ (http-get-error-code c)
+ (http-get-error-reason c))
+ (values '() '())))
+ (receive (package-sexp dependencies)
+ (go-module->guix-package* name #:goproxy goproxy
+ #:version version
+ #:pin-versions? pin-versions?)
+ (values package-sexp dependencies))))
+ #:guix-name go-module->guix-package-name
+ #:version version))
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
index afdba4e8f1..33d2470ce1 100644
--- a/guix/scripts/import/go.scm
+++ b/guix/scripts/import/go.scm
@@ -1,5 +1,6 @@
;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -27,28 +28,30 @@
#:use-module (srfi srfi-37)
#:use-module (ice-9 match)
#:use-module (ice-9 format)
+ #:use-module (ice-9 receive)
#:export (guix-import-go))
-
+\f
;;;
;;; Command-line options.
;;;
(define %default-options
- '())
+ '((goproxy . "https://proxy.golang.org")))
(define (show-help)
- (display (G_ "Usage: guix import go PACKAGE-PATH
-Import and convert the Go module for PACKAGE-PATH.\n"))
+ (display (G_ "Usage: guix import go PACKAGE-PATH[@VERSION]
+Import and convert the Go module for PACKAGE-PATH. Optionally, a version
+can be specified after the arobas (@) character.\n"))
(display (G_ "
-h, --help display this help and exit"))
(display (G_ "
- -V, --version display version information and exit"))
- (display (G_ "
- -r, --recursive generate package expressions for all Go modules\
- that are not yet in Guix"))
+ -r, --recursive generate package expressions for all Go modules
+that are not yet in Guix"))
(display (G_ "
-p, --goproxy=GOPROXY specify which goproxy server to use"))
+ (display (G_ "
+ --pin-versions use the exact versions of a module's dependencies"))
(newline)
(show-bug-report-information))
@@ -58,9 +61,6 @@ Import and convert the Go module for PACKAGE-PATH.\n"))
(lambda args
(show-help)
(exit 0)))
- (option '(#\V "version") #f #f
- (lambda args
- (show-version-and-exit "guix import go")))
(option '(#\r "recursive") #f #f
(lambda (opt name arg result)
(alist-cons 'recursive #t result)))
@@ -69,9 +69,12 @@ Import and convert the Go module for PACKAGE-PATH.\n"))
(alist-cons 'goproxy
(string->symbol arg)
(alist-delete 'goproxy result))))
+ (option '("pin-versions") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'pin-versions? #t result)))
%standard-import-options))
-
+\f
;;;
;;; Entry point.
;;;
@@ -93,25 +96,28 @@ Import and convert the Go module for PACKAGE-PATH.\n"))
(_ #f))
(reverse opts))))
(match args
- ((module-name)
- (if (assoc-ref opts 'recursive)
- (map (match-lambda
- ((and ('package ('name name) . rest) pkg)
- `(define-public ,(string->symbol name)
- ,pkg))
- (_ #f))
- (go-module-recursive-import module-name
- #:goproxy-url
- (or (assoc-ref opts 'goproxy)
- "https://proxy.golang.org")))
- (let ((sexp (go-module->guix-package module-name
- #:goproxy-url
- (or (assoc-ref opts 'goproxy)
- "https://proxy.golang.org"))))
- (unless sexp
- (leave (G_ "failed to download meta-data for module '~a'~%")
- module-name))
- sexp)))
+ ((spec) ;e.g., github.com/golang/protobuf@v1.3.1
+ (receive (name version)
+ (package-name->name+version spec)
+ (let ((arguments (list name
+ #:goproxy (assoc-ref opts 'goproxy)
+ #:version version
+ #:pin-versions?
+ (assoc-ref opts 'pin-versions?))))
+ (if (assoc-ref opts 'recursive)
+ ;; Recursive import.
+ (map (match-lambda
+ ((and ('package ('name name) . rest) pkg)
+ `(define-public ,(string->symbol name)
+ ,pkg))
+ (_ #f))
+ (apply go-module-recursive-import arguments))
+ ;; Single import.
+ (let ((sexp (apply go-module->guix-package arguments)))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for module '~a'~%")
+ module-name))
+ sexp)))))
(()
(leave (G_ "too few arguments~%")))
((many ...)
diff --git a/tests/go.scm b/tests/go.scm
index fa8fa7a2a6..e5780e68b0 100644
--- a/tests/go.scm
+++ b/tests/go.scm
@@ -19,7 +19,7 @@
;;; Summary
;; Tests for guix/import/go.scm
-(define-module (test-import-go)
+(define-module (tests-import-go)
#:use-module (guix base32)
#:use-module (guix build-system go)
#:use-module (guix import go)
@@ -147,7 +147,8 @@ require github.com/kr/pretty v0.2.1
("https://pkg.go.dev/github.com/go-check/check"
. ,pkg.go.dev)
("https://pkg.go.dev/github.com/go-check/check?tab=licenses"
- . ,pkg.go.dev-licence))))
+ . ,pkg.go.dev-licence)
+ ("https://proxy.golang.org/github.com/go-check/check/@v/list" . ""))))
(test-begin "go")
@@ -169,6 +170,12 @@ require github.com/kr/pretty v0.2.1
"daa7c04131f5"
(go-version->git-ref "v1.2.4-0.20191109021931-daa7c04131f5"))
+(test-assert "go-pseudo-version? multi-digit version number"
+ (go-pseudo-version? "v1.23.1-0.20200526195155-81db48ad09cc"))
+
+(test-assert "go-pseudo-version? semantic version with rc"
+ (go-pseudo-version? "v1.4.0-rc.4.0.20200313231945-b860323f09d0"))
+
;;; Unit tests for (guix import go)
(test-equal "go-path-escape"
@@ -185,37 +192,38 @@ require github.com/kr/pretty v0.2.1
(sort ((@@ (guix import go) parse-go.mod) input) inf?)))
(testing-parse-mod "parse-go.mod-simple"
- '(("good/thing" . "v1.4.5")
- ("new/thing/v2" . "v2.3.4")
- ("other/thing" . "v1.0.2"))
+ '(("good/thing" "v1.4.5")
+ ("new/thing/v2" "v2.3.4")
+ ("other/thing" "v1.0.2"))
fixture-go-mod-simple)
(testing-parse-mod "parse-go.mod-with-block"
- '(("A" . "v1")
- ("B" . "v1.0.0")
- ("C" . "v1.0.0")
- ("D" . "v1.2.3")
- ("E" . "dev"))
+ '(("A" "v1")
+ ("B" "v1.0.0")
+ ("C" "v1.0.0")
+ ("D" "v1.2.3")
+ ("E" "dev"))
fixture-go-mod-with-block)
-(testing-parse-mod "parse-go.mod-complete"
- '(("github.com/corp/arbitrary-repo" . "v0.0.2")
- ("quoted.example.com/abitrary/repo" . "v0.0.2")
- ("one.example.com/abitrary/repo" . "v1.1.111")
- ("hub.jazz.net/git/user/project/sub/directory" . "v1.1.19")
- ("hub.jazz.net/git/user/project" . "v1.1.18")
- ("launchpad.net/~user/project/branch/sub/directory" . "v1.1.17")
- ("launchpad.net/~user/project/branch" . "v1.1.16")
- ("launchpad.net/project/series/sub/directory" . "v1.1.15")
- ("launchpad.net/project/series" . "v1.1.14")
- ("launchpad.net/project" . "v1.1.13")
- ("bitbucket.org/user/project/sub/directory" . "v1.11.21")
- ("bitbucket.org/user/project" . "v1.11.20")
- ("k8s.io/kubernetes/subproject" . "v1.1.101")
- ("github.com/user/project/sub/directory" . "v1.1.12")
- ("github.com/user/project" . "v1.1.11")
- ("github.com/go-check/check" . "v0.0.0-20140225173054-eb6ee6f84d0a"))
- fixture-go-mod-complete)
+(testing-parse-mod
+ "parse-go.mod-complete"
+ '(("github.com/corp/arbitrary-repo" "v0.0.2")
+ ("quoted.example.com/abitrary/repo" "v0.0.2")
+ ("one.example.com/abitrary/repo" "v1.1.111")
+ ("hub.jazz.net/git/user/project/sub/directory" "v1.1.19")
+ ("hub.jazz.net/git/user/project" "v1.1.18")
+ ("launchpad.net/~user/project/branch/sub/directory" "v1.1.17")
+ ("launchpad.net/~user/project/branch" "v1.1.16")
+ ("launchpad.net/project/series/sub/directory" "v1.1.15")
+ ("launchpad.net/project/series" "v1.1.14")
+ ("launchpad.net/project" "v1.1.13")
+ ("bitbucket.org/user/project/sub/directory" "v1.11.21")
+ ("bitbucket.org/user/project" "v1.11.20")
+ ("k8s.io/kubernetes/subproject" "v1.1.101")
+ ("github.com/user/project/sub/directory" "v1.1.12")
+ ("github.com/user/project" "v1.1.11")
+ ("github.com/go-check/check" "v0.0.0-20140225173054-eb6ee6f84d0a"))
+ fixture-go-mod-complete)
;;; End-to-end tests for (guix import go)
(define (mock-http-fetch testcase)
--
2.30.1
^ permalink raw reply related [flat|nested] 12+ messages in thread* [bug#47310] [PATCH 4/4] import: go: Append version to symbol name in the pinned version mode.
2021-03-22 4:20 ` [bug#47310] [PATCH 1/4] import: utils: Refactor maybe-inputs and add maybe-propagated-inputs Maxim Cournoyer
2021-03-22 4:20 ` [bug#47310] [PATCH 2/4] import: go: Improve synopsis and description parsing Maxim Cournoyer
2021-03-22 4:20 ` [bug#47310] [PATCH 3/4] import: go: Add an option to use pinned versions Maxim Cournoyer
@ 2021-03-22 4:20 ` Maxim Cournoyer
2 siblings, 0 replies; 12+ messages in thread
From: Maxim Cournoyer @ 2021-03-22 4:20 UTC (permalink / raw)
To: 47310; +Cc: Maxim Cournoyer
This allows importing packages with complicated version specific dependency
chains without the package symbol names colliding.
* doc/guix.texi (Invoking guix import): Document the --pin-versions option.
Mention that a specific version can be imported. Remove the experimental
warning.
* guix/import/go.scm (go-module->guix-package-name)[version]: Add optional
argument.
(go-module->guix-package): Conditionally use dependencies whose symbol include
their version, based no the value of the PIN-VERSIONS? argument.
* guix/import/utils.scm (package->definition): Add a new case where the full
version string is appended to the package symbol.
* guix/scripts/import.scm (guix-import): Correctly print forms starting
with '(define-public [...]'.
* guix/scripts/import/go.scm (guix-import-go): Conditionally include the
version in the package symbols defined.
---
doc/guix.texi | 14 +++++++++++---
guix/import/go.scm | 34 +++++++++++++++++++++++-----------
guix/import/utils.scm | 7 +++++--
guix/scripts/import.scm | 3 ++-
guix/scripts/import/go.scm | 17 ++++++++++-------
5 files changed, 51 insertions(+), 24 deletions(-)
diff --git a/doc/guix.texi b/doc/guix.texi
index 386169b2a5..be20215638 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -11520,13 +11520,13 @@ Select the given repository (a repository name). Possible values include:
Import metadata for a Go module using
@uref{https://proxy.golang.org, proxy.golang.org}.
-This importer is highly experimental. See the source code for more info
-about the current state.
-
@example
guix import go gopkg.in/yaml.v2
@end example
+It is possible to use a package specification with a @code{@VERSION}
+suffix to import a specific version.
+
Additional options include:
@table @code
@@ -11535,6 +11535,14 @@ Additional options include:
Traverse the dependency graph of the given upstream package recursively
and generate package expressions for all those packages that are not yet
in Guix.
+@item --pin-versions
+When using this option, the importer preserves the exact versions of the
+Go modules dependencies instead of using their latest available
+versions. This can be useful when attempting to import packages that
+recursively depend on former versions of themselves to build. When
+using this mode, the symbol of the package is made by appending the
+version to its name, so that multiple versions of the same package can
+coexist.
@end table
@end table
diff --git a/guix/import/go.scm b/guix/import/go.scm
index 2376336265..d8b6fddac5 100644
--- a/guix/import/go.scm
+++ b/guix/import/go.scm
@@ -427,13 +427,17 @@ hence the need to derive this information."
(vcs-qualified-module-path->root-repo-url module-path)
module-path))
-(define (go-module->guix-package-name module-path)
- "Converts a module's path to the canonical Guix format for Go packages."
+(define* (go-module->guix-package-name module-path #:optional version)
+ "Converts a module's path to the canonical Guix format for Go packages.
+Optionally include a VERSION string to append to the name."
(string-downcase (string-append "go-" (string-replace-substring
(string-replace-substring
module-path
"." "-")
- "/" "-"))))
+ "/" "-")
+ (if version
+ (string-append "-" version)
+ ""))))
(define (strip-.git-suffix/maybe repo-url)
"Strip a repository URL '.git' suffix from REPO-URL if hosted at GitHub."
@@ -572,6 +576,8 @@ When VERSION is unspecified, the latest version available is used."
(let* ((available-versions (go-module-available-versions goproxy module-path))
(version* (or version
(go-module-version-string goproxy module-path))) ;latest
+ ;; Elide the "v" prefix Go uses.
+ (strip-v-prefix (cut string-trim <> #\v))
;; Pseudo-versions do not appear in the versions list; skip the
;; following check.
(_ (unless (or (go-pseudo-version? version*)
@@ -581,7 +587,9 @@ hint: use one of the following available versions ~a\n"
version* available-versions))))
(content (fetch-go.mod goproxy module-path version*))
(dependencies+versions (parse-go.mod content))
- (dependencies (map car dependencies+versions))
+ (dependencies (if pin-versions?
+ dependencies+versions
+ (map car dependencies+versions)))
(guix-name (go-module->guix-package-name module-path))
(root-module-path (module-path->repository-root module-path))
;; The VCS type and URL are not included in goproxy information. For
@@ -595,23 +603,27 @@ hint: use one of the following available versions ~a\n"
(values
`(package
(name ,guix-name)
- ;; Elide the "v" prefix Go uses
- (version ,(string-trim version* #\v))
+ (version ,(strip-v-prefix version*))
(source
,(vcs->origin vcs-type vcs-repo-url version*))
(build-system go-build-system)
(arguments
'(#:import-path ,root-module-path))
- ,@(maybe-propagated-inputs (map go-module->guix-package-name
- dependencies))
+ ,@(maybe-propagated-inputs
+ (map (match-lambda
+ ((name version)
+ (go-module->guix-package-name name (strip-v-prefix version)))
+ (name
+ (go-module->guix-package-name name)))
+ dependencies))
(home-page ,(format #f "https://~a" root-module-path))
(synopsis ,synopsis)
(description ,(and=> description beautify-description))
(license ,(match (list->licenses licenses)
- (() #f) ;unknown license
- ((license) ;a single license
+ (() #f) ;unknown license
+ ((license) ;a single license
license)
- ((license ...) ;a list of licenses
+ ((license ...) ;a list of licenses
`(list ,@license)))))
(if pin-versions?
dependencies+versions
diff --git a/guix/import/utils.scm b/guix/import/utils.scm
index 21fafad388..21352841b8 100644
--- a/guix/import/utils.scm
+++ b/guix/import/utils.scm
@@ -273,8 +273,9 @@ snippet generated is for regular inputs."
(maybe-inputs package-names output #:type 'propagated))
(define* (package->definition guix-package #:optional append-version?/string)
- "If APPEND-VERSION?/STRING is #t, append the package's major+minor
-version. If APPEND-VERSION?/string is a string, append this string."
+ "If APPEND-VERSION?/STRING is #t, append the package's major+minor version.
+If it is the symbol 'full, append the package's complete version. If
+APPEND-VERSION?/string is a string, append this string."
(match guix-package
((or
('package ('name name) ('version version) . rest)
@@ -286,6 +287,8 @@ version. If APPEND-VERSION?/string is a string, append this string."
(string-append name "-" append-version?/string))
((eq? append-version?/string #t)
(string-append name "-" (version-major+minor version)))
+ ((eq? 'full append-version?/string)
+ (string-append name "-" version))
(else name)))
,guix-package))))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 1d2b45d942..98554ef79b 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -119,7 +119,8 @@ Run IMPORTER with ARGS.\n"))
(current-output-port))))))
(match (apply (resolve-importer importer) args)
((and expr (or ('package _ ...)
- ('let _ ...)))
+ ('let _ ...)
+ ('define-public _ ...)))
(print expr))
((? list? expressions)
(for-each (lambda (expr)
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
index 33d2470ce1..04b07f80cc 100644
--- a/guix/scripts/import/go.scm
+++ b/guix/scripts/import/go.scm
@@ -22,9 +22,11 @@
#:use-module (guix utils)
#:use-module (guix scripts)
#:use-module (guix import go)
+ #:use-module (guix import utils)
#:use-module (guix scripts import)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-26)
#:use-module (srfi srfi-37)
#:use-module (ice-9 match)
#:use-module (ice-9 format)
@@ -94,7 +96,12 @@ that are not yet in Guix"))
(('argument . value)
value)
(_ #f))
- (reverse opts))))
+ (reverse opts)))
+ ;; Append the full version to the package symbol name when using
+ ;; pinned versions.
+ (package->definition* (if (assoc-ref opts 'pin-versions?)
+ (cut package->definition <> 'full)
+ package->definition)))
(match args
((spec) ;e.g., github.com/golang/protobuf@v1.3.1
(receive (name version)
@@ -106,18 +113,14 @@ that are not yet in Guix"))
(assoc-ref opts 'pin-versions?))))
(if (assoc-ref opts 'recursive)
;; Recursive import.
- (map (match-lambda
- ((and ('package ('name name) . rest) pkg)
- `(define-public ,(string->symbol name)
- ,pkg))
- (_ #f))
+ (map package->definition*
(apply go-module-recursive-import arguments))
;; Single import.
(let ((sexp (apply go-module->guix-package arguments)))
(unless sexp
(leave (G_ "failed to download meta-data for module '~a'~%")
module-name))
- sexp)))))
+ (package->definition* sexp))))))
(()
(leave (G_ "too few arguments~%")))
((many ...)
--
2.30.1
^ permalink raw reply related [flat|nested] 12+ messages in thread