* [bug#44178] Add a Go Module Importer
@ 2020-10-23 14:06 Katherine Cox-Buday
2020-10-28 10:41 ` Ludovic Courtès
` (4 more replies)
0 siblings, 5 replies; 27+ messages in thread
From: Katherine Cox-Buday @ 2020-10-23 14:06 UTC (permalink / raw)
To: 44178
[-- Attachment #1: 0001-guix-import-go.scm-Created-Go-Importer.patch --]
[-- Type: text/x-patch, Size: 18075 bytes --]
From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001
From: Katherine Cox-Buday <cox.katherine.e@gmail.com>
Date: Thu, 22 Oct 2020 19:40:17 -0500
Subject: [PATCH] * guix/import/go.scm: Created Go Importer *
guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm
(importers): Added Go Importer Subcommand
---
guix/import/go.scm | 276 +++++++++++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/go.scm | 118 ++++++++++++++++
3 files changed, 395 insertions(+), 1 deletion(-)
create mode 100644 guix/import/go.scm
create mode 100644 guix/scripts/import/go.scm
diff --git a/guix/import/go.scm b/guix/import/go.scm
new file mode 100644
index 0000000000..61009f3565
--- /dev/null
+++ b/guix/import/go.scm
@@ -0,0 +1,276 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix import go)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 rdelim)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (guix json)
+ #:use-module ((guix download) #:prefix download:)
+ #:use-module (guix import utils)
+ #:use-module (guix import json)
+ #:use-module (guix packages)
+ #:use-module (guix upstream)
+ #:use-module (guix utils)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix base16)
+ #:use-module (guix base32)
+ #:use-module (guix build download)
+ #:use-module (web uri)
+
+ #:export (go-module->guix-package
+ go-module-recursive-import
+ infer-module-root))
+
+(define (escape-capital-letters s)
+ "To avoid ambiguity when serving from case-insensitive file systems, the
+$module and $version elements are case-encoded by replacing every uppercase
+letter with an exclamation mark followed by the corresponding lower-case
+letter."
+ (let ((escaped-string (string)))
+ (string-for-each-index
+ (lambda (i)
+ (let ((c (string-ref s i)))
+ (set! escaped-string
+ (string-concatenate
+ (list escaped-string
+ (if (char-upper-case? c) "!" "")
+ (string (char-downcase c)))))))
+ s)
+ escaped-string))
+
+(define (fetch-latest-version goproxy-url module-path)
+ "Fetches the version number of the latest version for MODULE-PATH from the
+given GOPROXY-URL server."
+ (assoc-ref
+ (json-fetch (format #f "~a/~a/@latest" goproxy-url
+ (escape-capital-letters module-path)))
+ "Version"))
+
+(define (fetch-go.mod goproxy-url module-path version file)
+ "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
+and VERSION."
+ (url-fetch (format #f "~a/~a/@v/~a.mod" goproxy-url
+ (escape-capital-letters module-path)
+ (escape-capital-letters version))
+ file
+ #:print-build-trace? #f))
+
+(define (parse-go.mod go.mod-path)
+ "Parses a go.mod file and returns an alist of module path to version."
+ (with-input-from-file go.mod-path
+ (lambda ()
+ (let ((in-require? #f)
+ (requirements (list)))
+ (do ((line (read-line) (read-line)))
+ ((eof-object? line))
+ (set! line (string-trim line))
+ ;; The parser is either entering, within, exiting, or after the
+ ;; require block. The Go toolchain is trustworthy so edge-cases like
+ ;; double-entry, etc. need not complect the parser.
+ (cond
+ ((string=? line "require (")
+ (set! in-require? #t))
+ ((and in-require? (string=? line ")"))
+ (set! in-require? #f))
+ (in-require?
+ (let* ((requirement (string-split line #\space))
+ ;; Modules should be unquoted
+ (module-path (string-delete #\" (car requirement)))
+ (version (list-ref requirement 1)))
+ (set! requirements (acons module-path version requirements))))
+ ((string-prefix? "replace" line)
+ (let* ((requirement (string-split line #\space))
+ (module-path (list-ref requirement 1))
+ (new-module-path (list-ref requirement 3))
+ (version (list-ref requirement 4)))
+ (set! requirements (assoc-remove! requirements module-path))
+ (set! requirements (acons new-module-path version requirements))))))
+ requirements))))
+
+(define (module-path-without-major-version module-path)
+ "Go modules can be appended with a major version indicator,
+e.g. /v3. Sometimes it is desirable to work with the root module path. For
+instance, for a module path github.com/foo/bar/v3 this function returns
+github.com/foo/bar."
+ (let ((m (string-match "(.*)\\/v[0-9]+$" module-path)))
+ (if m
+ (match:substring m 1)
+ module-path)))
+
+(define (infer-module-root module-path)
+ "Go modules can be defined at any level of a repository's tree, but querying
+for the meta tag usually can only be done at the webpage at the root of the
+repository. Therefore, it is sometimes necessary to try and derive a module's
+root path from its path. For a set of well-known forges, the pattern of what
+consists of a module's root page is known before hand."
+ ;; See the following URL for the official Go equivalent:
+ ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
+ (define-record-type <scs>
+ (make-scs url-prefix root-regex type)
+ scs?
+ (url-prefix scs-url-prefix)
+ (root-regex scs-root-regex)
+ (type scs-type))
+ (let* ((known-scs
+ (list
+ (make-scs
+ "github.com"
+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-scs
+ "bitbucket.org"
+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$`"
+ 'unknown)
+ (make-scs
+ "hub.jazz.net/git/"
+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-scs
+ "git.apache.org"
+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-scs
+ "git.openstack.org"
+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)))
+ (scs (find (lambda (scs) (string-prefix? (scs-url-prefix scs) module-path))
+ known-scs)))
+ (if scs
+ (match:substring (string-match (scs-root-regex scs) module-path) 1)
+ module-path)))
+
+(define (to-guix-package-name module-path)
+ "Converts a module's path to the canonical Guix format for Go packages."
+ (string-downcase
+ (string-append "go-"
+ (string-replace-substring
+ (string-replace-substring
+ ;; Guix has its own field for version
+ (module-path-without-major-version module-path)
+ "." "-")
+ "/" "-"))))
+
+(define (fetch-module-meta-data module-path)
+ "Fetches module meta-data from a module's landing page. This is necessary
+because goproxy servers don't currently provide all the information needed to
+build a package."
+ (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
+ (module-metadata #f)
+ (meta-tag-prefix "<meta name=\"go-import\" content=\"")
+ (meta-tag-prefix-length (string-length meta-tag-prefix)))
+ (do ((line (read-line port) (read-line port)))
+ ((or (eof-object? line)
+ module-metadata))
+ (let ((meta-tag-index (string-contains line meta-tag-prefix)))
+ (when meta-tag-index
+ (let* ((start (+ meta-tag-index meta-tag-prefix-length))
+ (end (string-index line #\" start)))
+ (set! module-metadata
+ (string-split (substring/shared line start end) #\space))))))
+ (close-port port)
+ module-metadata))
+
+(define (module-meta-data-scs meta-data)
+ "Return the source control system specified by a module's meta-data."
+ (string->symbol (list-ref meta-data 1)))
+
+(define (module-meta-data-repo-url meta-data goproxy-url)
+ "Return the URL where the fetcher which will be used can download the source
+control."
+ (if (member (module-meta-data-scs meta-data) '(fossil mod))
+ goproxy-url
+ (list-ref meta-data 2)))
+
+(define (source-uri scs-type scs-repo-url file)
+ "Generate the `origin' block of a package depending on what type of source
+control system is being used."
+ (case scs-type
+ ((git)
+ `(origin
+ (method git-fetch)
+ (uri (git-reference
+ (url ,scs-repo-url)
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ ((hg)
+ `(origin
+ (method hg-fetch)
+ (uri (hg-reference
+ (url ,scs-repo-url)
+ (changeset ,version)))
+ (file-name (format #f "~a-~a-checkout" name version))))
+ ((svn)
+ `(origin
+ (method svn-fetch)
+ (uri (svn-reference
+ (url ,scs-repo-url)
+ (revision (string->number version))
+ (recursive? #f)))
+ (file-name (format #f "~a-~a-checkout" name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ (else
+ (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))
+
+(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (let* ((latest-version (fetch-latest-version goproxy-url module-path))
+ (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
+ temp))
+ (dependencies (map car (parse-go.mod temp)))
+ (guix-name (to-guix-package-name module-path))
+ (root-module-path (infer-module-root module-path))
+ ;; SCS type and URL are not included in goproxy information. For
+ ;; this we need to fetch it from the official module page.
+ (meta-data (fetch-module-meta-data root-module-path))
+ (scs-type (module-meta-data-scs meta-data))
+ (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
+ (values
+ `(package
+ (name ,guix-name)
+ ;; Elide the "v" prefix Go uses
+ (version ,(string-trim latest-version #\v))
+ (source
+ ,(source-uri scs-type scs-repo-url temp))
+ (build-system go-build-system)
+ ,@(maybe-inputs (map to-guix-package-name dependencies))
+ ;; TODO(katco): It would be nice to make an effort to fetch this
+ ;; from known forges, e.g. GitHub
+ (home-page ,(format #f "https://~a" root-module-path))
+ (synopsis "A Go package")
+ (description ,(format #f "~a is a Go package." guix-name))
+ (license #f))
+ dependencies)))))
+
+(define* (go-module-recursive-import package-name
+ #:key (goproxy-url "https://proxy.golang.org"))
+ (recursive-import package-name #f
+ #:repo->guix-package
+ (lambda (name _)
+ (go-module->guix-package name
+ #:goproxy-url goproxy-url))
+ #:guix-name to-guix-package-name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 0a3863f965..1d2b45d942 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -77,7 +77,7 @@ rather than \\n."
;;;
(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"
- "cran" "crate" "texlive" "json" "opam"))
+ "go" "cran" "crate" "texlive" "json" "opam"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
new file mode 100644
index 0000000000..000039769c
--- /dev/null
+++ b/guix/scripts/import/go.scm
@@ -0,0 +1,118 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import go)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import go)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-go))
+
+\f
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import go PACKAGE-PATH
+Import and convert the Go module for PACKAGE-PATH.\n"))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (display (G_ "
+ -r, --recursive generate package expressions for all Go modules\
+ that are not yet in Guix"))
+ (display (G_ "
+ -p, --goproxy=GOPROXY specify which goproxy server to use"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import go")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ (option '(#\p "goproxy") #t #f
+ (lambda (opt name arg result)
+ (alist-cons 'goproxy
+ (string->symbol arg)
+ (alist-delete 'goproxy result))))
+ %standard-import-options))
+
+\f
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-go . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (match args
+ ((module-name)
+ (if (assoc-ref opts 'recursive)
+ (map (match-lambda
+ ((and ('package ('name name) . rest) pkg)
+ `(define-public ,(string->symbol name)
+ ,pkg))
+ (_ #f))
+ (go-module-recursive-import module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org")))
+ (let ((sexp (go-module->guix-package module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org"))))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for module '~a'~%")
+ module-name))
+ sexp)))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%"))))))
--
2.28.0
[-- Attachment #2: Type: text/plain, Size: 15 bytes --]
--
Katherine
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [bug#44178] Add a Go Module Importer
2020-10-23 14:06 [bug#44178] Add a Go Module Importer Katherine Cox-Buday
@ 2020-10-28 10:41 ` Ludovic Courtès
2020-10-28 10:42 ` Ludovic Courtès
` (3 subsequent siblings)
4 siblings, 0 replies; 27+ messages in thread
From: Ludovic Courtès @ 2020-10-28 10:41 UTC (permalink / raw)
To: Katherine Cox-Buday; +Cc: 44178
Hi Katherine,
Katherine Cox-Buday <cox.katherine.e@gmail.com> skribis:
>>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001
> From: Katherine Cox-Buday <cox.katherine.e@gmail.com>
> Date: Thu, 22 Oct 2020 19:40:17 -0500
> Subject: [PATCH] * guix/import/go.scm: Created Go Importer *
> guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm
> (importers): Added Go Importer Subcommand
Nice! I think that can make a lot of people happy. :-)
Here’s a quick review. I won’t promise I can reply to followups in the
coming days because with the release preparation going on, I’d rather
focus on that. So perhaps this patch will have to wait until after this
release, but certainly before the next one!
> +(define (escape-capital-letters s)
> + "To avoid ambiguity when serving from case-insensitive file systems, the
> +$module and $version elements are case-encoded by replacing every uppercase
> +letter with an exclamation mark followed by the corresponding lower-case
> +letter."
> + (let ((escaped-string (string)))
> + (string-for-each-index
> + (lambda (i)
> + (let ((c (string-ref s i)))
> + (set! escaped-string
> + (string-concatenate
> + (list escaped-string
> + (if (char-upper-case? c) "!" "")
> + (string (char-downcase c)))))))
> + s)
> + escaped-string))
As a general comment, the coding style in Guix is functional “by
default” (info "(guix) Coding Style"). That means we almost never use
‘set!’ and procedures that modify their arguments.
We also avoid idioms like car/cdr and ‘do’, which are more commonly used
in other Lisps, as you know very well. ;-)
In the case above, I’d probably use ‘string-fold’. The resulting code
should be easier to reason about and likely more efficient.
> +(define (fetch-latest-version goproxy-url module-path)
> + "Fetches the version number of the latest version for MODULE-PATH from the
> +given GOPROXY-URL server."
> + (assoc-ref
> + (json-fetch (format #f "~a/~a/@latest" goproxy-url
> + (escape-capital-letters module-path)))
> + "Version"))
I’d suggest using ‘define-json-mapping’ from (json) like in the other
importers.
> +(define (infer-module-root module-path)
> + "Go modules can be defined at any level of a repository's tree, but querying
> +for the meta tag usually can only be done at the webpage at the root of the
> +repository. Therefore, it is sometimes necessary to try and derive a module's
> +root path from its path. For a set of well-known forges, the pattern of what
> +consists of a module's root page is known before hand."
> + ;; See the following URL for the official Go equivalent:
> + ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
> + (define-record-type <scs>
> + (make-scs url-prefix root-regex type)
> + scs?
> + (url-prefix scs-url-prefix)
> + (root-regex scs-root-regex)
> + (type scs-type))
Maybe VCS as “version control system”? (It took me a while to guess
what “SCS” meant.)
> +(define (fetch-module-meta-data module-path)
> + "Fetches module meta-data from a module's landing page. This is necessary
> +because goproxy servers don't currently provide all the information needed to
> +build a package."
> + (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
> + (module-metadata #f)
> + (meta-tag-prefix "<meta name=\"go-import\" content=\"")
> + (meta-tag-prefix-length (string-length meta-tag-prefix)))
> + (do ((line (read-line port) (read-line port)))
> + ((or (eof-object? line)
> + module-metadata))
> + (let ((meta-tag-index (string-contains line meta-tag-prefix)))
> + (when meta-tag-index
> + (let* ((start (+ meta-tag-index meta-tag-prefix-length))
> + (end (string-index line #\" start)))
> + (set! module-metadata
> + (string-split (substring/shared line start end) #\space))))))
I’d suggest a named ‘let’ or ‘fold’ here.
Likewise, instead of concatenating XML strings (which could lead to
malformed XML), I recommend using SXML: you would create an sexp like
(meta (@ (name "go-import") (content …)))
and at the end pass it to ‘sxml->sxml’ (info "(guile) Reading and
Writing XML").
> + (else
> + (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))
‘raise-exception’ takes an error condition. In this case, we should use
(srfi srfi-34) for ‘raise’ write something like:
(raise (condition (formatted-message (G_ "…" …))))
> +(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
> + (call-with-temporary-output-file
> + (lambda (temp port)
> + (let* ((latest-version (fetch-latest-version goproxy-url module-path))
> + (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
> + temp))
It seems that ‘go.mod-path’ isn’t used, and thus ‘fetch-go.mod’ &
co. aren’t used either, or am I overlooking something?
> + (dependencies (map car (parse-go.mod temp)))
Please use ‘match’ instead, or perhaps define a record type for the
abstraction at hand.
> + (guix-name (to-guix-package-name module-path))
> + (root-module-path (infer-module-root module-path))
> + ;; SCS type and URL are not included in goproxy information. For
> + ;; this we need to fetch it from the official module page.
> + (meta-data (fetch-module-meta-data root-module-path))
> + (scs-type (module-meta-data-scs meta-data))
> + (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
> + (values
> + `(package
> + (name ,guix-name)
> + ;; Elide the "v" prefix Go uses
> + (version ,(string-trim latest-version #\v))
> + (source
> + ,(source-uri scs-type scs-repo-url temp))
> + (build-system go-build-system)
> + ,@(maybe-inputs (map to-guix-package-name dependencies))
> + ;; TODO(katco): It would be nice to make an effort to fetch this
> + ;; from known forges, e.g. GitHub
> + (home-page ,(format #f "https://~a" root-module-path))
> + (synopsis "A Go package")
> + (description ,(format #f "~a is a Go package." guix-name))
Maybe something like “fill it out!” so we don’t get patch submissions
with the default synopsis/description. :-)
> + (license #f))
Likewise.
Two more things: could you (1) and an entry under “Invoking guix import”
in doc/guix.texi, and (2) add tests, taking inspiration from the
existing importer tests?
Thank you!
Ludo’.
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] Add a Go Module Importer
2020-10-23 14:06 [bug#44178] Add a Go Module Importer Katherine Cox-Buday
2020-10-28 10:41 ` Ludovic Courtès
@ 2020-10-28 10:42 ` Ludovic Courtès
2020-11-10 20:26 ` Marius Bakke
` (2 subsequent siblings)
4 siblings, 0 replies; 27+ messages in thread
From: Ludovic Courtès @ 2020-10-28 10:42 UTC (permalink / raw)
To: Katherine Cox-Buday; +Cc: 44178
Hi Katherine,
Katherine Cox-Buday <cox.katherine.e@gmail.com> skribis:
>>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001
> From: Katherine Cox-Buday <cox.katherine.e@gmail.com>
> Date: Thu, 22 Oct 2020 19:40:17 -0500
> Subject: [PATCH] * guix/import/go.scm: Created Go Importer *
> guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm
> (importers): Added Go Importer Subcommand
Nice! I think that can make a lot of people happy. :-)
Here’s a quick review. I won’t promise I can reply to followups in the
coming days because with the release preparation going on, I’d rather
focus on that. So perhaps this patch will have to wait until after this
release, but certainly before the next one!
> +(define (escape-capital-letters s)
> + "To avoid ambiguity when serving from case-insensitive file systems, the
> +$module and $version elements are case-encoded by replacing every uppercase
> +letter with an exclamation mark followed by the corresponding lower-case
> +letter."
> + (let ((escaped-string (string)))
> + (string-for-each-index
> + (lambda (i)
> + (let ((c (string-ref s i)))
> + (set! escaped-string
> + (string-concatenate
> + (list escaped-string
> + (if (char-upper-case? c) "!" "")
> + (string (char-downcase c)))))))
> + s)
> + escaped-string))
As a general comment, the coding style in Guix is functional “by
default” (info "(guix) Coding Style"). That means we almost never use
‘set!’ and procedures that modify their arguments.
We also avoid idioms like car/cdr and ‘do’, which are more commonly used
in other Lisps, as you know very well. ;-)
In the case above, I’d probably use ‘string-fold’. The resulting code
should be easier to reason about and likely more efficient.
> +(define (fetch-latest-version goproxy-url module-path)
> + "Fetches the version number of the latest version for MODULE-PATH from the
> +given GOPROXY-URL server."
> + (assoc-ref
> + (json-fetch (format #f "~a/~a/@latest" goproxy-url
> + (escape-capital-letters module-path)))
> + "Version"))
I’d suggest using ‘define-json-mapping’ from (json) like in the other
importers.
> +(define (infer-module-root module-path)
> + "Go modules can be defined at any level of a repository's tree, but querying
> +for the meta tag usually can only be done at the webpage at the root of the
> +repository. Therefore, it is sometimes necessary to try and derive a module's
> +root path from its path. For a set of well-known forges, the pattern of what
> +consists of a module's root page is known before hand."
> + ;; See the following URL for the official Go equivalent:
> + ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
> + (define-record-type <scs>
> + (make-scs url-prefix root-regex type)
> + scs?
> + (url-prefix scs-url-prefix)
> + (root-regex scs-root-regex)
> + (type scs-type))
Maybe VCS as “version control system”? (It took me a while to guess
what “SCS” meant.)
> +(define (fetch-module-meta-data module-path)
> + "Fetches module meta-data from a module's landing page. This is necessary
> +because goproxy servers don't currently provide all the information needed to
> +build a package."
> + (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
> + (module-metadata #f)
> + (meta-tag-prefix "<meta name=\"go-import\" content=\"")
> + (meta-tag-prefix-length (string-length meta-tag-prefix)))
> + (do ((line (read-line port) (read-line port)))
> + ((or (eof-object? line)
> + module-metadata))
> + (let ((meta-tag-index (string-contains line meta-tag-prefix)))
> + (when meta-tag-index
> + (let* ((start (+ meta-tag-index meta-tag-prefix-length))
> + (end (string-index line #\" start)))
> + (set! module-metadata
> + (string-split (substring/shared line start end) #\space))))))
I’d suggest a named ‘let’ or ‘fold’ here.
Likewise, instead of concatenating XML strings (which could lead to
malformed XML), I recommend using SXML: you would create an sexp like
(meta (@ (name "go-import") (content …)))
and at the end pass it to ‘sxml->sxml’ (info "(guile) Reading and
Writing XML").
> + (else
> + (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))
‘raise-exception’ takes an error condition. In this case, we should use
(srfi srfi-34) for ‘raise’ write something like:
(raise (condition (formatted-message (G_ "…" …))))
> +(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
> + (call-with-temporary-output-file
> + (lambda (temp port)
> + (let* ((latest-version (fetch-latest-version goproxy-url module-path))
> + (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
> + temp))
It seems that ‘go.mod-path’ isn’t used, and thus ‘fetch-go.mod’ &
co. aren’t used either, or am I overlooking something?
> + (dependencies (map car (parse-go.mod temp)))
Please use ‘match’ instead, or perhaps define a record type for the
abstraction at hand.
> + (guix-name (to-guix-package-name module-path))
> + (root-module-path (infer-module-root module-path))
> + ;; SCS type and URL are not included in goproxy information. For
> + ;; this we need to fetch it from the official module page.
> + (meta-data (fetch-module-meta-data root-module-path))
> + (scs-type (module-meta-data-scs meta-data))
> + (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
> + (values
> + `(package
> + (name ,guix-name)
> + ;; Elide the "v" prefix Go uses
> + (version ,(string-trim latest-version #\v))
> + (source
> + ,(source-uri scs-type scs-repo-url temp))
> + (build-system go-build-system)
> + ,@(maybe-inputs (map to-guix-package-name dependencies))
> + ;; TODO(katco): It would be nice to make an effort to fetch this
> + ;; from known forges, e.g. GitHub
> + (home-page ,(format #f "https://~a" root-module-path))
> + (synopsis "A Go package")
> + (description ,(format #f "~a is a Go package." guix-name))
Maybe something like “fill it out!” so we don’t get patch submissions
with the default synopsis/description. :-)
> + (license #f))
Likewise.
Two more things: could you (1) and an entry under “Invoking guix import”
in doc/guix.texi, and (2) add tests, taking inspiration from the
existing importer tests?
Thank you!
Ludo’.
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] Add a Go Module Importer
2020-10-23 14:06 [bug#44178] Add a Go Module Importer Katherine Cox-Buday
2020-10-28 10:41 ` Ludovic Courtès
2020-10-28 10:42 ` Ludovic Courtès
@ 2020-11-10 20:26 ` Marius Bakke
[not found] ` <CANe01w55ZO=_9v0HcDv248UsoLUXb_9WVAgM4LqiZ4E-r1XgXg@mail.gmail.com>
2020-11-11 20:48 ` [bug#44178] Add a Go Module Importer Katherine Cox-Buday
2020-12-09 14:22 ` [bug#44178] dftxbs3e
2021-01-28 7:29 ` [bug#44178] [PATCH] Create importer for Go modules guix-patches--- via
4 siblings, 2 replies; 27+ messages in thread
From: Marius Bakke @ 2020-11-10 20:26 UTC (permalink / raw)
To: Katherine Cox-Buday, 44178; +Cc: Helio Machado
[-- Attachment #1: Type: text/plain, Size: 576 bytes --]
Katherine Cox-Buday <cox.katherine.e@gmail.com> writes:
>>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001
> From: Katherine Cox-Buday <cox.katherine.e@gmail.com>
> Date: Thu, 22 Oct 2020 19:40:17 -0500
> Subject: [PATCH] * guix/import/go.scm: Created Go Importer *
> guix/scripts/import.scm: Created Go Importer Subcommand * guix/import/go.scm
> (importers): Added Go Importer Subcommand
I just want to say thanks a lot for this! I tested it, and it pretty
much works as advertised.
Cc'ing Helio who was working on a Go importer as well recently.
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 507 bytes --]
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] Add a Go Module Importer
[not found] ` <CANe01w55ZO=_9v0HcDv248UsoLUXb_9WVAgM4LqiZ4E-r1XgXg@mail.gmail.com>
@ 2020-11-11 1:23 ` Helio Machado
2021-01-23 21:35 ` [bug#44178] [PATCH] Create importer for Go modules guix-patches--- via
0 siblings, 1 reply; 27+ messages in thread
From: Helio Machado @ 2020-11-11 1:23 UTC (permalink / raw)
To: 44178
[-- Attachment #1: Type: text/plain, Size: 2703 bytes --]
Thanks for the ping, Marius! I've been quite busy with some yak shaving
tasks, but my importer is already working and has some interesting
improvements, like elegant module fetching from the official module proxy,
license extraction and recursive import support.
I need to fix an esoteric bug that trips the kernel's out-of-memory killer
when building a derivation with dependencies, but the importer part works
pretty well.
You can take a look to [my changes][1] for some readily backportable ideas,
like [the compact algorithm for uppercase path escaping][2].
Please forgive the code quality and the possible backwards-compatibility
mistakes; this is an unfinished proof of concept.
$ guix import go-modules -r github.com/FiloSottile/age # Please refer to
the issue 43872 for more information about the resting environment
[1]:
https://github.com/0x2b3bfa0/guix-go-modules/commit/5defe897065c5d3e63740932b360474132c77877
[2]:
https://github.com/0x2b3bfa0/guix-go-modules/blob/main/guix/build-system/go.scm#L65-L71
On Wed, 11 Nov 2020 at 02:19, Helio Machado <0x2b3bfa0@gmail.com> wrote:
> Thanks for the ping, Marius! I've been quite busy with some yak shaving
> tasks, but my importer is already working and has some interesting
> improvements, like elegant module fetching from the official module proxy,
> license extraction and recursive import support.
>
> I need to fix an esoteric bug that trips the kernel's out-of-memory killer
> when building a derivation with dependencies, but the importer part works
> pretty well.
>
> You can take a look to [my changes][1] for some readily backportable
> ideas, like [the compact algorithm for uppercase path escaping][2].
>
> Please forgive the code quality and the possible backwards-compatibility
> mistakes; this is an unfinished proof of concept.
>
> [1]:
> https://github.com/0x2b3bfa0/guix-go-modules/commit/5defe897065c5d3e63740932b360474132c77877
> [2]:
> https://github.com/0x2b3bfa0/guix-go-modules/blob/main/guix/build-system/go.scm#L65-L71
>
> On Tue, 10 Nov 2020 at 21:26, Marius Bakke <marius@gnu.org> wrote:
>
>> Katherine Cox-Buday <cox.katherine.e@gmail.com> writes:
>>
>> >>From cc92cbcf5ae89891f478f319e955419800bdfcf9 Mon Sep 17 00:00:00 2001
>> > From: Katherine Cox-Buday <cox.katherine.e@gmail.com>
>> > Date: Thu, 22 Oct 2020 19:40:17 -0500
>> > Subject: [PATCH] * guix/import/go.scm: Created Go Importer *
>> > guix/scripts/import.scm: Created Go Importer Subcommand *
>> guix/import/go.scm
>> > (importers): Added Go Importer Subcommand
>>
>> I just want to say thanks a lot for this! I tested it, and it pretty
>> much works as advertised.
>>
>> Cc'ing Helio who was working on a Go importer as well recently.
>>
>
[-- Attachment #2: Type: text/html, Size: 4512 bytes --]
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] Add a Go Module Importer
2020-11-10 20:26 ` Marius Bakke
[not found] ` <CANe01w55ZO=_9v0HcDv248UsoLUXb_9WVAgM4LqiZ4E-r1XgXg@mail.gmail.com>
@ 2020-11-11 20:48 ` Katherine Cox-Buday
1 sibling, 0 replies; 27+ messages in thread
From: Katherine Cox-Buday @ 2020-11-11 20:48 UTC (permalink / raw)
To: Marius Bakke; +Cc: 44178, Helio Machado
Marius Bakke <marius@gnu.org> writes:
> I just want to say thanks a lot for this! I tested it, and it pretty
> much works as advertised.
You're very welcome!
I have more changes locally which fix some edge-cases. I'm using
`go-ethereum` as my test case since someone mentioned that. Plus I need
to make some of the changes Ludovic pointed out. Still, we're underway!
--
Katherine
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178]
2020-10-23 14:06 [bug#44178] Add a Go Module Importer Katherine Cox-Buday
` (2 preceding siblings ...)
2020-11-10 20:26 ` Marius Bakke
@ 2020-12-09 14:22 ` dftxbs3e
2020-12-10 2:42 ` [bug#44178] dftxbs3e
2021-01-28 7:29 ` [bug#44178] [PATCH] Create importer for Go modules guix-patches--- via
4 siblings, 1 reply; 27+ messages in thread
From: dftxbs3e @ 2020-12-09 14:22 UTC (permalink / raw)
To: 44178
Thanks a lot for this!
I'm getting some error trying to use it (patching on top of
8e2aad26ae9b7365db83d4f6c74e9e79c57766a6), maybe that's fixed in your
local changes?
$ ./pre-inst-env guix import go -r github.com/syncthing/syncthing
WARNING: (guix import go): `url-fetch' imported from both (guix import
utils) and (guix build download)
Backtrace:
In ice-9/boot-9.scm:
1736:10 7 (with-exception-handler _ _ #:unwind? _ # _)
In unknown file:
6 (apply-smob/0 #<thunk 7ff10807d9a0>)
In ice-9/boot-9.scm:
718:2 5 (call-with-prompt _ _ #<procedure default-prompt-handle…>)
In ice-9/eval.scm:
619:8 4 (_ #(#(#<directory (guile-user) 7ff107cadf00>)))
In guix/ui.scm:
2127:12 3 (run-guix-command _ . _)
In guix/scripts/import.scm:
120:11 2 (guix-import . _)
In ice-9/eval.scm:
159:9 1 (_ #(#(#(#(#(#(#(#(#(#<directo…>) …) …) …) …) …) …) …) …))
In guix/import/utils.scm:
429:0 0 (recursive-import _ #:repo->guix-package _ #:guix-name _
…)
guix/import/utils.scm:429:0: In procedure recursive-import:
Invalid keyword: #f
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178]
2020-12-09 14:22 ` [bug#44178] dftxbs3e
@ 2020-12-10 2:42 ` dftxbs3e
2020-12-10 3:14 ` [bug#44178] dftxbs3e
0 siblings, 1 reply; 27+ messages in thread
From: dftxbs3e @ 2020-12-10 2:42 UTC (permalink / raw)
To: 44178
[-- Attachment #1: Type: text/plain, Size: 1456 bytes --]
On Wed, 2020-12-09 at 15:22 +0100, dftxbs3e wrote:
> Thanks a lot for this!
>
> I'm getting some error trying to use it (patching on top of
> 8e2aad26ae9b7365db83d4f6c74e9e79c57766a6), maybe that's fixed in your
> local changes?
>
> $ ./pre-inst-env guix import go -r github.com/syncthing/syncthing
> WARNING: (guix import go): `url-fetch' imported from both (guix
> import
> utils) and (guix build download)
> Backtrace:
> In ice-9/boot-9.scm:
> 1736:10 7 (with-exception-handler _ _ #:unwind? _ # _)
> In unknown file:
> 6 (apply-smob/0 #<thunk 7ff10807d9a0>)
> In ice-9/boot-9.scm:
> 718:2 5 (call-with-prompt _ _ #<procedure default-prompt-
> handle…>)
> In ice-9/eval.scm:
> 619:8 4 (_ #(#(#<directory (guile-user) 7ff107cadf00>)))
> In guix/ui.scm:
> 2127:12 3 (run-guix-command _ . _)
> In guix/scripts/import.scm:
> 120:11 2 (guix-import . _)
> In ice-9/eval.scm:
> 159:9 1 (_ #(#(#(#(#(#(#(#(#(#<directo…>) …) …) …) …) …) …) …)
> …))
> In guix/import/utils.scm:
> 429:0 0 (recursive-import _ #:repo->guix-package _ #:guix-name _
> …)
>
> guix/import/utils.scm:429:0: In procedure recursive-import:
> Invalid keyword: #f
I could fix it using the attached patch!
However, I noticed it doesnt pin versions in GNU Guix to what they are
in go.mod file, is that expected? It always takes the latest. It might
work but I am thinking it might cause breakage at some point?
Thank you!
[-- Attachment #2: go-module-importer-fix-guix-8e2aad26ae9b7365db83d4f6c74e9e79c57766a6.patch --]
[-- Type: text/x-patch, Size: 1004 bytes --]
diff --git a/guix/import/go.scm b/guix/import/go.scm
index 61009f3565..c7a1b1a5d4 100644
--- a/guix/import/go.scm
+++ b/guix/import/go.scm
@@ -23,7 +23,7 @@
#:use-module (ice-9 regex)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-9)
- #:use-module (guix json)
+ #:use-module (json)
#:use-module ((guix download) #:prefix download:)
#:use-module (guix import utils)
#:use-module (guix import json)
@@ -268,9 +268,9 @@ control system is being used."
(define* (go-module-recursive-import package-name
#:key (goproxy-url "https://proxy.golang.org"))
- (recursive-import package-name #f
+ (recursive-import package-name
#:repo->guix-package
- (lambda (name _)
+ (lambda* (name #:key repo version)
(go-module->guix-package name
#:goproxy-url goproxy-url))
#:guix-name to-guix-package-name))
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [bug#44178]
2020-12-10 2:42 ` [bug#44178] dftxbs3e
@ 2020-12-10 3:14 ` dftxbs3e
0 siblings, 0 replies; 27+ messages in thread
From: dftxbs3e @ 2020-12-10 3:14 UTC (permalink / raw)
To: 44178
It now fails with:
$ ./pre-inst-env guix import go -r github.com/hashicorp/consul/api
WARNING: (guix import go): `url-fetch' imported from both (guix import
utils) and (guix build download)
Starting download of /tmp/guix-file.i8tqa2
From
https://proxy.golang.org/github.com/hashicorp/consul/api/@v/v1.8.0.mod.
..
v1.8.0.mod 424B 334KiB/s 00:00
[##################] 100.0%
Backtrace:
In ice-9/boot-9.scm:
1736:10 17 (with-exception-handler _ _ #:unwind? _ # _)
In unknown file:
16 (apply-smob/0 #<thunk 7fc5871054a0>)
In ice-9/boot-9.scm:
718:2 15 (call-with-prompt _ _ #<procedure default-prompt-handle…>)
In ice-9/eval.scm:
619:8 14 (_ #(#(#<directory (guile-user) 7fc586d40f00>)))
In guix/ui.scm:
2127:12 13 (run-guix-command _ . _)
In guix/scripts/import.scm:
120:11 12 (guix-import . _)
In ice-9/eval.scm:
159:9 11 (_ _)
In guix/import/utils.scm:
458:31 10 (recursive-import _ #:repo->guix-package _ #:guix-name _
…)
449:33 9 (lookup-node "github.com/hashicorp/consul/api" #f)
In guix/utils.scm:
697:8 8 (call-with-temporary-output-file _)
In ice-9/eval.scm:
293:34 7 (_ #(#(#(#(#<directory (guix import go) 7fc…> …) …) …) …))
159:9 6 (_ #(#(#(#(#<directory (guix import go) 7fc…> …) …) …) …))
In ice-9/ports.scm:
445:17 5 (call-with-input-file _ _ #:binary _ #:encoding _ # _)
470:4 4 (_ _)
In ice-9/eval.scm:
619:8 3 (_ #(#(#(#<directory (guix import go) 7fc584cbef00>)) …))
619:8 2 (_ #(#(#<directory (guix import go) 7fc584cbef00> # …) …))
293:34 1 (_ #(#(#(#(#(#<directory (guix import go…> …) …) …) …) …))
In unknown file:
0 (list-ref ("replace" "github.com/hashicorp/consul/s…" …)
…)
ERROR: In procedure list-ref:
In procedure list-ref: Argument 2 out of range: 4
It's probably because the go.mod file contains a self-referencing
replace line (seems unsupported by the code):
module github.com/hashicorp/consul/api
go 1.12
replace github.com/hashicorp/consul/sdk => ../sdk
require (
github.com/hashicorp/consul/sdk v0.7.0
github.com/hashicorp/go-cleanhttp v0.5.1
github.com/hashicorp/go-hclog v0.12.0
github.com/hashicorp/go-rootcerts v1.0.2
github.com/hashicorp/go-uuid v1.0.1
github.com/hashicorp/serf v0.9.5
github.com/mitchellh/mapstructure v1.1.2
github.com/stretchr/testify v1.4.0
)
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2020-11-11 1:23 ` Helio Machado
@ 2021-01-23 21:35 ` guix-patches--- via
2021-01-23 22:41 ` Katherine Cox-Buday
0 siblings, 1 reply; 27+ messages in thread
From: guix-patches--- via @ 2021-01-23 21:35 UTC (permalink / raw)
To: 44178@debbugs.gnu.org; +Cc: Helio Machado, Katherine Cox-Buday
This patch add a `guix import go` command.
It was tested with several big repositories and seems to mostly work for
the import part (because building Guix packages is an other story). There
is still bugs blocking e.g. use of any k8s.io modules.
* guix/import/go.scm: Created Go Importer
* guix/scripts/import.scm: Created Go Importer Subcommand
* guix/import/go.scm (importers): Added Go Importer Subcommand
Signed-off-by: Francois Joulaud <francois.joulaud@radiofrance.com>
---
The patch is a rebased and modified version of the one proposed by
Katherine Cox-Buday.
Notable modifications are :
- move from (guix json) to (json)
- new parse-go.mod with no "set!" and parsing some go.mod which were in
error before
- adding comments (maybe too much comments)
- renamed SCS to VCS to be in accordance with vocabulary in use in Guix
and in Go worlds
- replacing escape-capital-letters by Helio Machado's go-path-escape
- no pruning of major version in go module names as they are considered
as completely different artefacts by Go programmers
- fixed recursive-import probably broken by the rebase
- force usage of url-fetch from (guix build download)
I would be happy to hear about problems and perspective for this patch and
will now focus on my next step which is actually building any package.
Hope I CCed the right persons, I am not really aware of applicable
netiquette here.
Interdiff :
diff --git a/guix/import/go.scm b/guix/import/go.scm
index 61009f3565..7f5f300f0a 100644
--- a/guix/import/go.scm
+++ b/guix/import/go.scm
@@ -1,5 +1,7 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -16,6 +18,21 @@
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+;;; (guix import golang) wants to make easier to create Guix package
+;;; declaration for Go modules.
+;;;
+;;; Modules in Go are "collection of related Go packages" which are
+;;; "the unit of source code interchange and versioning".
+;;; Modules are generally hosted in a repository.
+;;;
+;;; At this point it should handle correctly modules which
+;;; - have only Go dependencies;
+;;; - use go.mod;
+;;; - and are accessible from proxy.golang.org (or configured GOPROXY).
+;;;
+;;; We translate Go module paths to a Guix package name under the
+;;; assumption that there will be no collision.
+
(define-module (guix import go)
#:use-module (ice-9 match)
#:use-module (ice-9 rdelim)
@@ -23,7 +40,7 @@
#:use-module (ice-9 regex)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-9)
- #:use-module (guix json)
+ #:use-module (json)
#:use-module ((guix download) #:prefix download:)
#:use-module (guix import utils)
#:use-module (guix import json)
@@ -33,88 +50,129 @@
#:use-module ((guix licenses) #:prefix license:)
#:use-module (guix base16)
#:use-module (guix base32)
- #:use-module (guix build download)
+ #:use-module ((guix build download) #:prefix build-download:)
#:use-module (web uri)
#:export (go-module->guix-package
go-module-recursive-import
infer-module-root))
-(define (escape-capital-letters s)
- "To avoid ambiguity when serving from case-insensitive file systems, the
-$module and $version elements are case-encoded by replacing every uppercase
-letter with an exclamation mark followed by the corresponding lower-case
-letter."
- (let ((escaped-string (string)))
- (string-for-each-index
- (lambda (i)
- (let ((c (string-ref s i)))
- (set! escaped-string
- (string-concatenate
- (list escaped-string
- (if (char-upper-case? c) "!" "")
- (string (char-downcase c)))))))
- s)
- escaped-string))
+(define (go-path-escape path)
+ "Escape a module path by replacing every uppercase letter with an exclamation
+mark followed with its lowercase equivalent, as per the module Escaped Paths
+specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
+ (define (escape occurrence)
+ (string-append "!" (string-downcase (match:substring occurrence))))
+ (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
+
(define (fetch-latest-version goproxy-url module-path)
"Fetches the version number of the latest version for MODULE-PATH from the
given GOPROXY-URL server."
(assoc-ref
(json-fetch (format #f "~a/~a/@latest" goproxy-url
- (escape-capital-letters module-path)))
+ (go-path-escape module-path)))
"Version"))
(define (fetch-go.mod goproxy-url module-path version file)
"Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
and VERSION."
- (url-fetch (format #f "~a/~a/@v/~a.mod" goproxy-url
- (escape-capital-letters module-path)
- (escape-capital-letters version))
- file
- #:print-build-trace? #f))
+ (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
+ (go-path-escape module-path)
+ (go-path-escape version))))
+ (parameterize ((current-output-port (current-error-port)))
+ (build-download:url-fetch url
+ file
+ #:print-build-trace? #f))))
(define (parse-go.mod go.mod-path)
- "Parses a go.mod file and returns an alist of module path to version."
+ "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
+requirements from it."
+ ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
+ ;; which we think necessary for our use case.
+ (define (toplevel results)
+ "Main parser, RESULTS is a pair of alist serving as accumulator for
+ all encountered requirements and replacements."
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; parsing ended, give back the result
+ results)
+ ((string=? line "require (")
+ ;; a require block begins, delegate parsing to IN-REQUIRE
+ (in-require results))
+ ((string-prefix? "require " line)
+ ;; a require directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (require-directive results stripped-line)))
+ (toplevel new-results)))
+ ((string-prefix? "replace " line)
+ ;; a replace directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (replace-directive results stripped-line)))
+ (toplevel new-results)))
+ (#t
+ ;; unrecognised line, ignore silently
+ (toplevel results)))))
+ (define (in-require results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-require (require-directive results line))))))
+ (define (replace-directive results line)
+ "Extract replaced modules and new requirements from replace directive
+ in LINE and add to RESULTS."
+ ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
+ ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ (re (string-concatenate
+ '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
+ "[[:blank:]]+" "=>" "[[:blank:]]+"
+ "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 3))
+ (new-module-path (match:substring match 4))
+ (new-version (match:substring match 6))
+ (new-replaced (acons module-path version replaced))
+ (new-requirements
+ (if (string-match "^\\.?\\./" new-module-path)
+ requirements
+ (acons new-module-path new-version requirements))))
+ (cons new-requirements new-replaced)))
+ (define (require-directive results line)
+ "Extract requirement from LINE and add it to RESULTS."
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ ;; A line in a require directive is composed of a module path and
+ ;; a version separated by whitespace and an optionnal '//' comment at
+ ;; the end.
+ (re (string-concatenate
+ '("^[[:blank:]]*"
+ "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
+ "([[:blank:]]+//.*)?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 2)))
+ (cons (acons module-path version requirements) replaced)))
(with-input-from-file go.mod-path
(lambda ()
- (let ((in-require? #f)
- (requirements (list)))
- (do ((line (read-line) (read-line)))
- ((eof-object? line))
- (set! line (string-trim line))
- ;; The parser is either entering, within, exiting, or after the
- ;; require block. The Go toolchain is trustworthy so edge-cases like
- ;; double-entry, etc. need not complect the parser.
- (cond
- ((string=? line "require (")
- (set! in-require? #t))
- ((and in-require? (string=? line ")"))
- (set! in-require? #f))
- (in-require?
- (let* ((requirement (string-split line #\space))
- ;; Modules should be unquoted
- (module-path (string-delete #\" (car requirement)))
- (version (list-ref requirement 1)))
- (set! requirements (acons module-path version requirements))))
- ((string-prefix? "replace" line)
- (let* ((requirement (string-split line #\space))
- (module-path (list-ref requirement 1))
- (new-module-path (list-ref requirement 3))
- (version (list-ref requirement 4)))
- (set! requirements (assoc-remove! requirements module-path))
- (set! requirements (acons new-module-path version requirements))))))
- requirements))))
-
-(define (module-path-without-major-version module-path)
- "Go modules can be appended with a major version indicator,
-e.g. /v3. Sometimes it is desirable to work with the root module path. For
-instance, for a module path github.com/foo/bar/v3 this function returns
-github.com/foo/bar."
- (let ((m (string-match "(.*)\\/v[0-9]+$" module-path)))
- (if m
- (match:substring m 1)
- module-path)))
+ (let* ((results (toplevel '(() . ())))
+ (requirements (car results))
+ (replaced (cdr results)))
+ ;; At last we remove replaced modules from the requirements list
+ (fold
+ (lambda (replacedelem requirements)
+ (alist-delete! (car replacedelem) requirements))
+ requirements
+ replaced)))))
(define (infer-module-root module-path)
"Go modules can be defined at any level of a repository's tree, but querying
@@ -124,38 +182,42 @@ root path from its path. For a set of well-known forges, the pattern of what
consists of a module's root page is known before hand."
;; See the following URL for the official Go equivalent:
;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
- (define-record-type <scs>
- (make-scs url-prefix root-regex type)
- scs?
- (url-prefix scs-url-prefix)
- (root-regex scs-root-regex)
- (type scs-type))
- (let* ((known-scs
+ ;;
+ ;; FIXME: handle module path with VCS qualifier as described in
+ ;; https://golang.org/ref/mod#vcs-find and
+ ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
+ (define-record-type <vcs>
+ (make-vcs url-prefix root-regex type)
+ vcs?
+ (url-prefix vcs-url-prefix)
+ (root-regex vcs-root-regex)
+ (type vcs-type))
+ (let* ((known-vcs
(list
- (make-scs
+ (make-vcs
"github.com"
"^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
'git)
- (make-scs
+ (make-vcs
"bitbucket.org"
"^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$`"
'unknown)
- (make-scs
+ (make-vcs
"hub.jazz.net/git/"
"^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
'git)
- (make-scs
+ (make-vcs
"git.apache.org"
"^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
'git)
- (make-scs
+ (make-vcs
"git.openstack.org"
"^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
'git)))
- (scs (find (lambda (scs) (string-prefix? (scs-url-prefix scs) module-path))
- known-scs)))
- (if scs
- (match:substring (string-match (scs-root-regex scs) module-path) 1)
+ (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
+ known-vcs)))
+ (if vcs
+ (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
module-path)))
(define (to-guix-package-name module-path)
@@ -164,8 +226,7 @@ consists of a module's root page is known before hand."
(string-append "go-"
(string-replace-substring
(string-replace-substring
- ;; Guix has its own field for version
- (module-path-without-major-version module-path)
+ module-path
"." "-")
"/" "-"))))
@@ -173,7 +234,9 @@ consists of a module's root page is known before hand."
"Fetches module meta-data from a module's landing page. This is necessary
because goproxy servers don't currently provide all the information needed to
build a package."
- (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
+ ;; FIXME: This code breaks on k8s.io which have a meta tag splitted
+ ;; on several lines
+ (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
(module-metadata #f)
(meta-tag-prefix "<meta name=\"go-import\" content=\"")
(meta-tag-prefix-length (string-length meta-tag-prefix)))
@@ -185,7 +248,7 @@ build a package."
(let* ((start (+ meta-tag-index meta-tag-prefix-length))
(end (string-index line #\" start)))
(set! module-metadata
- (string-split (substring/shared line start end) #\space))))))
+ (string-split (substring/shared line start end) #\space))))))
(close-port port)
module-metadata))
@@ -244,7 +307,7 @@ control system is being used."
(dependencies (map car (parse-go.mod temp)))
(guix-name (to-guix-package-name module-path))
(root-module-path (infer-module-root module-path))
- ;; SCS type and URL are not included in goproxy information. For
+ ;; VCS type and URL are not included in goproxy information. For
;; this we need to fetch it from the official module page.
(meta-data (fetch-module-meta-data root-module-path))
(scs-type (module-meta-data-scs meta-data))
@@ -268,9 +331,10 @@ control system is being used."
(define* (go-module-recursive-import package-name
#:key (goproxy-url "https://proxy.golang.org"))
- (recursive-import package-name #f
- #:repo->guix-package
- (lambda (name _)
- (go-module->guix-package name
- #:goproxy-url goproxy-url))
- #:guix-name to-guix-package-name))
+ (recursive-import
+ package-name
+ #:repo->guix-package (lambda* (name . _)
+ (go-module->guix-package
+ name
+ #:goproxy-url goproxy-url))
+ #:guix-name to-guix-package-name))
guix/import/go.scm | 340 +++++++++++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/go.scm | 118 +++++++++++++
3 files changed, 459 insertions(+), 1 deletion(-)
create mode 100644 guix/import/go.scm
create mode 100644 guix/scripts/import/go.scm
diff --git a/guix/import/go.scm b/guix/import/go.scm
new file mode 100644
index 0000000000..7f5f300f0a
--- /dev/null
+++ b/guix/import/go.scm
@@ -0,0 +1,340 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; (guix import golang) wants to make easier to create Guix package
+;;; declaration for Go modules.
+;;;
+;;; Modules in Go are "collection of related Go packages" which are
+;;; "the unit of source code interchange and versioning".
+;;; Modules are generally hosted in a repository.
+;;;
+;;; At this point it should handle correctly modules which
+;;; - have only Go dependencies;
+;;; - use go.mod;
+;;; - and are accessible from proxy.golang.org (or configured GOPROXY).
+;;;
+;;; We translate Go module paths to a Guix package name under the
+;;; assumption that there will be no collision.
+
+(define-module (guix import go)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 rdelim)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (json)
+ #:use-module ((guix download) #:prefix download:)
+ #:use-module (guix import utils)
+ #:use-module (guix import json)
+ #:use-module (guix packages)
+ #:use-module (guix upstream)
+ #:use-module (guix utils)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix base16)
+ #:use-module (guix base32)
+ #:use-module ((guix build download) #:prefix build-download:)
+ #:use-module (web uri)
+
+ #:export (go-module->guix-package
+ go-module-recursive-import
+ infer-module-root))
+
+(define (go-path-escape path)
+ "Escape a module path by replacing every uppercase letter with an exclamation
+mark followed with its lowercase equivalent, as per the module Escaped Paths
+specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
+ (define (escape occurrence)
+ (string-append "!" (string-downcase (match:substring occurrence))))
+ (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
+
+
+(define (fetch-latest-version goproxy-url module-path)
+ "Fetches the version number of the latest version for MODULE-PATH from the
+given GOPROXY-URL server."
+ (assoc-ref
+ (json-fetch (format #f "~a/~a/@latest" goproxy-url
+ (go-path-escape module-path)))
+ "Version"))
+
+(define (fetch-go.mod goproxy-url module-path version file)
+ "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
+and VERSION."
+ (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
+ (go-path-escape module-path)
+ (go-path-escape version))))
+ (parameterize ((current-output-port (current-error-port)))
+ (build-download:url-fetch url
+ file
+ #:print-build-trace? #f))))
+
+(define (parse-go.mod go.mod-path)
+ "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
+requirements from it."
+ ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
+ ;; which we think necessary for our use case.
+ (define (toplevel results)
+ "Main parser, RESULTS is a pair of alist serving as accumulator for
+ all encountered requirements and replacements."
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; parsing ended, give back the result
+ results)
+ ((string=? line "require (")
+ ;; a require block begins, delegate parsing to IN-REQUIRE
+ (in-require results))
+ ((string-prefix? "require " line)
+ ;; a require directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (require-directive results stripped-line)))
+ (toplevel new-results)))
+ ((string-prefix? "replace " line)
+ ;; a replace directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (replace-directive results stripped-line)))
+ (toplevel new-results)))
+ (#t
+ ;; unrecognised line, ignore silently
+ (toplevel results)))))
+ (define (in-require results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-require (require-directive results line))))))
+ (define (replace-directive results line)
+ "Extract replaced modules and new requirements from replace directive
+ in LINE and add to RESULTS."
+ ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
+ ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ (re (string-concatenate
+ '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
+ "[[:blank:]]+" "=>" "[[:blank:]]+"
+ "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 3))
+ (new-module-path (match:substring match 4))
+ (new-version (match:substring match 6))
+ (new-replaced (acons module-path version replaced))
+ (new-requirements
+ (if (string-match "^\\.?\\./" new-module-path)
+ requirements
+ (acons new-module-path new-version requirements))))
+ (cons new-requirements new-replaced)))
+ (define (require-directive results line)
+ "Extract requirement from LINE and add it to RESULTS."
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ ;; A line in a require directive is composed of a module path and
+ ;; a version separated by whitespace and an optionnal '//' comment at
+ ;; the end.
+ (re (string-concatenate
+ '("^[[:blank:]]*"
+ "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
+ "([[:blank:]]+//.*)?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 2)))
+ (cons (acons module-path version requirements) replaced)))
+ (with-input-from-file go.mod-path
+ (lambda ()
+ (let* ((results (toplevel '(() . ())))
+ (requirements (car results))
+ (replaced (cdr results)))
+ ;; At last we remove replaced modules from the requirements list
+ (fold
+ (lambda (replacedelem requirements)
+ (alist-delete! (car replacedelem) requirements))
+ requirements
+ replaced)))))
+
+(define (infer-module-root module-path)
+ "Go modules can be defined at any level of a repository's tree, but querying
+for the meta tag usually can only be done at the webpage at the root of the
+repository. Therefore, it is sometimes necessary to try and derive a module's
+root path from its path. For a set of well-known forges, the pattern of what
+consists of a module's root page is known before hand."
+ ;; See the following URL for the official Go equivalent:
+ ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
+ ;;
+ ;; FIXME: handle module path with VCS qualifier as described in
+ ;; https://golang.org/ref/mod#vcs-find and
+ ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
+ (define-record-type <vcs>
+ (make-vcs url-prefix root-regex type)
+ vcs?
+ (url-prefix vcs-url-prefix)
+ (root-regex vcs-root-regex)
+ (type vcs-type))
+ (let* ((known-vcs
+ (list
+ (make-vcs
+ "github.com"
+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "bitbucket.org"
+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$`"
+ 'unknown)
+ (make-vcs
+ "hub.jazz.net/git/"
+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.apache.org"
+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.openstack.org"
+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)))
+ (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
+ known-vcs)))
+ (if vcs
+ (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
+ module-path)))
+
+(define (to-guix-package-name module-path)
+ "Converts a module's path to the canonical Guix format for Go packages."
+ (string-downcase
+ (string-append "go-"
+ (string-replace-substring
+ (string-replace-substring
+ module-path
+ "." "-")
+ "/" "-"))))
+
+(define (fetch-module-meta-data module-path)
+ "Fetches module meta-data from a module's landing page. This is necessary
+because goproxy servers don't currently provide all the information needed to
+build a package."
+ ;; FIXME: This code breaks on k8s.io which have a meta tag splitted
+ ;; on several lines
+ (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
+ (module-metadata #f)
+ (meta-tag-prefix "<meta name=\"go-import\" content=\"")
+ (meta-tag-prefix-length (string-length meta-tag-prefix)))
+ (do ((line (read-line port) (read-line port)))
+ ((or (eof-object? line)
+ module-metadata))
+ (let ((meta-tag-index (string-contains line meta-tag-prefix)))
+ (when meta-tag-index
+ (let* ((start (+ meta-tag-index meta-tag-prefix-length))
+ (end (string-index line #\" start)))
+ (set! module-metadata
+ (string-split (substring/shared line start end) #\space))))))
+ (close-port port)
+ module-metadata))
+
+(define (module-meta-data-scs meta-data)
+ "Return the source control system specified by a module's meta-data."
+ (string->symbol (list-ref meta-data 1)))
+
+(define (module-meta-data-repo-url meta-data goproxy-url)
+ "Return the URL where the fetcher which will be used can download the source
+control."
+ (if (member (module-meta-data-scs meta-data) '(fossil mod))
+ goproxy-url
+ (list-ref meta-data 2)))
+
+(define (source-uri scs-type scs-repo-url file)
+ "Generate the `origin' block of a package depending on what type of source
+control system is being used."
+ (case scs-type
+ ((git)
+ `(origin
+ (method git-fetch)
+ (uri (git-reference
+ (url ,scs-repo-url)
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ ((hg)
+ `(origin
+ (method hg-fetch)
+ (uri (hg-reference
+ (url ,scs-repo-url)
+ (changeset ,version)))
+ (file-name (format #f "~a-~a-checkout" name version))))
+ ((svn)
+ `(origin
+ (method svn-fetch)
+ (uri (svn-reference
+ (url ,scs-repo-url)
+ (revision (string->number version))
+ (recursive? #f)))
+ (file-name (format #f "~a-~a-checkout" name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ (else
+ (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))
+
+(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (let* ((latest-version (fetch-latest-version goproxy-url module-path))
+ (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
+ temp))
+ (dependencies (map car (parse-go.mod temp)))
+ (guix-name (to-guix-package-name module-path))
+ (root-module-path (infer-module-root module-path))
+ ;; VCS type and URL are not included in goproxy information. For
+ ;; this we need to fetch it from the official module page.
+ (meta-data (fetch-module-meta-data root-module-path))
+ (scs-type (module-meta-data-scs meta-data))
+ (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
+ (values
+ `(package
+ (name ,guix-name)
+ ;; Elide the "v" prefix Go uses
+ (version ,(string-trim latest-version #\v))
+ (source
+ ,(source-uri scs-type scs-repo-url temp))
+ (build-system go-build-system)
+ ,@(maybe-inputs (map to-guix-package-name dependencies))
+ ;; TODO(katco): It would be nice to make an effort to fetch this
+ ;; from known forges, e.g. GitHub
+ (home-page ,(format #f "https://~a" root-module-path))
+ (synopsis "A Go package")
+ (description ,(format #f "~a is a Go package." guix-name))
+ (license #f))
+ dependencies)))))
+
+(define* (go-module-recursive-import package-name
+ #:key (goproxy-url "https://proxy.golang.org"))
+ (recursive-import
+ package-name
+ #:repo->guix-package (lambda* (name . _)
+ (go-module->guix-package
+ name
+ #:goproxy-url goproxy-url))
+ #:guix-name to-guix-package-name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 0a3863f965..1d2b45d942 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -77,7 +77,7 @@ rather than \\n."
;;;
(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"
- "cran" "crate" "texlive" "json" "opam"))
+ "go" "cran" "crate" "texlive" "json" "opam"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
new file mode 100644
index 0000000000..fde7555973
--- /dev/null
+++ b/guix/scripts/import/go.scm
@@ -0,0 +1,118 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import go)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import go)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-go))
+
+
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import go PACKAGE-PATH
+Import and convert the Go module for PACKAGE-PATH.\n"))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (display (G_ "
+ -r, --recursive generate package expressions for all Go modules\
+ that are not yet in Guix"))
+ (display (G_ "
+ -p, --goproxy=GOPROXY specify which goproxy server to use"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import go")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ (option '(#\p "goproxy") #t #f
+ (lambda (opt name arg result)
+ (alist-cons 'goproxy
+ (string->symbol arg)
+ (alist-delete 'goproxy result))))
+ %standard-import-options))
+
+
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-go . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (match args
+ ((module-name)
+ (if (assoc-ref opts 'recursive)
+ (map (match-lambda
+ ((and ('package ('name name) . rest) pkg)
+ `(define-public ,(string->symbol name)
+ ,pkg))
+ (_ #f))
+ (go-module-recursive-import module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org")))
+ (let ((sexp (go-module->guix-package module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org"))))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for module '~a'~%")
+ module-name))
+ sexp)))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%"))))))
--
2.28.0
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-23 21:35 ` [bug#44178] [PATCH] Create importer for Go modules guix-patches--- via
@ 2021-01-23 22:41 ` Katherine Cox-Buday
2021-01-25 21:03 ` guix-patches--- via
0 siblings, 1 reply; 27+ messages in thread
From: Katherine Cox-Buday @ 2021-01-23 22:41 UTC (permalink / raw)
To: JOULAUD François; +Cc: 44178@debbugs.gnu.org, Helio Machado
Thanks so much for the patches, Helio, Joulaud!
I apologize for the long delay before looking at this again. My time
right now is extremely limited due to COVID-19 related childcare and
activities. I was negligent and left a patch to bitrot on my
computer[1]. This patch supersedes it.
In addition to the things this patch corrects, I was/am working on a few
other bugs:
- There are valid Go Module paths which when queried will not serve the
requisite meta tag. I had modified `fetch-module-meta-data` to
recursively walk up the module path searching for a valid meta tag
(see diff[1]).
- I think Joulaud's patch covers this, but replacements relative to a
module are a possibility.
- For reasons Joulaud calls out, a simple line-parser of the HTML for a
module is not sufficient. Since we are pulling pages down from the
wider internet, we should fall back on a library made for parsing HTML
so we handle any edge-cases (e.g. meta tags split across lines). I am
currently looking at `sxml`, and if that doesn't pan out `htmlprag`
- Some module homepages issue HTTP redirects. Last time I tested this,
`http-fetch` does not handle this properly.
I think that covers everything.
I have pushed everything (including Joulaud's patch with appropriate
attribution) here[2]. I am admittedly new at using email to organize
code changes, but using a forge seems easier.
[1] https://github.com/guix-mirror/guix/commit/cce35c6d68a9bddf9558e85d2cb88be323da9247
[2] https://github.com/kat-co/guix/tree/create-go-importer
Can I suggest we coordinate there, or is that too much of an imposition?
-
Katherine
JOULAUD François <Francois.JOULAUD@radiofrance.com> writes:
> This patch add a `guix import go` command.
>
> It was tested with several big repositories and seems to mostly work for
> the import part (because building Guix packages is an other story). There
> is still bugs blocking e.g. use of any k8s.io modules.
>
> * guix/import/go.scm: Created Go Importer
> * guix/scripts/import.scm: Created Go Importer Subcommand
> * guix/import/go.scm (importers): Added Go Importer Subcommand
>
> Signed-off-by: Francois Joulaud <francois.joulaud@radiofrance.com>
> ---
> The patch is a rebased and modified version of the one proposed by
> Katherine Cox-Buday.
>
> Notable modifications are :
> - move from (guix json) to (json)
> - new parse-go.mod with no "set!" and parsing some go.mod which were in
> error before
> - adding comments (maybe too much comments)
> - renamed SCS to VCS to be in accordance with vocabulary in use in Guix
> and in Go worlds
> - replacing escape-capital-letters by Helio Machado's go-path-escape
> - no pruning of major version in go module names as they are considered
> as completely different artefacts by Go programmers
> - fixed recursive-import probably broken by the rebase
> - force usage of url-fetch from (guix build download)
>
> I would be happy to hear about problems and perspective for this patch and
> will now focus on my next step which is actually building any package.
>
> Hope I CCed the right persons, I am not really aware of applicable
> netiquette here.
>
> Interdiff :
> diff --git a/guix/import/go.scm b/guix/import/go.scm
> index 61009f3565..7f5f300f0a 100644
> --- a/guix/import/go.scm
> +++ b/guix/import/go.scm
> @@ -1,5 +1,7 @@
> ;;; GNU Guix --- Functional package management for GNU
> ;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
> +;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
> +;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
> ;;;
> ;;; This file is part of GNU Guix.
> ;;;
> @@ -16,6 +18,21 @@
> ;;; You should have received a copy of the GNU General Public License
> ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
>
> +;;; (guix import golang) wants to make easier to create Guix package
> +;;; declaration for Go modules.
> +;;;
> +;;; Modules in Go are "collection of related Go packages" which are
> +;;; "the unit of source code interchange and versioning".
> +;;; Modules are generally hosted in a repository.
> +;;;
> +;;; At this point it should handle correctly modules which
> +;;; - have only Go dependencies;
> +;;; - use go.mod;
> +;;; - and are accessible from proxy.golang.org (or configured GOPROXY).
> +;;;
> +;;; We translate Go module paths to a Guix package name under the
> +;;; assumption that there will be no collision.
> +
> (define-module (guix import go)
> #:use-module (ice-9 match)
> #:use-module (ice-9 rdelim)
> @@ -23,7 +40,7 @@
> #:use-module (ice-9 regex)
> #:use-module (srfi srfi-1)
> #:use-module (srfi srfi-9)
> - #:use-module (guix json)
> + #:use-module (json)
> #:use-module ((guix download) #:prefix download:)
> #:use-module (guix import utils)
> #:use-module (guix import json)
> @@ -33,88 +50,129 @@
> #:use-module ((guix licenses) #:prefix license:)
> #:use-module (guix base16)
> #:use-module (guix base32)
> - #:use-module (guix build download)
> + #:use-module ((guix build download) #:prefix build-download:)
> #:use-module (web uri)
>
> #:export (go-module->guix-package
> go-module-recursive-import
> infer-module-root))
>
> -(define (escape-capital-letters s)
> - "To avoid ambiguity when serving from case-insensitive file systems, the
> -$module and $version elements are case-encoded by replacing every uppercase
> -letter with an exclamation mark followed by the corresponding lower-case
> -letter."
> - (let ((escaped-string (string)))
> - (string-for-each-index
> - (lambda (i)
> - (let ((c (string-ref s i)))
> - (set! escaped-string
> - (string-concatenate
> - (list escaped-string
> - (if (char-upper-case? c) "!" "")
> - (string (char-downcase c)))))))
> - s)
> - escaped-string))
> +(define (go-path-escape path)
> + "Escape a module path by replacing every uppercase letter with an exclamation
> +mark followed with its lowercase equivalent, as per the module Escaped Paths
> +specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
> + (define (escape occurrence)
> + (string-append "!" (string-downcase (match:substring occurrence))))
> + (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
> +
>
> (define (fetch-latest-version goproxy-url module-path)
> "Fetches the version number of the latest version for MODULE-PATH from the
> given GOPROXY-URL server."
> (assoc-ref
> (json-fetch (format #f "~a/~a/@latest" goproxy-url
> - (escape-capital-letters module-path)))
> + (go-path-escape module-path)))
> "Version"))
>
> (define (fetch-go.mod goproxy-url module-path version file)
> "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
> and VERSION."
> - (url-fetch (format #f "~a/~a/@v/~a.mod" goproxy-url
> - (escape-capital-letters module-path)
> - (escape-capital-letters version))
> - file
> - #:print-build-trace? #f))
> + (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
> + (go-path-escape module-path)
> + (go-path-escape version))))
> + (parameterize ((current-output-port (current-error-port)))
> + (build-download:url-fetch url
> + file
> + #:print-build-trace? #f))))
>
> (define (parse-go.mod go.mod-path)
> - "Parses a go.mod file and returns an alist of module path to version."
> + "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
> +requirements from it."
> + ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
> + ;; which we think necessary for our use case.
> + (define (toplevel results)
> + "Main parser, RESULTS is a pair of alist serving as accumulator for
> + all encountered requirements and replacements."
> + (let ((line (read-line)))
> + (cond
> + ((eof-object? line)
> + ;; parsing ended, give back the result
> + results)
> + ((string=? line "require (")
> + ;; a require block begins, delegate parsing to IN-REQUIRE
> + (in-require results))
> + ((string-prefix? "require " line)
> + ;; a require directive by itself
> + (let* ((stripped-line (string-drop line 8))
> + (new-results (require-directive results stripped-line)))
> + (toplevel new-results)))
> + ((string-prefix? "replace " line)
> + ;; a replace directive by itself
> + (let* ((stripped-line (string-drop line 8))
> + (new-results (replace-directive results stripped-line)))
> + (toplevel new-results)))
> + (#t
> + ;; unrecognised line, ignore silently
> + (toplevel results)))))
> + (define (in-require results)
> + (let ((line (read-line)))
> + (cond
> + ((eof-object? line)
> + ;; this should never happen here but we ignore silently
> + results)
> + ((string=? line ")")
> + ;; end of block, coming back to toplevel
> + (toplevel results))
> + (#t
> + (in-require (require-directive results line))))))
> + (define (replace-directive results line)
> + "Extract replaced modules and new requirements from replace directive
> + in LINE and add to RESULTS."
> + ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
> + ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
> + (let* ((requirements (car results))
> + (replaced (cdr results))
> + (re (string-concatenate
> + '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
> + "[[:blank:]]+" "=>" "[[:blank:]]+"
> + "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
> + (match (string-match re line))
> + (module-path (match:substring match 1))
> + (version (match:substring match 3))
> + (new-module-path (match:substring match 4))
> + (new-version (match:substring match 6))
> + (new-replaced (acons module-path version replaced))
> + (new-requirements
> + (if (string-match "^\\.?\\./" new-module-path)
> + requirements
> + (acons new-module-path new-version requirements))))
> + (cons new-requirements new-replaced)))
> + (define (require-directive results line)
> + "Extract requirement from LINE and add it to RESULTS."
> + (let* ((requirements (car results))
> + (replaced (cdr results))
> + ;; A line in a require directive is composed of a module path and
> + ;; a version separated by whitespace and an optionnal '//' comment at
> + ;; the end.
> + (re (string-concatenate
> + '("^[[:blank:]]*"
> + "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
> + "([[:blank:]]+//.*)?")))
> + (match (string-match re line))
> + (module-path (match:substring match 1))
> + (version (match:substring match 2)))
> + (cons (acons module-path version requirements) replaced)))
> (with-input-from-file go.mod-path
> (lambda ()
> - (let ((in-require? #f)
> - (requirements (list)))
> - (do ((line (read-line) (read-line)))
> - ((eof-object? line))
> - (set! line (string-trim line))
> - ;; The parser is either entering, within, exiting, or after the
> - ;; require block. The Go toolchain is trustworthy so edge-cases like
> - ;; double-entry, etc. need not complect the parser.
> - (cond
> - ((string=? line "require (")
> - (set! in-require? #t))
> - ((and in-require? (string=? line ")"))
> - (set! in-require? #f))
> - (in-require?
> - (let* ((requirement (string-split line #\space))
> - ;; Modules should be unquoted
> - (module-path (string-delete #\" (car requirement)))
> - (version (list-ref requirement 1)))
> - (set! requirements (acons module-path version requirements))))
> - ((string-prefix? "replace" line)
> - (let* ((requirement (string-split line #\space))
> - (module-path (list-ref requirement 1))
> - (new-module-path (list-ref requirement 3))
> - (version (list-ref requirement 4)))
> - (set! requirements (assoc-remove! requirements module-path))
> - (set! requirements (acons new-module-path version requirements))))))
> - requirements))))
> -
> -(define (module-path-without-major-version module-path)
> - "Go modules can be appended with a major version indicator,
> -e.g. /v3. Sometimes it is desirable to work with the root module path. For
> -instance, for a module path github.com/foo/bar/v3 this function returns
> -github.com/foo/bar."
> - (let ((m (string-match "(.*)\\/v[0-9]+$" module-path)))
> - (if m
> - (match:substring m 1)
> - module-path)))
> + (let* ((results (toplevel '(() . ())))
> + (requirements (car results))
> + (replaced (cdr results)))
> + ;; At last we remove replaced modules from the requirements list
> + (fold
> + (lambda (replacedelem requirements)
> + (alist-delete! (car replacedelem) requirements))
> + requirements
> + replaced)))))
>
> (define (infer-module-root module-path)
> "Go modules can be defined at any level of a repository's tree, but querying
> @@ -124,38 +182,42 @@ root path from its path. For a set of well-known forges, the pattern of what
> consists of a module's root page is known before hand."
> ;; See the following URL for the official Go equivalent:
> ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
> - (define-record-type <scs>
> - (make-scs url-prefix root-regex type)
> - scs?
> - (url-prefix scs-url-prefix)
> - (root-regex scs-root-regex)
> - (type scs-type))
> - (let* ((known-scs
> + ;;
> + ;; FIXME: handle module path with VCS qualifier as described in
> + ;; https://golang.org/ref/mod#vcs-find and
> + ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
> + (define-record-type <vcs>
> + (make-vcs url-prefix root-regex type)
> + vcs?
> + (url-prefix vcs-url-prefix)
> + (root-regex vcs-root-regex)
> + (type vcs-type))
> + (let* ((known-vcs
> (list
> - (make-scs
> + (make-vcs
> "github.com"
> "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
> 'git)
> - (make-scs
> + (make-vcs
> "bitbucket.org"
> "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$`"
> 'unknown)
> - (make-scs
> + (make-vcs
> "hub.jazz.net/git/"
> "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
> 'git)
> - (make-scs
> + (make-vcs
> "git.apache.org"
> "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
> 'git)
> - (make-scs
> + (make-vcs
> "git.openstack.org"
> "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
> 'git)))
> - (scs (find (lambda (scs) (string-prefix? (scs-url-prefix scs) module-path))
> - known-scs)))
> - (if scs
> - (match:substring (string-match (scs-root-regex scs) module-path) 1)
> + (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
> + known-vcs)))
> + (if vcs
> + (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
> module-path)))
>
> (define (to-guix-package-name module-path)
> @@ -164,8 +226,7 @@ consists of a module's root page is known before hand."
> (string-append "go-"
> (string-replace-substring
> (string-replace-substring
> - ;; Guix has its own field for version
> - (module-path-without-major-version module-path)
> + module-path
> "." "-")
> "/" "-"))))
>
> @@ -173,7 +234,9 @@ consists of a module's root page is known before hand."
> "Fetches module meta-data from a module's landing page. This is necessary
> because goproxy servers don't currently provide all the information needed to
> build a package."
> - (let* ((port (http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
> + ;; FIXME: This code breaks on k8s.io which have a meta tag splitted
> + ;; on several lines
> + (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
> (module-metadata #f)
> (meta-tag-prefix "<meta name=\"go-import\" content=\"")
> (meta-tag-prefix-length (string-length meta-tag-prefix)))
> @@ -185,7 +248,7 @@ build a package."
> (let* ((start (+ meta-tag-index meta-tag-prefix-length))
> (end (string-index line #\" start)))
> (set! module-metadata
> - (string-split (substring/shared line start end) #\space))))))
> + (string-split (substring/shared line start end) #\space))))))
> (close-port port)
> module-metadata))
>
> @@ -244,7 +307,7 @@ control system is being used."
> (dependencies (map car (parse-go.mod temp)))
> (guix-name (to-guix-package-name module-path))
> (root-module-path (infer-module-root module-path))
> - ;; SCS type and URL are not included in goproxy information. For
> + ;; VCS type and URL are not included in goproxy information. For
> ;; this we need to fetch it from the official module page.
> (meta-data (fetch-module-meta-data root-module-path))
> (scs-type (module-meta-data-scs meta-data))
> @@ -268,9 +331,10 @@ control system is being used."
>
> (define* (go-module-recursive-import package-name
> #:key (goproxy-url "https://proxy.golang.org"))
> - (recursive-import package-name #f
> - #:repo->guix-package
> - (lambda (name _)
> - (go-module->guix-package name
> - #:goproxy-url goproxy-url))
> - #:guix-name to-guix-package-name))
> + (recursive-import
> + package-name
> + #:repo->guix-package (lambda* (name . _)
> + (go-module->guix-package
> + name
> + #:goproxy-url goproxy-url))
> + #:guix-name to-guix-package-name))
>
> guix/import/go.scm | 340 +++++++++++++++++++++++++++++++++++++
> guix/scripts/import.scm | 2 +-
> guix/scripts/import/go.scm | 118 +++++++++++++
> 3 files changed, 459 insertions(+), 1 deletion(-)
> create mode 100644 guix/import/go.scm
> create mode 100644 guix/scripts/import/go.scm
>
> diff --git a/guix/import/go.scm b/guix/import/go.scm
> new file mode 100644
> index 0000000000..7f5f300f0a
> --- /dev/null
> +++ b/guix/import/go.scm
> @@ -0,0 +1,340 @@
> +;;; GNU Guix --- Functional package management for GNU
> +;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
> +;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
> +;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
> +;;;
> +;;; This file is part of GNU Guix.
> +;;;
> +;;; GNU Guix is free software; you can redistribute it and/or modify it
> +;;; under the terms of the GNU General Public License as published by
> +;;; the Free Software Foundation; either version 3 of the License, or (at
> +;;; your option) any later version.
> +;;;
> +;;; GNU Guix is distributed in the hope that it will be useful, but
> +;;; WITHOUT ANY WARRANTY; without even the implied warranty of
> +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +;;; GNU General Public License for more details.
> +;;;
> +;;; You should have received a copy of the GNU General Public License
> +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
> +
> +;;; (guix import golang) wants to make easier to create Guix package
> +;;; declaration for Go modules.
> +;;;
> +;;; Modules in Go are "collection of related Go packages" which are
> +;;; "the unit of source code interchange and versioning".
> +;;; Modules are generally hosted in a repository.
> +;;;
> +;;; At this point it should handle correctly modules which
> +;;; - have only Go dependencies;
> +;;; - use go.mod;
> +;;; - and are accessible from proxy.golang.org (or configured GOPROXY).
> +;;;
> +;;; We translate Go module paths to a Guix package name under the
> +;;; assumption that there will be no collision.
> +
> +(define-module (guix import go)
> + #:use-module (ice-9 match)
> + #:use-module (ice-9 rdelim)
> + #:use-module (ice-9 receive)
> + #:use-module (ice-9 regex)
> + #:use-module (srfi srfi-1)
> + #:use-module (srfi srfi-9)
> + #:use-module (json)
> + #:use-module ((guix download) #:prefix download:)
> + #:use-module (guix import utils)
> + #:use-module (guix import json)
> + #:use-module (guix packages)
> + #:use-module (guix upstream)
> + #:use-module (guix utils)
> + #:use-module ((guix licenses) #:prefix license:)
> + #:use-module (guix base16)
> + #:use-module (guix base32)
> + #:use-module ((guix build download) #:prefix build-download:)
> + #:use-module (web uri)
> +
> + #:export (go-module->guix-package
> + go-module-recursive-import
> + infer-module-root))
> +
> +(define (go-path-escape path)
> + "Escape a module path by replacing every uppercase letter with an exclamation
> +mark followed with its lowercase equivalent, as per the module Escaped Paths
> +specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
> + (define (escape occurrence)
> + (string-append "!" (string-downcase (match:substring occurrence))))
> + (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
> +
> +
> +(define (fetch-latest-version goproxy-url module-path)
> + "Fetches the version number of the latest version for MODULE-PATH from the
> +given GOPROXY-URL server."
> + (assoc-ref
> + (json-fetch (format #f "~a/~a/@latest" goproxy-url
> + (go-path-escape module-path)))
> + "Version"))
> +
> +(define (fetch-go.mod goproxy-url module-path version file)
> + "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
> +and VERSION."
> + (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
> + (go-path-escape module-path)
> + (go-path-escape version))))
> + (parameterize ((current-output-port (current-error-port)))
> + (build-download:url-fetch url
> + file
> + #:print-build-trace? #f))))
> +
> +(define (parse-go.mod go.mod-path)
> + "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
> +requirements from it."
> + ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
> + ;; which we think necessary for our use case.
> + (define (toplevel results)
> + "Main parser, RESULTS is a pair of alist serving as accumulator for
> + all encountered requirements and replacements."
> + (let ((line (read-line)))
> + (cond
> + ((eof-object? line)
> + ;; parsing ended, give back the result
> + results)
> + ((string=? line "require (")
> + ;; a require block begins, delegate parsing to IN-REQUIRE
> + (in-require results))
> + ((string-prefix? "require " line)
> + ;; a require directive by itself
> + (let* ((stripped-line (string-drop line 8))
> + (new-results (require-directive results stripped-line)))
> + (toplevel new-results)))
> + ((string-prefix? "replace " line)
> + ;; a replace directive by itself
> + (let* ((stripped-line (string-drop line 8))
> + (new-results (replace-directive results stripped-line)))
> + (toplevel new-results)))
> + (#t
> + ;; unrecognised line, ignore silently
> + (toplevel results)))))
> + (define (in-require results)
> + (let ((line (read-line)))
> + (cond
> + ((eof-object? line)
> + ;; this should never happen here but we ignore silently
> + results)
> + ((string=? line ")")
> + ;; end of block, coming back to toplevel
> + (toplevel results))
> + (#t
> + (in-require (require-directive results line))))))
> + (define (replace-directive results line)
> + "Extract replaced modules and new requirements from replace directive
> + in LINE and add to RESULTS."
> + ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
> + ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
> + (let* ((requirements (car results))
> + (replaced (cdr results))
> + (re (string-concatenate
> + '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
> + "[[:blank:]]+" "=>" "[[:blank:]]+"
> + "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
> + (match (string-match re line))
> + (module-path (match:substring match 1))
> + (version (match:substring match 3))
> + (new-module-path (match:substring match 4))
> + (new-version (match:substring match 6))
> + (new-replaced (acons module-path version replaced))
> + (new-requirements
> + (if (string-match "^\\.?\\./" new-module-path)
> + requirements
> + (acons new-module-path new-version requirements))))
> + (cons new-requirements new-replaced)))
> + (define (require-directive results line)
> + "Extract requirement from LINE and add it to RESULTS."
> + (let* ((requirements (car results))
> + (replaced (cdr results))
> + ;; A line in a require directive is composed of a module path and
> + ;; a version separated by whitespace and an optionnal '//' comment at
> + ;; the end.
> + (re (string-concatenate
> + '("^[[:blank:]]*"
> + "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
> + "([[:blank:]]+//.*)?")))
> + (match (string-match re line))
> + (module-path (match:substring match 1))
> + (version (match:substring match 2)))
> + (cons (acons module-path version requirements) replaced)))
> + (with-input-from-file go.mod-path
> + (lambda ()
> + (let* ((results (toplevel '(() . ())))
> + (requirements (car results))
> + (replaced (cdr results)))
> + ;; At last we remove replaced modules from the requirements list
> + (fold
> + (lambda (replacedelem requirements)
> + (alist-delete! (car replacedelem) requirements))
> + requirements
> + replaced)))))
> +
> +(define (infer-module-root module-path)
> + "Go modules can be defined at any level of a repository's tree, but querying
> +for the meta tag usually can only be done at the webpage at the root of the
> +repository. Therefore, it is sometimes necessary to try and derive a module's
> +root path from its path. For a set of well-known forges, the pattern of what
> +consists of a module's root page is known before hand."
> + ;; See the following URL for the official Go equivalent:
> + ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
> + ;;
> + ;; FIXME: handle module path with VCS qualifier as described in
> + ;; https://golang.org/ref/mod#vcs-find and
> + ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
> + (define-record-type <vcs>
> + (make-vcs url-prefix root-regex type)
> + vcs?
> + (url-prefix vcs-url-prefix)
> + (root-regex vcs-root-regex)
> + (type vcs-type))
> + (let* ((known-vcs
> + (list
> + (make-vcs
> + "github.com"
> + "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)
> + (make-vcs
> + "bitbucket.org"
> + "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$`"
> + 'unknown)
> + (make-vcs
> + "hub.jazz.net/git/"
> + "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)
> + (make-vcs
> + "git.apache.org"
> + "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)
> + (make-vcs
> + "git.openstack.org"
> + "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)))
> + (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
> + known-vcs)))
> + (if vcs
> + (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
> + module-path)))
> +
> +(define (to-guix-package-name module-path)
> + "Converts a module's path to the canonical Guix format for Go packages."
> + (string-downcase
> + (string-append "go-"
> + (string-replace-substring
> + (string-replace-substring
> + module-path
> + "." "-")
> + "/" "-"))))
> +
> +(define (fetch-module-meta-data module-path)
> + "Fetches module meta-data from a module's landing page. This is necessary
> +because goproxy servers don't currently provide all the information needed to
> +build a package."
> + ;; FIXME: This code breaks on k8s.io which have a meta tag splitted
> + ;; on several lines
> + (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
> + (module-metadata #f)
> + (meta-tag-prefix "<meta name=\"go-import\" content=\"")
> + (meta-tag-prefix-length (string-length meta-tag-prefix)))
> + (do ((line (read-line port) (read-line port)))
> + ((or (eof-object? line)
> + module-metadata))
> + (let ((meta-tag-index (string-contains line meta-tag-prefix)))
> + (when meta-tag-index
> + (let* ((start (+ meta-tag-index meta-tag-prefix-length))
> + (end (string-index line #\" start)))
> + (set! module-metadata
> + (string-split (substring/shared line start end) #\space))))))
> + (close-port port)
> + module-metadata))
> +
> +(define (module-meta-data-scs meta-data)
> + "Return the source control system specified by a module's meta-data."
> + (string->symbol (list-ref meta-data 1)))
> +
> +(define (module-meta-data-repo-url meta-data goproxy-url)
> + "Return the URL where the fetcher which will be used can download the source
> +control."
> + (if (member (module-meta-data-scs meta-data) '(fossil mod))
> + goproxy-url
> + (list-ref meta-data 2)))
> +
> +(define (source-uri scs-type scs-repo-url file)
> + "Generate the `origin' block of a package depending on what type of source
> +control system is being used."
> + (case scs-type
> + ((git)
> + `(origin
> + (method git-fetch)
> + (uri (git-reference
> + (url ,scs-repo-url)
> + (commit (string-append "v" version))))
> + (file-name (git-file-name name version))
> + (sha256
> + (base32
> + ,(guix-hash-url file)))))
> + ((hg)
> + `(origin
> + (method hg-fetch)
> + (uri (hg-reference
> + (url ,scs-repo-url)
> + (changeset ,version)))
> + (file-name (format #f "~a-~a-checkout" name version))))
> + ((svn)
> + `(origin
> + (method svn-fetch)
> + (uri (svn-reference
> + (url ,scs-repo-url)
> + (revision (string->number version))
> + (recursive? #f)))
> + (file-name (format #f "~a-~a-checkout" name version))
> + (sha256
> + (base32
> + ,(guix-hash-url file)))))
> + (else
> + (raise-exception (format #f "unsupported scs type: ~a" scs-type)))))
> +
> +(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
> + (call-with-temporary-output-file
> + (lambda (temp port)
> + (let* ((latest-version (fetch-latest-version goproxy-url module-path))
> + (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
> + temp))
> + (dependencies (map car (parse-go.mod temp)))
> + (guix-name (to-guix-package-name module-path))
> + (root-module-path (infer-module-root module-path))
> + ;; VCS type and URL are not included in goproxy information. For
> + ;; this we need to fetch it from the official module page.
> + (meta-data (fetch-module-meta-data root-module-path))
> + (scs-type (module-meta-data-scs meta-data))
> + (scs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
> + (values
> + `(package
> + (name ,guix-name)
> + ;; Elide the "v" prefix Go uses
> + (version ,(string-trim latest-version #\v))
> + (source
> + ,(source-uri scs-type scs-repo-url temp))
> + (build-system go-build-system)
> + ,@(maybe-inputs (map to-guix-package-name dependencies))
> + ;; TODO(katco): It would be nice to make an effort to fetch this
> + ;; from known forges, e.g. GitHub
> + (home-page ,(format #f "https://~a" root-module-path))
> + (synopsis "A Go package")
> + (description ,(format #f "~a is a Go package." guix-name))
> + (license #f))
> + dependencies)))))
> +
> +(define* (go-module-recursive-import package-name
> + #:key (goproxy-url "https://proxy.golang.org"))
> + (recursive-import
> + package-name
> + #:repo->guix-package (lambda* (name . _)
> + (go-module->guix-package
> + name
> + #:goproxy-url goproxy-url))
> + #:guix-name to-guix-package-name))
> diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
> index 0a3863f965..1d2b45d942 100644
> --- a/guix/scripts/import.scm
> +++ b/guix/scripts/import.scm
> @@ -77,7 +77,7 @@ rather than \\n."
> ;;;
>
> (define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"
> - "cran" "crate" "texlive" "json" "opam"))
> + "go" "cran" "crate" "texlive" "json" "opam"))
>
> (define (resolve-importer name)
> (let ((module (resolve-interface
> diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
> new file mode 100644
> index 0000000000..fde7555973
> --- /dev/null
> +++ b/guix/scripts/import/go.scm
> @@ -0,0 +1,118 @@
> +;;; GNU Guix --- Functional package management for GNU
> +;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
> +;;;
> +;;; This file is part of GNU Guix.
> +;;;
> +;;; GNU Guix is free software; you can redistribute it and/or modify it
> +;;; under the terms of the GNU General Public License as published by
> +;;; the Free Software Foundation; either version 3 of the License, or (at
> +;;; your option) any later version.
> +;;;
> +;;; GNU Guix is distributed in the hope that it will be useful, but
> +;;; WITHOUT ANY WARRANTY; without even the implied warranty of
> +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +;;; GNU General Public License for more details.
> +;;;
> +;;; You should have received a copy of the GNU General Public License
> +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
> +
> +(define-module (guix scripts import go)
> + #:use-module (guix ui)
> + #:use-module (guix utils)
> + #:use-module (guix scripts)
> + #:use-module (guix import go)
> + #:use-module (guix scripts import)
> + #:use-module (srfi srfi-1)
> + #:use-module (srfi srfi-11)
> + #:use-module (srfi srfi-37)
> + #:use-module (ice-9 match)
> + #:use-module (ice-9 format)
> + #:export (guix-import-go))
> +
> +
> +;;;
> +;;; Command-line options.
> +;;;
> +
> +(define %default-options
> + '())
> +
> +(define (show-help)
> + (display (G_ "Usage: guix import go PACKAGE-PATH
> +Import and convert the Go module for PACKAGE-PATH.\n"))
> + (display (G_ "
> + -h, --help display this help and exit"))
> + (display (G_ "
> + -V, --version display version information and exit"))
> + (display (G_ "
> + -r, --recursive generate package expressions for all Go modules\
> + that are not yet in Guix"))
> + (display (G_ "
> + -p, --goproxy=GOPROXY specify which goproxy server to use"))
> + (newline)
> + (show-bug-report-information))
> +
> +(define %options
> + ;; Specification of the command-line options.
> + (cons* (option '(#\h "help") #f #f
> + (lambda args
> + (show-help)
> + (exit 0)))
> + (option '(#\V "version") #f #f
> + (lambda args
> + (show-version-and-exit "guix import go")))
> + (option '(#\r "recursive") #f #f
> + (lambda (opt name arg result)
> + (alist-cons 'recursive #t result)))
> + (option '(#\p "goproxy") #t #f
> + (lambda (opt name arg result)
> + (alist-cons 'goproxy
> + (string->symbol arg)
> + (alist-delete 'goproxy result))))
> + %standard-import-options))
> +
> +
> +;;;
> +;;; Entry point.
> +;;;
> +
> +(define (guix-import-go . args)
> + (define (parse-options)
> + ;; Return the alist of option values.
> + (args-fold* args %options
> + (lambda (opt name arg result)
> + (leave (G_ "~A: unrecognized option~%") name))
> + (lambda (arg result)
> + (alist-cons 'argument arg result))
> + %default-options))
> +
> + (let* ((opts (parse-options))
> + (args (filter-map (match-lambda
> + (('argument . value)
> + value)
> + (_ #f))
> + (reverse opts))))
> + (match args
> + ((module-name)
> + (if (assoc-ref opts 'recursive)
> + (map (match-lambda
> + ((and ('package ('name name) . rest) pkg)
> + `(define-public ,(string->symbol name)
> + ,pkg))
> + (_ #f))
> + (go-module-recursive-import module-name
> + #:goproxy-url
> + (or (assoc-ref opts 'goproxy)
> + "https://proxy.golang.org")))
> + (let ((sexp (go-module->guix-package module-name
> + #:goproxy-url
> + (or (assoc-ref opts 'goproxy)
> + "https://proxy.golang.org"))))
> + (unless sexp
> + (leave (G_ "failed to download meta-data for module '~a'~%")
> + module-name))
> + sexp)))
> + (()
> + (leave (G_ "too few arguments~%")))
> + ((many ...)
> + (leave (G_ "too many arguments~%"))))))
--
Katherine
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-23 22:41 ` Katherine Cox-Buday
@ 2021-01-25 21:03 ` guix-patches--- via
2021-01-27 14:38 ` Katherine Cox-Buday
2021-01-28 5:01 ` [bug#44178] [PATCH] Create importer for Go modules Timmy Douglas
0 siblings, 2 replies; 27+ messages in thread
From: guix-patches--- via @ 2021-01-25 21:03 UTC (permalink / raw)
To: Katherine Cox-Buday; +Cc: 44178@debbugs.gnu.org, Helio Machado
Hello,
On Sat, Jan 23, 2021 at 04:41:18PM -0600, Katherine Cox-Buday wrote:
> Thanks so much for the patches, Helio, Joulaud!
You're welcome! As a side note I prefer to be adressed as "François" ;-)
> I have pushed everything (including Joulaud's patch with appropriate
> attribution) here[2]. I am admittedly new at using email to organize
> code changes, but using a forge seems easier.
> Can I suggest we coordinate there, or is that too much of an imposition?
I have no problem to coordinate in a forge and can push in a shared
branch if you give me access.
Even if I must say I found it refreshing to be able to work with mails. It
has the nice property of fully-offline and asynchronous communication
and it helped me to better articulate my problems.
Best regards,
François
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-25 21:03 ` guix-patches--- via
@ 2021-01-27 14:38 ` Katherine Cox-Buday
2021-01-28 13:27 ` Ludovic Courtès
2021-01-28 5:01 ` [bug#44178] [PATCH] Create importer for Go modules Timmy Douglas
1 sibling, 1 reply; 27+ messages in thread
From: Katherine Cox-Buday @ 2021-01-27 14:38 UTC (permalink / raw)
To: JOULAUD François; +Cc: 44178@debbugs.gnu.org, Helio Machado
JOULAUD François <Francois.JOULAUD@radiofrance.com> writes:
> Hello,
>
> On Sat, Jan 23, 2021 at 04:41:18PM -0600, Katherine Cox-Buday wrote:
>> Thanks so much for the patches, Helio, Joulaud!
>
> You're welcome! As a side note I prefer to be adressed as "François"
> ;-)
I'm very sorry, François!
>> I have pushed everything (including Joulaud's patch with appropriate
>> attribution) here[2]. I am admittedly new at using email to organize
>> code changes, but using a forge seems easier.
>
>> Can I suggest we coordinate there, or is that too much of an imposition?
>
> I have no problem to coordinate in a forge and can push in a shared
> branch if you give me access.
>
> Even if I must say I found it refreshing to be able to work with mails. It
> has the nice property of fully-offline and asynchronous communication
> and it helped me to better articulate my problems.
OK, how about we stick to the preferred Guix approach then and
coordinate here. Maybe you can teach me some things along the way :) I
do like the idea of an email-based forge!
The main problems I've had thus far are:
- The inter-diff patch you sent was not formatted correctly because it
contained some extra leading whitespace. I found myself juggling a few
different tools just to understand your changes.
- The conversation about this seems to be happening in 3 different
places: here, help-guix, and guix-devel. I guess we should be
centralizing here.
- I don't have any experience coordinating with people in this
email-style forge. I don't know how the inter-diff patches work with
attribution, nor how to get everyone on the same page. It seems like
multiple people are making conflicting changes at once (maybe that's
just my perception).
--
Katherine
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-25 21:03 ` guix-patches--- via
2021-01-27 14:38 ` Katherine Cox-Buday
@ 2021-01-28 5:01 ` Timmy Douglas
1 sibling, 0 replies; 27+ messages in thread
From: Timmy Douglas @ 2021-01-28 5:01 UTC (permalink / raw)
To: JOULAUD François, Katherine Cox-Buday
Cc: 44178@debbugs.gnu.org, Helio Machado
I checked out https://github.com/kat-co/guix/tree/create-go-importer:
$ ./pre-inst-env guix import go -r github.com/coredns/coredns
;;; note: source file /s/timmy/guix/guix/import/go.scm
;;; newer than compiled /home/timmy/.cache/guile/ccache/3.0-LE-8-4.3/s/timmy/guix/guix/import/go.scm.go
Starting download of /tmp/guix-file.jkncVo
From https://proxy.golang.org/github.com/coredns/coredns/@v/v1.8.1.mod...
v1.8.1.mod 2KiB 870KiB/s 00:00 [##################] 100.0%
Starting download of /tmp/guix-file.GtI9fs
From https://proxy.golang.org/k8s.io/klog/@v/v1.0.0.mod...
v1.0.0.mod 68B 189KiB/s 00:00 [##################] 100.0%
Backtrace:
In ice-9/boot-9.scm:
1736:10 13 (with-exception-handler _ _ #:unwind? _ # _)
In unknown file:
12 (apply-smob/0 #<thunk 7fdc57f964e0>)
In ice-9/boot-9.scm:
718:2 11 (call-with-prompt _ _ #<procedure default-prompt-handle?>)
In ice-9/eval.scm:
619:8 10 (_ #(#(#<directory (guile-user) 7fdc57be3f00>)))
In guix/ui.scm:
2154:12 9 (run-guix-command _ . _)
In guix/scripts/import.scm:
120:11 8 (guix-import . _)
In ice-9/eval.scm:
159:9 7 (_ _)
In guix/import/utils.scm:
464:27 6 (recursive-import _ #:repo->guix-package _ #:guix-name _ ?)
In srfi/srfi-1.scm:
586:17 5 (map1 (("k8s.io/klog" #f) ("k8s.io/client-go" #f) (?) ?))
In guix/import/utils.scm:
453:33 4 (lookup-node "k8s.io/klog" #f)
In guix/utils.scm:
700:8 3 (call-with-temporary-output-file #<procedure 7fdc464b03?>)
In ice-9/eval.scm:
293:34 2 (_ #(#(#(#(#(#(#(#(#<directory ?> ?) ?) ?) ?) ?) ?) ?) ?))
155:9 1 (_ #(#(#<directory (guix import go) 7fdc55b65f00>) #f))
In unknown file:
0 (list-ref #f 1)
ERROR: In procedure list-ref:
In procedure list-ref: Wrong type argument in position 1: #f
This is due to:
(go-module->guix-package "k8s.io/klog")
The temp file looks like this:
module k8s.io/klog
go 1.12
require github.com/go-logr/logr v0.1.0
-> (fetch-module-meta-data '("github.com/go-logr/logr"))
-> (string->uri (format #f "https://~a?go-get=1" module-path)) -> #f
-> (http-fetch #f)
Is there a better way to debug this? `guix import` kicked me back to the
cmd line instead of the guile debugger, which is understandable for
users, but the stacktrace is missing a lot of information. I opened
emacs and geiser and had to eval a bunch of things to narrow it down. It
feels like I'm doing it wrong. (I don't have much experience with scheme)
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2020-10-23 14:06 [bug#44178] Add a Go Module Importer Katherine Cox-Buday
` (3 preceding siblings ...)
2020-12-09 14:22 ` [bug#44178] dftxbs3e
@ 2021-01-28 7:29 ` guix-patches--- via
4 siblings, 0 replies; 27+ messages in thread
From: guix-patches--- via @ 2021-01-28 7:29 UTC (permalink / raw)
To: Timmy Douglas; +Cc: 44178@debbugs.gnu.org
[-- Attachment #1: Type: text/plain, Size: 218 bytes --]
Hello,
Problem with k8s.io is known.
I tested htmlprag quickly yesterday and I think I can come up with a fix soon.
Regards,
François
Envoyé depuis un ordiphone. Veuillez excuser la brièveté.
[-- Attachment #2: Type: text/html, Size: 786 bytes --]
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-27 14:38 ` Katherine Cox-Buday
@ 2021-01-28 13:27 ` Ludovic Courtès
2021-01-29 16:43 ` guix-patches--- via
0 siblings, 1 reply; 27+ messages in thread
From: Ludovic Courtès @ 2021-01-28 13:27 UTC (permalink / raw)
To: Katherine Cox-Buday
Cc: JOULAUD François, 44178@debbugs.gnu.org, Helio Machado
Hello!
Katherine Cox-Buday <cox.katherine.e@gmail.com> skribis:
> - The conversation about this seems to be happening in 3 different
> places: here, help-guix, and guix-devel. I guess we should be
> centralizing here.
+1
> - I don't have any experience coordinating with people in this
> email-style forge. I don't know how the inter-diff patches work with
> attribution, nor how to get everyone on the same page. It seems like
> multiple people are making conflicting changes at once (maybe that's
> just my perception).
I think whoever among you is available to work on it these days could
take the lead and prepare a final version of the patches. It looks like
it’s approaching a first “committable” version (perhaps just missing an
addition to doc/guix.texi and test cases like we have ‘tests/cpan.scm’ &
co.)
For attribution, I’d keep Katherine as the commit author and add a
‘Co-authored-by’ line for François and for Helio (that’s how we usually
handle that given that Git assumes each commit has a single author).
When that first version is committed, you can all submit patches for
improvements. For now, the focus should be on getting the first version
in. :-)
My 2¢,
Ludo’.
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-28 13:27 ` Ludovic Courtès
@ 2021-01-29 16:43 ` guix-patches--- via
2021-01-29 16:52 ` [bug#44178] [PATCHv2] " guix-patches--- via
2021-01-31 16:23 ` [bug#44178] [PATCH] " Ludovic Courtès
0 siblings, 2 replies; 27+ messages in thread
From: guix-patches--- via @ 2021-01-29 16:43 UTC (permalink / raw)
To: Ludovic Courtès, 44178@debbugs.gnu.org; +Cc: Katherine Cox-Buday
Hello!
On Thu, Jan 28, 2021 at 02:27:57PM +0100, Ludovic Courtès wrote:
> I think whoever among you is available to work on it these days could
> take the lead and prepare a final version of the patches. It looks like
> it’s approaching a first “committable” version (perhaps just missing an
> addition to doc/guix.texi and test cases like we have ‘tests/cpan.scm’ &
> co.)
I thought I would be able to send a working v2 of this patch today but
it seems I was too optimistic.
I found that some go.mod out there uses quoted string
which our ad-hoc parser don't know how to parse. cf.
https://github.com/go-yaml/yaml/blob/496545a6307b2a7d7a710fd516e5e16e8ab62dbc/go.mod
I don't know if this is a blocker for a merge or not.
Apart from that I don't know how to add guile-lib to the dependencies of
Guix (in order to use htmlprag). Help needed.
I tested it recursively with github.com/hashicorp/consul (which was one
of those with the most dependencies I found) and it mostly works.
Regards,
François
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCHv2] Create importer for Go modules
2021-01-29 16:43 ` guix-patches--- via
@ 2021-01-29 16:52 ` guix-patches--- via
2021-01-31 16:23 ` [bug#44178] [PATCH] " Ludovic Courtès
1 sibling, 0 replies; 27+ messages in thread
From: guix-patches--- via @ 2021-01-29 16:52 UTC (permalink / raw)
To: 44178@debbugs.gnu.org
This patch add a `guix import go` command.
It was tested with several big repositories and seems to mostly work for
the import part (because building Guix packages is an other story).
* guix/import/go.sc: Created Go Importerm
* guix/scripts/import.scm: Added Go Importer Subcommand
* guix/scripts/import/go.scm: Created Go Importer Subcommand
* doc/guix.texi: add a paragraph about `guix import go`
* tests/import-go.scm: tests for parse-go.mod procedure
Signed-off-by: Francois Joulaud <francois.joulaud@radiofrance.com>
---
doc/guix.texi | 25 +++
guix/import/go.scm | 384 +++++++++++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/go.scm | 118 ++++++++++++
tests/import-go.scm | 143 ++++++++++++++
5 files changed, 671 insertions(+), 1 deletion(-)
create mode 100644 guix/import/go.scm
create mode 100644 guix/scripts/import/go.scm
create mode 100644 tests/import-go.scm
diff --git a/doc/guix.texi b/doc/guix.texi
index 6ea782fd23..d77e2811ae 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -860,6 +860,10 @@ substitutes (@pxref{Invoking guix publish}).
@uref{https://ngyro.com/software/guile-semver.html, Guile-Semver} for
the @code{crate} importer (@pxref{Invoking guix import}).
+@item
+@uref{https://www.nongnu.org/guile-lib/doc/ref/htmlprag/, guile-lib} for
+the @code{crate} importer (@pxref{Invoking guix import}).
+
@item
When @url{http://www.bzip.org, libbz2} is available,
@command{guix-daemon} can use it to compress build logs.
@@ -11370,6 +11374,27 @@ Select the given repository (a repository name). Possible values include:
of coq packages.
@end itemize
@end table
+
+@item go
+@cindex go
+Import metadata for a Go module using
+@uref{https://proxy.golang.org, proxy.golang.org}.
+
+This importer is highly experimental.
+
+@example
+guix import go gopkg.in/yaml.v2
+@end example
+
+Additional options include:
+
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
@end table
The structure of the @command{guix import} code is modular. It would be
diff --git a/guix/import/go.scm b/guix/import/go.scm
new file mode 100644
index 0000000000..cf2d31ce12
--- /dev/null
+++ b/guix/import/go.scm
@@ -0,0 +1,384 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; (guix import golang) wants to make easier to create Guix package
+;;; declaration for Go modules.
+;;;
+;;; Modules in Go are "collection of related Go packages" which are
+;;; "the unit of source code interchange and versioning".
+;;; Modules are generally hosted in a repository.
+;;;
+;;; At this point it should handle correctly modules which
+;;; - have only Go dependencies;
+;;; - use go.mod;
+;;; - and are accessible from proxy.golang.org (or configured GOPROXY).
+;;;
+;;; We translate Go module paths to a Guix package name under the
+;;; assumption that there will be no collision.
+
+(define-module (guix import go)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 rdelim)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (htmlprag)
+ #:use-module (sxml xpath)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (srfi srfi-11)
+ #:use-module (json)
+ #:use-module ((guix download) #:prefix download:)
+ #:use-module (guix import utils)
+ #:use-module (guix import json)
+ #:use-module (guix packages)
+ #:use-module (guix upstream)
+ #:use-module (guix utils)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix base16)
+ #:use-module (guix base32)
+ #:use-module ((guix build download) #:prefix build-download:)
+ #:use-module (web uri)
+
+ #:export (go-module->guix-package
+ go-module-recursive-import
+ infer-module-root))
+
+(define (go-path-escape path)
+ "Escape a module path by replacing every uppercase letter with an exclamation
+mark followed with its lowercase equivalent, as per the module Escaped Paths
+specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
+ (define (escape occurrence)
+ (string-append "!" (string-downcase (match:substring occurrence))))
+ (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
+
+
+(define (fetch-latest-version goproxy-url module-path)
+ "Fetches the version number of the latest version for MODULE-PATH from the
+given GOPROXY-URL server."
+ (assoc-ref
+ (json-fetch (format #f "~a/~a/@latest" goproxy-url
+ (go-path-escape module-path)))
+ "Version"))
+
+(define (fetch-go.mod goproxy-url module-path version file)
+ "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
+and VERSION."
+ (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
+ (go-path-escape module-path)
+ (go-path-escape version))))
+ (parameterize ((current-output-port (current-error-port)))
+ (build-download:url-fetch url
+ file
+ #:print-build-trace? #f))))
+
+(define (parse-go.mod go.mod-path)
+ (parse-go.mod-port (open-input-file go.mod-path)))
+
+(define (parse-go.mod-port go.mod-port)
+ "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
+requirements from it."
+ ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
+ ;; which we think necessary for our use case.
+ (define (toplevel results)
+ "Main parser, RESULTS is a pair of alist serving as accumulator for
+ all encountered requirements and replacements."
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; parsing ended, give back the result
+ results)
+ ((string=? line "require (")
+ ;; a require block begins, delegate parsing to IN-REQUIRE
+ (in-require results))
+ ((string=? line "replace (")
+ ;; a replace block begins, delegate parsing to IN-REPLACE
+ (in-replace results))
+ ((string-prefix? "require " line)
+ ;; a require directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (require-directive results stripped-line)))
+ (toplevel new-results)))
+ ((string-prefix? "replace " line)
+ ;; a replace directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (replace-directive results stripped-line)))
+ (toplevel new-results)))
+ (#t
+ ;; unrecognised line, ignore silently
+ (toplevel results)))))
+ (define (in-require results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-require (require-directive results line))))))
+ (define (in-replace results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-replace (replace-directive results line))))))
+ (define (replace-directive results line)
+ "Extract replaced modules and new requirements from replace directive
+ in LINE and add to RESULTS."
+ ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
+ ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ (re (string-concatenate
+ '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
+ "[[:blank:]]+" "=>" "[[:blank:]]+"
+ "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 3))
+ (new-module-path (match:substring match 4))
+ (new-version (match:substring match 6))
+ (new-replaced (acons module-path version replaced))
+ (new-requirements
+ (if (string-match "^\\.?\\./" new-module-path)
+ requirements
+ (acons new-module-path new-version requirements))))
+ (cons new-requirements new-replaced)))
+ (define (require-directive results line)
+ "Extract requirement from LINE and add it to RESULTS."
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ ;; A line in a require directive is composed of a module path and
+ ;; a version separated by whitespace and an optionnal '//' comment at
+ ;; the end.
+ (re (string-concatenate
+ '("^[[:blank:]]*"
+ "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
+ "([[:blank:]]+//.*)?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 2)))
+ (cons (acons module-path version requirements) replaced)))
+ (with-input-from-port go.mod-port
+ (lambda ()
+ (let* ((results (toplevel '(() . ())))
+ (requirements (car results))
+ (replaced (cdr results)))
+ ;; At last we remove replaced modules from the requirements list
+ (fold
+ (lambda (replacedelem requirements)
+ (alist-delete! (car replacedelem) requirements))
+ requirements
+ replaced)))))
+
+(define (infer-module-root module-path)
+ "Go modules can be defined at any level of a repository's tree, but querying
+for the meta tag usually can only be done at the webpage at the root of the
+repository. Therefore, it is sometimes necessary to try and derive a module's
+root path from its path. For a set of well-known forges, the pattern of what
+consists of a module's root page is known before hand."
+ ;; See the following URL for the official Go equivalent:
+ ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
+ ;;
+ ;; TODO: handle module path with VCS qualifier as described in
+ ;; https://golang.org/ref/mod#vcs-find and
+ ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
+ (define-record-type <vcs>
+ (make-vcs url-prefix root-regex type)
+ vcs?
+ (url-prefix vcs-url-prefix)
+ (root-regex vcs-root-regex)
+ (type vcs-type))
+ (let* ((known-vcs
+ (list
+ (make-vcs
+ "github.com"
+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "bitbucket.org"
+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$"
+ 'unknown)
+ (make-vcs
+ "hub.jazz.net/git/"
+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.apache.org"
+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.openstack.org"
+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)))
+ (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
+ known-vcs)))
+ (if vcs
+ (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
+ module-path)))
+
+(define (go-module->guix-package-name module-path)
+ "Converts a module's path to the canonical Guix format for Go packages."
+ (string-downcase
+ (string-append "go-"
+ (string-replace-substring
+ (string-replace-substring
+ module-path
+ "." "-")
+ "/" "-"))))
+
+(define-record-type <module-meta>
+ (make-module-meta import-prefix vcs repo-root)
+ module-meta?
+ (import-prefix module-meta-import-prefix)
+ ;; VCS field is a symbol
+ (vcs module-meta-vcs)
+ (repo-root module-meta-repo-root))
+
+(define (fetch-module-meta-data module-path)
+ "Fetches module meta-data from a module's landing page. This is
+ necessary because goproxy servers don't currently provide all the
+ information needed to build a package."
+ ;; <meta name="go-import" content="import-prefix vcs repo-root">
+ (define (meta-go-import->module-meta text)
+ "Takes the content of the go-import meta tag as TEXT and gives back
+ a MODULE-META record"
+ (define (get-component s start)
+ (let*
+ ((start (string-skip s char-set:whitespace start))
+ (end (string-index s char-set:whitespace start))
+ (end (if end end (string-length s)))
+ (result (substring s start end)))
+ (values result end)))
+ (let*-values (((import-prefix end) (get-component text 0))
+ ((vcs end) (get-component text end))
+ ((repo-root end) (get-component text end)))
+ (make-module-meta import-prefix (string->symbol vcs) repo-root)))
+ (define (html->meta-go-import port)
+ "Read PORT with HTML content. Find the go-import meta tag and gives
+ back its content as a string."
+ (let* ((parsedhtml (html->sxml port))
+ (extract-content (node-join
+ (select-kids (node-typeof? 'html))
+ (select-kids (node-typeof? 'head))
+ (select-kids (node-typeof? 'meta))
+ (select-kids (node-typeof? '@))
+ (node-self
+ (node-join
+ (select-kids (node-typeof? 'name))
+ (select-kids (node-equal? "go-import"))))
+ (select-kids (node-typeof? 'content))
+ (select-kids (lambda (_) #t))))
+ (content (car (extract-content parsedhtml))))
+ content))
+ (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
+ (meta-go-import (html->meta-go-import port))
+ (module-metadata (meta-go-import->module-meta meta-go-import)))
+ (close-port port)
+ module-metadata))
+
+(define (module-meta-data-repo-url meta-data goproxy-url)
+ "Return the URL where the fetcher which will be used can download the source
+control."
+ (if (member (module-meta-vcs meta-data)'(fossil mod))
+ goproxy-url
+ (module-meta-repo-root meta-data)))
+
+(define (source-uri vcs-type vcs-repo-url file)
+ "Generate the `origin' block of a package depending on what type of source
+control system is being used."
+ (case vcs-type
+ ((git)
+ `(origin
+ (method git-fetch)
+ (uri (git-reference
+ (url ,vcs-repo-url)
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ ((hg)
+ `(origin
+ (method hg-fetch)
+ (uri (hg-reference
+ (url ,vcs-repo-url)
+ (changeset ,version)))
+ (file-name (format #f "~a-~a-checkout" name version))))
+ ((svn)
+ `(origin
+ (method svn-fetch)
+ (uri (svn-reference
+ (url ,vcs-repo-url)
+ (revision (string->number version))
+ (recursive? #f)))
+ (file-name (format #f "~a-~a-checkout" name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ (else
+ (raise-exception (format #f "unsupported vcs type: ~a" vcs-type)))))
+
+(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (let* ((latest-version (fetch-latest-version goproxy-url module-path))
+ (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
+ temp))
+ (dependencies (map car (parse-go.mod temp)))
+ (guix-name (go-module->guix-package-name module-path))
+ (root-module-path (infer-module-root module-path))
+ ;; VCS type and URL are not included in goproxy information. For
+ ;; this we need to fetch it from the official module page.
+ (meta-data (fetch-module-meta-data root-module-path))
+ (vcs-type (module-meta-vcs meta-data))
+ (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
+ (values
+ `(package
+ (name ,guix-name)
+ ;; Elide the "v" prefix Go uses
+ (version ,(string-trim latest-version #\v))
+ (source
+ ,(source-uri vcs-type vcs-repo-url temp))
+ (build-system go-build-system)
+ ,@(maybe-inputs (map go-module->guix-package-name dependencies))
+ ;; TODO(katco): It would be nice to make an effort to fetch this
+ ;; from known forges, e.g. GitHub
+ (home-page ,(format #f "https://~a" root-module-path))
+ (synopsis "A Go package")
+ (description ,(format #f "~a is a Go package." guix-name))
+ (license #f))
+ dependencies)))))
+
+(define* (go-module-recursive-import package-name
+ #:key (goproxy-url "https://proxy.golang.org"))
+ (recursive-import
+ package-name
+ #:repo->guix-package (lambda* (name . _)
+ (go-module->guix-package
+ name
+ #:goproxy-url goproxy-url))
+ #:guix-name go-module->guix-package-name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 0a3863f965..1d2b45d942 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -77,7 +77,7 @@ rather than \\n."
;;;
(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"
- "cran" "crate" "texlive" "json" "opam"))
+ "go" "cran" "crate" "texlive" "json" "opam"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
new file mode 100644
index 0000000000..fde7555973
--- /dev/null
+++ b/guix/scripts/import/go.scm
@@ -0,0 +1,118 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import go)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import go)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-go))
+
+
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import go PACKAGE-PATH
+Import and convert the Go module for PACKAGE-PATH.\n"))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (display (G_ "
+ -r, --recursive generate package expressions for all Go modules\
+ that are not yet in Guix"))
+ (display (G_ "
+ -p, --goproxy=GOPROXY specify which goproxy server to use"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import go")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ (option '(#\p "goproxy") #t #f
+ (lambda (opt name arg result)
+ (alist-cons 'goproxy
+ (string->symbol arg)
+ (alist-delete 'goproxy result))))
+ %standard-import-options))
+
+
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-go . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (match args
+ ((module-name)
+ (if (assoc-ref opts 'recursive)
+ (map (match-lambda
+ ((and ('package ('name name) . rest) pkg)
+ `(define-public ,(string->symbol name)
+ ,pkg))
+ (_ #f))
+ (go-module-recursive-import module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org")))
+ (let ((sexp (go-module->guix-package module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org"))))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for module '~a'~%")
+ module-name))
+ sexp)))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%"))))))
diff --git a/tests/import-go.scm b/tests/import-go.scm
new file mode 100644
index 0000000000..7c59bf2d7c
--- /dev/null
+++ b/tests/import-go.scm
@@ -0,0 +1,143 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; Summary
+;; Tests for guix/import/go.scm
+
+(define-module (test-import-go)
+ #:use-module (guix import go)
+ #:use-module (guix base32)
+ ;#:use-module (guix tests)
+ #:use-module (ice-9 iconv)
+ #:use-module (ice-9 match)
+ #:use-module (srfi srfi-64))
+
+(define fixture-go-mod-simple
+ "module my/thing
+go 1.12
+require other/thing v1.0.2
+require new/thing/v2 v2.3.4
+exclude old/thing v1.2.3
+replace bad/thing v1.4.5 => good/thing v1.4.5
+")
+
+(define fixture-go-mod-with-block
+ "module M
+
+require (
+ A v1
+ B v1.0.0
+ C v1.0.0
+ D v1.2.3
+ E dev
+)
+
+exclude D v1.2.3
+")
+
+
+(define fixture-go-mod-complete
+ "module M
+
+go 1.13
+
+replace github.com/myname/myproject/myapi => ./api
+
+replace github.com/mymname/myproject/thissdk => ../sdk
+
+replace launchpad.net/gocheck => github.com/go-check/check v0.0.0-20140225173054-eb6ee6f84d0a
+
+require (
+ github.com/user/project v1.1.11
+ github.com/user/project/sub/directory v1.1.12
+ bitbucket.org/user/project v1.11.20
+ bitbucket.org/user/project/sub/directory v1.11.21
+ launchpad.net/project v1.1.13
+ launchpad.net/project/series v1.1.14
+ launchpad.net/project/series/sub/directory v1.1.15
+ launchpad.net/~user/project/branch v1.1.16
+ launchpad.net/~user/project/branch/sub/directory v1.1.17
+ hub.jazz.net/git/user/project v1.1.18
+ hub.jazz.net/git/user/project/sub/directory v1.1.19
+ k8s.io/kubernetes/subproject v1.1.101
+ one.example.com/abitrary/repo v1.1.111
+ two.example.com/abitrary/repo v0.0.2
+)
+
+replace two.example.com/abitrary/repo => github.com/corp/arbitrary-repo v0.0.2
+
+replace (
+ golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13
+ golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13
+)
+
+")
+
+(test-begin "import go")
+
+(test-equal "go-path-escape"
+ "github.com/!azure/!avere"
+ ((@@ (guix import go) go-path-escape) "github.com/Azure/Avere"))
+
+
+
+;; We define a function for all similar tests with different go.mod files
+(define (testing-parse-mod name expected input)
+ (define (inf? p1 p2)
+ (string<? (car p1) (car p2)))
+ (let ((input-port (open-input-string input)))
+ (test-equal name
+ (sort expected inf?)
+ (sort
+ ( (@@ (guix import go) parse-go.mod-port)
+ input-port)
+ inf?))))
+
+(testing-parse-mod "parse-go.mod-simple"
+ '(("good/thing" . "v1.4.5")
+ ("new/thing/v2" . "v2.3.4")
+ ("other/thing" . "v1.0.2"))
+ fixture-go-mod-simple)
+
+(testing-parse-mod "parse-go.mod-with-block"
+ '(("A" . "v1")
+ ("B" . "v1.0.0")
+ ("C" . "v1.0.0")
+ ("D" . "v1.2.3")
+ ("E" . "dev"))
+ fixture-go-mod-with-block)
+
+(testing-parse-mod "parse-go.mod-complete"
+ '(("github.com/corp/arbitrary-repo" . "v0.0.2")
+ ("one.example.com/abitrary/repo" . "v1.1.111")
+ ("hub.jazz.net/git/user/project/sub/directory" . "v1.1.19")
+ ("hub.jazz.net/git/user/project" . "v1.1.18")
+ ("launchpad.net/~user/project/branch/sub/directory" . "v1.1.17")
+ ("launchpad.net/~user/project/branch" . "v1.1.16")
+ ("launchpad.net/project/series/sub/directory" . "v1.1.15")
+ ("launchpad.net/project/series" . "v1.1.14")
+ ("launchpad.net/project" . "v1.1.13")
+ ("bitbucket.org/user/project/sub/directory" . "v1.11.21")
+ ("bitbucket.org/user/project" . "v1.11.20")
+ ("k8s.io/kubernetes/subproject" . "v1.1.101")
+ ("github.com/user/project/sub/directory" . "v1.1.12")
+ ("github.com/user/project" . "v1.1.11")
+ ("github.com/go-check/check" . "v0.0.0-20140225173054-eb6ee6f84d0a"))
+ fixture-go-mod-complete)
+
+(test-end "import go")
--
2.30.0
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-29 16:43 ` guix-patches--- via
2021-01-29 16:52 ` [bug#44178] [PATCHv2] " guix-patches--- via
@ 2021-01-31 16:23 ` Ludovic Courtès
2021-02-19 15:51 ` JOULAUD François via Guix-patches via
1 sibling, 1 reply; 27+ messages in thread
From: Ludovic Courtès @ 2021-01-31 16:23 UTC (permalink / raw)
To: JOULAUD François; +Cc: 44178@debbugs.gnu.org, Katherine Cox-Buday
Hi,
JOULAUD François <Francois.JOULAUD@radiofrance.com> skribis:
> On Thu, Jan 28, 2021 at 02:27:57PM +0100, Ludovic Courtès wrote:
>> I think whoever among you is available to work on it these days could
>> take the lead and prepare a final version of the patches. It looks like
>> it’s approaching a first “committable” version (perhaps just missing an
>> addition to doc/guix.texi and test cases like we have ‘tests/cpan.scm’ &
>> co.)
>
> I thought I would be able to send a working v2 of this patch today but
> it seems I was too optimistic.
>
> I found that some go.mod out there uses quoted string
> which our ad-hoc parser don't know how to parse. cf.
> https://github.com/go-yaml/yaml/blob/496545a6307b2a7d7a710fd516e5e16e8ab62dbc/go.mod
>
> I don't know if this is a blocker for a merge or not.
Your call; if it’s an infrequent problem, we could commit it and leave a
FIXME in the code. We could also use guile-yaml (or maybe some Go
code?) to parse it correctly.
> Apart from that I don't know how to add guile-lib to the dependencies of
> Guix (in order to use htmlprag). Help needed.
So ‘xml->sxml’ isn’t good enough? (If we can avoid the guile-lib
dependency, the better.)
To depend on Guile-Lib, you would:
1. Add it to (guix self) — this is the code used by ‘guix pull’;
2. Add it to the ‘inputs’ field of the ‘guix’ package;
3. Maybe add a configure check in ‘configure.ac’, though it would be
best if we could arrange to make it an optional dependency.
> I tested it recursively with github.com/hashicorp/consul (which was one
> of those with the most dependencies I found) and it mostly works.
Yay, sounds promising!
Thanks,
Ludo’.
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH] Create importer for Go modules
2021-01-31 16:23 ` [bug#44178] [PATCH] " Ludovic Courtès
@ 2021-02-19 15:51 ` JOULAUD François via Guix-patches via
2021-02-19 16:21 ` [bug#44178] [PATCHv3] " JOULAUD François via Guix-patches via
0 siblings, 1 reply; 27+ messages in thread
From: JOULAUD François via Guix-patches via @ 2021-02-19 15:51 UTC (permalink / raw)
To: Ludovic Courtès; +Cc: 44178@debbugs.gnu.org, Katherine Cox-Buday
Hello,
I will send a v3 of the patch very soon which I hope will be mergeable.
It is still very experimental and I noted it as such in documentation.
I had problems with the hash in origin which did not work as expected. I
prefered to drop this completely for now and replaced it by a full 0
placeholder. At least it is consistently and conspicuously bad. I have a
working version of something which download git repo and generates guix
hash for it but I'd rather to push this in a subsequent patch as it is
very rough for now.
On Sun, Jan 31, 2021 at 05:23:59PM +0100, Ludovic Courtès wrote:
> JOULAUD François <Francois.JOULAUD@radiofrance.com> skribis:
> > I found that some go.mod out there uses quoted string
> > which our ad-hoc parser don't know how to parse. cf.
>
> Your call; if it’s an infrequent problem, we could commit it and leave a
> FIXME in the code. We could also use guile-yaml (or maybe some Go
> code?) to parse it correctly.
I found a way to work around the problem.
Indeed using "go mod" to parse the go.mod file could perhaps be easier
and have been explored[1].
It works for now with the ad-hoc parser. Let's revisit the choice later
if neeeded.
> > Apart from that I don't know how to add guile-lib to the dependencies of
> > Guix (in order to use htmlprag). Help needed.
>
> So ‘xml->sxml’ isn’t good enough? (If we can avoid the guile-lib
> dependency, the better.)
HTML is not well-formed XML (and the hopes given by XHTML have faded)
so no, xml->sxml is unfortunately not good enough.
> To depend on Guile-Lib, you would:
>
> 1. Add it to (guix self) — this is the code used by ‘guix pull’;
>
> 2. Add it to the ‘inputs’ field of the ‘guix’ package;
Done 1 and 2 in the patch. Mainly by copy-paste without understanding
anything. I hope it will work.
> 3. Maybe add a configure check in ‘configure.ac’, though it would be
> best if we could arrange to make it an optional dependency.
I did not touch to configure.ac which is a strange beast to me. Hope that
for optional dependency (only used in "guix import go") it is sufficient.
> > I tested it recursively with github.com/hashicorp/consul (which was one
> > of those with the most dependencies I found) and it mostly works.
>
> Yay, sounds promising!
Promising but not there. I have now several recursive dependencies
between generated packages I must investigate.
Still, 234 package definitions generated on one recursive import (even
if I had to retry because of intermittent failure of fetching from
proxy.golang.org), so Yay!
[1]: https://git.sr.ht/~elais/orange/tree/master/item/guix/import/go.scm#L78
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCHv3] Create importer for Go modules
2021-02-19 15:51 ` JOULAUD François via Guix-patches via
@ 2021-02-19 16:21 ` JOULAUD François via Guix-patches via
2021-03-02 21:54 ` [bug#44178] Add a Go Module Importer Ludovic Courtès
0 siblings, 1 reply; 27+ messages in thread
From: JOULAUD François via Guix-patches via @ 2021-02-19 16:21 UTC (permalink / raw)
To: 44178@debbugs.gnu.org; +Cc: Ludovic Courtès, Katherine Cox-Buday
This patch add a `guix import go` command.
It was tested with several big repositories and mostly works. Several
features are lacking (see TODO in source code) but we will do the
improvments step-by-step in future patches.
* doc/guix.texi: doc about go importer and guile-lib dependency
* gnu/packages/package-management.scm: added guile-lib dependency
* guix/self.scm: add guile-lib dependency
* guix/build-system/go.scm: go-version->git-ref function
* guix/import/go.scm: Created Go importer
* guix/scripts/import/go.scm: Subcommand for Go importer
* guix/scripts/import.scm: Declare subcommand guix import go
* tests/import-go.scm: Tests for parse-go.mod procedure
Signed-off-by: Francois Joulaud <francois.joulaud@radiofrance.com>
---
doc/guix.texi | 26 ++
gnu/packages/package-management.scm | 2 +
guix/build-system/go.scm | 35 ++-
guix/import/go.scm | 416 ++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/go.scm | 118 ++++++++
guix/self.scm | 5 +-
tests/import-go.scm | 144 ++++++++++
8 files changed, 745 insertions(+), 3 deletions(-)
create mode 100644 guix/import/go.scm
create mode 100644 guix/scripts/import/go.scm
create mode 100644 tests/import-go.scm
diff --git a/doc/guix.texi b/doc/guix.texi
index 5d28fca837..89c8abd261 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -861,6 +861,10 @@ substitutes (@pxref{Invoking guix publish}).
@uref{https://ngyro.com/software/guile-semver.html, Guile-Semver} for
the @code{crate} importer (@pxref{Invoking guix import}).
+@item
+@uref{https://www.nongnu.org/guile-lib/doc/ref/htmlprag/, guile-lib} for
+the @code{crate} importer (@pxref{Invoking guix import}).
+
@item
When @url{http://www.bzip.org, libbz2} is available,
@command{guix-daemon} can use it to compress build logs.
@@ -11493,6 +11497,28 @@ Select the given repository (a repository name). Possible values include:
of coq packages.
@end itemize
@end table
+
+@item go
+@cindex go
+Import metadata for a Go module using
+@uref{https://proxy.golang.org, proxy.golang.org}.
+
+This importer is highly experimental. See the source code for more info
+about the current state.
+
+@example
+guix import go gopkg.in/yaml.v2
+@end example
+
+Additional options include:
+
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
@end table
The structure of the @command{guix import} code is modular. It would be
diff --git a/gnu/packages/package-management.scm b/gnu/packages/package-management.scm
index 9fb8c40a31..06bb5bd2df 100644
--- a/gnu/packages/package-management.scm
+++ b/gnu/packages/package-management.scm
@@ -304,6 +304,7 @@ $(prefix)/etc/init.d\n")))
'((assoc-ref inputs "guile"))))
(avahi (assoc-ref inputs "guile-avahi"))
(gcrypt (assoc-ref inputs "guile-gcrypt"))
+ (guile-lib (assoc-ref inputs "guile-lib"))
(json (assoc-ref inputs "guile-json"))
(sqlite (assoc-ref inputs "guile-sqlite3"))
(zlib (assoc-ref inputs "guile-zlib"))
@@ -367,6 +368,7 @@ $(prefix)/etc/init.d\n")))
`(("guile-avahi" ,guile-avahi)))
("guile-gcrypt" ,guile-gcrypt)
("guile-json" ,guile-json-4)
+ ("guile-lib" ,guile-lib)
("guile-sqlite3" ,guile-sqlite3)
("guile-zlib" ,guile-zlib)
("guile-lzlib" ,guile-lzlib)
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm
index f8ebaefb27..594e0cb4f3 100644
--- a/guix/build-system/go.scm
+++ b/guix/build-system/go.scm
@@ -26,9 +26,42 @@
#:use-module (guix build-system gnu)
#:use-module (guix packages)
#:use-module (ice-9 match)
+ #:use-module (ice-9 regex)
#:export (%go-build-system-modules
go-build
- go-build-system))
+ go-build-system
+
+ go-version->git-ref))
+
+(define (go-version->git-ref version)
+ "GO-VERSION->GIT-REF parse pseudo-versions and extract the commit
+ hash from it, defaulting to full VERSION if we don't recognise a
+ pseudo-version pattern."
+ ;; A module version like v1.2.3 is introduced by tagging a revision in
+ ;; the underlying source repository. Untagged revisions can be referred
+ ;; to using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef,
+ ;; where the time is the commit time in UTC and the final suffix is the
+ ;; prefix of the commit hash.
+ ;; cf. https://golang.org/cmd/go/#hdr-Pseudo_versions
+ (let* ((version
+ ;; if a source code repository has a v2.0.0 or later tag for
+ ;; a file tree with no go.mod, the version is considered to be
+ ;; part of the v1 module's available versions and is given an
+ ;; +incompatible suffix
+ ;; https://golang.org/cmd/go/#hdr-Module_compatibility_and_semantic_versioning
+ (if (string-suffix? "+incompatible" version)
+ (string-drop-right version 13)
+ version))
+ (re (string-concatenate
+ (list
+ "(v?[0-9]\\.[0-9]\\.[0-9])" ; "v" prefix can be omitted in version prefix
+ "(-|-pre\\.0\\.|-0\\.)" ; separator
+ "([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])-" ; timestamp
+ "([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])"))) ; commit hash
+ (match (string-match re version)))
+ (if match
+ (match:substring match 4)
+ version)))
;; Commentary:
;;
diff --git a/guix/import/go.scm b/guix/import/go.scm
new file mode 100644
index 0000000000..fead355bd2
--- /dev/null
+++ b/guix/import/go.scm
@@ -0,0 +1,416 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; (guix import golang) wants to make easier to create Guix package
+;;; declaration for Go modules.
+;;;
+;;; Modules in Go are "collection of related Go packages" which are
+;;; "the unit of source code interchange and versioning".
+;;; Modules are generally hosted in a repository.
+;;;
+;;; At this point it should handle correctly modules which
+;;; have only Go dependencies and are accessible from proxy.golang.org
+;;; (or configured GOPROXY).
+;;;
+;;; We want it to work more or less this way:
+;;; - get latest version for the module from GOPROXY
+;;; - infer VCS root repo from which we will check-out source by
+;;; + recognising known patterns (like github.com)
+;;; + or (TODO) recognising .vcs suffix
+;;; + or parsing meta tag in html served at the URL
+;;; + or (TODO) if nothing else works by using zip file served by GOPROXY
+;;; - get go.mod from GOPROXY (which is able to synthetize one if needed)
+;;; - extract list of dependencies from this go.mod
+;;;
+;;; We translate Go module paths to a Guix package name under the
+;;; assumption that there will be no collision.
+
+;;; TODO list
+;;; - get correct hash in vcs->origin
+;;; - print partial result during recursive imports (need to catch
+;;; exceptions)
+;;; - infer repo from module path with VCS qualifier
+;;; (e.g. site.example/my/path/to/repo.git/and/subdir/module)
+;;; - don't print fetch messages to stdout
+;;; - pre-fill synopsis, description and license
+
+(define-module (guix import go)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 rdelim)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (guix build-system go)
+ #:use-module (htmlprag)
+ #:use-module (sxml xpath)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (srfi srfi-11)
+ #:use-module (json)
+ #:use-module ((guix download) #:prefix download:)
+ #:use-module (guix git)
+ #:use-module (guix import utils)
+ #:use-module (guix import json)
+ #:use-module (guix packages)
+ #:use-module (guix upstream)
+ #:use-module (guix utils)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix base16)
+ #:use-module (guix base32)
+ #:use-module (guix memoization)
+ #:use-module ((guix build download) #:prefix build-download:)
+ #:use-module (web uri)
+
+ #:export (go-module->guix-package
+ go-module-recursive-import
+ infer-module-root-repo))
+
+
+(define (go-path-escape path)
+ "Escape a module path by replacing every uppercase letter with an exclamation
+mark followed with its lowercase equivalent, as per the module Escaped Paths
+specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
+ (define (escape occurrence)
+ (string-append "!" (string-downcase (match:substring occurrence))))
+ (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
+
+
+(define (go-module-latest-version goproxy-url module-path)
+ "Fetches the version number of the latest version for MODULE-PATH from the
+given GOPROXY-URL server."
+ (assoc-ref
+ (json-fetch (format #f "~a/~a/@latest" goproxy-url
+ (go-path-escape module-path)))
+ "Version"))
+
+(define go-module-latest-version* (memoize go-module-latest-version))
+
+(define (fetch-go.mod goproxy-url module-path version file)
+ "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
+and VERSION."
+ (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
+ (go-path-escape module-path)
+ (go-path-escape version))))
+ (parameterize ((current-output-port (current-error-port)))
+ (build-download:url-fetch url
+ file
+ #:print-build-trace? #f))))
+
+(define (parse-go.mod go.mod-path)
+ (parse-go.mod-port (open-input-file go.mod-path)))
+
+(define (parse-go.mod-port go.mod-port)
+ "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
+requirements from it."
+ ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
+ ;; which we think necessary for our use case.
+ (define (toplevel results)
+ "Main parser, RESULTS is a pair of alist serving as accumulator for
+ all encountered requirements and replacements."
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; parsing ended, give back the result
+ results)
+ ((string=? line "require (")
+ ;; a require block begins, delegate parsing to IN-REQUIRE
+ (in-require results))
+ ((string=? line "replace (")
+ ;; a replace block begins, delegate parsing to IN-REPLACE
+ (in-replace results))
+ ((string-prefix? "require " line)
+ ;; a require directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (require-directive results stripped-line)))
+ (toplevel new-results)))
+ ((string-prefix? "replace " line)
+ ;; a replace directive by itself
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (replace-directive results stripped-line)))
+ (toplevel new-results)))
+ (#t
+ ;; unrecognised line, ignore silently
+ (toplevel results)))))
+ (define (in-require results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-require (require-directive results line))))))
+ (define (in-replace results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-replace (replace-directive results line))))))
+ (define (replace-directive results line)
+ "Extract replaced modules and new requirements from replace directive
+ in LINE and add to RESULTS."
+ ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
+ ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ (re (string-concatenate
+ '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
+ "[[:blank:]]+" "=>" "[[:blank:]]+"
+ "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ (version (match:substring match 3))
+ (new-module-path (match:substring match 4))
+ (new-version (match:substring match 6))
+ (new-replaced (acons module-path version replaced))
+ (new-requirements
+ (if (string-match "^\\.?\\./" new-module-path)
+ requirements
+ (acons new-module-path new-version requirements))))
+ (cons new-requirements new-replaced)))
+ (define (require-directive results line)
+ "Extract requirement from LINE and add it to RESULTS."
+ (let* ((requirements (car results))
+ (replaced (cdr results))
+ ;; A line in a require directive is composed of a module path and
+ ;; a version separated by whitespace and an optionnal '//' comment at
+ ;; the end.
+ (re (string-concatenate
+ '("^[[:blank:]]*"
+ "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
+ "([[:blank:]]+//.*)?")))
+ (match (string-match re line))
+ (module-path (match:substring match 1))
+ ;; we saw double-quoted string in the wild without escape
+ ;; sequences so we just trim the quotes
+ (module-path (string-trim-both module-path #\"))
+ (version (match:substring match 2)))
+ (cons (acons module-path version requirements) replaced)))
+ (with-input-from-port go.mod-port
+ (lambda ()
+ (let* ((results (toplevel '(() . ())))
+ (requirements (car results))
+ (replaced (cdr results)))
+ ;; At last we remove replaced modules from the requirements list
+ (fold
+ (lambda (replacedelem requirements)
+ (alist-delete! (car replacedelem) requirements))
+ requirements
+ replaced)))))
+
+(define (infer-module-root-repo module-path)
+ "Go modules can be defined at any level of a repository's tree, but querying
+for the meta tag usually can only be done at the webpage at the root of the
+repository. Therefore, it is sometimes necessary to try and derive a module's
+root path from its path. For a set of well-known forges, the pattern of what
+consists of a module's root page is known before hand."
+ ;; See the following URL for the official Go equivalent:
+ ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
+ ;;
+ ;; TODO: handle module path with VCS qualifier as described in
+ ;; https://golang.org/ref/mod#vcs-find and
+ ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
+ (define-record-type <vcs>
+ (make-vcs url-prefix root-regex type)
+ vcs?
+ (url-prefix vcs-url-prefix)
+ (root-regex vcs-root-regex)
+ (type vcs-type))
+ (let* ((known-vcs
+ (list
+ (make-vcs
+ "github.com"
+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "bitbucket.org"
+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$"
+ 'unknown)
+ (make-vcs
+ "hub.jazz.net/git/"
+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.apache.org"
+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.openstack.org"
+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)))
+ (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
+ known-vcs)))
+ (if vcs
+ (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
+ module-path)))
+
+(define (go-module->guix-package-name module-path)
+ "Converts a module's path to the canonical Guix format for Go packages."
+ (string-downcase
+ (string-append "go-"
+ (string-replace-substring
+ (string-replace-substring
+ module-path
+ "." "-")
+ "/" "-"))))
+
+(define-record-type <module-meta>
+ (make-module-meta import-prefix vcs repo-root)
+ module-meta?
+ (import-prefix module-meta-import-prefix)
+ ;; VCS field is a symbol
+ (vcs module-meta-vcs)
+ (repo-root module-meta-repo-root))
+
+(define (fetch-module-meta-data module-path)
+ "Fetches module meta-data from a module's landing page. This is
+ necessary because goproxy servers don't currently provide all the
+ information needed to build a package."
+ ;; <meta name="go-import" content="import-prefix vcs repo-root">
+ (define (meta-go-import->module-meta text)
+ "Takes the content of the go-import meta tag as TEXT and gives back
+ a MODULE-META record"
+ (define (get-component s start)
+ (let*
+ ((start (string-skip s char-set:whitespace start))
+ (end (string-index s char-set:whitespace start))
+ (end (if end end (string-length s)))
+ (result (substring s start end)))
+ (values result end)))
+ (let*-values (((import-prefix end) (get-component text 0))
+ ((vcs end) (get-component text end))
+ ((repo-root end) (get-component text end)))
+ (make-module-meta import-prefix (string->symbol vcs) repo-root)))
+ (define (html->meta-go-import port)
+ "Read PORT with HTML content. Find the go-import meta tag and gives
+ back its content as a string."
+ (let* ((parsedhtml (html->sxml port))
+ (extract-content (node-join
+ (select-kids (node-typeof? 'html))
+ (select-kids (node-typeof? 'head))
+ (select-kids (node-typeof? 'meta))
+ (select-kids (node-typeof? '@))
+ (node-self
+ (node-join
+ (select-kids (node-typeof? 'name))
+ (select-kids (node-equal? "go-import"))))
+ (select-kids (node-typeof? 'content))
+ (select-kids (lambda (_) #t))))
+ (content (car (extract-content parsedhtml))))
+ content))
+ (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
+ (meta-go-import (html->meta-go-import port))
+ (module-metadata (meta-go-import->module-meta meta-go-import)))
+ (close-port port)
+ module-metadata))
+
+(define (module-meta-data-repo-url meta-data goproxy-url)
+ "Return the URL where the fetcher which will be used can download the source
+control."
+ (if (member (module-meta-vcs meta-data)'(fossil mod))
+ goproxy-url
+ (module-meta-repo-root meta-data)))
+
+(define (vcs->origin vcs-type vcs-repo-url version file)
+ "Generate the `origin' block of a package depending on what type of source
+control system is being used."
+ (case vcs-type
+ ((git)
+ `(origin
+ (method git-fetch)
+ (uri (git-reference
+ (url ,vcs-repo-url)
+ (commit (go-version->git-ref version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ ;; FIXME: get hash for git repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
+ ((hg)
+ `(origin
+ (method hg-fetch)
+ (uri (hg-reference
+ (url ,vcs-repo-url)
+ (changeset ,version)))
+ (file-name (format #f "~a-~a-checkout" name version))))
+ ((svn)
+ `(origin
+ (method svn-fetch)
+ (uri (svn-reference
+ (url ,vcs-repo-url)
+ (revision (string->number version))
+ (recursive? #f)))
+ (file-name (format #f "~a-~a-checkout" name version))
+ (sha256
+ (base32
+ ,(guix-hash-url file)))))
+ (else
+ (raise-exception (format #f "unsupported vcs type: ~a" vcs-type)))))
+
+(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
+ (call-with-temporary-output-file
+ (lambda (temp port)
+ (let* ((latest-version (go-module-latest-version* goproxy-url module-path))
+ (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
+ temp))
+ (dependencies (map car (parse-go.mod temp)))
+ (guix-name (go-module->guix-package-name module-path))
+ (root-module-path (infer-module-root-repo module-path))
+ ;; VCS type and URL are not included in goproxy information. For
+ ;; this we need to fetch it from the official module page.
+ (meta-data (fetch-module-meta-data root-module-path))
+ (vcs-type (module-meta-vcs meta-data))
+ (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
+ (values
+ `(package
+ (name ,guix-name)
+ ;; Elide the "v" prefix Go uses
+ (version ,(string-trim latest-version #\v))
+ (source
+ ,(vcs->origin vcs-type vcs-repo-url latest-version temp))
+ (build-system go-build-system)
+ (arguments
+ '(#:import-path ,root-module-path))
+ ,@(maybe-inputs (map go-module->guix-package-name dependencies))
+ ;; TODO(katco): It would be nice to make an effort to fetch this
+ ;; from known forges, e.g. GitHub
+ (home-page ,(format #f "https://~a" root-module-path))
+ (synopsis "A Go package")
+ (description ,(format #f "~a is a Go package." guix-name))
+ (license #f))
+ dependencies)))))
+
+(define go-module->guix-package* (memoize go-module->guix-package))
+
+(define* (go-module-recursive-import package-name
+ #:key (goproxy-url "https://proxy.golang.org"))
+ (recursive-import
+ package-name
+ #:repo->guix-package (lambda* (name . _)
+ (go-module->guix-package*
+ name
+ #:goproxy-url goproxy-url))
+ #:guix-name go-module->guix-package-name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 0a3863f965..1d2b45d942 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -77,7 +77,7 @@ rather than \\n."
;;;
(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"
- "cran" "crate" "texlive" "json" "opam"))
+ "go" "cran" "crate" "texlive" "json" "opam"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
new file mode 100644
index 0000000000..fde7555973
--- /dev/null
+++ b/guix/scripts/import/go.scm
@@ -0,0 +1,118 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import go)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import go)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-go))
+
+
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import go PACKAGE-PATH
+Import and convert the Go module for PACKAGE-PATH.\n"))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (display (G_ "
+ -r, --recursive generate package expressions for all Go modules\
+ that are not yet in Guix"))
+ (display (G_ "
+ -p, --goproxy=GOPROXY specify which goproxy server to use"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import go")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ (option '(#\p "goproxy") #t #f
+ (lambda (opt name arg result)
+ (alist-cons 'goproxy
+ (string->symbol arg)
+ (alist-delete 'goproxy result))))
+ %standard-import-options))
+
+
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-go . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (match args
+ ((module-name)
+ (if (assoc-ref opts 'recursive)
+ (map (match-lambda
+ ((and ('package ('name name) . rest) pkg)
+ `(define-public ,(string->symbol name)
+ ,pkg))
+ (_ #f))
+ (go-module-recursive-import module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org")))
+ (let ((sexp (go-module->guix-package module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org"))))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for module '~a'~%")
+ module-name))
+ sexp)))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%"))))))
diff --git a/guix/self.scm b/guix/self.scm
index 35fba1152d..ed5ee9ddea 100644
--- a/guix/self.scm
+++ b/guix/self.scm
@@ -814,6 +814,9 @@ itself."
(define guile-ssh
(specification->package "guile-ssh"))
+ (define guile-lib
+ (specification->package "guile-lib"))
+
(define guile-git
(specification->package "guile-git"))
@@ -842,7 +845,7 @@ itself."
(append-map transitive-package-dependencies
(list guile-gcrypt gnutls guile-git guile-avahi
guile-json guile-semver guile-ssh guile-sqlite3
- guile-zlib guile-lzlib guile-zstd)))
+ guile-lib guile-zlib guile-lzlib guile-zstd)))
(define *core-modules*
(scheme-node "guix-core"
diff --git a/tests/import-go.scm b/tests/import-go.scm
new file mode 100644
index 0000000000..ad4185684c
--- /dev/null
+++ b/tests/import-go.scm
@@ -0,0 +1,144 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; Summary
+;; Tests for guix/import/go.scm
+
+(define-module (test-import-go)
+ #:use-module (guix import go)
+ #:use-module (guix base32)
+ #:use-module (ice-9 iconv)
+ #:use-module (ice-9 match)
+ #:use-module (srfi srfi-64))
+
+(define fixture-go-mod-simple
+ "module my/thing
+go 1.12
+require other/thing v1.0.2
+require new/thing/v2 v2.3.4
+exclude old/thing v1.2.3
+replace bad/thing v1.4.5 => good/thing v1.4.5
+")
+
+(define fixture-go-mod-with-block
+ "module M
+
+require (
+ A v1
+ B v1.0.0
+ C v1.0.0
+ D v1.2.3
+ E dev
+)
+
+exclude D v1.2.3
+")
+
+
+(define fixture-go-mod-complete
+ "module M
+
+go 1.13
+
+replace github.com/myname/myproject/myapi => ./api
+
+replace github.com/mymname/myproject/thissdk => ../sdk
+
+replace launchpad.net/gocheck => github.com/go-check/check v0.0.0-20140225173054-eb6ee6f84d0a
+
+require (
+ github.com/user/project v1.1.11
+ github.com/user/project/sub/directory v1.1.12
+ bitbucket.org/user/project v1.11.20
+ bitbucket.org/user/project/sub/directory v1.11.21
+ launchpad.net/project v1.1.13
+ launchpad.net/project/series v1.1.14
+ launchpad.net/project/series/sub/directory v1.1.15
+ launchpad.net/~user/project/branch v1.1.16
+ launchpad.net/~user/project/branch/sub/directory v1.1.17
+ hub.jazz.net/git/user/project v1.1.18
+ hub.jazz.net/git/user/project/sub/directory v1.1.19
+ k8s.io/kubernetes/subproject v1.1.101
+ one.example.com/abitrary/repo v1.1.111
+ two.example.com/abitrary/repo v0.0.2
+ \"quoted.example.com/abitrary/repo\" v0.0.2
+)
+
+replace two.example.com/abitrary/repo => github.com/corp/arbitrary-repo v0.0.2
+
+replace (
+ golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13
+ golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13
+)
+
+")
+
+(test-begin "import go")
+
+(test-equal "go-path-escape"
+ "github.com/!azure/!avere"
+ ((@@ (guix import go) go-path-escape) "github.com/Azure/Avere"))
+
+
+
+;; We define a function for all similar tests with different go.mod files
+(define (testing-parse-mod name expected input)
+ (define (inf? p1 p2)
+ (string<? (car p1) (car p2)))
+ (let ((input-port (open-input-string input)))
+ (test-equal name
+ (sort expected inf?)
+ (sort
+ ( (@@ (guix import go) parse-go.mod-port)
+ input-port)
+ inf?))))
+
+(testing-parse-mod "parse-go.mod-simple"
+ '(("good/thing" . "v1.4.5")
+ ("new/thing/v2" . "v2.3.4")
+ ("other/thing" . "v1.0.2"))
+ fixture-go-mod-simple)
+
+(testing-parse-mod "parse-go.mod-with-block"
+ '(("A" . "v1")
+ ("B" . "v1.0.0")
+ ("C" . "v1.0.0")
+ ("D" . "v1.2.3")
+ ("E" . "dev"))
+ fixture-go-mod-with-block)
+
+(testing-parse-mod "parse-go.mod-complete"
+ '(("github.com/corp/arbitrary-repo" . "v0.0.2")
+ ("quoted.example.com/abitrary/repo" . "v0.0.2")
+ ("one.example.com/abitrary/repo" . "v1.1.111")
+ ("hub.jazz.net/git/user/project/sub/directory" . "v1.1.19")
+ ("hub.jazz.net/git/user/project" . "v1.1.18")
+ ("launchpad.net/~user/project/branch/sub/directory" . "v1.1.17")
+ ("launchpad.net/~user/project/branch" . "v1.1.16")
+ ("launchpad.net/project/series/sub/directory" . "v1.1.15")
+ ("launchpad.net/project/series" . "v1.1.14")
+ ("launchpad.net/project" . "v1.1.13")
+ ("bitbucket.org/user/project/sub/directory" . "v1.11.21")
+ ("bitbucket.org/user/project" . "v1.11.20")
+ ("k8s.io/kubernetes/subproject" . "v1.1.101")
+ ("github.com/user/project/sub/directory" . "v1.1.12")
+ ("github.com/user/project" . "v1.1.11")
+ ("github.com/go-check/check" . "v0.0.0-20140225173054-eb6ee6f84d0a"))
+ fixture-go-mod-complete)
+
+(test-end "import go")
--
2.28.0
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [bug#44178] Add a Go Module Importer
2021-02-19 16:21 ` [bug#44178] [PATCHv3] " JOULAUD François via Guix-patches via
@ 2021-03-02 21:54 ` Ludovic Courtès
2021-03-04 5:40 ` [bug#44178] [PATCH v4] Re: bug#44178: " Maxim Cournoyer
2021-03-08 13:54 ` [bug#44178] " JOULAUD François via Guix-patches via
0 siblings, 2 replies; 27+ messages in thread
From: Ludovic Courtès @ 2021-03-02 21:54 UTC (permalink / raw)
To: JOULAUD François; +Cc: 44178@debbugs.gnu.org, Katherine Cox-Buday
Hi,
JOULAUD François <Francois.JOULAUD@radiofrance.com> skribis:
> This patch add a `guix import go` command.
>
> It was tested with several big repositories and mostly works. Several
> features are lacking (see TODO in source code) but we will do the
> improvments step-by-step in future patches.
>
> * doc/guix.texi: doc about go importer and guile-lib dependency
> * gnu/packages/package-management.scm: added guile-lib dependency
> * guix/self.scm: add guile-lib dependency
> * guix/build-system/go.scm: go-version->git-ref function
> * guix/import/go.scm: Created Go importer
> * guix/scripts/import/go.scm: Subcommand for Go importer
> * guix/scripts/import.scm: Declare subcommand guix import go
> * tests/import-go.scm: Tests for parse-go.mod procedure
Nitpick: please mention the sections (for documentation) or variables
changed (see
<https://guix.gnu.org/manual/en/html_node/Submitting-Patches.html>).
Some comments below, mostly stylistic as I’m not familiar with the
actual file formats etc. that the importer implements.
> +@item
> +@uref{https://www.nongnu.org/guile-lib/doc/ref/htmlprag/, guile-lib} for
> +the @code{crate} importer (@pxref{Invoking guix import}).
s/crate/go/
s/guile-lib/Guile-Lib/
> + (re (string-concatenate
> + (list
> + "(v?[0-9]\\.[0-9]\\.[0-9])" ; "v" prefix can be omitted in version prefix
> + "(-|-pre\\.0\\.|-0\\.)" ; separator
> + "([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])-" ; timestamp
> + "([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])"))) ; commit hash
You can use ‘string-append’ instead of (string-concatenate (list …)).
Use [[:xdigit:]] instead of [0-9A-Fa-f] for clarity and
locale-independence.
Also, you can arrange to use ‘make-regexp’ so that the regexp is
compiled once for all, and then just ‘regexp-exec’:
(define %go-version-rx (make-regexp …))
(define (go-version->git-ref version)
(… (regexp-exec %go-version-rx …) …))
It’s not critical though.
> + (define (replace-directive results line)
> + "Extract replaced modules and new requirements from replace directive
> + in LINE and add to RESULTS."
> + ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
> + ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
> + (let* ((requirements (car results))
> + (replaced (cdr results))
Please use ‘match’ instead of car/cdr (throughout):
https://guix.gnu.org/manual/en/html_node/Coding-Style.html
> + (re (string-concatenate
> + '("([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
> + "[[:blank:]]+" "=>" "[[:blank:]]+"
> + "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
> + (match (string-match re line))
As above, you should arrange to pre-compile the regexp.
> + (module-path (match:substring match 1))
> + (version (match:substring match 3))
> + (new-module-path (match:substring match 4))
> + (new-version (match:substring match 6))
> + (new-replaced (acons module-path version replaced))
s/acons/alist-cons/ for consistency with the rest of the code.
> + (re (string-concatenate
> + '("^[[:blank:]]*"
> + "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
> + "([[:blank:]]+//.*)?")))
> + (match (string-match re line))
Same as above.
> + ;; TODO: handle module path with VCS qualifier as described in
> + ;; https://golang.org/ref/mod#vcs-find and
> + ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
> + (define-record-type <vcs>
> + (make-vcs url-prefix root-regex type)
> + vcs?
> + (url-prefix vcs-url-prefix)
> + (root-regex vcs-root-regex)
> + (type vcs-type))
You could rename ‘make-vcs’ above to ‘%make-vcs’ and do:
(define (make-vcs prefix regexp type)
(%make-vcs prefix (make-regex regexp) type))
so that again you can rely on pre-compiled regexps.
> + (let* ((known-vcs
> + (list
> + (make-vcs
> + "github.com"
> + "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)
> + (make-vcs
> + "bitbucket.org"
> + "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$"
> + 'unknown)
> + (make-vcs
> + "hub.jazz.net/git/"
> + "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)
> + (make-vcs
> + "git.apache.org"
> + "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)
> + (make-vcs
> + "git.openstack.org"
> + "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
> + 'git)))
> + (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
> + known-vcs)))
Keep ‘known-vcs’ in a global variable so it doesn’t have to be
recomputed every time.
> + `(package
> + (name ,guix-name)
> + ;; Elide the "v" prefix Go uses
> + (version ,(string-trim latest-version #\v))
> + (source
> + ,(vcs->origin vcs-type vcs-repo-url latest-version temp))
> + (build-system go-build-system)
> + (arguments
> + '(#:import-path ,root-module-path))
> + ,@(maybe-inputs (map go-module->guix-package-name dependencies))
> + ;; TODO(katco): It would be nice to make an effort to fetch this
> + ;; from known forges, e.g. GitHub
> + (home-page ,(format #f "https://~a" root-module-path))
> + (synopsis "A Go package")
^
‘guix lint’ wouldn’t like it. :-) Maybe "Write synopsis here" instead?
> + (description ,(format #f "~a is a Go package." guix-name))
> + (license #f))
Is there no info about the license?
> +++ b/guix/self.scm
> @@ -814,6 +814,9 @@ itself."
> (define guile-ssh
> (specification->package "guile-ssh"))
>
> + (define guile-lib
> + (specification->package "guile-lib"))
> +
> (define guile-git
> (specification->package "guile-git"))
>
> @@ -842,7 +845,7 @@ itself."
> (append-map transitive-package-dependencies
> (list guile-gcrypt gnutls guile-git guile-avahi
> guile-json guile-semver guile-ssh guile-sqlite3
> - guile-zlib guile-lzlib guile-zstd)))
> + guile-lib guile-zlib guile-lzlib guile-zstd)))
New dependency; it’s a bit of a commitment, but hopefully Guile-Lib is
stable enough and works on all the supported architectures.
Please add guix/scripts/import/go.scm to ‘po/guix/POTFILES.in’ so it can
be translated.
> +++ b/tests/import-go.scm
Looks nice! It should be called ‘tests/go.scm’ for consistency, with:
(test-begin "go")
…
(test-end "go")
Would it be an option to also have an end-to-end test (checking the
resulting ‘package’ sexp)? That’d be nice, but perhaps we can add it
afterwards if you prefer.
Let’s see how much of the comments above you can address for a v4, and
then we can get that in and improve it from there if needed!
Thanks again,
Ludo’.
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH v4] Re: bug#44178: Add a Go Module Importer
2021-03-02 21:54 ` [bug#44178] Add a Go Module Importer Ludovic Courtès
@ 2021-03-04 5:40 ` Maxim Cournoyer
2021-03-04 14:14 ` JOULAUD François via Guix-patches via
2021-03-08 13:54 ` [bug#44178] " JOULAUD François via Guix-patches via
1 sibling, 1 reply; 27+ messages in thread
From: Maxim Cournoyer @ 2021-03-04 5:40 UTC (permalink / raw)
To: JOULAUD François
Cc: Ludovic Courtès, 44178@debbugs.gnu.org, Katherine Cox-Buday
[-- Attachment #1: Type: text/plain, Size: 3179 bytes --]
Hi François, Ludovic, et al!
Sorry for bumping in the review, but I have been experimenting with this
importer, and it looks promising; thanks for everyone involved! I made
a couple changes, mostly with regard to integrating support for the
synopsis, description and license field of the package, plus other
cosmetic changes. I thought I should share it quickly so that it can be
used as the basis for a v5, so here's the patch, attached.
I hope you don't mind!
I tested it with:
$ ./pre-inst-env guix environment guix
$ ./pre-inst-env guix import go -r github.com/dgraph-io/badger/v2
--8<---------------cut here---------------start------------->8---
[...]
(define-public go-github-com-dgraph-io-badger-v2
(package
(name "go-github-com-dgraph-io-badger-v2")
(version "2.2007.2")
(source
(origin
(method git-fetch)
(uri (git-reference
(url "https://github.com/dgraph-io/badger.git")
(commit (go-version->git-ref version))))
(file-name (git-file-name name version))
(sha256
(base32
"0000000000000000000000000000000000000000000000000000"))))
(build-system go-build-system)
(arguments
'(#:import-path "github.com/dgraph-io/badger"))
(inputs
`(("go-gopkg-in-check-v1" ,go-gopkg-in-check-v1)
("go-golang-org-x-sys" ,go-golang-org-x-sys)
("go-golang-org-x-net" ,go-golang-org-x-net)
("go-github-com-stretchr-testify"
,go-github-com-stretchr-testify)
("go-github-com-spf13-cobra"
,go-github-com-spf13-cobra)
("go-github-com-spaolacci-murmur3"
,go-github-com-spaolacci-murmur3)
("go-github-com-pkg-errors"
,go-github-com-pkg-errors)
("go-github-com-kr-pretty"
,go-github-com-kr-pretty)
("go-github-com-golang-snappy"
,go-github-com-golang-snappy)
("go-github-com-golang-protobuf"
,go-github-com-golang-protobuf)
("go-github-com-dustin-go-humanize"
,go-github-com-dustin-go-humanize)
("go-github-com-dgryski-go-farm"
,go-github-com-dgryski-go-farm)
("go-github-com-dgraph-io-ristretto"
,go-github-com-dgraph-io-ristretto)
("go-github-com-cespare-xxhash"
,go-github-com-cespare-xxhash)
("go-github-com-datadog-zstd"
,go-github-com-datadog-zstd)))
(home-page "https://github.com/dgraph-io/badger")
(synopsis "BadgerDB")
(description
"Package badger implements an embeddable, simple and fast key-value database, written in pure Go. It is designed to be highly performant for both reads and writes simultaneously. Badger uses Multi-Version Concurrency Control (MVCC), and supports transactions. It runs transactions concurrently, with serializable snapshot isolation guarantees.")
(license (license:asl2.0))))
--8<---------------cut here---------------end--------------->8---
Attached is the fixup commit which should apply cleanly on top of your
v3 patch on master, along a (now required) commit to use a temporary
fork of guile-lib:
[-- Attachment #2: 0001-gnu-guile-lib-Update-to-a-temporary-fork.patch --]
[-- Type: text/x-patch, Size: 5679 bytes --]
From 16c07537375ab5d18ee76a5fdfb2b8ed7192b395 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Wed, 3 Mar 2021 16:20:22 -0500
Subject: [PATCH] gnu: guile-lib: Update to a temporary fork.
This fork add support to enable stricter/more correct parsing of HTML in
htmlprag, which is used by the go importer.
* gnu/packages/guile-xyz.scm (guile-lib)[source]: Fetch from git.
Remove snippet and modules field.
[native-inputs]: Add autoconf, automake, gettext and texinfo.
---
gnu/packages/guile-xyz.scm | 96 ++++++++++++++++++--------------------
1 file changed, 46 insertions(+), 50 deletions(-)
diff --git a/gnu/packages/guile-xyz.scm b/gnu/packages/guile-xyz.scm
index ce5aad8ec7..c14193921b 100644
--- a/gnu/packages/guile-xyz.scm
+++ b/gnu/packages/guile-xyz.scm
@@ -16,7 +16,7 @@
;;; Copyright © 2017 Theodoros Foradis <theodoros@foradis.org>
;;; Copyright © 2017 Nikita <nikita@n0.is>
;;; Copyright © 2017, 2018 Tobias Geerinckx-Rice <me@tobias.gr>
-;;; Copyright © 2018 Maxim Cournoyer <maxim.cournoyer@gmail.com>
+;;; Copyright © 2018, 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;; Copyright © 2018, 2019, 2020 Arun Isaac <arunisaac@systemreboot.net>
;;; Copyright © 2018 Pierre-Antoine Rouby <pierre-antoine.rouby@inria.fr>
;;; Copyright © 2018 Eric Bavier <bavier@member.fsf.org>
@@ -2167,59 +2167,55 @@ library.")
("guile" ,guile-3.0)))))
(define-public guile-lib
- (package
- (name "guile-lib")
- (version "0.2.6.1")
- (source (origin
- (method url-fetch)
- (uri (string-append "mirror://savannah/guile-lib/guile-lib-"
- version ".tar.gz"))
- (sha256
- (base32
- "0aizxdif5dpch9cvs8zz5g8ds5s4xhfnwza2il5ji7fv2h7ks7bd"))
- (modules '((guix build utils)))
- (snippet
- '(begin
- ;; Work around miscompilation on Guile 3.0.0 at -O2:
- ;; <https://bugs.gnu.org/39251>.
- (substitute* "src/md5.scm"
- (("\\(define f-ash ash\\)")
- "(define f-ash (@ (guile) ash))\n")
- (("\\(define f-add \\+\\)")
- "(define f-add (@ (guile) +))\n"))
- #t))))
- (build-system gnu-build-system)
- (arguments
- '(#:make-flags
- '("GUILE_AUTO_COMPILE=0") ; to prevent guild errors
- #:phases
- (modify-phases %standard-phases
- (add-before 'configure 'patch-module-dir
- (lambda _
- (substitute* "src/Makefile.in"
- (("^moddir = ([[:graph:]]+)")
- "moddir = $(datadir)/guile/site/@GUILE_EFFECTIVE_VERSION@\n")
- (("^godir = ([[:graph:]]+)")
- "godir = \
-$(libdir)/guile/@GUILE_EFFECTIVE_VERSION@/site-ccache\n"))
- #t)))))
- (native-inputs
- `(("guile" ,guile-3.0)
- ("pkg-config" ,pkg-config)))
- (inputs
- `(("guile" ,guile-3.0)))
- (home-page "https://www.nongnu.org/guile-lib/")
- (synopsis "Collection of useful Guile Scheme modules")
- (description
- "Guile-Lib is intended as an accumulation place for pure-scheme Guile
+ (let ((revision "1")
+ (commit "c059f13e332347201eaa4a32ef27c53d064f2d17"))
+ (package
+ (name "guile-lib")
+ (version (git-version "0.2.6.1" revision commit))
+ (source (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://notabug.org/apteryx/guile-lib/")
+ (commit commit)))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "1dl2f53p737n637n2805slci5i32s6cy0bq1j0xkmzd5piymg4f8"))))
+ (build-system gnu-build-system)
+ (arguments
+ '(#:make-flags
+ '("GUILE_AUTO_COMPILE=0") ;to prevent guild errors
+ #:phases
+ (modify-phases %standard-phases
+ (add-before 'configure 'patch-module-dir
+ (lambda _
+ (substitute* "src/Makefile.in"
+ (("^moddir = ([[:graph:]]+)")
+ "moddir = $(datadir)/guile/site/@GUILE_EFFECTIVE_VERSION@\n")
+ (("^godir = ([[:graph:]]+)")
+ "godir = \
+$(libdir)/guile/@GUILE_EFFECTIVE_VERSION@/site-ccache\n")))))))
+ (native-inputs
+ `(("autoconf" ,autoconf)
+ ("automake" ,automake)
+ ("gettext" ,gettext-minimal)
+ ("guile" ,guile-3.0)
+ ("pkg-config" ,pkg-config)
+ ("texinfo" ,texinfo)))
+ (inputs
+ `(("guile" ,guile-3.0)))
+ (home-page "https://www.nongnu.org/guile-lib/")
+ (synopsis "Collection of useful Guile Scheme modules")
+ (description
+ "Guile-Lib is intended as an accumulation place for pure-scheme Guile
modules, allowing for people to cooperate integrating their generic Guile
modules into a coherent library. Think \"a down-scaled, limited-scope CPAN
for Guile\".")
- ;; The whole is under GPLv3+, but some modules are under laxer
- ;; distribution terms such as LGPL and public domain. See `COPYING' for
- ;; details.
- (license license:gpl3+)))
+ ;; The whole is under GPLv3+, but some modules are under laxer
+ ;; distribution terms such as LGPL and public domain. See `COPYING' for
+ ;; details.
+ (license license:gpl3+))))
(define-public guile2.0-lib
(package
--
2.30.1
[-- Attachment #3: 0002-fixup-Create-importer-for-Go-modules.patch --]
[-- Type: text/x-patch, Size: 29727 bytes --]
From f3a6130577252e3d079a6209ec2e21bf5d8baf25 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Wed, 3 Mar 2021 16:45:11 -0500
Subject: [PATCH] fixup! Create importer for Go modules
---
guix/build-system/go.scm | 34 ++--
guix/import/go.scm | 420 ++++++++++++++++++++++-----------------
2 files changed, 257 insertions(+), 197 deletions(-)
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm
index 594e0cb4f3..d07c703a6a 100644
--- a/guix/build-system/go.scm
+++ b/guix/build-system/go.scm
@@ -34,30 +34,28 @@
go-version->git-ref))
(define (go-version->git-ref version)
- "GO-VERSION->GIT-REF parse pseudo-versions and extract the commit
- hash from it, defaulting to full VERSION if we don't recognise a
- pseudo-version pattern."
- ;; A module version like v1.2.3 is introduced by tagging a revision in
- ;; the underlying source repository. Untagged revisions can be referred
- ;; to using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef,
- ;; where the time is the commit time in UTC and the final suffix is the
- ;; prefix of the commit hash.
- ;; cf. https://golang.org/cmd/go/#hdr-Pseudo_versions
+ "GO-VERSION->GIT-REF parse pseudo-versions and extract the commit hash from
+it, defaulting to full VERSION if a pseudo-version pattern is not recognized."
+ ;; A module version like v1.2.3 is introduced by tagging a revision in the
+ ;; underlying source repository. Untagged revisions can be referred to
+ ;; using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef, where
+ ;; the time is the commit time in UTC and the final suffix is the prefix of
+ ;; the commit hash (see: https://golang.org/cmd/go/#hdr-Pseudo_versions).
(let* ((version
- ;; if a source code repository has a v2.0.0 or later tag for
- ;; a file tree with no go.mod, the version is considered to be
- ;; part of the v1 module's available versions and is given an
- ;; +incompatible suffix
- ;; https://golang.org/cmd/go/#hdr-Module_compatibility_and_semantic_versioning
+ ;; If a source code repository has a v2.0.0 or later tag for a file
+ ;; tree with no go.mod, the version is considered to be part of the
+ ;; v1 module's available versions and is given an +incompatible
+ ;; suffix
+ ;; (see:https://golang.org/cmd/go/#hdr-Module_compatibility_and_semantic_versioning).
(if (string-suffix? "+incompatible" version)
(string-drop-right version 13)
version))
(re (string-concatenate
(list
- "(v?[0-9]\\.[0-9]\\.[0-9])" ; "v" prefix can be omitted in version prefix
- "(-|-pre\\.0\\.|-0\\.)" ; separator
- "([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])-" ; timestamp
- "([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])"))) ; commit hash
+ "(v?[0-9]\\.[0-9]\\.[0-9])" ;"v" prefix can be omitted in version prefix
+ "(-|-pre\\.0\\.|-0\\.)" ;separator
+ "([0-9]{14})-" ;timestamp
+ "([0-9A-Fa-f]{12})"))) ;commit hash
(match (string-match re version)))
(if match
(match:substring match 4)
diff --git a/guix/import/go.scm b/guix/import/go.scm
index fead355bd2..7bc97c5c92 100644
--- a/guix/import/go.scm
+++ b/guix/import/go.scm
@@ -2,6 +2,7 @@
;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -18,51 +19,37 @@
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
-;;; (guix import golang) wants to make easier to create Guix package
-;;; declaration for Go modules.
+;;; (guix import golang) attempts to make it easier to create Guix package
+;;; declarations for Go modules.
;;;
-;;; Modules in Go are "collection of related Go packages" which are
-;;; "the unit of source code interchange and versioning".
-;;; Modules are generally hosted in a repository.
+;;; Modules in Go are a "collection of related Go packages" which are "the
+;;; unit of source code interchange and versioning". Modules are generally
+;;; hosted in a repository.
;;;
-;;; At this point it should handle correctly modules which
-;;; have only Go dependencies and are accessible from proxy.golang.org
-;;; (or configured GOPROXY).
+;;; At this point it should handle correctly modules which have only Go
+;;; dependencies and are accessible from proxy.golang.org (or configured via
+;;; GOPROXY).
;;;
;;; We want it to work more or less this way:
;;; - get latest version for the module from GOPROXY
;;; - infer VCS root repo from which we will check-out source by
;;; + recognising known patterns (like github.com)
-;;; + or (TODO) recognising .vcs suffix
-;;; + or parsing meta tag in html served at the URL
+;;; + or recognizing .vcs suffix
+;;; + or parsing meta tag in HTML served at the URL
;;; + or (TODO) if nothing else works by using zip file served by GOPROXY
;;; - get go.mod from GOPROXY (which is able to synthetize one if needed)
;;; - extract list of dependencies from this go.mod
;;;
-;;; We translate Go module paths to a Guix package name under the
+;;; The Go module paths are translated to a Guix package name under the
;;; assumption that there will be no collision.
;;; TODO list
;;; - get correct hash in vcs->origin
;;; - print partial result during recursive imports (need to catch
;;; exceptions)
-;;; - infer repo from module path with VCS qualifier
-;;; (e.g. site.example/my/path/to/repo.git/and/subdir/module)
-;;; - don't print fetch messages to stdout
-;;; - pre-fill synopsis, description and license
(define-module (guix import go)
- #:use-module (ice-9 match)
- #:use-module (ice-9 rdelim)
- #:use-module (ice-9 receive)
- #:use-module (ice-9 regex)
#:use-module (guix build-system go)
- #:use-module (htmlprag)
- #:use-module (sxml xpath)
- #:use-module (srfi srfi-1)
- #:use-module (srfi srfi-9)
- #:use-module (srfi srfi-11)
- #:use-module (json)
#:use-module ((guix download) #:prefix download:)
#:use-module (guix git)
#:use-module (guix import utils)
@@ -75,49 +62,134 @@
#:use-module (guix base32)
#:use-module (guix memoization)
#:use-module ((guix build download) #:prefix build-download:)
+ #:use-module (htmlprag)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 rdelim)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (json)
+ #:use-module (rnrs io ports)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-26)
+ #:use-module (sxml xpath)
+ #:use-module (web client)
+ #:use-module (web response)
#:use-module (web uri)
#:export (go-module->guix-package
- go-module-recursive-import
- infer-module-root-repo))
+ go-module-recursive-import))
+;;; Parameterize htmlprag to parse valid HTML more reliably.
+(%strict-tokenizer? #t)
(define (go-path-escape path)
- "Escape a module path by replacing every uppercase letter with an exclamation
-mark followed with its lowercase equivalent, as per the module Escaped Paths
-specification. https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths"
+ "Escape a module path by replacing every uppercase letter with an
+exclamation mark followed with its lowercase equivalent, as per the module
+Escaped Paths specification (see:
+https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths)."
(define (escape occurrence)
(string-append "!" (string-downcase (match:substring occurrence))))
(regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
-
(define (go-module-latest-version goproxy-url module-path)
- "Fetches the version number of the latest version for MODULE-PATH from the
+ "Fetch the version number of the latest version for MODULE-PATH from the
given GOPROXY-URL server."
- (assoc-ref
- (json-fetch (format #f "~a/~a/@latest" goproxy-url
- (go-path-escape module-path)))
- "Version"))
+ (assoc-ref (json-fetch (format #f "~a/~a/@latest" goproxy-url
+ (go-path-escape module-path)))
+ "Version"))
+
+(define (go-package-licenses name)
+ "Retrieve the list of licenses that apply to NAME, a Go package or module
+name (e.g. \"github.com/golang/protobuf/proto\"). The data is scraped from
+the https://pkg.go.dev/ web site."
+ (let*-values (((url) (string-append "https://pkg.go.dev/" name
+ "?tab=licenses"))
+ ((response body) (http-get url))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ ((select) (sxpath `(// (* (@ (equal? (class "License"))))
+ h2 // *text*))))
+ (and (eq? (response-code response) 200)
+ (match (select (html->sxml body))
+ (() #f) ;nothing selected
+ (licenses licenses)))))
+
+(define (go-package-description name)
+ "Retrieve a short description for NAME, a Go package name,
+e.g. \"google.golang.org/protobuf/proto\". The data is scraped from the
+https://pkg.go.dev/ web site."
+ (let*-values (((url) (string-append "https://pkg.go.dev/" name))
+ ((response body) (http-get url))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ ((select) (sxpath
+ `(// (section
+ (@ (equal? (class "Documentation-overview"))))
+ (p 1)))))
+ (and (eq? (response-code response) 200)
+ (match (select (html->sxml body))
+ (() #f) ;nothing selected
+ (((p . strings))
+ ;; The paragraph text is returned as a list of strings embedding
+ ;; newline characters. Join them and strip the newline
+ ;; characters.
+ (string-delete #\newline (string-join strings)))))))
+
+(define (go-package-synopsis module-name)
+ "Retrieve a short synopsis for a Go module named MODULE-NAME,
+e.g. \"google.golang.org/protobuf\". The data is scraped from
+the https://pkg.go.dev/ web site."
+ ;; Note: Only the *module* (rather than package) page has the README title
+ ;; used as a synopsis on the https://pkg.go.dev web site.
+ (let*-values (((url) (string-append "https://pkg.go.dev/" module-name))
+ ((response body) (http-get url))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ ((select) (sxpath
+ `(// (div (@ (equal? (class "UnitReadme-content"))))
+ // h3 *text*))))
+ (and (eq? (response-code response) 200)
+ (match (select (html->sxml body))
+ (() #f) ;nothing selected
+ ((title more ...) ;title is the first string of the list
+ (string-trim-both title))))))
-(define go-module-latest-version* (memoize go-module-latest-version))
+(define (list->licenses licenses)
+ "Given a list of LICENSES mostly following the SPDX conventions, return the
+corresponding Guix license or 'unknown-license!"
+ (filter-map (lambda (license)
+ (and (not (string-null? license))
+ (not (any (cut string=? <> license)
+ '("AND" "OR" "WITH")))
+ ;; Adjust the license names scraped from
+ ;; https://pkg.go.dev to an equivalent SPDX identifier,
+ ;; if they differ (see: https://github.com/golang/pkgsite
+ ;; /internal/licenses/licenses.go#L174).
+ (or (spdx-string->license
+ (match license
+ ("BlueOak-1.0" "BlueOak-1.0.0")
+ ("BSD-0-Clause" "0BSD")
+ ("BSD-2-Clause" "BSD-2-Clause-FreeBSD")
+ ("GPL2" "GPL-2.0")
+ ("GPL3" "GPL-3.0")
+ ("NIST" "NIST-PD")
+ (_ license)))
+ 'unknown-license!)))
+ licenses))
-(define (fetch-go.mod goproxy-url module-path version file)
- "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
-and VERSION."
+(define (fetch-go.mod goproxy-url module-path version)
+ "Fetch go.mod from the given GOPROXY-URL server for the given MODULE-PATH
+and VERSION and return an input port."
(let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
(go-path-escape module-path)
(go-path-escape version))))
- (parameterize ((current-output-port (current-error-port)))
- (build-download:url-fetch url
- file
- #:print-build-trace? #f))))
+ (build-download:http-fetch (string->uri url))))
-(define (parse-go.mod go.mod-path)
- (parse-go.mod-port (open-input-file go.mod-path)))
-
-(define (parse-go.mod-port go.mod-port)
- "PARSE-GO.MOD takes a filename in GO.MOD-PATH and extract a list of
-requirements from it."
+(define (parse-go.mod port)
+ "Parse the go.mod file accessible via the input PORT, returning a list of
+requirements."
;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
;; which we think necessary for our use case.
(define (toplevel results)
@@ -147,6 +219,7 @@ requirements from it."
(#t
;; unrecognised line, ignore silently
(toplevel results)))))
+
(define (in-require results)
(let ((line (read-line)))
(cond
@@ -158,6 +231,7 @@ requirements from it."
(toplevel results))
(#t
(in-require (require-directive results line))))))
+
(define (in-replace results)
(let ((line (read-line)))
(cond
@@ -169,6 +243,7 @@ requirements from it."
(toplevel results))
(#t
(in-replace (replace-directive results line))))))
+
(define (replace-directive results line)
"Extract replaced modules and new requirements from replace directive
in LINE and add to RESULTS."
@@ -191,6 +266,7 @@ requirements from it."
requirements
(acons new-module-path new-version requirements))))
(cons new-requirements new-replaced)))
+
(define (require-directive results line)
"Extract requirement from LINE and add it to RESULTS."
(let* ((requirements (car results))
@@ -209,7 +285,8 @@ requirements from it."
(module-path (string-trim-both module-path #\"))
(version (match:substring match 2)))
(cons (acons module-path version requirements) replaced)))
- (with-input-from-port go.mod-port
+
+ (with-input-from-port port
(lambda ()
(let* ((results (toplevel '(() . ())))
(requirements (car results))
@@ -221,120 +298,102 @@ requirements from it."
requirements
replaced)))))
-(define (infer-module-root-repo module-path)
- "Go modules can be defined at any level of a repository's tree, but querying
-for the meta tag usually can only be done at the webpage at the root of the
-repository. Therefore, it is sometimes necessary to try and derive a module's
-root path from its path. For a set of well-known forges, the pattern of what
-consists of a module's root page is known before hand."
+(define (module-path->repository-root module-path)
+ "Infer the repository root from a module path. Go modules can be
+defined at any level of a repository tree, but querying for the meta tag
+usually can only be done from the web page at the root of the repository,
+hence the need to derive this information."
;; See the following URL for the official Go equivalent:
;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
- ;;
- ;; TODO: handle module path with VCS qualifier as described in
- ;; https://golang.org/ref/mod#vcs-find and
- ;; https://golang.org/cmd/go/#hdr-Remote_import_paths
+
(define-record-type <vcs>
(make-vcs url-prefix root-regex type)
vcs?
(url-prefix vcs-url-prefix)
(root-regex vcs-root-regex)
(type vcs-type))
- (let* ((known-vcs
- (list
- (make-vcs
- "github.com"
- "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
- 'git)
- (make-vcs
- "bitbucket.org"
- "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$"
- 'unknown)
- (make-vcs
- "hub.jazz.net/git/"
- "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
- 'git)
- (make-vcs
- "git.apache.org"
- "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
- 'git)
- (make-vcs
- "git.openstack.org"
- "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?(/[A-Za-z0-9_.\\-]+)*$"
- 'git)))
- (vcs (find (lambda (vcs) (string-prefix? (vcs-url-prefix vcs) module-path))
- known-vcs)))
- (if vcs
- (match:substring (string-match (vcs-root-regex vcs) module-path) 1)
- module-path)))
+
+ (define known-vcs
+ (list
+ (make-vcs
+ "github.com"
+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "bitbucket.org"
+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$"
+ 'unknown)
+ (make-vcs
+ "hub.jazz.net/git/"
+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.apache.org"
+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.openstack.org"
+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?\
+(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)))
+
+ ;; For reference, see: https://golang.org/ref/mod#vcs-find.
+ (define vcs-qualifiers '(".bzr" ".fossil" ".git" ".hg" ".svn"))
+
+ (define (vcs-qualified-module-path->root-repo-url module-path)
+ (let* ((vcs-qualifiers-group (string-join vcs-qualifiers "|"))
+ (pattern (format #f "^(.*(~a))(/|$)" vcs-qualifiers-group))
+ (m (string-match pattern module-path)))
+ (and=> m (cut match:substring <> 1))))
+
+ (or (and=> (find (lambda (vcs)
+ (string-prefix? (vcs-url-prefix vcs) module-path))
+ known-vcs)
+ (lambda (vcs)
+ (match:substring (string-match (vcs-root-regex vcs)
+ module-path) 1)))
+ (vcs-qualified-module-path->root-repo-url module-path)
+ module-path))
(define (go-module->guix-package-name module-path)
"Converts a module's path to the canonical Guix format for Go packages."
- (string-downcase
- (string-append "go-"
- (string-replace-substring
- (string-replace-substring
- module-path
- "." "-")
- "/" "-"))))
+ (string-downcase (string-append "go-" (string-replace-substring
+ (string-replace-substring
+ module-path
+ "." "-")
+ "/" "-"))))
(define-record-type <module-meta>
(make-module-meta import-prefix vcs repo-root)
module-meta?
(import-prefix module-meta-import-prefix)
- ;; VCS field is a symbol
- (vcs module-meta-vcs)
+ (vcs module-meta-vcs) ;a symbol
(repo-root module-meta-repo-root))
(define (fetch-module-meta-data module-path)
- "Fetches module meta-data from a module's landing page. This is
- necessary because goproxy servers don't currently provide all the
- information needed to build a package."
+ "Retrieve the module meta-data from its landing page. This is necessary
+because goproxy servers don't currently provide all the information needed to
+build a package."
;; <meta name="go-import" content="import-prefix vcs repo-root">
- (define (meta-go-import->module-meta text)
- "Takes the content of the go-import meta tag as TEXT and gives back
- a MODULE-META record"
- (define (get-component s start)
- (let*
- ((start (string-skip s char-set:whitespace start))
- (end (string-index s char-set:whitespace start))
- (end (if end end (string-length s)))
- (result (substring s start end)))
- (values result end)))
- (let*-values (((import-prefix end) (get-component text 0))
- ((vcs end) (get-component text end))
- ((repo-root end) (get-component text end)))
- (make-module-meta import-prefix (string->symbol vcs) repo-root)))
- (define (html->meta-go-import port)
- "Read PORT with HTML content. Find the go-import meta tag and gives
- back its content as a string."
- (let* ((parsedhtml (html->sxml port))
- (extract-content (node-join
- (select-kids (node-typeof? 'html))
- (select-kids (node-typeof? 'head))
- (select-kids (node-typeof? 'meta))
- (select-kids (node-typeof? '@))
- (node-self
- (node-join
- (select-kids (node-typeof? 'name))
- (select-kids (node-equal? "go-import"))))
- (select-kids (node-typeof? 'content))
- (select-kids (lambda (_) #t))))
- (content (car (extract-content parsedhtml))))
- content))
- (let* ((port (build-download:http-fetch (string->uri (format #f "https://~a?go-get=1" module-path))))
- (meta-go-import (html->meta-go-import port))
- (module-metadata (meta-go-import->module-meta meta-go-import)))
- (close-port port)
- module-metadata))
+ (let* ((port (build-download:http-fetch
+ (string->uri (format #f "https://~a?go-get=1" module-path))))
+ (select (sxpath `(// head (meta (@ (equal? (name "go-import"))))
+ // content))))
+ (match (select (call-with-port port html->sxml))
+ (() #f) ;nothing selected
+ (((content content-text))
+ (match (string-split content-text #\space)
+ ((root-path vcs repo-url)
+ (make-module-meta root-path (string->symbol vcs) repo-url)))))))
(define (module-meta-data-repo-url meta-data goproxy-url)
- "Return the URL where the fetcher which will be used can download the source
-control."
- (if (member (module-meta-vcs meta-data)'(fossil mod))
+ "Return the URL where the fetcher which will be used can download the
+source."
+ (if (member (module-meta-vcs meta-data) '(fossil mod))
goproxy-url
(module-meta-repo-root meta-data)))
-(define (vcs->origin vcs-type vcs-repo-url version file)
+(define (vcs->origin vcs-type vcs-repo-url version)
"Generate the `origin' block of a package depending on what type of source
control system is being used."
(case vcs-type
@@ -347,61 +406,64 @@ control system is being used."
(file-name (git-file-name name version))
(sha256
(base32
- ;; FIXME: get hash for git repo checkout
- "0000000000000000000000000000000000000000000000000000"))))
+ ;; FIXME: populate hash for git repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
((hg)
`(origin
(method hg-fetch)
(uri (hg-reference
(url ,vcs-repo-url)
(changeset ,version)))
- (file-name (format #f "~a-~a-checkout" name version))))
+ (file-name (string-append name "-" version "-checkout"))
+ (sha256
+ (base32
+ ;; FIXME: populate hash for hg repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
((svn)
`(origin
(method svn-fetch)
(uri (svn-reference
(url ,vcs-repo-url)
- (revision (string->number version))
- (recursive? #f)))
- (file-name (format #f "~a-~a-checkout" name version))
+ (revision (string->number version))))
+ (file-name (string-append name "-" version "-checkout"))
(sha256
(base32
- ,(guix-hash-url file)))))
+ ;; FIXME: populate hash for svn repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
(else
(raise-exception (format #f "unsupported vcs type: ~a" vcs-type)))))
-(define* (go-module->guix-package module-path #:key (goproxy-url "https://proxy.golang.org"))
- (call-with-temporary-output-file
- (lambda (temp port)
- (let* ((latest-version (go-module-latest-version* goproxy-url module-path))
- (go.mod-path (fetch-go.mod goproxy-url module-path latest-version
- temp))
- (dependencies (map car (parse-go.mod temp)))
- (guix-name (go-module->guix-package-name module-path))
- (root-module-path (infer-module-root-repo module-path))
- ;; VCS type and URL are not included in goproxy information. For
- ;; this we need to fetch it from the official module page.
- (meta-data (fetch-module-meta-data root-module-path))
- (vcs-type (module-meta-vcs meta-data))
- (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url)))
- (values
- `(package
- (name ,guix-name)
- ;; Elide the "v" prefix Go uses
- (version ,(string-trim latest-version #\v))
- (source
- ,(vcs->origin vcs-type vcs-repo-url latest-version temp))
- (build-system go-build-system)
- (arguments
- '(#:import-path ,root-module-path))
- ,@(maybe-inputs (map go-module->guix-package-name dependencies))
- ;; TODO(katco): It would be nice to make an effort to fetch this
- ;; from known forges, e.g. GitHub
- (home-page ,(format #f "https://~a" root-module-path))
- (synopsis "A Go package")
- (description ,(format #f "~a is a Go package." guix-name))
- (license #f))
- dependencies)))))
+(define* (go-module->guix-package module-path #:key
+ (goproxy-url "https://proxy.golang.org"))
+ (let* ((latest-version (go-module-latest-version goproxy-url module-path))
+ (port (fetch-go.mod goproxy-url module-path latest-version))
+ (dependencies (map car (call-with-port port parse-go.mod)))
+ (guix-name (go-module->guix-package-name module-path))
+ (root-module-path (module-path->repository-root module-path))
+ ;; The VCS type and URL are not included in goproxy information. For
+ ;; this we need to fetch it from the official module page.
+ (meta-data (fetch-module-meta-data root-module-path))
+ (vcs-type (module-meta-vcs meta-data))
+ (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url))
+ (synopsis (go-package-synopsis root-module-path))
+ (description (go-package-description module-path))
+ (licenses (go-package-licenses module-path)))
+ (values
+ `(package
+ (name ,guix-name)
+ ;; Elide the "v" prefix Go uses
+ (version ,(string-trim latest-version #\v))
+ (source
+ ,(vcs->origin vcs-type vcs-repo-url latest-version))
+ (build-system go-build-system)
+ (arguments
+ '(#:import-path ,root-module-path))
+ ,@(maybe-inputs (map go-module->guix-package-name dependencies))
+ (home-page ,(format #f "https://~a" root-module-path))
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,(and=> licenses list->licenses)))
+ dependencies)))
(define go-module->guix-package* (memoize go-module->guix-package))
--
2.30.1
[-- Attachment #4: Type: text/plain, Size: 95 bytes --]
I hope I'm not making things more difficult for you!
Thank you for working on it! :-)
Maxim
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH v4] Re: bug#44178: Add a Go Module Importer
2021-03-04 5:40 ` [bug#44178] [PATCH v4] Re: bug#44178: " Maxim Cournoyer
@ 2021-03-04 14:14 ` JOULAUD François via Guix-patches via
2021-03-04 15:47 ` Maxim Cournoyer
0 siblings, 1 reply; 27+ messages in thread
From: JOULAUD François via Guix-patches via @ 2021-03-04 14:14 UTC (permalink / raw)
To: Maxim Cournoyer
Cc: Ludovic Courtès, 44178@debbugs.gnu.org, Katherine Cox-Buday
Hi,
On Thu, Mar 04, 2021 at 12:40:36AM -0500, Maxim Cournoyer wrote:
> I made a couple changes, mostly with regard to integrating support for the
> synopsis, description and license field of the package, plus other
> cosmetic changes. I thought I should share it quickly so that it can be
> used as the basis for a v5, so here's the patch, attached.
First quick glance and the code look a lot better after your work.
I will rebase my work in progress on top of it and will provide a v5
this week.
Thanks a lot.
François
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] [PATCH v4] Re: bug#44178: Add a Go Module Importer
2021-03-04 14:14 ` JOULAUD François via Guix-patches via
@ 2021-03-04 15:47 ` Maxim Cournoyer
0 siblings, 0 replies; 27+ messages in thread
From: Maxim Cournoyer @ 2021-03-04 15:47 UTC (permalink / raw)
To: JOULAUD François
Cc: Ludovic Courtès, 44178@debbugs.gnu.org, Katherine Cox-Buday
Hi François,
JOULAUD François <Francois.JOULAUD@radiofrance.com> writes:
> Hi,
>
> On Thu, Mar 04, 2021 at 12:40:36AM -0500, Maxim Cournoyer wrote:
>> I made a couple changes, mostly with regard to integrating support for the
>> synopsis, description and license field of the package, plus other
>> cosmetic changes. I thought I should share it quickly so that it can be
>> used as the basis for a v5, so here's the patch, attached.
>
> First quick glance and the code look a lot better after your work.
>
> I will rebase my work in progress on top of it and will provide a v5
> this week.
>
> Thanks a lot.
>
> François
Sounds good! Thanks to you!
Maxim
^ permalink raw reply [flat|nested] 27+ messages in thread
* [bug#44178] Add a Go Module Importer
2021-03-02 21:54 ` [bug#44178] Add a Go Module Importer Ludovic Courtès
2021-03-04 5:40 ` [bug#44178] [PATCH v4] Re: bug#44178: " Maxim Cournoyer
@ 2021-03-08 13:54 ` JOULAUD François via Guix-patches via
2021-03-10 17:12 ` bug#44178: " Ludovic Courtès
1 sibling, 1 reply; 27+ messages in thread
From: JOULAUD François via Guix-patches via @ 2021-03-08 13:54 UTC (permalink / raw)
To: Ludovic Courtès
Cc: 44178@debbugs.gnu.org, Katherine Cox-Buday, Maxim Cournoyer
[-- Attachment #1: Type: text/plain, Size: 4760 bytes --]
Hello,
Please find attached v5 version of the patch. Hopefully this is the last one.
I took quite all changes from Maxim's proposal.
Things I did not took are related to html parsing. I did not use of
"%strict-tokenizer" because it needs a yet-to-be-packaged version of
Guile-Libe and did not change the result on any of my tests. I did not
take either the short-form sxpath expression for go-import meta parsing
as it is buggy on my test cases. We can revisit those choices on future
patches but for now I think I have a working version.
Other changes are mainly responses to Ludovic review.
On Tue, Mar 02, 2021 at 10:54:49PM +0100, Ludovic Courtès wrote:
> Nitpick: please mention the sections (for documentation) or variables
> changed
I tried to do it. Don't hesitate to modify message if needed before
commiting.
> (see <https://guix.gnu.org/manual/en/html_node/Submitting-Patches.html).
I finally understood that the document refers to GNU Guidelines for
Changelogs. Some examples specific to Guix would be nice for noobs
like me.
> Some comments below, mostly stylistic as I’m not familiar with the
> actual file formats etc. that the importer implements.
I am not yet completely familiar with it either. All languages now try
to live in their own ecosystem with their own set of incompatible build
and distribution tools. I am just beginning to grasp how Go fo it.
> s/crate/go/
> s/guile-lib/Guile-Lib/
done.
> You can use ‘string-append’ instead of (string-concatenate (list …)).
> Use [[:xdigit:]] instead of [0-9A-Fa-f] for clarity and
> locale-independence.
Thanks for the string-append tip.
>
> Also, you can arrange to use ‘make-regexp’ so that the regexp is
> compiled once for all, and then just ‘regexp-exec’:
I thought about it but was lazy. Thanks to your remark it is now done.
> Please use ‘match’ instead of car/cdr (throughout):
This one was more difficult than I thought. It lead me to create some
specific record type, probably for the better.
> s/acons/alist-cons/ for consistency with the rest of the code.
I still must look at the difference between different type of alists. I
trusted you and just applied the substitution.
> You could rename ‘make-vcs’ above to ‘%make-vcs’ and do:
>
> (define (make-vcs prefix regexp type)
> (%make-vcs prefix (make-regex regexp) type))
>
> so that again you can rely on pre-compiled regexps.
Thanks for the tip.
I wonder when we use "%" prefix versus "*" suffix. I was under the
impression that "%" prefix was more for global (possibly mutable)
variables but you don't use it that way here.
> Keep ‘known-vcs’ in a global variable so it doesn’t have to be
> recomputed every time.
known-vcs is now a top-level variable with precompiled regexs.
> ‘guix lint’ wouldn’t like it. :-) Maybe "Write synopsis here" instead?
>
> Is there no info about the license?
Maxim's patch parse pkg.go.dev for synopsis, license and description.
It is not without flaws (Human review badly needed as it uses README
for trying to extract synopsis) but still better than before.
> New dependency; it’s a bit of a commitment, but hopefully Guile-Lib is
> stable enough and works on all the supported architectures.
It is a bit of commitment but we really needed a library for parsing
HTML. It is only useful on "import go" as of now so nothing critical
for using Guix itself if we can keep it optionnal.
> Please add guix/scripts/import/go.scm to ‘po/guix/POTFILES.in’ so it can
> be translated.
Done.
> > +++ b/tests/import-go.scm
>
> Looks nice! It should be called ‘tests/go.scm’ for consistency, with:
I renamed it. I also put in it one test for guix/build-system/go.scm.
I still am not satisfied with the overall look of this file which is
really difficult to read, but at least we have some basic tests.
> Would it be an option to also have an end-to-end test (checking the
> resulting ‘package’ sexp)? That’d be nice, but perhaps we can add it
> afterwards if you prefer.
I added one end-to-end test loosely based on github.com/go-check/check
example.
For end-to-end tests I reused the "mock" syntax from guix/tests.scm by
doing copy-paste because use-module of "(guix tests)" was really too
slow for me. I don't know what's going on here (it seems to rebuild all
of "gnu" scheme modules) but feel free to delete the copy and import
"(guix tests)" if you prefer.
> Let’s see how much of the comments above you can address for a v4, and
> then we can get that in and improve it from there if needed!
I hope all needed to get that in the tree is done now ;-)
François
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Create-importer-for-Go-modules-good-message.patch --]
[-- Type: text/x-diff; name="0001-Create-importer-for-Go-modules-good-message.patch", Size: 50117 bytes --]
From dab6612fecd809279f6fc4710176c78a0fdf28a8 Mon Sep 17 00:00:00 2001
From: Katherine Cox-Buday <cox.katherine.e@gmail.com>
Date: Thu, 22 Oct 2020 19:40:17 -0500
Subject: [PATCHv5] Create importer for Go modules (good message)
This patch add a `guix import go` command.
* doc/guix.texi: doc about go importer and guile-lib dependency
* gnu/packages/package-management.scm: added guile-lib dependency
* guix/self.scm: add guile-lib dependency
* guix/build-system/go.scm (go-version->git-ref): new procedure
* guix/import/go.scm: Created Go importer
* guix/scripts/import/go.scm: Subcommand for Go importer
* guix/scripts/import.scm: Declare subcommand guix import go
* tests/import-go.scm: Tests for Go importer and build-system
Signed-off-by: Francois Joulaud <francois.joulaud@radiofrance.com>
Co-Authored-By: Helio Machado <0x2b3bfa0@gmail.com>
Co-Authored-By: Francois Joulaud <francois.joulaud@radiofrance.com>
Co-Authored-By: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Closes: https://issues.guix.gnu.org/issue/44178
---
doc/guix.texi | 26 ++
gnu/packages/package-management.scm | 2 +
guix/build-system/go.scm | 34 +-
guix/import/go.scm | 490 ++++++++++++++++++++++++++++
guix/scripts/import.scm | 2 +-
guix/scripts/import/go.scm | 118 +++++++
guix/self.scm | 5 +-
po/guix/POTFILES.in | 1 +
tests/go.scm | 289 ++++++++++++++++
9 files changed, 964 insertions(+), 3 deletions(-)
create mode 100644 guix/import/go.scm
create mode 100644 guix/scripts/import/go.scm
create mode 100644 tests/go.scm
diff --git a/doc/guix.texi b/doc/guix.texi
index 5d28fca837..bb5b64b2fc 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -861,6 +861,10 @@ substitutes (@pxref{Invoking guix publish}).
@uref{https://ngyro.com/software/guile-semver.html, Guile-Semver} for
the @code{crate} importer (@pxref{Invoking guix import}).
+@item
+@uref{https://www.nongnu.org/guile-lib/doc/ref/htmlprag/, Guile-Lib} for
+the @code{go} importer (@pxref{Invoking guix import}).
+
@item
When @url{http://www.bzip.org, libbz2} is available,
@command{guix-daemon} can use it to compress build logs.
@@ -11493,6 +11497,28 @@ Select the given repository (a repository name). Possible values include:
of coq packages.
@end itemize
@end table
+
+@item go
+@cindex go
+Import metadata for a Go module using
+@uref{https://proxy.golang.org, proxy.golang.org}.
+
+This importer is highly experimental. See the source code for more info
+about the current state.
+
+@example
+guix import go gopkg.in/yaml.v2
+@end example
+
+Additional options include:
+
+@table @code
+@item --recursive
+@itemx -r
+Traverse the dependency graph of the given upstream package recursively
+and generate package expressions for all those packages that are not yet
+in Guix.
+@end table
@end table
The structure of the @command{guix import} code is modular. It would be
diff --git a/gnu/packages/package-management.scm b/gnu/packages/package-management.scm
index 9fb8c40a31..06bb5bd2df 100644
--- a/gnu/packages/package-management.scm
+++ b/gnu/packages/package-management.scm
@@ -304,6 +304,7 @@ $(prefix)/etc/init.d\n")))
'((assoc-ref inputs "guile"))))
(avahi (assoc-ref inputs "guile-avahi"))
(gcrypt (assoc-ref inputs "guile-gcrypt"))
+ (guile-lib (assoc-ref inputs "guile-lib"))
(json (assoc-ref inputs "guile-json"))
(sqlite (assoc-ref inputs "guile-sqlite3"))
(zlib (assoc-ref inputs "guile-zlib"))
@@ -367,6 +368,7 @@ $(prefix)/etc/init.d\n")))
`(("guile-avahi" ,guile-avahi)))
("guile-gcrypt" ,guile-gcrypt)
("guile-json" ,guile-json-4)
+ ("guile-lib" ,guile-lib)
("guile-sqlite3" ,guile-sqlite3)
("guile-zlib" ,guile-zlib)
("guile-lzlib" ,guile-lzlib)
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm
index f8ebaefb27..ad3df9cc4e 100644
--- a/guix/build-system/go.scm
+++ b/guix/build-system/go.scm
@@ -26,9 +26,41 @@
#:use-module (guix build-system gnu)
#:use-module (guix packages)
#:use-module (ice-9 match)
+ #:use-module (ice-9 regex)
#:export (%go-build-system-modules
go-build
- go-build-system))
+ go-build-system
+
+ go-version->git-ref))
+
+(define %go-version-rx
+ (make-regexp (string-append
+ "(v?[0-9]\\.[0-9]\\.[0-9])" ;"v" prefix can be omitted in version prefix
+ "(-|-pre\\.0\\.|-0\\.)" ;separator
+ "([0-9]{14})-" ;timestamp
+ "([0-9A-Fa-f]{12})"))) ;commit hash
+
+(define (go-version->git-ref version)
+ "GO-VERSION->GIT-REF parse pseudo-versions and extract the commit hash from
+it, defaulting to full VERSION if a pseudo-version pattern is not recognized."
+ ;; A module version like v1.2.3 is introduced by tagging a revision in the
+ ;; underlying source repository. Untagged revisions can be referred to
+ ;; using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef, where
+ ;; the time is the commit time in UTC and the final suffix is the prefix of
+ ;; the commit hash (see: https://golang.org/ref/mod#pseudo-versions).
+ (let* ((version
+ ;; If a source code repository has a v2.0.0 or later tag for a file
+ ;; tree with no go.mod, the version is considered to be part of the
+ ;; v1 module's available versions and is given an +incompatible
+ ;; suffix
+ ;; (see:https://golang.org/cmd/go/#hdr-Module_compatibility_and_semantic_versioning).
+ (if (string-suffix? "+incompatible" version)
+ (string-drop-right version 13)
+ version))
+ (match (regexp-exec %go-version-rx version)))
+ (if match
+ (match:substring match 4)
+ version)))
;; Commentary:
;;
diff --git a/guix/import/go.scm b/guix/import/go.scm
new file mode 100644
index 0000000000..c452d81b4a
--- /dev/null
+++ b/guix/import/go.scm
@@ -0,0 +1,490 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com>
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; (guix import golang) attempts to make it easier to create Guix package
+;;; declarations for Go modules.
+;;;
+;;; Modules in Go are a "collection of related Go packages" which are "the
+;;; unit of source code interchange and versioning". Modules are generally
+;;; hosted in a repository.
+;;;
+;;; At this point it should handle correctly modules which have only Go
+;;; dependencies and are accessible from proxy.golang.org (or configured via
+;;; GOPROXY).
+;;;
+;;; We want it to work more or less this way:
+;;; - get latest version for the module from GOPROXY
+;;; - infer VCS root repo from which we will check-out source by
+;;; + recognising known patterns (like github.com)
+;;; + or recognizing .vcs suffix
+;;; + or parsing meta tag in HTML served at the URL
+;;; + or (TODO) if nothing else works by using zip file served by GOPROXY
+;;; - get go.mod from GOPROXY (which is able to synthetize one if needed)
+;;; - extract list of dependencies from this go.mod
+;;;
+;;; The Go module paths are translated to a Guix package name under the
+;;; assumption that there will be no collision.
+
+;;; TODO list
+;;; - get correct hash in vcs->origin
+;;; - print partial result during recursive imports (need to catch
+;;; exceptions)
+
+(define-module (guix import go)
+ #:use-module (guix build-system go)
+ #:use-module (guix git)
+ #:use-module (guix i18n)
+ #:use-module (guix import utils)
+ #:use-module (guix import json)
+ #:use-module (guix packages)
+ #:use-module (guix upstream)
+ #:use-module (guix utils)
+ #:use-module (guix http-client)
+ #:use-module ((guix licenses) #:prefix license:)
+ #:use-module (guix base16)
+ #:use-module (guix base32)
+ #:use-module (guix memoization)
+ #:use-module (htmlprag)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 rdelim)
+ #:use-module (ice-9 receive)
+ #:use-module (ice-9 regex)
+ #:use-module (json)
+ #:use-module (rnrs io ports)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-9)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-26)
+ #:use-module (sxml xpath)
+ #:use-module (web client)
+ #:use-module (web response)
+ #:use-module (web uri)
+
+ #:export (go-module->guix-package
+ go-module-recursive-import))
+
+
+(define (go-path-escape path)
+ "Escape a module path by replacing every uppercase letter with an
+exclamation mark followed with its lowercase equivalent, as per the module
+Escaped Paths specification (see:
+https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths)."
+ (define (escape occurrence)
+ (string-append "!" (string-downcase (match:substring occurrence))))
+ (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post))
+
+(define (go-module-latest-version goproxy-url module-path)
+ "Fetch the version number of the latest version for MODULE-PATH from the
+given GOPROXY-URL server."
+ (assoc-ref (json-fetch (format #f "~a/~a/@latest" goproxy-url
+ (go-path-escape module-path)))
+ "Version"))
+
+
+(define (go-package-licenses name)
+ "Retrieve the list of licenses that apply to NAME, a Go package or module
+name (e.g. \"github.com/golang/protobuf/proto\"). The data is scraped from
+the https://pkg.go.dev/ web site."
+ (let*-values (((url) (string-append "https://pkg.go.dev/" name
+ "?tab=licenses"))
+ ((response body) (http-get url))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ ((select) (sxpath `(// (* (@ (equal? (class "License"))))
+ h2 // *text*))))
+ (and (eq? (response-code response) 200)
+ (match (select (html->sxml body))
+ (() #f) ;nothing selected
+ (licenses licenses)))))
+
+(define (go.pkg.dev-info name)
+ (http-get (string-append "https://pkg.go.dev/" name)))
+(define go.pkg.dev-info*
+ (memoize go.pkg.dev-info))
+
+(define (go-package-description name)
+ "Retrieve a short description for NAME, a Go package name,
+e.g. \"google.golang.org/protobuf/proto\". The data is scraped from the
+https://pkg.go.dev/ web site."
+ (let*-values (((response body) (go.pkg.dev-info* name))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ ((select) (sxpath
+ `(// (section
+ (@ (equal? (class "Documentation-overview"))))
+ (p 1)))))
+ (and (eq? (response-code response) 200)
+ (match (select (html->sxml body))
+ (() #f) ;nothing selected
+ (((p . strings))
+ ;; The paragraph text is returned as a list of strings embedding
+ ;; newline characters. Join them and strip the newline
+ ;; characters.
+ (string-delete #\newline (string-join strings)))))))
+
+(define (go-package-synopsis module-name)
+ "Retrieve a short synopsis for a Go module named MODULE-NAME,
+e.g. \"google.golang.org/protobuf\". The data is scraped from
+the https://pkg.go.dev/ web site."
+ ;; Note: Only the *module* (rather than package) page has the README title
+ ;; used as a synopsis on the https://pkg.go.dev web site.
+ (let*-values (((response body) (go.pkg.dev-info* module-name))
+ ;; Extract the text contained in a h2 child node of any
+ ;; element marked with a "License" class attribute.
+ ((select) (sxpath
+ `(// (div (@ (equal? (class "UnitReadme-content"))))
+ // h3 *text*))))
+ (and (eq? (response-code response) 200)
+ (match (select (html->sxml body))
+ (() #f) ;nothing selected
+ ((title more ...) ;title is the first string of the list
+ (string-trim-both title))))))
+
+(define (list->licenses licenses)
+ "Given a list of LICENSES mostly following the SPDX conventions, return the
+corresponding Guix license or 'unknown-license!"
+ (filter-map (lambda (license)
+ (and (not (string-null? license))
+ (not (any (cut string=? <> license)
+ '("AND" "OR" "WITH")))
+ ;; Adjust the license names scraped from
+ ;; https://pkg.go.dev to an equivalent SPDX identifier,
+ ;; if they differ (see: https://github.com/golang/pkgsite
+ ;; /internal/licenses/licenses.go#L174).
+ (or (spdx-string->license
+ (match license
+ ("BlueOak-1.0" "BlueOak-1.0.0")
+ ("BSD-0-Clause" "0BSD")
+ ("BSD-2-Clause" "BSD-2-Clause-FreeBSD")
+ ("GPL2" "GPL-2.0")
+ ("GPL3" "GPL-3.0")
+ ("NIST" "NIST-PD")
+ (_ license)))
+ 'unknown-license!)))
+ licenses))
+
+(define (fetch-go.mod goproxy-url module-path version)
+ "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH
+and VERSION."
+ (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url
+ (go-path-escape module-path)
+ (go-path-escape version))))
+ (http-fetch url)))
+
+(define %go.mod-require-directive-rx
+ ;; A line in a require directive is composed of a module path and
+ ;; a version separated by whitespace and an optionnal '//' comment at
+ ;; the end.
+ (make-regexp
+ (string-append
+ "^[[:blank:]]*"
+ "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)"
+ "([[:blank:]]+//.*)?")))
+
+(define %go.mod-replace-directive-rx
+ ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
+ ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
+ (make-regexp
+ (string-append
+ "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"
+ "[[:blank:]]+" "=>" "[[:blank:]]+"
+ "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?")))
+
+(define (parse-go.mod port)
+ "Parse the go.mod file accessible via the input PORT, returning a list of
+requirements."
+ (define-record-type <results>
+ (make-results requirements replacements)
+ results?
+ (requirements results-requirements)
+ (replacements results-replacements))
+ ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
+ ;; which we think necessary for our use case.
+ (define (toplevel results)
+ "Main parser, RESULTS is a pair of alist serving as accumulator for
+ all encountered requirements and replacements."
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; parsing ended, give back the result
+ results)
+ ((string=? line "require (")
+ ;; a require block begins, delegate parsing to IN-REQUIRE
+ (in-require results))
+ ((string=? line "replace (")
+ ;; a replace block begins, delegate parsing to IN-REPLACE
+ (in-replace results))
+ ((string-prefix? "require " line)
+ ;; a standalone require directive
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (require-directive results stripped-line)))
+ (toplevel new-results)))
+ ((string-prefix? "replace " line)
+ ;; a standalone replace directive
+ (let* ((stripped-line (string-drop line 8))
+ (new-results (replace-directive results stripped-line)))
+ (toplevel new-results)))
+ (#t
+ ;; unrecognised line, ignore silently
+ (toplevel results)))))
+
+ (define (in-require results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-require (require-directive results line))))))
+
+ (define (in-replace results)
+ (let ((line (read-line)))
+ (cond
+ ((eof-object? line)
+ ;; this should never happen here but we ignore silently
+ results)
+ ((string=? line ")")
+ ;; end of block, coming back to toplevel
+ (toplevel results))
+ (#t
+ (in-replace (replace-directive results line))))))
+
+ (define (replace-directive results line)
+ "Extract replaced modules and new requirements from replace directive
+ in LINE and add to RESULTS."
+ (match results
+ (($ <results> requirements replaced)
+ (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line))
+ (module-path (match:substring rx-match 1))
+ (version (match:substring rx-match 3))
+ (new-module-path (match:substring rx-match 4))
+ (new-version (match:substring rx-match 6))
+ (new-replaced (alist-cons module-path version replaced))
+ (new-requirements
+ (if (string-match "^\\.?\\./" new-module-path)
+ requirements
+ (alist-cons new-module-path new-version requirements))))
+ (make-results new-requirements new-replaced)))))
+ (define (require-directive results line)
+ "Extract requirement from LINE and add it to RESULTS."
+ (let* ((rx-match (regexp-exec %go.mod-require-directive-rx line))
+ (module-path (match:substring rx-match 1))
+ ;; we saw double-quoted string in the wild without escape
+ ;; sequences so we just trim the quotes
+ (module-path (string-trim-both module-path #\"))
+ (version (match:substring rx-match 2)))
+ (match results
+ (($ <results> requirements replaced)
+ (make-results (alist-cons module-path version requirements) replaced)))))
+
+ (with-input-from-port port
+ (lambda ()
+ (let ((results (toplevel (make-results '() '()))))
+ (match results
+ (($ <results> requirements replaced)
+ ;; At last we remove replaced modules from the requirements list
+ (fold
+ (lambda (replacedelem requirements)
+ (alist-delete! (car replacedelem) requirements))
+ requirements
+ replaced)))))))
+
+(define-record-type <vcs>
+ (%make-vcs url-prefix root-regex type)
+ vcs?
+ (url-prefix vcs-url-prefix)
+ (root-regex vcs-root-regex)
+ (type vcs-type))
+(define (make-vcs prefix regexp type)
+ (%make-vcs prefix (make-regexp regexp) type))
+(define known-vcs
+ ;; See the following URL for the official Go equivalent:
+ ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087
+ (list
+ (make-vcs
+ "github.com"
+ "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "bitbucket.org"
+ "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$"
+ 'unknown)
+ (make-vcs
+ "hub.jazz.net/git/"
+ "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.apache.org"
+ "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)
+ (make-vcs
+ "git.openstack.org"
+ "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?\
+(/[A-Za-z0-9_.\\-]+)*$"
+ 'git)))
+
+(define (module-path->repository-root module-path)
+ "Infer the repository root from a module path. Go modules can be
+defined at any level of a repository tree, but querying for the meta tag
+usually can only be done from the web page at the root of the repository,
+hence the need to derive this information."
+
+ ;; For reference, see: https://golang.org/ref/mod#vcs-find.
+ (define vcs-qualifiers '(".bzr" ".fossil" ".git" ".hg" ".svn"))
+
+ (define (vcs-qualified-module-path->root-repo-url module-path)
+ (let* ((vcs-qualifiers-group (string-join vcs-qualifiers "|"))
+ (pattern (format #f "^(.*(~a))(/|$)" vcs-qualifiers-group))
+ (m (string-match pattern module-path)))
+ (and=> m (cut match:substring <> 1))))
+
+ (or (and=> (find (lambda (vcs)
+ (string-prefix? (vcs-url-prefix vcs) module-path))
+ known-vcs)
+ (lambda (vcs)
+ (match:substring (regexp-exec (vcs-root-regex vcs)
+ module-path) 1)))
+ (vcs-qualified-module-path->root-repo-url module-path)
+ module-path))
+
+(define (go-module->guix-package-name module-path)
+ "Converts a module's path to the canonical Guix format for Go packages."
+ (string-downcase (string-append "go-" (string-replace-substring
+ (string-replace-substring
+ module-path
+ "." "-")
+ "/" "-"))))
+
+(define-record-type <module-meta>
+ (make-module-meta import-prefix vcs repo-root)
+ module-meta?
+ (import-prefix module-meta-import-prefix)
+ (vcs module-meta-vcs) ;a symbol
+ (repo-root module-meta-repo-root))
+
+(define (fetch-module-meta-data module-path)
+ "Retrieve the module meta-data from its landing page. This is necessary
+because goproxy servers don't currently provide all the information needed to
+build a package."
+ ;; <meta name="go-import" content="import-prefix vcs repo-root">
+ (let* ((port (http-fetch (format #f "https://~a?go-get=1" module-path)))
+ (select (sxpath `(// head (meta (@ (equal? (name "go-import"))))
+ // content))))
+ (match (select (call-with-port port html->sxml))
+ (() #f) ;nothing selected
+ (((content content-text))
+ (match (string-split content-text #\space)
+ ((root-path vcs repo-url)
+ (make-module-meta root-path (string->symbol vcs) repo-url)))))))
+
+(define (module-meta-data-repo-url meta-data goproxy-url)
+ "Return the URL where the fetcher which will be used can download the
+source."
+ (if (member (module-meta-vcs meta-data) '(fossil mod))
+ goproxy-url
+ (module-meta-repo-root meta-data)))
+
+(define (vcs->origin vcs-type vcs-repo-url version)
+ "Generate the `origin' block of a package depending on what type of source
+control system is being used."
+ (case vcs-type
+ ((git)
+ `(origin
+ (method git-fetch)
+ (uri (git-reference
+ (url ,vcs-repo-url)
+ (commit (go-version->git-ref version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ ;; FIXME: populate hash for git repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
+ ((hg)
+ `(origin
+ (method hg-fetch)
+ (uri (hg-reference
+ (url ,vcs-repo-url)
+ (changeset ,version)))
+ (file-name (string-append name "-" version "-checkout"))
+ (sha256
+ (base32
+ ;; FIXME: populate hash for hg repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
+ ((svn)
+ `(origin
+ (method svn-fetch)
+ (uri (svn-reference
+ (url ,vcs-repo-url)
+ (revision (string->number version))))
+ (file-name (string-append name "-" version "-checkout"))
+ (sha256
+ (base32
+ ;; FIXME: populate hash for svn repo checkout
+ "0000000000000000000000000000000000000000000000000000"))))
+ (else
+ (raise (formatted-message (G_ "unsupported vcs type '~a' for package '~a'" vcs-type vcs-repo-url))))))
+
+(define* (go-module->guix-package module-path #:key
+ (goproxy-url "https://proxy.golang.org"))
+ (let* ((latest-version (go-module-latest-version goproxy-url module-path))
+ (port (fetch-go.mod goproxy-url module-path latest-version))
+ (dependencies (map car (call-with-port port parse-go.mod)))
+ (guix-name (go-module->guix-package-name module-path))
+ (root-module-path (module-path->repository-root module-path))
+ ;; The VCS type and URL are not included in goproxy information. For
+ ;; this we need to fetch it from the official module page.
+ (meta-data (fetch-module-meta-data root-module-path))
+ (vcs-type (module-meta-vcs meta-data))
+ (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url))
+ (synopsis (go-package-synopsis root-module-path))
+ (description (go-package-description module-path))
+ (licenses (go-package-licenses module-path)))
+ (values
+ `(package
+ (name ,guix-name)
+ ;; Elide the "v" prefix Go uses
+ (version ,(string-trim latest-version #\v))
+ (source
+ ,(vcs->origin vcs-type vcs-repo-url latest-version))
+ (build-system go-build-system)
+ (arguments
+ '(#:import-path ,root-module-path))
+ ,@(maybe-inputs (map go-module->guix-package-name dependencies))
+ (home-page ,(format #f "https://~a" root-module-path))
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,(and=> licenses list->licenses)))
+ dependencies)))
+
+(define go-module->guix-package* (memoize go-module->guix-package))
+
+(define* (go-module-recursive-import package-name
+ #:key (goproxy-url "https://proxy.golang.org"))
+ (recursive-import
+ package-name
+ #:repo->guix-package (lambda* (name . _)
+ (go-module->guix-package*
+ name
+ #:goproxy-url goproxy-url))
+ #:guix-name go-module->guix-package-name))
diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm
index 0a3863f965..1d2b45d942 100644
--- a/guix/scripts/import.scm
+++ b/guix/scripts/import.scm
@@ -77,7 +77,7 @@ rather than \\n."
;;;
(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem"
- "cran" "crate" "texlive" "json" "opam"))
+ "go" "cran" "crate" "texlive" "json" "opam"))
(define (resolve-importer name)
(let ((module (resolve-interface
diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm
new file mode 100644
index 0000000000..fde7555973
--- /dev/null
+++ b/guix/scripts/import/go.scm
@@ -0,0 +1,118 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix scripts import go)
+ #:use-module (guix ui)
+ #:use-module (guix utils)
+ #:use-module (guix scripts)
+ #:use-module (guix import go)
+ #:use-module (guix scripts import)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
+ #:use-module (srfi srfi-37)
+ #:use-module (ice-9 match)
+ #:use-module (ice-9 format)
+ #:export (guix-import-go))
+
+
+;;;
+;;; Command-line options.
+;;;
+
+(define %default-options
+ '())
+
+(define (show-help)
+ (display (G_ "Usage: guix import go PACKAGE-PATH
+Import and convert the Go module for PACKAGE-PATH.\n"))
+ (display (G_ "
+ -h, --help display this help and exit"))
+ (display (G_ "
+ -V, --version display version information and exit"))
+ (display (G_ "
+ -r, --recursive generate package expressions for all Go modules\
+ that are not yet in Guix"))
+ (display (G_ "
+ -p, --goproxy=GOPROXY specify which goproxy server to use"))
+ (newline)
+ (show-bug-report-information))
+
+(define %options
+ ;; Specification of the command-line options.
+ (cons* (option '(#\h "help") #f #f
+ (lambda args
+ (show-help)
+ (exit 0)))
+ (option '(#\V "version") #f #f
+ (lambda args
+ (show-version-and-exit "guix import go")))
+ (option '(#\r "recursive") #f #f
+ (lambda (opt name arg result)
+ (alist-cons 'recursive #t result)))
+ (option '(#\p "goproxy") #t #f
+ (lambda (opt name arg result)
+ (alist-cons 'goproxy
+ (string->symbol arg)
+ (alist-delete 'goproxy result))))
+ %standard-import-options))
+
+
+;;;
+;;; Entry point.
+;;;
+
+(define (guix-import-go . args)
+ (define (parse-options)
+ ;; Return the alist of option values.
+ (args-fold* args %options
+ (lambda (opt name arg result)
+ (leave (G_ "~A: unrecognized option~%") name))
+ (lambda (arg result)
+ (alist-cons 'argument arg result))
+ %default-options))
+
+ (let* ((opts (parse-options))
+ (args (filter-map (match-lambda
+ (('argument . value)
+ value)
+ (_ #f))
+ (reverse opts))))
+ (match args
+ ((module-name)
+ (if (assoc-ref opts 'recursive)
+ (map (match-lambda
+ ((and ('package ('name name) . rest) pkg)
+ `(define-public ,(string->symbol name)
+ ,pkg))
+ (_ #f))
+ (go-module-recursive-import module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org")))
+ (let ((sexp (go-module->guix-package module-name
+ #:goproxy-url
+ (or (assoc-ref opts 'goproxy)
+ "https://proxy.golang.org"))))
+ (unless sexp
+ (leave (G_ "failed to download meta-data for module '~a'~%")
+ module-name))
+ sexp)))
+ (()
+ (leave (G_ "too few arguments~%")))
+ ((many ...)
+ (leave (G_ "too many arguments~%"))))))
diff --git a/guix/self.scm b/guix/self.scm
index 35fba1152d..ed5ee9ddea 100644
--- a/guix/self.scm
+++ b/guix/self.scm
@@ -814,6 +814,9 @@ itself."
(define guile-ssh
(specification->package "guile-ssh"))
+ (define guile-lib
+ (specification->package "guile-lib"))
+
(define guile-git
(specification->package "guile-git"))
@@ -842,7 +845,7 @@ itself."
(append-map transitive-package-dependencies
(list guile-gcrypt gnutls guile-git guile-avahi
guile-json guile-semver guile-ssh guile-sqlite3
- guile-zlib guile-lzlib guile-zstd)))
+ guile-lib guile-zlib guile-lzlib guile-zstd)))
(define *core-modules*
(scheme-node "guix-core"
diff --git a/po/guix/POTFILES.in b/po/guix/POTFILES.in
index 666e630adf..cbbfe1e76b 100644
--- a/po/guix/POTFILES.in
+++ b/po/guix/POTFILES.in
@@ -101,6 +101,7 @@ guix/scripts/import/cpan.scm
guix/scripts/import/crate.scm
guix/scripts/import/gem.scm
guix/scripts/import/gnu.scm
+guix/scripts/import/go.scm
guix/scripts/import/hackage.scm
guix/scripts/import/json.scm
guix/scripts/import/nix.scm
diff --git a/tests/go.scm b/tests/go.scm
new file mode 100644
index 0000000000..d9fe4dac81
--- /dev/null
+++ b/tests/go.scm
@@ -0,0 +1,289 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+;;; Summary
+;; Tests for guix/import/go.scm
+
+(define-module (test-import-go)
+ #:use-module (guix base32)
+ #:use-module (guix build-system go)
+ #:use-module (guix import go)
+ #:use-module (ice-9 iconv)
+ #:use-module (ice-9 match)
+ #:use-module (srfi srfi-19)
+ #:use-module (srfi srfi-64)
+ #:use-module (web response))
+
+;; This is a copy of the syntax-rule in (guix tests) because I don't
+;; want to load all (guix tests) just for this macro.
+(define-syntax-rule (mock (module proc replacement) body ...)
+ "Within BODY, replace the definition of PROC from MODULE with the definition
+given by REPLACEMENT."
+ (let* ((m (resolve-module 'module))
+ (original (module-ref m 'proc)))
+ (dynamic-wind
+ (lambda () (module-set! m 'proc replacement))
+ (lambda () body ...)
+ (lambda () (module-set! m 'proc original)))))
+
+(define fixture-go-mod-simple
+ "module my/thing
+go 1.12
+require other/thing v1.0.2
+require new/thing/v2 v2.3.4
+exclude old/thing v1.2.3
+replace bad/thing v1.4.5 => good/thing v1.4.5
+")
+
+(define fixture-go-mod-with-block
+ "module M
+
+require (
+ A v1
+ B v1.0.0
+ C v1.0.0
+ D v1.2.3
+ E dev
+)
+
+exclude D v1.2.3
+")
+
+
+(define fixture-go-mod-complete
+ "module M
+
+go 1.13
+
+replace github.com/myname/myproject/myapi => ./api
+
+replace github.com/mymname/myproject/thissdk => ../sdk
+
+replace launchpad.net/gocheck => github.com/go-check/check v0.0.0-20140225173054-eb6ee6f84d0a
+
+require (
+ github.com/user/project v1.1.11
+ github.com/user/project/sub/directory v1.1.12
+ bitbucket.org/user/project v1.11.20
+ bitbucket.org/user/project/sub/directory v1.11.21
+ launchpad.net/project v1.1.13
+ launchpad.net/project/series v1.1.14
+ launchpad.net/project/series/sub/directory v1.1.15
+ launchpad.net/~user/project/branch v1.1.16
+ launchpad.net/~user/project/branch/sub/directory v1.1.17
+ hub.jazz.net/git/user/project v1.1.18
+ hub.jazz.net/git/user/project/sub/directory v1.1.19
+ k8s.io/kubernetes/subproject v1.1.101
+ one.example.com/abitrary/repo v1.1.111
+ two.example.com/abitrary/repo v0.0.2
+ \"quoted.example.com/abitrary/repo\" v0.0.2
+)
+
+replace two.example.com/abitrary/repo => github.com/corp/arbitrary-repo v0.0.2
+
+replace (
+ golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13
+ golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13
+)
+
+")
+
+
+
+(define fixture-latest-for-go-check
+ "{\"Version\":\"v0.0.0-20201130134442-10cb98267c6c\",\"Time\":\"2020-11-30T13:44:42Z\"}")
+
+
+(define fixtures-go-check-test
+ (let ((version
+ "{\"Version\":\"v0.0.0-20201130134442-10cb98267c6c\",\"Time\":\"2020-11-30T13:44:42Z\"}")
+ (go.mod
+ "module gopkg.in/check.v1
+
+go 1.11
+
+require github.com/kr/pretty v0.2.1
+")
+ (go-get
+ "<!DOCTYPE html>
+<html lang=\"en\" >
+ <head>
+ <meta charset=\"utf-8\">
+ <link rel=\"dns-prefetch\" href=\"https://github.githubassets.com\">
+ <script crossorigin=\"anonymous\" defer=\"defer\" integrity=\"sha512-aw5tciVT0IsECUmMuwp9ez60QReE2/yFNL1diLgZnOom6RhU8+0lG3RlAKto4JwbCoEP15E41Pksd7rK5BKfCQ==\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/topic-suggestions-6b0e6d72.js\"></script>
+ <meta name=\"viewport\" content=\"width=device-width\">
+
+ <title>GitHub - go-check/check: Rich testing for the Go language</title>
+ <meta name=\"description\" content=\"Rich testing for the Go language. Contribute to go-check/check development by creating an account on GitHub.\">
+ <link rel=\"search\" type=\"application/opensearchdescription+xml\" href=\"/opensearch.xml\" title=\"GitHub\">
+ <link rel=\"fluid-icon\" href=\"https://github.com/fluidicon.png\" title=\"GitHub\">
+ <!-- To prevent page flashing, the optimizely JS needs to be loaded in the
+ <head> tag before the DOM renders -->
+ <meta name=\"hostname\" content=\"github.com\">
+ <meta name=\"user-login\" content=\"\">
+ <link href=\"https://github.com/go-check/check/commits/v1.atom\" rel=\"alternate\" title=\"Recent Commits to check:v1\" type=\"application/atom+xml\">
+ <meta name=\"go-import\" content=\"github.com/go-check/check git https://github.com/go-check/check.git\">
+ </head>
+ <body class=\"logged-out env-production page-responsive\" style=\"word-wrap: break-word;\">
+ </body>
+</html>
+")
+ (pkg.go.dev "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"utf-8\">\n</head>\n<body class=\"Site Site--wide Site--redesign\">\n <div class=\"Site-content\">\n <div class=\"Container\">\n <div class=\"UnitDetails\" data-test-id=\"UnitDetails\">\n <div class=\"UnitDetails-content js-unitDetailsContent\" role=\"main\" data-test-id=\"UnitDetails-content\">\n <div class=\"UnitReadme js-readme\">\n <h2 class=\"UnitReadme-title\" id=\"section-readme\"><img height=\"25px\" width=\"20px\" src=\"/static/img/pkg-icon-readme_20x16.svg\" alt=\"\">README</h2>\n <div class=\"UnitReadme-content\" data-test-id=\"Unit-readmeContent\">\n <div class=\"Overview-readmeContent js-readmeContent\">\n <h3 class=\"h1\" id=\"readme-instructions\">Instructions</h3>\n <p>Install the package with:</p>\n <pre><code>go get gopkg.in/check.v1\n</code></pre>\n </div>\n <div class=\"UnitReadme-fadeOut\"></div>\n </div>\n </div>\n <div class=\"UnitDoc\">\n <h2 class=\"UnitDoc-title\" id=\"section-documentation\"><img height=\"25px\" width=\"20px\" src=\"/static/img/pkg-icon-doc_20x12.svg\" alt=\"\">Documentation</h2>\n <div class=\"Documentation js-documentation\">\n <div class=\"Documentation-content js-docContent\">\n <section class=\"Documentation-overview\">\n <h3 tabindex=\"-1\" id=\"pkg-overview\" class=\"Documentation-overviewHeader\">Overview <a href=\"#pkg-overview\">¶</a></h3>\n <div role=\"navigation\" aria-label=\"Table of Contents\">\n <ul class=\"Documentation-toc\"></ul>\n </div>\n <p>Package check is a rich testing extension for Go's testing package.</p>\n <p>For details about the project, see:</p>\n <pre><a href=\"http://labix.org/gocheck\">http://labix.org/gocheck</a>\n</pre>\n </section>\n <h3 tabindex=\"-1\" id=\"pkg-constants\" class=\"Documentation-constantsHeader\">Constants <a href=\"#pkg-constants\">¶</a></h3>\n </div>\n </div>\n </div>\n </div>\n </div>\n </div>\n </div>\n</body>\n</html>\n")
+ (pkg.go.dev-licence "<!DOCTYPE html>\n<html lang=\"en\">\n<meta charset=\"utf-8\">\n<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n<body class=\"Site Site--wide Site--redesign\">\n <div class=\"Unit-content\" role=\"main\">\n <section class=\"License\" id=\"lic-0\">\n <h2><div id=\"#lic-0\">BSD-2-Clause</div></h2>\n <p>This is not legal advice. <a href=\"/license-policy\">Read disclaimer.</a></p>\n <pre class=\"License-contents\">Gocheck - A rich testing framework for Go\n \nCopyright line\n\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met: \n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer. \n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution. \n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR\nANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n</pre>\n </section>\n <div class=\"License-source\">Source: github.com/go-check/check@v0.0.0-20201128035030-22ab2dfb190c/LICENSE</div>\n </div>\n </div>\n"))
+ `(("https://proxy.golang.org/github.com/go-check/check/@v/v0.0.0-20201130134442-10cb98267c6c.mod"
+ . ,go.mod)
+ ("https://proxy.golang.org/github.com/go-check/check/@latest"
+ . ,version)
+ ("https://github.com/go-check/check?go-get=1"
+ . ,go-get)
+ ("https://pkg.go.dev/github.com/go-check/check"
+ . ,pkg.go.dev)
+ ("https://pkg.go.dev/github.com/go-check/check?tab=licenses"
+ . ,pkg.go.dev-licence))))
+
+(test-begin "go")
+
+;;; Unit tests for go build-system
+
+(test-equal "go-version basic"
+ "v1.0.2"
+ (go-version->git-ref "v1.0.2"))
+
+(test-equal "go-version omited 'v' character"
+ "v1.0.2"
+ (go-version->git-ref "v1.0.2"))
+
+(test-equal "go-version with embeded git-ref"
+ "65e3620a7ae7"
+ (go-version->git-ref "v0.0.0-20190821162956-65e3620a7ae7"))
+
+(test-equal "go-version with complex embeded git-ref"
+ "daa7c04131f5"
+ (go-version->git-ref "v1.2.4-0.20191109021931-daa7c04131f5"))
+
+;;; Unit tests for (guix import go)
+
+(test-equal "go-path-escape"
+ "github.com/!azure/!avere"
+ ((@@ (guix import go) go-path-escape) "github.com/Azure/Avere"))
+
+
+;; We define a function for all similar tests with different go.mod files
+(define (testing-parse-mod name expected input)
+ (define (inf? p1 p2)
+ (string<? (car p1) (car p2)))
+ (let ((input-port (open-input-string input)))
+ (test-equal name
+ (sort expected inf?)
+ (sort
+ ( (@@ (guix import go) parse-go.mod)
+ input-port)
+ inf?))))
+
+(testing-parse-mod "parse-go.mod-simple"
+ '(("good/thing" . "v1.4.5")
+ ("new/thing/v2" . "v2.3.4")
+ ("other/thing" . "v1.0.2"))
+ fixture-go-mod-simple)
+
+(testing-parse-mod "parse-go.mod-with-block"
+ '(("A" . "v1")
+ ("B" . "v1.0.0")
+ ("C" . "v1.0.0")
+ ("D" . "v1.2.3")
+ ("E" . "dev"))
+ fixture-go-mod-with-block)
+
+(testing-parse-mod "parse-go.mod-complete"
+ '(("github.com/corp/arbitrary-repo" . "v0.0.2")
+ ("quoted.example.com/abitrary/repo" . "v0.0.2")
+ ("one.example.com/abitrary/repo" . "v1.1.111")
+ ("hub.jazz.net/git/user/project/sub/directory" . "v1.1.19")
+ ("hub.jazz.net/git/user/project" . "v1.1.18")
+ ("launchpad.net/~user/project/branch/sub/directory" . "v1.1.17")
+ ("launchpad.net/~user/project/branch" . "v1.1.16")
+ ("launchpad.net/project/series/sub/directory" . "v1.1.15")
+ ("launchpad.net/project/series" . "v1.1.14")
+ ("launchpad.net/project" . "v1.1.13")
+ ("bitbucket.org/user/project/sub/directory" . "v1.11.21")
+ ("bitbucket.org/user/project" . "v1.11.20")
+ ("k8s.io/kubernetes/subproject" . "v1.1.101")
+ ("github.com/user/project/sub/directory" . "v1.1.12")
+ ("github.com/user/project" . "v1.1.11")
+ ("github.com/go-check/check" . "v0.0.0-20140225173054-eb6ee6f84d0a"))
+ fixture-go-mod-complete)
+
+;;; End-to-end tests for (guix import go)
+(define (mock-http-fetch testcase)
+ (lambda (url . rest)
+ (let ((body (assoc-ref testcase url)))
+ (if body
+ (open-input-string body)
+ (error "mocked http-fetch Unexpected URL: " url)))))
+
+(define (mock-http-get testcase)
+ (lambda (url . rest)
+ (let ((body (assoc-ref testcase url))
+ (response-header
+ (build-response
+ #:version '(1 . 1)
+ #:code 200
+ #:reason-phrase "Ok"
+ #:headers `(
+ (content-type text/html (charset . "utf-8"))
+ (date . ,(make-date 0 10 58 12 6 3 2021 0))
+ (transfer-encoding (chunked)))
+ #:port #f
+ #:validate-headers? #t)))
+ (if body
+ (values response-header body)
+ (error "mocked http-get Unexpected URL: " url)))))
+
+(test-equal "go-module->guix-package"
+ '(package
+ (name "go-github-com-go-check-check")
+ (version "0.0.0-20201130134442-10cb98267c6c")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/go-check/check.git")
+ (commit (go-version->git-ref version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32
+ "0000000000000000000000000000000000000000000000000000"))))
+ (build-system go-build-system)
+ (arguments
+ (quote (#:import-path "github.com/go-check/check")))
+ (inputs
+ (quasiquote (("go-github-com-kr-pretty"
+ (unquote go-github-com-kr-pretty)))))
+ (home-page "https://github.com/go-check/check")
+ (synopsis "Instructions")
+ (description #f)
+ (license (license:bsd-2)))
+ ;; Replace network resources with sample data.
+ (mock ((web client) http-get
+ (mock-http-get fixtures-go-check-test))
+ (mock ((guix http-client) http-fetch
+ (mock-http-fetch fixtures-go-check-test))
+ (go-module->guix-package "github.com/go-check/check"))))
+
+(test-end "go")
+
--
2.30.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* bug#44178: Add a Go Module Importer
2021-03-08 13:54 ` [bug#44178] " JOULAUD François via Guix-patches via
@ 2021-03-10 17:12 ` Ludovic Courtès
0 siblings, 0 replies; 27+ messages in thread
From: Ludovic Courtès @ 2021-03-10 17:12 UTC (permalink / raw)
To: JOULAUD François
Cc: Katherine Cox-Buday, 44178@debbugs.gnu.org, Maxim Cournoyer
Hi François, Katherine, & all!
I’m happy to say that it’s finally pushed, on behalf of Katherine and
the rest of you!
https://git.savannah.gnu.org/cgit/guix.git/commit/?id=02e2e093e858e8a0ca7bd66c1f1f6fd0a1705edb
I had to make a number of changes, among which (off the top of my head):
• Add files to Makefile.am. One can now run:
make check TESTS=tests/go.scm
See
<https://guix.gnu.org/manual/en/html_node/Running-the-Test-Suite.html>.
• Update ‘specification->package’ in (guix self).
• Fix version handling in the generated sexp (thanks Maxim for helping
out on IRC!).
• Fix the generated ‘license’ field.
• Fix minor issues reported by compiler warnings.
• Compute the hash of Git checkouts (done in a followup commit) since
that’s part of the minimum one expects from importers.
• Recode (guix import go) as UTF-8 rather than Latin-1.
• Move commentary below ‘define-module’ form.
Let me know if I broke anything on the way or if anything’s unclear!
Now, you’ve already identified things that could be improved, so feel
free to send focused patches addressing specific issues.
Thanks everyone for the great team work! :-)
Ludo’.
^ permalink raw reply [flat|nested] 27+ messages in thread
end of thread, other threads:[~2021-03-10 17:36 UTC | newest]
Thread overview: 27+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2020-10-23 14:06 [bug#44178] Add a Go Module Importer Katherine Cox-Buday
2020-10-28 10:41 ` Ludovic Courtès
2020-10-28 10:42 ` Ludovic Courtès
2020-11-10 20:26 ` Marius Bakke
[not found] ` <CANe01w55ZO=_9v0HcDv248UsoLUXb_9WVAgM4LqiZ4E-r1XgXg@mail.gmail.com>
2020-11-11 1:23 ` Helio Machado
2021-01-23 21:35 ` [bug#44178] [PATCH] Create importer for Go modules guix-patches--- via
2021-01-23 22:41 ` Katherine Cox-Buday
2021-01-25 21:03 ` guix-patches--- via
2021-01-27 14:38 ` Katherine Cox-Buday
2021-01-28 13:27 ` Ludovic Courtès
2021-01-29 16:43 ` guix-patches--- via
2021-01-29 16:52 ` [bug#44178] [PATCHv2] " guix-patches--- via
2021-01-31 16:23 ` [bug#44178] [PATCH] " Ludovic Courtès
2021-02-19 15:51 ` JOULAUD François via Guix-patches via
2021-02-19 16:21 ` [bug#44178] [PATCHv3] " JOULAUD François via Guix-patches via
2021-03-02 21:54 ` [bug#44178] Add a Go Module Importer Ludovic Courtès
2021-03-04 5:40 ` [bug#44178] [PATCH v4] Re: bug#44178: " Maxim Cournoyer
2021-03-04 14:14 ` JOULAUD François via Guix-patches via
2021-03-04 15:47 ` Maxim Cournoyer
2021-03-08 13:54 ` [bug#44178] " JOULAUD François via Guix-patches via
2021-03-10 17:12 ` bug#44178: " Ludovic Courtès
2021-01-28 5:01 ` [bug#44178] [PATCH] Create importer for Go modules Timmy Douglas
2020-11-11 20:48 ` [bug#44178] Add a Go Module Importer Katherine Cox-Buday
2020-12-09 14:22 ` [bug#44178] dftxbs3e
2020-12-10 2:42 ` [bug#44178] dftxbs3e
2020-12-10 3:14 ` [bug#44178] dftxbs3e
2021-01-28 7:29 ` [bug#44178] [PATCH] Create importer for Go modules guix-patches--- via
Code repositories for project(s) associated with this public inbox
https://git.savannah.gnu.org/cgit/guix.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).