[PATCH] build-system: cargo: Add #:cargo-build-flags keyword argument.

[PATCH] build-system: cargo: Add #:cargo-build-flags keyword argument.

[Danny Milosavljevic]

* guix/build-system/cargo.scm (cargo-build): Add #:cargo-build-flags keyword argument.
---
 guix/build-system/cargo.scm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm
index ffc0afda3..011ea9c57 100644
--- a/guix/build-system/cargo.scm
+++ b/guix/build-system/cargo.scm
@@ -65,6 +65,7 @@ to NAME and VERSION."
                       (tests? #t)
                       (test-target #f)
                       (configure-flags #f)
+                      (cargo-build-flags ''("--release" "--frozen"))
                       (phases '(@ (guix build cargo-build-system)
                                   %standard-phases))
                       (outputs '("out"))
@@ -88,6 +89,7 @@ to NAME and VERSION."
                                 (source
                                  source))
                     #:system ,system
+                    #:cargo-build-flags ,cargo-build-flags
                     #:test-target ,test-target
                     #:tests? ,tests?
                     #:phases ,phases

[PATCH v2] build-system: cargo: Add #:cargo-build-flags keyword argument.

[Danny Milosavljevic]

* guix/build-system/cargo.scm (cargo-build): Add #:cargo-build-flags keyword argument.
---
 guix/build-system/cargo.scm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm
index ffc0afda3..011ea9c57 100644
--- a/guix/build-system/cargo.scm
+++ b/guix/build-system/cargo.scm
@@ -65,6 +65,7 @@ to NAME and VERSION."
                       (tests? #t)
                       (test-target #f)
                       (configure-flags #f)
+                      (cargo-build-flags ''("--release"))
                       (phases '(@ (guix build cargo-build-system)
                                   %standard-phases))
                       (outputs '("out"))
@@ -88,6 +89,7 @@ to NAME and VERSION."
                                 (source
                                  source))
                     #:system ,system
+                    #:cargo-build-flags ,cargo-build-flags
                     #:test-target ,test-target
                     #:tests? ,tests?
                     #:phases ,phases

WIP patch: build-system: cargo: Make lots and lots of rust packages work

[Danny Milosavljevic]

For it to work, you also have to back out the (cons "src" outputs) (in ./guix/build-system/cargo.scm) and the (assoc-ref outputs "src") (in guix/build/cargo-build-system.scm) starting from current master.

--- guix/build/cargo-build-system.scm	2017-01-03 04:31:01.691543854 +0100
+++ /home/dannym/src/guix/guix/build/cargo-build-system.scm	2017-01-03 04:55:26.810917585 +0100
@@ -19,6 +19,8 @@
 (define-module (guix build cargo-build-system)
   #:use-module ((guix build gnu-build-system) #:prefix gnu:)
   #:use-module (guix build utils)
+  #:use-module (ice-9 popen)
+  #:use-module (ice-9 rdelim)
   #:use-module (ice-9 ftw)
   #:use-module (ice-9 format)
   #:use-module (ice-9 match)
@@ -43,35 +45,86 @@
 
 (define* (configure #:key inputs #:allow-other-keys)
   "Replace Cargo.toml [dependencies] section with guix inputs."
-  ;; Make sure Cargo.toml is writeable when the crate uses git-fetch.
-  (chmod "Cargo.toml" #o644)
-  (let ((port (open-file "Cargo.toml" "a" #:encoding "utf-8")))
-    (format port "~%[replace]~%")
-    (for-each
-     (match-lambda
-       ((name . path)
-        (let ((crate (package-name->crate-name name)))
-          (when (and crate path)
-            (match (string-split (basename path) #\-)
-              ((_ ... version)
-               (format port "\"~a:~a\" = { path = \"~a/share/rust-source\" }~%"
-                       crate version path)))))))
-     inputs)
-    (close-port port))
+  (system* "chmod" "+w" "Cargo.toml")
+  (system* "chmod" "+w" ".")
+  (if (not (file-exists? "vendor"))
+    (begin
+      (if (not (file-exists? "Cargo.lock"))
+        (substitute* "Cargo.toml"
+          ((".*32-sys.*") "
+")
+          ((".*winapi.*") "
+")))
+      (mkdir "vendor")
+      (for-each
+        (match-lambda
+          ((name . path)
+           (let ((crate (package-name->crate-name name)))
+             (when (and crate path)
+               (match (string-split (basename path) #\-)
+                 ((_ ... version)
+                  (symlink (string-append path "/share/rust-source") (string-append "vendor/" (basename path)))))))))
+        inputs)
+      (mkdir-p ".cargo")
+      (let ((port (open-file ".cargo/config" "w" #:encoding "utf-8")))
+        (display "
+[source.crates-io]
+registry = 'https://github.com/rust-lang/crates.io-index'
+replace-with = 'vendored-sources'
+
+[source.vendored-sources]
+directory = '" port)
+        (display (getcwd) port)
+        (display "/vendor" port)
+        (display "'
+" port)
+        (close-port port))))
+    (setenv "CC" (string-append (assoc-ref inputs "gcc") "/bin/gcc"))
+
+    ;(setenv "CARGO_HOME" "/gnu/store")
+    ; (setenv "CMAKE_C_COMPILER" cc)
   #t)
 
-(define* (build #:key (cargo-build-flags '("--release" "--frozen"))
-                #:allow-other-keys)
+(define* (build #:key (cargo-build-flags '("--release")) #:allow-other-keys)
   "Build a given Cargo package."
   (if (file-exists? "Cargo.lock")
-      (zero? (apply system* `("cargo" "build" ,@cargo-build-flags)))
-      #t))
+    (zero? (apply system* `("cargo" "build" ,@cargo-build-flags)))
+    #t))
 
 (define* (check #:key tests? #:allow-other-keys)
   "Run tests for a given Cargo package."
-  (if (and tests? (file-exists? "Cargo.lock"))
-      (zero? (system* "cargo" "test"))
-      #t))
+  (when tests?
+    (zero? (system* "cargo" "test"))))
+
+(define (file-sha256 file-name)
+    (let ((port (open-pipe* OPEN_READ
+                            "sha256sum"
+                            "--"
+                            file-name)))
+      (let ((result (read-delimited " " port)))
+        (close-pipe port)
+        result)))
+
+;; Example /gnu/store/hwlr49riz3la33m6in2n898ly045ylld-rust-rand-0.3.15: ; generally not a lot of store items.
+(define (generate-checksums dir-name src-name)
+  (let* ((file-names (find-files dir-name "."))
+         (dir-prefix-name (string-append dir-name "/"))
+         (dir-prefix-name-len (string-length dir-prefix-name))
+         (checksums-file-name (string-append dir-name "/.cargo-checksum.json")))
+    (call-with-output-file checksums-file-name
+      (lambda (port)
+        (display "{\"files\":{" port)
+        (let ((sep ""))
+          (for-each (lambda (file-name)
+            (let ((file-relative-name (string-drop file-name dir-prefix-name-len)))
+                  (display sep port)
+                  (set! sep ",")
+                  (write file-relative-name port)
+                  (display ":" port)
+                  (write (file-sha256 file-name) port))) file-names))
+        (display "},\"package\":" port)
+        (write (file-sha256 src-name) port)
+        (display "}" port)))))
 
 (define* (install #:key inputs outputs #:allow-other-keys)
   "Install a given Cargo package."
@@ -87,15 +140,17 @@
     ;; references in Cargo.toml with store paths.
     (copy-recursively "src" (string-append rsrc "/src"))
     (install-file "Cargo.toml" rsrc)
+    (system* "touch" (string-append rsrc "/.cargo-ok"))
+    (generate-checksums rsrc src)
     ;; When the package includes executables we install
     ;; it using cargo install. This fails when the crate
     ;; doesn't contain an executable.
     (if (file-exists? "Cargo.lock")
         (system* "cargo" "install" "--root" out)
-        (mkdir out))))
+        (mkdir out))
+    #t))
 
 (define %standard-phases
-  ;; 'configure' phase is not needed.
   (modify-phases gnu:%standard-phases
     (replace 'configure configure)
     (replace 'build build)

Re: WIP patch: build-system: cargo: Make lots and lots of rust packages work

[David Craven]

> For it to work, you also have to back out the (cons "src" outputs) (in ./guix/build-system/cargo.scm) and the (assoc-ref outputs "src") (in guix/build/cargo-build-system.scm) starting from current master.

Can you elaborate on why you think it's a bad idea to separate the
source from the binaries into different outputs?

I see that you are trying to mimic what cargo vendor does. Cool proof
of concept and nice solution to the computing sha256 hashes on the
build side.

The question is if this is the best approach. I was thinking of trying
[0] first and see how that works for the build system use case. The
cargo package example using vendoring is intended to be a temporary
solution, and I'm not sure we have finished exploring the "design
space" yet.

[0] https://github.com/alexcrichton/cargo-local-registry

Re: WIP patch: build-system: cargo: Make lots and lots of rust packages work

[Danny Milosavljevic]

Hi David,

On Tue, 3 Jan 2017 12:08:56 +0100
David Craven <david@craven.ch> wrote:

> Can you elaborate on why you think it's a bad idea to separate the
> source from the binaries into different outputs?

Huh? I think it's a good idea. It just doesn't work yet and I didn't find out why :)

> I see that you are trying to mimic what cargo vendor does. Cool proof
> of concept and nice solution to the computing sha256 hashes on the
> build side.

Yeah. I thought they can't exactly replace all the bundled files in all the repos (written by anyone) if they ever changed the file format - so it's there to stay "forever". There's no downside I can see.

That said, we can also do it via cargo vendor if it's easy. But it's not really required - we can always just change it to that if it ever breaks.

> The question is if this is the best approach. I was thinking of trying
> [0] first and see how that works for the build system use case. The
> cargo package example using vendoring is intended to be a temporary
> solution, and I'm not sure we have finished exploring the "design
> space" yet.
> 
> [0] https://github.com/alexcrichton/cargo-local-registry

That's interesting too! We can try both.

My patch is just a WIP patch of something that works. We should explore the other alternatives, too.

But about the current approach in master, I talked to Alex Crichton [1], he said that the "replace" approach won't work without a registry because it tries to replace crates by the exact same version and what you specify for "replace" is just an API version. I think by then you can just go all the way to a custom registry and dispense of the "replace".

[1] https://github.com/rust-lang/cargo/issues/3476

Re: WIP patch: build-system: cargo: Make lots and lots of rust packages work

[Danny Milosavljevic]

> > Can you elaborate on why you think it's a bad idea to separate the
> > source from the binaries into different outputs?  
> 
> Huh? I think it's a good idea. It just doesn't work yet and I didn't find out why :)

Hmm I think I would have to update all my imported crates' package recipes to depend on the "src" output for inputs. I didn't do that and that's probably (?) why it doesn't work.

WIP v2: build-system: cargo: Make lots and lots of rust packages work

[Danny Milosavljevic]

[-- Attachment #1: Type: text/plain, Size: 84 bytes --]

With minimal changes (bugfixes) compared to the last version... (see attachement)



[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: cargo-build-system.scm --]
[-- Type: text/x-scheme, Size: 6187 bytes --]

;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016 David Craven <david@craven.ch>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix build cargo-build-system)
  #:use-module ((guix build gnu-build-system) #:prefix gnu:)
  #:use-module (guix build utils)
  #:use-module (ice-9 popen)
  #:use-module (ice-9 rdelim)
  #:use-module (ice-9 ftw)
  #:use-module (ice-9 format)
  #:use-module (ice-9 match)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-26)
  #:export (%standard-phases
            cargo-build))

;; Commentary:
;;
;; Builder-side code of the standard Rust package build procedure.
;;
;; Code:

;; FIXME: Needs to be parsed from url not package name.
(define (package-name->crate-name name)
  "Return the crate name of NAME."
  (match (string-split name #\-)
    (("rust" rest ...)
     (string-join rest "-"))
    (_ #f)))

(define* (configure #:key inputs #:allow-other-keys)
  "Replace Cargo.toml [dependencies] section with guix inputs."
  (system* "chmod" "+w" "Cargo.toml")
  (system* "chmod" "+w" ".")
  (if (not (file-exists? "vendor"))
    (if (not (file-exists? "Cargo.lock"))
      (begin
      (substitute* "Cargo.toml"
          ((".*32-sys.*") "
")
          ((".*winapi.*") "
")
          ((".*core-foundation.*") "
"))
      (mkdir "vendor")
      (for-each
        (match-lambda
          ((name . path)
           (let ((crate (package-name->crate-name name)))
             (when (and crate path)
               (match (string-split (basename path) #\-)
                 ((_ ... version)
                  (symlink (string-append path "/share/rust-source") (string-append "vendor/" (basename path)))))))))
        inputs)
      (mkdir-p ".cargo")
      (let ((port (open-file ".cargo/config" "w" #:encoding "utf-8")))
        (display "
[source.crates-io]
registry = 'https://github.com/rust-lang/crates.io-index'
replace-with = 'vendored-sources'

[source.vendored-sources]
directory = '" port)
        (display (getcwd) port)
        (display "/vendor" port)
        (display "'
" port)
        (close-port port)))))
    (setenv "CC" (string-append (assoc-ref inputs "gcc") "/bin/gcc"))

    ;(setenv "CARGO_HOME" "/gnu/store")
    ; (setenv "CMAKE_C_COMPILER" cc)
  #t)

(define* (build #:key (cargo-build-flags '("--release")) #:allow-other-keys)
  "Build a given Cargo package."
  ;if (file-exists? "Cargo.lock")
  (zero? (apply system* `("cargo" "build" ,@cargo-build-flags))))
  ;  #t

(define* (check #:key tests? #:allow-other-keys)
  "Run tests for a given Cargo package."
  (when tests?
    (zero? (system* "cargo" "test"))))

(define (file-sha256 file-name)
    (let ((port (open-pipe* OPEN_READ
                            "sha256sum"
                            "--"
                            file-name)))
      (let ((result (read-delimited " " port)))
        (close-pipe port)
        result)))

;; Example /gnu/store/hwlr49riz3la33m6in2n898ly045ylld-rust-rand-0.3.15: ; generally not a lot of store items.
(define (generate-checksums dir-name src-name)
  (let* ((file-names (find-files dir-name "."))
         (dir-prefix-name (string-append dir-name "/"))
         (dir-prefix-name-len (string-length dir-prefix-name))
         (checksums-file-name (string-append dir-name "/.cargo-checksum.json")))
    (call-with-output-file checksums-file-name
      (lambda (port)
        (display "{\"files\":{" port)
        (let ((sep ""))
          (for-each (lambda (file-name)
            (let ((file-relative-name (string-drop file-name dir-prefix-name-len)))
                  (display sep port)
                  (set! sep ",")
                  (write file-relative-name port)
                  (display ":" port)
                  (write (file-sha256 file-name) port))) file-names))
        (display "},\"package\":" port)
        (write (file-sha256 src-name) port)
        (display "}" port)))))

(define* (install #:key inputs outputs #:allow-other-keys)
  "Install a given Cargo package."
  (let* ((out (assoc-ref outputs "out"))
         (src (assoc-ref inputs "source"))
         (rsrc (string-append out
                              "/share/rust-source")))
    (mkdir-p rsrc)
    ;; Rust doesn't have a stable ABI yet. Because of this
    ;; Cargo doesn't have a search path for binaries yet.
    ;; Until this changes we are working around this by
    ;; distributing crates as source and replacing
    ;; references in Cargo.toml with store paths.
    (copy-recursively "src" (string-append rsrc "/src"))
    (install-file "Cargo.toml" rsrc)
    (system* "touch" (string-append rsrc "/.cargo-ok"))
    (generate-checksums rsrc src)
    ;; When the package includes executables we install
    ;; it using cargo install. This fails when the crate
    ;; doesn't contain an executable.
    (if (file-exists? "Cargo.lock")
        (mkdir-p out)
        (system* "cargo" "install" "--root" out))
    #t))

(define %standard-phases
  (modify-phases gnu:%standard-phases
    (replace 'configure configure)
    (replace 'build build)
    (replace 'check check)
    (replace 'install install)))

(define* (cargo-build #:key inputs (phases %standard-phases)
                      #:allow-other-keys #:rest args)
  "Build the given Cargo package, applying all of PHASES in order."
  (apply gnu:gnu-build #:inputs inputs #:phases phases args))

;;; cargo-build-system.scm ends here

WIP v3: build-system: cargo: Make lots and lots of rust packages work

[Danny Milosavljevic]

[-- Attachment #1: Type: text/plain, Size: 45 bytes --]

Now also uses "src" output. See attachment.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: cargo-build-system.scm --]
[-- Type: text/x-scheme, Size: 6241 bytes --]

;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016 David Craven <david@craven.ch>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix build cargo-build-system)
  #:use-module ((guix build gnu-build-system) #:prefix gnu:)
  #:use-module (guix build utils)
  #:use-module (ice-9 popen)
  #:use-module (ice-9 rdelim)
  #:use-module (ice-9 ftw)
  #:use-module (ice-9 format)
  #:use-module (ice-9 match)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-26)
  #:export (%standard-phases
            cargo-build))

;; Commentary:
;;
;; Builder-side code of the standard Rust package build procedure.
;;
;; Code:

;; FIXME: Needs to be parsed from url not package name.
(define (package-name->crate-name name)
  "Return the crate name of NAME."
  (match (string-split name #\-)
    (("rust" rest ...)
     (string-join rest "-"))
    (_ #f)))

(define* (configure #:key inputs #:allow-other-keys)
  "Replace Cargo.toml [dependencies] section with guix inputs."
  (system* "chmod" "+w" "Cargo.toml")
  (system* "chmod" "+w" ".")
  (if (not (file-exists? "vendor"))
    (if (or (not (file-exists? "Cargo.lock")) (file-exists? "libc-test"))
      (begin
      (substitute* "Cargo.toml"
          ((".*32-sys.*") "
")
          ((".*winapi.*") "
")
          ((".*core-foundation.*") "
"))
      (mkdir "vendor")
      (for-each
        (match-lambda
          ((name . path)
           (let ((crate (package-name->crate-name name)))
             (when (and crate path)
               (match (string-split (basename path) #\-)
                 ((_ ... version)
                  (symlink (string-append path "/share/rust-source") (string-append "vendor/" (basename path)))))))))
        inputs)
      (mkdir-p ".cargo")
      (let ((port (open-file ".cargo/config" "w" #:encoding "utf-8")))
        (display "
[source.crates-io]
registry = 'https://github.com/rust-lang/crates.io-index'
replace-with = 'vendored-sources'

[source.vendored-sources]
directory = '" port)
        (display (getcwd) port)
        (display "/vendor" port)
        (display "'
" port)
        (close-port port)))))
    (setenv "CC" (string-append (assoc-ref inputs "gcc") "/bin/gcc"))

    ;(setenv "CARGO_HOME" "/gnu/store")
    ; (setenv "CMAKE_C_COMPILER" cc)
  #t)

(define* (build #:key (cargo-build-flags '("--release")) #:allow-other-keys)
  "Build a given Cargo package."
  ;if (file-exists? "Cargo.lock")
  (zero? (apply system* `("cargo" "build" ,@cargo-build-flags))))
  ;  #t

(define* (check #:key tests? #:allow-other-keys)
  "Run tests for a given Cargo package."
  (when tests?
    (zero? (system* "cargo" "test"))))

(define (file-sha256 file-name)
    (let ((port (open-pipe* OPEN_READ
                            "sha256sum"
                            "--"
                            file-name)))
      (let ((result (read-delimited " " port)))
        (close-pipe port)
        result)))

;; Example /gnu/store/hwlr49riz3la33m6in2n898ly045ylld-rust-rand-0.3.15: ; generally not a lot of store items.
(define (generate-checksums dir-name src-name)
  (let* ((file-names (find-files dir-name "."))
         (dir-prefix-name (string-append dir-name "/"))
         (dir-prefix-name-len (string-length dir-prefix-name))
         (checksums-file-name (string-append dir-name "/.cargo-checksum.json")))
    (call-with-output-file checksums-file-name
      (lambda (port)
        (display "{\"files\":{" port)
        (let ((sep ""))
          (for-each (lambda (file-name)
            (let ((file-relative-name (string-drop file-name dir-prefix-name-len)))
                  (display sep port)
                  (set! sep ",")
                  (write file-relative-name port)
                  (display ":" port)
                  (write (file-sha256 file-name) port))) file-names))
        (display "},\"package\":" port)
        (write (file-sha256 src-name) port)
        (display "}" port)))))

(define* (install #:key inputs outputs #:allow-other-keys)
  "Install a given Cargo package."
  (let* ((out (assoc-ref outputs "out"))
         (src (assoc-ref inputs "source"))
         (rsrc (string-append (assoc-ref outputs "src")
                              "/share/rust-source")))
    (mkdir-p rsrc)
    ;; Rust doesn't have a stable ABI yet. Because of this
    ;; Cargo doesn't have a search path for binaries yet.
    ;; Until this changes we are working around this by
    ;; distributing crates as source and replacing
    ;; references in Cargo.toml with store paths.
    (copy-recursively "src" (string-append rsrc "/src"))
    (install-file "Cargo.toml" rsrc)
    (system* "touch" (string-append rsrc "/.cargo-ok"))
    (generate-checksums rsrc src)
    ;; When the package includes executables we install
    ;; it using cargo install. This fails when the crate
    ;; doesn't contain an executable.
    (if (file-exists? "Cargo.lock")
        (mkdir-p out)
        (system* "cargo" "install" "--root" out))
    #t))

(define %standard-phases
  (modify-phases gnu:%standard-phases
    (replace 'configure configure)
    (replace 'build build)
    (replace 'check check)
    (replace 'install install)))

(define* (cargo-build #:key inputs (phases %standard-phases)
                      #:allow-other-keys #:rest args)
  "Build the given Cargo package, applying all of PHASES in order."
  (apply gnu:gnu-build #:inputs inputs #:phases phases args))

;;; cargo-build-system.scm ends here