[bug#35318] [PATCH] Update cargo-build-system to expand package inputs

[Ivan Petkov <ivanppetkov@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 74 bytes --]

Updated the patch series as per Ludo’s feedback.

Thanks,
—Ivan


[-- Attachment #2: 0001-build-system-cargo-expand-transitive-package-inputs.patch --]
[-- Type: application/octet-stream, Size: 6864 bytes --]

From 3ac8a092fad93754e323f7d9bec2af1a3912b36f Mon Sep 17 00:00:00 2001
From: Ivan Petkov <ivanppetkov@gmail.com>
Date: Tue, 16 Apr 2019 03:37:44 -0700
Subject: [PATCH 1/3] build-system/cargo: expand transitive package inputs

* guix/build/cargo: (package-target-inputs): Add it.
(expand-inputs): Add it.
(expand-native-inputs): Add it.
(package-transitive-dependencies): Add it.
(lower): Use new variables.
---
 guix/build-system/cargo.scm | 105 +++++++++++++++++++++++++++++++++++-
 1 file changed, 103 insertions(+), 2 deletions(-)

diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm
index dc137421e9..811a044a8c 100644
--- a/guix/build-system/cargo.scm
+++ b/guix/build-system/cargo.scm
@@ -29,6 +29,8 @@
   #:use-module (guix build-system)
   #:use-module (guix build-system gnu)
   #:use-module (ice-9 match)
+  #:use-module (ice-9 vlist)
+  #:use-module (srfi srfi-1)
   #:use-module (srfi srfi-26)
   #:export (%cargo-build-system-modules
             %cargo-utils-modules
@@ -121,6 +123,105 @@ to NAME and VERSION."
                                 #:outputs (cons "src" outputs)
                                 #:guile-for-build guile-for-build))
 
+(define (package-transitive-dependencies inputs)
+  "Return the closure of INPUTS when considering the 'inputs' and
+'propagated-inputs' edges.  Omit duplicate inputs, except for those
+already present in INPUTS itself.
+
+This is implemented as a breadth-first traversal such that INPUTS is
+preserved, and only duplicate extracted inputs are removed.
+
+Forked from ((guix packages) transitive-inputs) since this extraction
+uses slightly different rules compared to the rest of guix (i.e. we
+extract more than just propagated-inputs)."
+  (define (seen? seen item outputs)
+    ;; FIXME: We're using pointer identity here, which is extremely sensitive
+    ;; to memoization in package-producing procedures; see
+    ;; <https://bugs.gnu.org/30155>.
+    (match (vhash-assq item seen)
+      ((_ . o) (equal? o outputs))
+      (_       #f)))
+
+  (let loop ((inputs     inputs)
+             (result     '())
+             (propagated '())
+             (first?     #t)
+             (seen       vlist-null))
+    (match inputs
+      (()
+       (if (null? propagated)
+           (reverse result)
+           (loop (reverse (concatenate propagated)) result '() #f seen)))
+      (((and input (label (? package? package) outputs ...)) rest ...)
+       (if (and (not first?) (seen? seen package outputs))
+           (loop rest result propagated first? seen)
+           (loop rest
+                 (cons input result)
+                 (cons (append (package-inputs package)
+                               (package-propagated-inputs package))
+                       propagated)
+                 first?
+                 (vhash-consq package outputs seen))))
+      ((input rest ...)
+       (loop rest (cons input result) propagated first? seen)))))
+
+(define* (expand-inputs inputs)
+  "Return the transitive target inputs for each package in INPUTS.
+
+Cargo requires all transitive crate dependencies to be available in its index,
+even if they are optional (this is so it can generate deterministic
+Cargo.lock files regardless of the target platform or enabled features).
+
+Rather than burden the package definition to expand these, we'll automate
+that process in the build system."
+  (package-transitive-dependencies inputs))
+
+(define* (expand-native-inputs native-inputs)
+  "Return the transitive target inputs for each package in NATIVE-INPUTS.
+
+We need all transitive crate dependencies for any cargo dev-dependencies,
+but this is only needed when building/testing a crate directly
+(i.e. we will never need transitive dev-dependencies for any dependency
+crates).
+
+In addition, we need to modify any native-inputs to avoid any apparent
+cycles. Cargo does not permit cyclic dependencies between crates, however,
+it permits cycles to occur via dev-dependencies. For example, if crate X
+depends on crate Y, crate Y's tests could pull in crate X to to verify
+everything builds properly (this is a rare scenario, but it it happens for
+example with the `proc-macro2` and `quote` crates). This is allowed by cargo
+because tests are built as a pseudo-crate which happens to depend on the
+X and Y crates, forming an acyclic graph.
+
+Cargo ultimately needs only the crate sources available when doing a build.
+Ultimately, if a cycle is detected, we can break it by replacing the current
+crate we're trying to build with a no-op build which only packages the crate
+source; cargo will then find the source available in its index and build things
+properly."
+  (map
+    (lambda (dep)
+      (match dep
+       ((name input rest ...)
+        ;; TODO: Starting out real simple by ensuring no native-inputs are
+        ;; actually built so we don't accidentally hit a cycle. We can revisit
+        ;; this later and more intelligently replace any dependency on the cycle
+        ;; root with a version of the root with #:skip-build? enabled.
+        ;;
+        ;; Right now this means that if a package shows up in both input
+        ;; and native-input closures, we'll try to build it's src output twice.
+        ;;
+        ;; This also means that we're potentially setting the #:skip-build?
+        ;; flag on non cargo-build-system packages, but this should be okay
+        ;; for now.
+        (let* ((translated-input (package
+                                   (inherit input)
+                                   (inputs '())
+                                   (propagated-inputs '())
+                                   (native-inputs '())
+                                   (arguments '(#:skip-build? #t)))))
+          `(,name ,translated-input ,@rest)))))
+   (package-transitive-dependencies native-inputs)))
+
 (define* (lower name
                 #:key source inputs native-inputs outputs system target
                 (rust (default-rust))
@@ -139,13 +240,13 @@ to NAME and VERSION."
          (host-inputs `(,@(if source
                               `(("source" ,source))
                               '())
-                        ,@inputs
+                        ,@(expand-inputs inputs)
 
                         ;; Keep the standard inputs of 'gnu-build-system'
                         ,@(standard-packages)))
          (build-inputs `(("cargo" ,rust "cargo")
                          ("rustc" ,rust)
-                         ,@native-inputs))
+                         ,@(expand-native-inputs native-inputs)))
          (outputs outputs)
          (build cargo-build)
          (arguments (strip-keyword-arguments private-keywords arguments)))))
-- 
2.21.0


[-- Attachment #3: 0002-gnu-crate-add-unicode-xid.patch --]
[-- Type: application/octet-stream, Size: 2957 bytes --]

From ec8509623a3b59a25a4ae995dca0f18cd5b1ce48 Mon Sep 17 00:00:00 2001
From: Ivan Petkov <ivanppetkov@gmail.com>
Date: Tue, 16 Apr 2019 03:40:14 -0700
Subject: [PATCH 2/3] gnu: crate: add unicode-xid

gnu/local.mk: (GNU_SYSTEM_MODULES): Add new file.
gnu/packages/crates-io.scm: (rust-unicode-xid): New variable.
---
 gnu/local.mk               |  1 +
 gnu/packages/crates-io.scm | 45 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 gnu/packages/crates-io.scm

diff --git a/gnu/local.mk b/gnu/local.mk
index 3f97397c4a..f92db5821c 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -121,6 +121,7 @@ GNU_SYSTEM_MODULES =				\
   %D%/packages/cpp.scm 				\
   %D%/packages/cppi.scm				\
   %D%/packages/cran.scm				\
+  %D%/packages/crates-io.scm			\
   %D%/packages/cross-base.scm			\
   %D%/packages/crypto.scm			\
   %D%/packages/cryptsetup.scm			\
diff --git a/gnu/packages/crates-io.scm b/gnu/packages/crates-io.scm
new file mode 100644
index 0000000000..533fe0d21e
--- /dev/null
+++ b/gnu/packages/crates-io.scm
@@ -0,0 +1,45 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2019 Ivan Petkov <ivanppetkov@gmail.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (gnu packages crates-io)
+  #:use-module (guix build-system cargo)
+  #:use-module (guix download)
+  #:use-module ((guix licenses) #:prefix license:)
+  #:use-module (guix packages))
+
+(define-public rust-unicode-xid
+  (package
+    (name "rust-unicode-xid")
+    (version "0.1.0")
+    (source
+      (origin
+        (method url-fetch)
+        (uri (crate-uri "unicode-xid" version))
+        (file-name
+          (string-append name "-" version ".tar.gz"))
+        (sha256
+          (base32
+            "1z57lqh4s18rr4x0j4fw4fmp9hf9346h0kmdgqsqx0fhjr3k0wpw"))))
+    (build-system cargo-build-system)
+    (home-page
+      "https://github.com/unicode-rs/unicode-xid")
+    (synopsis "Determine Unicode XID related properties")
+    (description "Determine whether characters have the XID_Start
+or XID_Continue properties according to Unicode Standard Annex #31.")
+    ;; Dual licensed.
+    (license (list license:asl2.0 license:expat))))
-- 
2.21.0


[-- Attachment #4: 0003-gnu-crate-Add-proc-macro2-and-quote.patch --]
[-- Type: application/octet-stream, Size: 2534 bytes --]

From 1cff79e586664fd76bb7eb97fe5228986b32d8b4 Mon Sep 17 00:00:00 2001
From: Ivan Petkov <ivanppetkov@gmail.com>
Date: Tue, 16 Apr 2019 03:42:27 -0700
Subject: [PATCH 3/3] gnu: crate: Add proc-macro2 and quote

* gnu/packages/crates-io.scm: (rust-proc-macro2): New variable.
(rust-quote): New variable.
---
 gnu/packages/crates-io.scm | 48 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/gnu/packages/crates-io.scm b/gnu/packages/crates-io.scm
index 533fe0d21e..90e08731ea 100644
--- a/gnu/packages/crates-io.scm
+++ b/gnu/packages/crates-io.scm
@@ -43,3 +43,51 @@
 or XID_Continue properties according to Unicode Standard Annex #31.")
     ;; Dual licensed.
     (license (list license:asl2.0 license:expat))))
+
+(define-public rust-proc-macro2
+  (package
+    (name "rust-proc-macro2")
+    (version "0.4.27")
+    (source
+      (origin
+        (method url-fetch)
+        (uri (crate-uri "proc-macro2" version))
+        (file-name
+          (string-append name "-" version ".tar.gz"))
+        (sha256
+          (base32
+            "05c92v787snyaq4ss16vxc9mdv6zndfgsdq8k3hnnyffmsf7ycad"))))
+    (build-system cargo-build-system)
+    (native-inputs
+      `(("rust-quote" ,rust-quote "src")))
+    (inputs
+      `(("rust-unicode-xid" ,rust-unicode-xid "src")))
+    (home-page "https://github.com/alexcrichton/proc-macro2")
+    (synopsis "Stable implementation of the upcoming new `proc_macro` API")
+    (description "This package provides a stable implementation of the upcoming new
+`proc_macro` API.  Comes with an option, off by default, to also reimplement itself
+in terms of the upstream unstable API.")
+    ;; Dual licensed.
+    (license (list license:asl2.0 license:expat))))
+
+(define-public rust-quote
+  (package
+    (name "rust-quote")
+    (version "0.6.12")
+    (source
+      (origin
+        (method url-fetch)
+        (uri (crate-uri "quote" version))
+        (file-name
+          (string-append name "-" version ".tar.gz"))
+        (sha256
+          (base32
+            "1nw0klza45hf127kfyrpxsxd5jw2l6h21qxalil3hkr7bnf7kx7s"))))
+    (build-system cargo-build-system)
+    (inputs
+      `(("rust-proc-macro2" ,rust-proc-macro2 "src")))
+    (home-page "https://github.com/dtolnay/quote")
+    (synopsis "Quasi-quoting macro quote!(...)")
+    (description "Quasi-quoting macro quote!(...)")
+    ;; Dual licensed.
+    (license (list license:asl2.0 license:expat))))
-- 
2.21.0