[bug#31026] [PATCH 0/2] Split off rust@1.19.0.

[Danny Milosavljevic <dannym@scratchpost.org>]

[-- Attachment #1.1: Type: text/plain, Size: 1559 bytes --]

On Mon, 2 Apr 2018 18:19:21 +0000
Nils Gillmann <ng0@n0.is> wrote:

> Danny Milosavljevic transcribed 260 bytes:
> > Prepare for mrustc bootstrap.  
> 
> Woo, good news. Did I miss an email 

I've been trying to use mrustc to bootstrap rust on and off for quite
some time now.  The upstream authors have been busy too and it's far enough
that I have a working rust-1.19.0 compiler compiled by mrustc in /gnu/store :)

The libstd still had some rough edges - and one test takes about 30 h of
building - so it will still take some time until this is done.

I thought in the meantime I'd clean up and split up the patches.

These here are some of the patches that can be applied without switching to mrustc
yet (that is, they just refactor stuff).

>or does this still require some review?

Right now, mrustc officially *supports* x86_64 targets only.

aarch64 is under review upstream (support is there but unfinished).

We will only be able to use mrustc to bootstrap if it supports more architectures.

We will see when that comes to pass.

mrustc is written in a way that it can compile rust@1.19.0 so that's why this
exact version is in this patchset.  I hope in the future mrustc can compile
newer versions of Rust.

> I can reserve some time for building this tonight and review tomorrow at some
> point.

Nice!

I've attached my current gnu/packages/rust.scm here - which includes some things
not ready to merge yet.  But it should (famous last words) bootstrap rust 1.24
starting from mrustc in multiple steps.

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: rust.scm --]
[-- Type: text/x-scheme, Size: 22340 bytes --]

;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016 David Craven <david@craven.ch>
;;; Copyright © 2016 Eric Le Bihan <eric.le.bihan.dev@free.fr>
;;; Copyright © 2016 Nils Gillmann <ng0@n0.is>
;;; Copyright © 2017 Ben Woodcroft <donttrustben@gmail.com>
;;; Copyright © 2017, 2018 Nikolai Merinov <nikolai.merinov@member.fsf.org>
;;; Copyright © 2017 Efraim Flashner <efraim@flashner.co.il>
;;; Copyright © 2018 Tobias Geerinckx-Rice <me@tobias.gr>
;;; Copyright © 2018 Danny Milosavljevic <dannym+a@scratchpost.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (gnu packages rust)
  #:use-module (gnu packages base)
  #:use-module (gnu packages bison)
  #:use-module (gnu packages bootstrap)
  #:use-module (gnu packages cmake)
  #:use-module (gnu packages compression)
  #:use-module (gnu packages curl)
  #:use-module (gnu packages elf)
  #:use-module (gnu packages flex)
  #:use-module (gnu packages gcc)
  #:use-module (gnu packages gdb)
  #:use-module (gnu packages jemalloc)
  #:use-module (gnu packages linux)
  #:use-module (gnu packages llvm)
  #:use-module (gnu packages pkg-config)
  #:use-module (gnu packages python)
  #:use-module (gnu packages ssh)
  #:use-module (gnu packages tls)
  #:use-module (gnu packages version-control)
  #:use-module (gnu packages)
  #:use-module (guix build-system cargo)
  #:use-module (guix build-system gnu)
  #:use-module (guix build-system trivial)
  #:use-module (guix download)
  #:use-module (guix git-download)
  #:use-module ((guix licenses) #:prefix license:)
  #:use-module (guix packages)
  #:use-module ((guix build utils) #:select (alist-replace))
  #:use-module (guix utils)
  #:use-module (ice-9 match)
  #:use-module (srfi srfi-26))

(define %cargo-reference-project-file "/dev/null")
(define %cargo-reference-hash
  "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")

(define (rust-source version hash)
  (origin
    (method url-fetch)
    (uri (string-append "https://static.rust-lang.org/dist/"
                        "rustc-" version "-src.tar.gz"))
    (sha256 (base32 hash))
    (modules '((guix build utils)))
    (snippet
     '(begin
         (delete-file-recursively "src/llvm")
       #t))))

\f
(define-public mrustc
  (let ((commit "1a16def28935548e29be0fe5a632e25c83045924")
        (revision "0")
        (rustc-version "1.19.0"))
    (package
      (name "mrustc")
      (version (git-version "0.0.0" revision commit))
      (source (origin
                (method git-fetch)
                (uri (git-reference
                      (url "https://github.com/thepowersgang/mrustc.git")
                      (commit commit)))
                (file-name (git-file-name name version))
                (sha256
                 (base32
                  "0i5kqnzsd1rkj8qm147jx934nyn9sd1fz2b4achj9z0r00h84dh6"))))
      (outputs '("out" "cargo"))
      (build-system gnu-build-system)
      (native-inputs
       `(("bison" ,bison)
         ("flex" ,flex)
         ("llvm" ,llvm-3.9.1)
         ;; Required for the libstd sources.
         ("rustc"
          ,(rust-source "1.19.0" "0l8c14qsf42rmkqy92ahij4vf356dbyspxcips1aswpvad81y8qm"))))
      (arguments
       `(#:tests? #f
         #:make-flags (list (string-append "LLVM_CONFIG="
                                           (assoc-ref %build-inputs "llvm")
                                           "/bin/llvm-config"))
         #:phases
         (modify-phases %standard-phases
           (add-after 'unpack 'unpack-target-compiler
             (lambda* (#:key inputs outputs #:allow-other-keys)
               (substitute* "minicargo.mk"
                 ;; Don't try to build LLVM.
                 (("^[$][(]LLVM_CONFIG[)]:") "xxx:")
                 ;; Build for the correct target architecture.
                 (("^RUSTC_TARGET := x86_64-unknown-linux-gnu")
                  (string-append "RUSTC_TARGET := "
                                 ,(or (%current-target-system)
                                      (nix-system->gnu-triplet
                                       (%current-system))))))
               (invoke "tar" "xf" (assoc-ref inputs "rustc"))
               (chdir "rustc-1.19.0-src")
               (invoke "patch" "-p0" "../rust_src.patch")
               (chdir "..")
               #t))
           (delete 'configure)
           (add-after 'build 'build-minicargo
             (lambda _
               (for-each (lambda (target)
                           (invoke "make" "-f" "minicargo.mk" target))
                         '("output/libstd.hir" "output/libpanic_unwind.hir"
                           "output/libproc_macro.hir" "output/libtest.hir"))
               ;; Technically the above already does it - but we want to be clear.
               (invoke "make" "-C" "tools/minicargo")))
           (replace 'install
             (lambda* (#:key inputs outputs #:allow-other-keys)
               (let* ((out (assoc-ref outputs "out"))
                      (bin (string-append out "/bin"))
                      (tools-bin (string-append out "/tools/bin"))
                      (cargo-out (assoc-ref outputs "cargo"))
                      (cargo-bin (string-append cargo-out "/bin"))
                      (lib (string-append out "/lib"))
                      (lib/rust (string-append lib "/mrust"))
                      (gcc (assoc-ref inputs "gcc")))
                 ;; These files are not reproducible.
                 (for-each delete-file (find-files "output" "\\.txt$"))
                 (mkdir-p lib)
                 (copy-recursively "output" lib/rust)
                 (mkdir-p bin)
                 (mkdir-p tools-bin)
                 (install-file "bin/mrustc" bin)
                 ;; TODO: Try to generate config.toml and remove this.
                 (symlink (string-append gcc "/bin/gcc")
                          (string-append out "/bin/cc"))
                 ;; minicargo uses relative paths to resolve mrustc.
                 (install-file "tools/bin/minicargo" tools-bin)
                 (install-file "tools/bin/minicargo" cargo-bin)
                 #t))))))
      (synopsis "Compiler for the Rust progamming language")
      (description "Rust is a systems programming language that provides memory
safety and thread safety guarantees.")
      (home-page "https://github.com/thepowersgang/mrustc")
      ;; Dual licensed.
      (license (list license:asl2.0 license:expat)))))

;; FIXME rust 1.19: backtrace-sys/src/libbacktrace/configure: /usr/bin/file .
(define-public rust-1.19
  (package
    (name "rust")
    (version "1.19.0")
    (source (rust-source version "0l8c14qsf42rmkqy92ahij4vf356dbyspxcips1aswpvad81y8qm"))
    (outputs '("out" "cargo"))
    (arguments
     `(#:imported-modules ,%cargo-build-system-modules ;for `generate-checksums'
       #:modules ((guix build utils) (ice-9 match) (guix build gnu-build-system))
       #:phases
       (modify-phases %standard-phases
         (add-after 'unpack 'set-env
           (lambda _
             (setenv "SHELL" (which "sh"))
             (setenv "CONFIG_SHELL" (which "sh"))
             ;; guix llvm-3.9.1 package installs only shared libraries
             (setenv "LLVM_LINK_SHARED" "1")
             #t))
         (add-after 'unpack 'patch-tests
           (lambda* (#:key inputs #:allow-other-keys)
             (let ((bash (assoc-ref inputs "bash")))
               (substitute* "src/libstd/process.rs"
                 ;; The newline is intentional.
                 ;; There's a line length "tidy" check in Rust which would
                 ;; fail otherwise.
                 (("\"/bin/sh\"") (string-append "\n\"" bash "/bin/sh\"")))
               (substitute* "src/libstd/net/tcp.rs"
                 ;; There is no network in build environment
                 (("fn connect_timeout_unroutable")
                  "#[ignore]\nfn connect_timeout_unroutable"))
               ;; <https://lists.gnu.org/archive/html/guix-devel/2017-06/msg00222.html>
               (substitute* "src/libstd/sys/unix/process/process_common.rs"
                 (("#\\[cfg_attr\\(target_arch = \"arm\", ignore\\)\\]
    fn test_process_mask") "#[ignore]
    fn test_process_mask"))
               ;; Our ld-wrapper cannot process non-UTF8 bytes in LIBRARY_PATH.
               ;; <https://lists.gnu.org/archive/html/guix-devel/2017-06/msg00193.html>
               (delete-file-recursively "src/test/run-make/linker-output-non-utf8")
               #t)))
         (add-after 'patch-source-shebangs 'patch-cargo-checksums
           (lambda* _
             (substitute* "src/Cargo.lock"
               (("(\"checksum .* = )\".*\"" all name)
                (string-append name "\"" ,%cargo-reference-hash "\"")))
             (for-each
              (lambda (filename)
                (use-modules (guix build cargo-build-system))
                (delete-file filename)
                (let* ((dir (dirname filename)))
                  (display (string-append
                            "patch-cargo-checksums: generate-checksums for "
                            dir "\n"))
                  (generate-checksums dir ,%cargo-reference-project-file)))
              (find-files "src/vendor" ".cargo-checksum.json"))
             #t))
         (replace 'configure
           (const #t))
         (replace 'build
           (lambda* (#:key inputs outputs #:allow-other-keys)
             (let ((rustc-bootstrap (assoc-ref inputs "rustc-bootstrap")))
               (substitute* "src/librustc_errors/Cargo.toml"
                 (("[[]dependencies[]]") "
[dependencies]
term = \"0.4.4\"
"))
               (substitute* "src/librustc/Cargo.toml"
                 (("[[]dependencies[]]") "
[dependencies]
getopts = { path = \"../libgetopts\" }
"))
               (substitute* "src/librustdoc/Cargo.toml"
                 (("[[]dependencies[]]") "
[dependencies]
test = { path = \"../libtest\" }
"))

               ;; Required for the "cc" symlink.  TODO: Make unnecessary.
               (setenv "PATH" (string-append rustc-bootstrap "/bin/cc:"
                       (getenv "PATH")))
               (setenv "CFG_COMPILER_HOST_TRIPLE"
                       ,(nix-system->gnu-triplet (%current-system)))
               (setenv "CFG_RELEASE" "")
               (setenv "CFG_RELEASE_CHANNEL" "stable")
               (setenv "CFG_VERSION" "1.19.0-stable-mrustc")
               (setenv "CFG_PREFIX" "mrustc") ; FIXME output path.  FIXMEwhat's LIBDIR_RELATIVE?
               (setenv "CFG_LIBDIR_RELATIVE" "lib")
               (mkdir-p "output")
               (invoke (string-append rustc-bootstrap "/tools/bin/minicargo")
                       "src/rustc" "--vendor-dir" "src/vendor"
                       "--output-dir" (string-append ;(getcwd) ; FIXME: Remove.
                                       "output/rustc-build")
                       "-L" (string-append rustc-bootstrap "/lib/mrust")
                       "-j" "1")
               (install-file "output/rustc-build/rustc" "output") ; FIXME: Remove?
                                        ;(setenv "CC" "gcc")
               (setenv "CFG_COMPILER_HOST_TRIPLE" #f)
               (setenv "CFG_RELEASE" #f)
               (setenv "CFG_RELEASE_CHANNEL" #f)
               (setenv "CFG_VERSION" #f)
               (setenv "CFG_PREFIX" #f)
               (setenv "CFG_LIBDIR_RELATIVE" #f)
               (invoke (string-append rustc-bootstrap "/tools/bin/minicargo")
                       "src/tools/cargo" "--vendor-dir" "src/vendor"
                       "--output-dir" (string-append ;(getcwd) ; FIXME: Remove.
                                       "output/cargo-build")
                       "-L" "output/"
                       "-L" (string-append rustc-bootstrap "/lib/mrust")
                       "-j" "1")
                       ;; Now use the newly-built rustc to build the libraries.
                       ;; (Cargo has problems with libstd's circular dependencies)
               (for-each (match-lambda
                           ((name . flags)
                            (apply invoke
                                   "output/rust-build/rustc"
                                   "-L" "output/target-libs"
                                   (string-append "src/" name "/lib.rs")
                                   "-o"
                                   (string-append "output/target-libs/"
                                                  name ".rlib")
                                   flags)))
                         '(("libcore")
                           ("libcollections")
                                   ;; FIXME: Add libarena
                           ("libstd_unicode")
                           ("libcompiler_builtins")
                           ("liblibc" "--cfg" "stdbuild")
                           ("libunwind" "-l" "gcc_s")
                           ("liballoc")
                           ("libstd" "-l" "dl" "-l" "rt" "-l" "pthread")))
                                        ; toml "cargo", "rustc"; Alternative: CFG_LOCAL_RUST_ROOT in get_mk (config.mk). => /bin/rustc; or "cargo" in config.toml . ;FIXME (invoke "output/cargo-build/cargo" "build" "--manifest-path" "src/bootstrap/Cargo.toml" "--verbose") ; "--locked" "--frozen"
               #t)))
         (replace 'check
           (const #t))
         (replace 'install
           (lambda* (#:key inputs outputs #:allow-other-keys)
             (let* ((out (assoc-ref outputs "out"))
                    (target-system ,(or (%current-target-system)
                                        (nix-system->gnu-triplet
                                         (%current-system))))
                    (out-libs (string-append out "/lib/rustlib/"
                                             target-system "/lib")))
                                        ;(setenv "CFG_PREFIX" out)
               (mkdir-p out-libs)
               (copy-recursively "output/target-libs" out-libs)
               (install-file "output/rustc-build/rustc"
                             (string-append out "/bin"))
               (install-file "output/cargo-build/cargo"
                             (string-append (assoc-ref outputs "cargo")
                                            "/bin")))
             #t)))))
    (build-system gnu-build-system)
    (native-inputs
     `(("bison" ,bison)                 ; For the tests
       ("cmake" ,cmake)
       ("flex" ,flex)                   ; For the tests
       ("gdb" ,gdb)                     ; For the tests
       ("git" ,git)
       ("procps" ,procps)               ; For the tests
       ("strace" ,strace)               ; FIXME Remove.
       ("python-2" ,python-2)
       ("rustc-bootstrap" ,mrustc)
       ("cargo-bootstrap" ,mrustc "cargo")
       ("pkg-config" ,pkg-config)       ; For "cargo"
       ("which" ,which)))
    (inputs
     `(("jemalloc" ,jemalloc-4.5.0)
       ("llvm" ,llvm-3.9.1)
       ("openssl" ,openssl)
       ("libcurl" ,curl)))              ; For "cargo"
    (native-search-paths (package-native-search-paths gcc))
    (synopsis "Compiler for the Rust progamming language")
    (description "Rust is a systems programming language that provides memory
safety and thread safety guarantees.")
    (home-page "https://www.rust-lang.org")
    ;; Dual licensed.
    (license (list license:asl2.0 license:expat))))

(define (rust-bootstrapped-package base-rust version checksum)
  "Bootstrap rust VERSION with source checksum CHECKSUM using BASE-RUST."
  (package
    (inherit base-rust)
    (version version)
    (source
     (rust-source version checksum))
    (native-inputs
     (alist-replace "cargo-bootstrap" (list base-rust "cargo")
                    (alist-replace "rustc-bootstrap" (list base-rust)
                                   (package-native-inputs base-rust))))))

;; FIXME: Make private.
(define-public rust-1.20
  (let ((base-rust
         (rust-bootstrapped-package rust-1.19 "1.20.0"
                                    "0542y4rnzlsrricai130mqyxl8r6rd991frb4qsnwb27yigqg91a")))
    (package
      (inherit base-rust)
      (outputs '("out" "doc" "cargo"))
      (native-inputs
       (cons* ;(list "strace" strace)
             (package-native-inputs base-rust)))
      (arguments
       (substitute-keyword-arguments (package-arguments base-rust)
         ((#:phases phases)
          `(modify-phases ,phases
             (replace 'configure
               (lambda* (#:key inputs outputs #:allow-other-keys)
                 (let* ((out (assoc-ref outputs "out"))
                        (doc (assoc-ref outputs "doc"))
                        (gcc (assoc-ref inputs "gcc"))
                        (gdb (assoc-ref inputs "gdb"))
                        (binutils (assoc-ref inputs "binutils"))
                        (python (assoc-ref inputs "python-2"))
                        (rustc (assoc-ref inputs "rustc-bootstrap"))
                        (cargo (assoc-ref inputs "cargo-bootstrap"))
                        (llvm (assoc-ref inputs "llvm"))
                        (jemalloc (assoc-ref inputs "jemalloc")))
                   (call-with-output-file "config.toml"
                     (lambda (port)
                       (display (string-append "
[llvm]
[build]
cargo = \"" cargo "/bin/cargo" "\"
rustc = \"" rustc "/bin/rustc" "\"
docs = true
python = \"" python "/bin/python2" "\"
gdb = \"" gdb "/bin/gdb" "\"
vendor = true
submodules = false
[install]
prefix = \"" out "\"
docdir = \"" doc "/share/doc/rust" "\"
sysconfdir = \"etc\"
localstatedir = \"var/lib\"
[rust]
default-linker = \"" gcc "/bin/gcc" "\"
channel = \"stable\"
rpath = true
# There is 2 failed codegen tests:
# codegen/mainsubprogram.rs and codegen/mainsubprogramstart.rs
# These tests required patched LLVM
codegen-tests = false
[target." %host-type "]
llvm-config = \"" llvm "/bin/llvm-config" "\"
cc = \"" gcc "/bin/gcc" "\"
cxx = \"" gcc "/bin/g++" "\"
ar = \"" binutils "/bin/ar" "\"
jemalloc = \"" jemalloc "/lib/libjemalloc_pic.a" "\"
[dist]
") port)))
               #t)))
         (replace 'build
           (lambda* _
             (invoke "./x.py" "build")
             (invoke "./x.py" "build" "src/tools/cargo")))
         (add-before 'build 'reset-timestamps-after-changes
           (lambda* _
             (define ref (stat "README.md"))
             (for-each
              (lambda (filename)
                (set-file-time filename ref))
              (find-files "." #:directories? #t))
             #t))
         (replace 'check
           (lambda* _
             (invoke "./x.py" "test")))
         (replace 'install
           (lambda* (#:key outputs #:allow-other-keys)
             (invoke "./x.py" "install")
             (substitute* "config.toml"
               ;; replace prefix to specific output
               (("prefix = \"[^\"]*\"")
                (string-append "prefix = \"" (assoc-ref outputs "cargo") "\"")))
             (invoke "./x.py" "install" "cargo")
             #t))
         (add-after 'install 'wrap-rustc
           (lambda* (#:key inputs outputs #:allow-other-keys)
             (let ((out (assoc-ref outputs "out"))
                   (libc (assoc-ref inputs "libc"))
                   (ld-wrapper (assoc-ref inputs "ld-wrapper")))
               ;; Let gcc find ld and libc startup files.
               (wrap-program (string-append out "/bin/rustc")
                 `("PATH" ":" prefix (,(string-append ld-wrapper "/bin")))
                 `("LIBRARY_PATH" ":" suffix (,(string-append libc "/lib"))))
               #t))))))))))

(define-public rust-1.21
  (rust-bootstrapped-package rust-1.20 "1.21.0"
                             "1yj8lnxybjrybp00fqhxw8fpr641dh8wcn9mk44xjnsb4i1c21qp"))

(define-public rust-1.22
  (rust-bootstrapped-package rust-1.21 "1.22.0"
                             "0saaprfb01z95gxlqqki28a3rq7p5a7labxdn3w1d9n49yy6zj8a"))

(define-public rust-1.23
  (let ((base-rust
         (rust-bootstrapped-package rust-1.22 "1.23.0"
                                    "5vv10x2h9kq7fxh2v01damdq8pvlp5acyh1kzcda9sfjx12kv99y")))
    (package
      (inherit base-rust)
      (arguments
       (substitute-keyword-arguments (package-arguments base-rust)
         ((#:phases phases)
          `(modify-phases ,phases
             (add-after 'patch-tests 'fix-mtime-bug
               (lambda* _
                 (substitute* "src/build_helper/lib.rs"
                   ;; Bug in Rust code.
                   ;; Current implementation assume that if dst not exist then its mtime
                   ;; is 0, but in same time "src" have 0 mtime in guix build!
                   (("let threshold = mtime\\(dst\\);")
                    "if !dst.exists() {\nreturn false\n}\n let threshold = mtime(dst);"))
                 #t)))))))))

(define-public rust-1.24
  (let ((base-rust
         (rust-bootstrapped-package rust-1.23 "1.24.1"
                                    "1vv10x2h9kq7fxh2v01damdq8pvlp5acyh1kzcda9sfjx12kv99y")))
    (package
      (inherit base-rust)
      (arguments
       (substitute-keyword-arguments (package-arguments base-rust)
         ((#:phases phases)
          `(modify-phases ,phases
             (delete 'fix-mtime-bug))))))))

(define-public rust rust-1.24)

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 488 bytes --]