[bug#31026] [PATCH 0/2] Split off rust@1.19.0.

[Nils Gillmann <ng0@n0.is>]

Danny Milosavljevic transcribed 24K bytes:
> On Mon, 2 Apr 2018 18:19:21 +0000
> Nils Gillmann <ng0@n0.is> wrote:
> 
> > Danny Milosavljevic transcribed 260 bytes:
> > > Prepare for mrustc bootstrap.  
> > 
> > Woo, good news. Did I miss an email 
> 
> I've been trying to use mrustc to bootstrap rust on and off for quite
> some time now.  The upstream authors have been busy too and it's far enough
> that I have a working rust-1.19.0 compiler compiled by mrustc in /gnu/store :)
> 
> The libstd still had some rough edges - and one test takes about 30 h of
> building - so it will still take some time until this is done.
> 
> I thought in the meantime I'd clean up and split up the patches.
> 
> These here are some of the patches that can be applied without switching to mrustc
> yet (that is, they just refactor stuff).
> 
> >or does this still require some review?
> 
> Right now, mrustc officially *supports* x86_64 targets only.
> 
> aarch64 is under review upstream (support is there but unfinished).
> 
> We will only be able to use mrustc to bootstrap if it supports more architectures.
> 
> We will see when that comes to pass.
> 
> mrustc is written in a way that it can compile rust@1.19.0 so that's why this
> exact version is in this patchset.  I hope in the future mrustc can compile
> newer versions of Rust.
> 
> > I can reserve some time for building this tonight and review tomorrow at some
> > point.

I have to pass on this one, delay it for a couple of days. Too much to do.
Someone else should do it. What do describe reads like lots of time spend
building.

> Nice!
> 
> I've attached my current gnu/packages/rust.scm here - which includes some things
> not ready to merge yet.  But it should (famous last words) bootstrap rust 1.24
> starting from mrustc in multiple steps.

> ;;; GNU Guix --- Functional package management for GNU
> ;;; Copyright © 2016 David Craven <david@craven.ch>
> ;;; Copyright © 2016 Eric Le Bihan <eric.le.bihan.dev@free.fr>
> ;;; Copyright © 2016 Nils Gillmann <ng0@n0.is>
> ;;; Copyright © 2017 Ben Woodcroft <donttrustben@gmail.com>
> ;;; Copyright © 2017, 2018 Nikolai Merinov <nikolai.merinov@member.fsf.org>
> ;;; Copyright © 2017 Efraim Flashner <efraim@flashner.co.il>
> ;;; Copyright © 2018 Tobias Geerinckx-Rice <me@tobias.gr>
> ;;; Copyright © 2018 Danny Milosavljevic <dannym+a@scratchpost.org>
> ;;;
> ;;; This file is part of GNU Guix.
> ;;;
> ;;; GNU Guix is free software; you can redistribute it and/or modify it
> ;;; under the terms of the GNU General Public License as published by
> ;;; the Free Software Foundation; either version 3 of the License, or (at
> ;;; your option) any later version.
> ;;;
> ;;; GNU Guix is distributed in the hope that it will be useful, but
> ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
> ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> ;;; GNU General Public License for more details.
> ;;;
> ;;; You should have received a copy of the GNU General Public License
> ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
> 
> (define-module (gnu packages rust)
>   #:use-module (gnu packages base)
>   #:use-module (gnu packages bison)
>   #:use-module (gnu packages bootstrap)
>   #:use-module (gnu packages cmake)
>   #:use-module (gnu packages compression)
>   #:use-module (gnu packages curl)
>   #:use-module (gnu packages elf)
>   #:use-module (gnu packages flex)
>   #:use-module (gnu packages gcc)
>   #:use-module (gnu packages gdb)
>   #:use-module (gnu packages jemalloc)
>   #:use-module (gnu packages linux)
>   #:use-module (gnu packages llvm)
>   #:use-module (gnu packages pkg-config)
>   #:use-module (gnu packages python)
>   #:use-module (gnu packages ssh)
>   #:use-module (gnu packages tls)
>   #:use-module (gnu packages version-control)
>   #:use-module (gnu packages)
>   #:use-module (guix build-system cargo)
>   #:use-module (guix build-system gnu)
>   #:use-module (guix build-system trivial)
>   #:use-module (guix download)
>   #:use-module (guix git-download)
>   #:use-module ((guix licenses) #:prefix license:)
>   #:use-module (guix packages)
>   #:use-module ((guix build utils) #:select (alist-replace))
>   #:use-module (guix utils)
>   #:use-module (ice-9 match)
>   #:use-module (srfi srfi-26))
> 
> (define %cargo-reference-project-file "/dev/null")
> (define %cargo-reference-hash
>   "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
> 
> (define (rust-source version hash)
>   (origin
>     (method url-fetch)
>     (uri (string-append "https://static.rust-lang.org/dist/"
>                         "rustc-" version "-src.tar.gz"))
>     (sha256 (base32 hash))
>     (modules '((guix build utils)))
>     (snippet
>      '(begin
>          (delete-file-recursively "src/llvm")
>        #t))))
> 
> \f
> (define-public mrustc
>   (let ((commit "1a16def28935548e29be0fe5a632e25c83045924")
>         (revision "0")
>         (rustc-version "1.19.0"))
>     (package
>       (name "mrustc")
>       (version (git-version "0.0.0" revision commit))
>       (source (origin
>                 (method git-fetch)
>                 (uri (git-reference
>                       (url "https://github.com/thepowersgang/mrustc.git")
>                       (commit commit)))
>                 (file-name (git-file-name name version))
>                 (sha256
>                  (base32
>                   "0i5kqnzsd1rkj8qm147jx934nyn9sd1fz2b4achj9z0r00h84dh6"))))
>       (outputs '("out" "cargo"))
>       (build-system gnu-build-system)
>       (native-inputs
>        `(("bison" ,bison)
>          ("flex" ,flex)
>          ("llvm" ,llvm-3.9.1)
>          ;; Required for the libstd sources.
>          ("rustc"
>           ,(rust-source "1.19.0" "0l8c14qsf42rmkqy92ahij4vf356dbyspxcips1aswpvad81y8qm"))))
>       (arguments
>        `(#:tests? #f

Maybe add something like ";Tests require rustc"?
At least my ticket upstream wasn't closed last time I looked at it.

>          #:make-flags (list (string-append "LLVM_CONFIG="
>                                            (assoc-ref %build-inputs "llvm")
>                                            "/bin/llvm-config"))
>          #:phases
>          (modify-phases %standard-phases
>            (add-after 'unpack 'unpack-target-compiler
>              (lambda* (#:key inputs outputs #:allow-other-keys)
>                (substitute* "minicargo.mk"
>                  ;; Don't try to build LLVM.
>                  (("^[$][(]LLVM_CONFIG[)]:") "xxx:")
>                  ;; Build for the correct target architecture.
>                  (("^RUSTC_TARGET := x86_64-unknown-linux-gnu")
>                   (string-append "RUSTC_TARGET := "
>                                  ,(or (%current-target-system)
>                                       (nix-system->gnu-triplet
>                                        (%current-system))))))
>                (invoke "tar" "xf" (assoc-ref inputs "rustc"))
>                (chdir "rustc-1.19.0-src")
>                (invoke "patch" "-p0" "../rust_src.patch")
>                (chdir "..")
>                #t))
>            (delete 'configure)
>            (add-after 'build 'build-minicargo
>              (lambda _
>                (for-each (lambda (target)
>                            (invoke "make" "-f" "minicargo.mk" target))
>                          '("output/libstd.hir" "output/libpanic_unwind.hir"
>                            "output/libproc_macro.hir" "output/libtest.hir"))
>                ;; Technically the above already does it - but we want to be clear.
>                (invoke "make" "-C" "tools/minicargo")))
>            (replace 'install
>              (lambda* (#:key inputs outputs #:allow-other-keys)
>                (let* ((out (assoc-ref outputs "out"))
>                       (bin (string-append out "/bin"))
>                       (tools-bin (string-append out "/tools/bin"))
>                       (cargo-out (assoc-ref outputs "cargo"))
>                       (cargo-bin (string-append cargo-out "/bin"))
>                       (lib (string-append out "/lib"))
>                       (lib/rust (string-append lib "/mrust"))
>                       (gcc (assoc-ref inputs "gcc")))
>                  ;; These files are not reproducible.
>                  (for-each delete-file (find-files "output" "\\.txt$"))
>                  (mkdir-p lib)
>                  (copy-recursively "output" lib/rust)
>                  (mkdir-p bin)
>                  (mkdir-p tools-bin)
>                  (install-file "bin/mrustc" bin)
>                  ;; TODO: Try to generate config.toml and remove this.
>                  (symlink (string-append gcc "/bin/gcc")
>                           (string-append out "/bin/cc"))
>                  ;; minicargo uses relative paths to resolve mrustc.
>                  (install-file "tools/bin/minicargo" tools-bin)
>                  (install-file "tools/bin/minicargo" cargo-bin)
>                  #t))))))
>       (synopsis "Compiler for the Rust progamming language")
>       (description "Rust is a systems programming language that provides memory
> safety and thread safety guarantees.")
>       (home-page "https://github.com/thepowersgang/mrustc")
>       ;; Dual licensed.
>       (license (list license:asl2.0 license:expat)))))
> 
> ;; FIXME rust 1.19: backtrace-sys/src/libbacktrace/configure: /usr/bin/file .
> (define-public rust-1.19
>   (package
>     (name "rust")
>     (version "1.19.0")
>     (source (rust-source version "0l8c14qsf42rmkqy92ahij4vf356dbyspxcips1aswpvad81y8qm"))
>     (outputs '("out" "cargo"))
>     (arguments
>      `(#:imported-modules ,%cargo-build-system-modules ;for `generate-checksums'
>        #:modules ((guix build utils) (ice-9 match) (guix build gnu-build-system))
>        #:phases
>        (modify-phases %standard-phases
>          (add-after 'unpack 'set-env
>            (lambda _
>              (setenv "SHELL" (which "sh"))
>              (setenv "CONFIG_SHELL" (which "sh"))
>              ;; guix llvm-3.9.1 package installs only shared libraries
>              (setenv "LLVM_LINK_SHARED" "1")
>              #t))
>          (add-after 'unpack 'patch-tests
>            (lambda* (#:key inputs #:allow-other-keys)
>              (let ((bash (assoc-ref inputs "bash")))
>                (substitute* "src/libstd/process.rs"
>                  ;; The newline is intentional.
>                  ;; There's a line length "tidy" check in Rust which would
>                  ;; fail otherwise.
>                  (("\"/bin/sh\"") (string-append "\n\"" bash "/bin/sh\"")))
>                (substitute* "src/libstd/net/tcp.rs"
>                  ;; There is no network in build environment
>                  (("fn connect_timeout_unroutable")
>                   "#[ignore]\nfn connect_timeout_unroutable"))
>                ;; <https://lists.gnu.org/archive/html/guix-devel/2017-06/msg00222.html>
>                (substitute* "src/libstd/sys/unix/process/process_common.rs"
>                  (("#\\[cfg_attr\\(target_arch = \"arm\", ignore\\)\\]
>     fn test_process_mask") "#[ignore]
>     fn test_process_mask"))
>                ;; Our ld-wrapper cannot process non-UTF8 bytes in LIBRARY_PATH.
>                ;; <https://lists.gnu.org/archive/html/guix-devel/2017-06/msg00193.html>
>                (delete-file-recursively "src/test/run-make/linker-output-non-utf8")
>                #t)))
>          (add-after 'patch-source-shebangs 'patch-cargo-checksums
>            (lambda* _
>              (substitute* "src/Cargo.lock"
>                (("(\"checksum .* = )\".*\"" all name)
>                 (string-append name "\"" ,%cargo-reference-hash "\"")))
>              (for-each
>               (lambda (filename)
>                 (use-modules (guix build cargo-build-system))
>                 (delete-file filename)
>                 (let* ((dir (dirname filename)))
>                   (display (string-append
>                             "patch-cargo-checksums: generate-checksums for "
>                             dir "\n"))
>                   (generate-checksums dir ,%cargo-reference-project-file)))
>               (find-files "src/vendor" ".cargo-checksum.json"))
>              #t))
>          (replace 'configure
>            (const #t))
>          (replace 'build
>            (lambda* (#:key inputs outputs #:allow-other-keys)
>              (let ((rustc-bootstrap (assoc-ref inputs "rustc-bootstrap")))
>                (substitute* "src/librustc_errors/Cargo.toml"
>                  (("[[]dependencies[]]") "
> [dependencies]
> term = \"0.4.4\"
> "))
>                (substitute* "src/librustc/Cargo.toml"
>                  (("[[]dependencies[]]") "
> [dependencies]
> getopts = { path = \"../libgetopts\" }
> "))
>                (substitute* "src/librustdoc/Cargo.toml"
>                  (("[[]dependencies[]]") "
> [dependencies]
> test = { path = \"../libtest\" }
> "))
> 
>                ;; Required for the "cc" symlink.  TODO: Make unnecessary.
>                (setenv "PATH" (string-append rustc-bootstrap "/bin/cc:"
>                        (getenv "PATH")))
>                (setenv "CFG_COMPILER_HOST_TRIPLE"
>                        ,(nix-system->gnu-triplet (%current-system)))
>                (setenv "CFG_RELEASE" "")
>                (setenv "CFG_RELEASE_CHANNEL" "stable")
>                (setenv "CFG_VERSION" "1.19.0-stable-mrustc")
>                (setenv "CFG_PREFIX" "mrustc") ; FIXME output path.  FIXMEwhat's LIBDIR_RELATIVE?
>                (setenv "CFG_LIBDIR_RELATIVE" "lib")
>                (mkdir-p "output")
>                (invoke (string-append rustc-bootstrap "/tools/bin/minicargo")
>                        "src/rustc" "--vendor-dir" "src/vendor"
>                        "--output-dir" (string-append ;(getcwd) ; FIXME: Remove.
>                                        "output/rustc-build")
>                        "-L" (string-append rustc-bootstrap "/lib/mrust")
>                        "-j" "1")
>                (install-file "output/rustc-build/rustc" "output") ; FIXME: Remove?
>                                         ;(setenv "CC" "gcc")
>                (setenv "CFG_COMPILER_HOST_TRIPLE" #f)
>                (setenv "CFG_RELEASE" #f)
>                (setenv "CFG_RELEASE_CHANNEL" #f)
>                (setenv "CFG_VERSION" #f)
>                (setenv "CFG_PREFIX" #f)
>                (setenv "CFG_LIBDIR_RELATIVE" #f)
>                (invoke (string-append rustc-bootstrap "/tools/bin/minicargo")
>                        "src/tools/cargo" "--vendor-dir" "src/vendor"
>                        "--output-dir" (string-append ;(getcwd) ; FIXME: Remove.
>                                        "output/cargo-build")
>                        "-L" "output/"
>                        "-L" (string-append rustc-bootstrap "/lib/mrust")
>                        "-j" "1")
>                        ;; Now use the newly-built rustc to build the libraries.
>                        ;; (Cargo has problems with libstd's circular dependencies)
>                (for-each (match-lambda
>                            ((name . flags)
>                             (apply invoke
>                                    "output/rust-build/rustc"
>                                    "-L" "output/target-libs"
>                                    (string-append "src/" name "/lib.rs")
>                                    "-o"
>                                    (string-append "output/target-libs/"
>                                                   name ".rlib")
>                                    flags)))
>                          '(("libcore")
>                            ("libcollections")
>                                    ;; FIXME: Add libarena
>                            ("libstd_unicode")
>                            ("libcompiler_builtins")
>                            ("liblibc" "--cfg" "stdbuild")
>                            ("libunwind" "-l" "gcc_s")
>                            ("liballoc")
>                            ("libstd" "-l" "dl" "-l" "rt" "-l" "pthread")))
>                                         ; toml "cargo", "rustc"; Alternative: CFG_LOCAL_RUST_ROOT in get_mk (config.mk). => /bin/rustc; or "cargo" in config.toml . ;FIXME (invoke "output/cargo-build/cargo" "build" "--manifest-path" "src/bootstrap/Cargo.toml" "--verbose") ; "--locked" "--frozen"
>                #t)))
>          (replace 'check
>            (const #t))
>          (replace 'install
>            (lambda* (#:key inputs outputs #:allow-other-keys)
>              (let* ((out (assoc-ref outputs "out"))
>                     (target-system ,(or (%current-target-system)
>                                         (nix-system->gnu-triplet
>                                          (%current-system))))
>                     (out-libs (string-append out "/lib/rustlib/"
>                                              target-system "/lib")))
>                                         ;(setenv "CFG_PREFIX" out)
>                (mkdir-p out-libs)
>                (copy-recursively "output/target-libs" out-libs)
>                (install-file "output/rustc-build/rustc"
>                              (string-append out "/bin"))
>                (install-file "output/cargo-build/cargo"
>                              (string-append (assoc-ref outputs "cargo")
>                                             "/bin")))
>              #t)))))
>     (build-system gnu-build-system)
>     (native-inputs
>      `(("bison" ,bison)                 ; For the tests
>        ("cmake" ,cmake)
>        ("flex" ,flex)                   ; For the tests
>        ("gdb" ,gdb)                     ; For the tests
>        ("git" ,git)
>        ("procps" ,procps)               ; For the tests
>        ("strace" ,strace)               ; FIXME Remove.
>        ("python-2" ,python-2)
>        ("rustc-bootstrap" ,mrustc)
>        ("cargo-bootstrap" ,mrustc "cargo")
>        ("pkg-config" ,pkg-config)       ; For "cargo"
>        ("which" ,which)))
>     (inputs
>      `(("jemalloc" ,jemalloc-4.5.0)
>        ("llvm" ,llvm-3.9.1)
>        ("openssl" ,openssl)
>        ("libcurl" ,curl)))              ; For "cargo"
>     (native-search-paths (package-native-search-paths gcc))
>     (synopsis "Compiler for the Rust progamming language")
>     (description "Rust is a systems programming language that provides memory
> safety and thread safety guarantees.")
>     (home-page "https://www.rust-lang.org")
>     ;; Dual licensed.
>     (license (list license:asl2.0 license:expat))))
> 
> (define (rust-bootstrapped-package base-rust version checksum)
>   "Bootstrap rust VERSION with source checksum CHECKSUM using BASE-RUST."
>   (package
>     (inherit base-rust)
>     (version version)
>     (source
>      (rust-source version checksum))
>     (native-inputs
>      (alist-replace "cargo-bootstrap" (list base-rust "cargo")
>                     (alist-replace "rustc-bootstrap" (list base-rust)
>                                    (package-native-inputs base-rust))))))
> 
> ;; FIXME: Make private.
> (define-public rust-1.20
>   (let ((base-rust
>          (rust-bootstrapped-package rust-1.19 "1.20.0"
>                                     "0542y4rnzlsrricai130mqyxl8r6rd991frb4qsnwb27yigqg91a")))
>     (package
>       (inherit base-rust)
>       (outputs '("out" "doc" "cargo"))
>       (native-inputs
>        (cons* ;(list "strace" strace)
>              (package-native-inputs base-rust)))
>       (arguments
>        (substitute-keyword-arguments (package-arguments base-rust)
>          ((#:phases phases)
>           `(modify-phases ,phases
>              (replace 'configure
>                (lambda* (#:key inputs outputs #:allow-other-keys)
>                  (let* ((out (assoc-ref outputs "out"))
>                         (doc (assoc-ref outputs "doc"))
>                         (gcc (assoc-ref inputs "gcc"))
>                         (gdb (assoc-ref inputs "gdb"))
>                         (binutils (assoc-ref inputs "binutils"))
>                         (python (assoc-ref inputs "python-2"))
>                         (rustc (assoc-ref inputs "rustc-bootstrap"))
>                         (cargo (assoc-ref inputs "cargo-bootstrap"))
>                         (llvm (assoc-ref inputs "llvm"))
>                         (jemalloc (assoc-ref inputs "jemalloc")))
>                    (call-with-output-file "config.toml"
>                      (lambda (port)
>                        (display (string-append "
> [llvm]
> [build]
> cargo = \"" cargo "/bin/cargo" "\"
> rustc = \"" rustc "/bin/rustc" "\"
> docs = true
> python = \"" python "/bin/python2" "\"
> gdb = \"" gdb "/bin/gdb" "\"
> vendor = true
> submodules = false
> [install]
> prefix = \"" out "\"
> docdir = \"" doc "/share/doc/rust" "\"
> sysconfdir = \"etc\"
> localstatedir = \"var/lib\"
> [rust]
> default-linker = \"" gcc "/bin/gcc" "\"
> channel = \"stable\"
> rpath = true
> # There is 2 failed codegen tests:
> # codegen/mainsubprogram.rs and codegen/mainsubprogramstart.rs
> # These tests required patched LLVM
> codegen-tests = false
> [target." %host-type "]
> llvm-config = \"" llvm "/bin/llvm-config" "\"
> cc = \"" gcc "/bin/gcc" "\"
> cxx = \"" gcc "/bin/g++" "\"
> ar = \"" binutils "/bin/ar" "\"
> jemalloc = \"" jemalloc "/lib/libjemalloc_pic.a" "\"
> [dist]
> ") port)))
>                #t)))
>          (replace 'build
>            (lambda* _
>              (invoke "./x.py" "build")
>              (invoke "./x.py" "build" "src/tools/cargo")))
>          (add-before 'build 'reset-timestamps-after-changes
>            (lambda* _
>              (define ref (stat "README.md"))
>              (for-each
>               (lambda (filename)
>                 (set-file-time filename ref))
>               (find-files "." #:directories? #t))
>              #t))
>          (replace 'check
>            (lambda* _
>              (invoke "./x.py" "test")))
>          (replace 'install
>            (lambda* (#:key outputs #:allow-other-keys)
>              (invoke "./x.py" "install")
>              (substitute* "config.toml"
>                ;; replace prefix to specific output
>                (("prefix = \"[^\"]*\"")
>                 (string-append "prefix = \"" (assoc-ref outputs "cargo") "\"")))
>              (invoke "./x.py" "install" "cargo")
>              #t))
>          (add-after 'install 'wrap-rustc
>            (lambda* (#:key inputs outputs #:allow-other-keys)
>              (let ((out (assoc-ref outputs "out"))
>                    (libc (assoc-ref inputs "libc"))
>                    (ld-wrapper (assoc-ref inputs "ld-wrapper")))
>                ;; Let gcc find ld and libc startup files.
>                (wrap-program (string-append out "/bin/rustc")
>                  `("PATH" ":" prefix (,(string-append ld-wrapper "/bin")))
>                  `("LIBRARY_PATH" ":" suffix (,(string-append libc "/lib"))))
>                #t))))))))))
> 
> (define-public rust-1.21
>   (rust-bootstrapped-package rust-1.20 "1.21.0"
>                              "1yj8lnxybjrybp00fqhxw8fpr641dh8wcn9mk44xjnsb4i1c21qp"))
> 
> (define-public rust-1.22
>   (rust-bootstrapped-package rust-1.21 "1.22.0"
>                              "0saaprfb01z95gxlqqki28a3rq7p5a7labxdn3w1d9n49yy6zj8a"))
> 
> (define-public rust-1.23
>   (let ((base-rust
>          (rust-bootstrapped-package rust-1.22 "1.23.0"
>                                     "5vv10x2h9kq7fxh2v01damdq8pvlp5acyh1kzcda9sfjx12kv99y")))
>     (package
>       (inherit base-rust)
>       (arguments
>        (substitute-keyword-arguments (package-arguments base-rust)
>          ((#:phases phases)
>           `(modify-phases ,phases
>              (add-after 'patch-tests 'fix-mtime-bug
>                (lambda* _
>                  (substitute* "src/build_helper/lib.rs"
>                    ;; Bug in Rust code.
>                    ;; Current implementation assume that if dst not exist then its mtime
>                    ;; is 0, but in same time "src" have 0 mtime in guix build!
>                    (("let threshold = mtime\\(dst\\);")
>                     "if !dst.exists() {\nreturn false\n}\n let threshold = mtime(dst);"))
>                  #t)))))))))
> 
> (define-public rust-1.24
>   (let ((base-rust
>          (rust-bootstrapped-package rust-1.23 "1.24.1"
>                                     "1vv10x2h9kq7fxh2v01damdq8pvlp5acyh1kzcda9sfjx12kv99y")))
>     (package
>       (inherit base-rust)
>       (arguments
>        (substitute-keyword-arguments (package-arguments base-rust)
>          ((#:phases phases)
>           `(modify-phases ,phases
>              (delete 'fix-mtime-bug))))))))
> 
> (define-public rust rust-1.24)