Re: Preparing the reduced bootstrap tarballs

[Jan Nieuwenhuizen <janneke@gnu.org>]

[-- Attachment #1: Type: text/plain, Size: 1701 bytes --]

Ludovic Courtès writes:

> Jeremiah@pdp10.guru skribis:
>
>>> So if you like, please make that change.  There is only one little
>>> thing: I have no (scripted) recipe to create mescc-tools-seed-XYZ.  But
>>> wait: I have a great excuse for that...I was too lazy or too sloppy.
>>
>> I do, in mescc-tools-seed; the script bootstrap.sh when run with the
>> option "sin" will build the mescc-tools-seed binaries using mescc-tools.
>> The .M1 files are always generated by cc_x86.s using the C source files.
>
> I saw this script but it’s not entirely clear to me how to package the
> whole thing.  We don’t have a “stage0” package for instance in Guix, do
> we?

I had a look but it seems we need either binary seeds for stage0, or
binary seeds for mescc-tools and I'd rather build them within Guix; so
that's what I did now.

>>> WDYT?
>> I think we will end up having several versions of mescc-tools-seed; as
>> each architecture guix supports will end up needing a variant if we plan
>> on keeping them small. (I also have no idea how to make a multi-arch fat
>> elf binary)
>
> For now let’s focus on x86_64/i686.  :-)
>
> IMO we should change the seeds as rarely as possible because they are
> managed “out-of-band” and verifying them is difficult (you need to fetch
> the right Guix commit, run “guix build bootstrap-tarballs”, and compare
> the result—assuming this is all bit-reproducible.)
>
> The one we’re using today in Guix date back to 2013.

Okay, I am proposing the attached patches, also available on my 
core-updates-next at http://gitlab.com/janneke/guix

Still I hope these won't last for five more years ;-)
janneke.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-bootstrap-Add-mescc-tools-static-mescc-tools-static-.patch --]
[-- Type: text/x-patch, Size: 1884 bytes --]

From 5702107a31f52a615c516084b7a82d9f5e2967e0 Mon Sep 17 00:00:00 2001
From: Jan Nieuwenhuizen <janneke@gnu.org>
Date: Fri, 16 Nov 2018 19:29:55 +0100
Subject: [PATCH 1/3] bootstrap: Add mescc-tools-static,
 mescc-tools-static-tarball.

* gnu/packages/make-bootstrap.scm (%mescc-tools-static,
%mescc-tools-bootstrap-tarball):  New variable.
---
 gnu/packages/make-bootstrap.scm | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/gnu/packages/make-bootstrap.scm b/gnu/packages/make-bootstrap.scm
index 7bfba3c14..36a5f6904 100644
--- a/gnu/packages/make-bootstrap.scm
+++ b/gnu/packages/make-bootstrap.scm
@@ -48,6 +48,7 @@
             %glibc-bootstrap-tarball
             %gcc-bootstrap-tarball
             %guile-bootstrap-tarball
+            %mescc-tools-bootstrap-tarball
             %mes-bootstrap-tarball
             %bootstrap-tarballs
 
@@ -535,6 +536,16 @@ for `sh' in $PATH, and without nscd, and with static NSS modules."
            #t))))
     (inputs `(("gcc" ,%gcc-static)))))
 
+(define %mescc-tools-static
+  ;; A statically linked MesCC Tools for bootstrap.
+  (package
+    (inherit mescc-tools)
+    (name "mescc-tools-static")
+    (arguments
+     (substitute-keyword-arguments (package-arguments mescc-tools)
+       ((#:make-flags flags)
+        `(cons "CC=gcc -static" ,flags))))))
+
 (define %mes-stripped
   ;; The subset of Mes files needed for bootstrap.
   (package
@@ -731,6 +742,10 @@ for `sh' in $PATH, and without nscd, and with static NSS modules."
   ;; A tarball with the statically-linked, relocatable Guile.
   (tarball-package %guile-static-stripped))
 
+(define %mescc-tools-bootstrap-tarball
+  ;; A tarball with MesCC binary seed.
+  (tarball-package %mescc-tools-static))
+
 (define %mes-bootstrap-tarball
   ;; A tarball with Mes ASCII Seed and binary Mes C Library.
   (tarball-package %mes-stripped))
-- 
2.19.1


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #3: 0002-bootstrap-Add-bootstrap-mescc-tools.patch --]
[-- Type: text/x-patch, Size: 3163 bytes --]

From cc3782c5f9d2c606a265be0dfdca44d79ec25622 Mon Sep 17 00:00:00 2001
From: Jan Nieuwenhuizen <janneke@gnu.org>
Date: Fri, 16 Nov 2018 21:30:00 +0100
Subject: [PATCH 2/3] bootstrap: Add %bootstrap-mescc-tools.

Built with
    5702107a31f52a615c516084b7a82d9f5e2967e0 bootstrap: Add mescc-tools-static, mescc-tools-static-tarball.

* gnu/packages/bootstrap.scm (%bootstrap-mescc-tools): New variable.
---
 gnu/packages/bootstrap.scm | 49 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/gnu/packages/bootstrap.scm b/gnu/packages/bootstrap.scm
index 7e6200c57..1075325dc 100644
--- a/gnu/packages/bootstrap.scm
+++ b/gnu/packages/bootstrap.scm
@@ -49,6 +49,7 @@
             %bootstrap-gcc
             %bootstrap-glibc
             %bootstrap-inputs
+            %bootstrap-mescc-tools
             %bootstrap-mes
             %mescc-tools-seed
             %srfi-43))
@@ -615,6 +616,54 @@ exec ~a/bin/.gcc-wrapped -B~a/lib \
     (home-page #f)
     (license gpl3+)))
 
+(define %bootstrap-mescc-tools
+  ;; The initial MesCC tools.  Uses binaries from a tarball typically built by
+  ;; %MESCC-TOOLS-BOOTSTRAP-TARBALL.
+  (package
+    (name "bootstrap-mescc-tools")
+    (version "0.5.2")
+    (source #f)
+    (build-system trivial-build-system)
+    (arguments
+     `(#:guile ,%bootstrap-guile
+       #:modules ((guix build utils))
+       #:builder
+       (begin
+         (use-modules (guix build utils)
+                      (ice-9 popen))
+         (let ((out     (assoc-ref %outputs "out"))
+               (tar     (assoc-ref %build-inputs "tar"))
+               (xz      (assoc-ref %build-inputs "xz"))
+               (tarball (assoc-ref %build-inputs "tarball")))
+
+           (mkdir out)
+           (copy-file tarball "binaries.tar.xz")
+           (invoke xz "-d" "binaries.tar.xz")
+           (let ((builddir (getcwd))
+                 (bindir   (string-append out "/bin")))
+             (with-directory-excursion out
+               (invoke tar "xvf"
+                       (string-append builddir "/binaries.tar"))))))))
+    (inputs
+     `(("tar" ,(search-bootstrap-binary "tar" (%current-system)))
+       ("xz"  ,(search-bootstrap-binary "xz" (%current-system)))
+       ("tarball" ,(bootstrap-origin
+                    (origin
+                      (method url-fetch)
+                      (uri (string-append
+                            "http://lilypond.org/janneke/mes/"
+                            (match (%current-system)
+                              ((or "i686-linux" "x86_64-linux")
+                               "mescc-tools-static-0.5.2-0.bb062b0-i686-linux.tar.xz"))))
+                      (sha256
+                       (match (%current-system)
+                         ((or "i686-linux" "x86_64-linux")
+                          (base32 "1qcqz5lfl2blbfvh5ccg08pcxfhqrmlrsh6cyaxvm0hxsr1rvlm1")))))))))
+    (synopsis "Bootstrap binaries of MesCC Tools")
+    (description synopsis)
+    (home-page #f)
+    (license gpl3+)))
+
 (define %bootstrap-mes
   ;; The initial Mes.  Uses binaries from a tarball typically built by
   ;; %MES-BOOTSTRAP-TARBALL.
-- 
2.19.1


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #4: 0003-bootstrap-Replace-mescc-tools-seed-with-bootstrap-me.patch --]
[-- Type: text/x-patch, Size: 6662 bytes --]

From daa9ff9463917687aabed8330eacb304a7e4deae Mon Sep 17 00:00:00 2001
From: Jan Nieuwenhuizen <janneke@gnu.org>
Date: Fri, 16 Nov 2018 21:30:28 +0100
Subject: [PATCH 3/3] bootstrap: Replace %mescc-tools-seed with
 %bootstrap-mescc-tools.

* gnu/packages/bootstrap.scm (%mescc-tools-seed): Remove.
(%bootstrap-inputs): Replace %mescc-tools-seed with %bootstrap-mescc-tools.
* gnu/packages/commencement.scm (mes-boot): Likewise.
(tcc-boot0): Likewise.
(mescc-tools-boot): Remove.
---
 gnu/packages/bootstrap.scm    | 14 +------
 gnu/packages/commencement.scm | 79 ++---------------------------------
 2 files changed, 4 insertions(+), 89 deletions(-)

diff --git a/gnu/packages/bootstrap.scm b/gnu/packages/bootstrap.scm
index 1075325dc..2eac08cb4 100644
--- a/gnu/packages/bootstrap.scm
+++ b/gnu/packages/bootstrap.scm
@@ -51,7 +51,6 @@
             %bootstrap-inputs
             %bootstrap-mescc-tools
             %bootstrap-mes
-            %mescc-tools-seed
             %srfi-43))
 
 ;;; Commentary:
@@ -716,17 +715,6 @@ exec ~a/bin/.gcc-wrapped -B~a/lib \
     (home-page #f)
     (license gpl3+)))
 
-(define %mescc-tools-seed ; todo: add tarballs to alpha.gnu.org/pub/mes/bootstrap/
-  (let ((commit "dc4e20e74924a5c80a2b7a77b4d7b927234fa71c"))
-    (origin
-      (method url-fetch)
-      (uri (string-append "https://gitlab.com/janneke/mescc-tools-seed"
-                          "/-/archive/" commit
-                          "/mescc-tools-seed-" commit ".tar.gz"))
-      (sha256
-       (base32
-        "1lj7df73vxanmffmiwkhcn83r7yd9n8568nkki06bqq5zg526nyz")))))
-
 (define %srfi-43
   (origin
     (method url-fetch)
@@ -742,7 +730,7 @@ exec ~a/bin/.gcc-wrapped -B~a/lib \
   `(,@(match (%current-system)
         ((or "i686-linux" "x86_64-linux")
          `(("linux-libre-headers" ,%bootstrap-linux-libre-headers)
-           ("mescc-tools-seed" ,%mescc-tools-seed)
+           ("bootstrap-mescc-tools" ,%bootstrap-mescc-tools)
            ("mes" ,%bootstrap-mes)
            ("srfi-43" ,%srfi-43 )))
         (_
diff --git a/gnu/packages/commencement.scm b/gnu/packages/commencement.scm
index 4def178a8..43373b591 100644
--- a/gnu/packages/commencement.scm
+++ b/gnu/packages/commencement.scm
@@ -106,79 +106,6 @@
       (native-inputs '())
       (propagated-inputs '()))))
 
-(define mescc-tools-boot
-  (package-with-bootstrap-guile
-   (package
-     (inherit mescc-tools)
-     (name "mescc-tools-boot")
-     (version "0.5.2")
-     (source (origin
-               (method url-fetch)
-               (uri (string-append
-                     "http://git.savannah.nongnu.org/cgit/mescc-tools.git/snapshot/"
-                     name "-Release_" version
-                     ".tar.gz"))
-               (patches (search-patches "mescc-tools-boot.patch"))
-               (file-name (string-append "mescc-tools" "-" version ".tar.gz"))
-               (sha256
-                (base32
-                 "01x7bhmgwyf6mc2g1hcvibhps98nllacqm4f0j5l51b1mbi18pc2"))))
-     (inputs '())
-     (propagated-inputs '())
-     (native-inputs
-      `(("mescc-tools-seed" ,%mescc-tools-seed)
-        ("mes-source" ,(package-source mes-boot0))
-
-        ("bootstrap-mes" ,%bootstrap-mes)
-        ("coreutils" ,%bootstrap-coreutils&co)))
-     (build-system gnu-build-system)
-     (arguments
-      `(#:implicit-inputs? #f
-        #:guile ,%bootstrap-guile
-        #:strip-binaries? #f   ; binutil's strip b0rkes MesCC/M1/hex2 binaries
-        #:phases
-        (modify-phases %standard-phases
-          (add-after 'unpack 'unpack-seeds
-            (lambda* (#:key outputs #:allow-other-keys)
-              (let* ((coreutils (assoc-ref %build-inputs "coreutils"))
-                     (mescc-tools-seed (assoc-ref %build-inputs "mescc-tools-seed"))
-                     (mes-source (assoc-ref %build-inputs "mes-source"))
-                     (out (assoc-ref %outputs "out")))
-                (with-directory-excursion ".."
-                  (and
-                   (mkdir-p "mescc-tools-seed")
-                   (invoke "tar" "--strip=1" "-C" "mescc-tools-seed"
-                           "-xvf" mescc-tools-seed)
-                   (mkdir-p "mes-source")
-                   (invoke "tar" "--strip=1" "-C" "mes-source"
-                           "-xvf" mes-source)
-                  #t)))))
-          (replace 'configure
-            (lambda* (#:key outputs #:allow-other-keys)
-              (let ((coreutils (assoc-ref %build-inputs "coreutils"))
-                    (bootstrap-mes (assoc-ref %build-inputs "bootstrap-mes"))
-                    (out (assoc-ref %outputs "out")))
-                (setenv "PATH" (string-append coreutils "/bin"
-                                              ":" "../mescc-tools-seed"))
-                (format (current-error-port) "PATH=~s\n" (getenv "PATH"))
-                (setenv "PREFIX" out)
-                (setenv "MES_PREFIX" "../mes-source")
-                (setenv "MESCC_TOOLS_SEED" "../mescc-tools-seed")
-                (setenv "MES_SEED" (string-append bootstrap-mes "/lib"))
-                #t)))
-          (replace 'build
-            (lambda _
-              (invoke "sh" "build.sh")))
-          (replace 'check
-            (lambda _
-              ;; bootstrap build.sh lacks exec_enable, get_machine, and
-              ;; kaem_machine
-              ;; (invoke "sh" "check.sh")
-              #t))
-          (replace 'install
-            (lambda _
-              (invoke "sh" "install.sh")))))))))
-
 (define nyacc-boot
   (let ((version "0.86.0")
         (revision "0")
@@ -211,7 +138,7 @@
      (inputs '())
      (propagated-inputs '())
      (native-inputs
-      `(("mescc-tools" ,mescc-tools-boot)
+      `(("mescc-tools" ,%bootstrap-mescc-tools)
         ("nyacc-source" ,(package-source nyacc-boot))
 
         ("coreutils" , %bootstrap-coreutils&co)
@@ -284,7 +211,7 @@
                (setenv "MES_ARENA" "100000000")
                (setenv "DIFF" "sh scripts/diff.scm")
                ;; fail fast tests
-               ;; (invoke "sh" "-x" "build-aux/test.sh" "scaffold/tests/t")
+               (invoke "sh" "-x" "build-aux/test.sh" "scaffold/tests/t")
                ;; (invoke "sh" "-x" "build-aux/test.sh" "scaffold/tests/63-struct-cell")
                (invoke "sh" "check.sh"))))
           (replace 'install
@@ -332,7 +259,7 @@
        (propagated-inputs '())
        (native-inputs
         `(("mes" ,mes-boot)
-          ("mescc-tools" ,mescc-tools-boot)
+          ("mescc-tools" ,%bootstrap-mescc-tools)
           ("nyacc-source" ,(package-source nyacc-boot))
 
           ("coreutils" , %bootstrap-coreutils&co)
-- 
2.19.1


[-- Attachment #5: Type: text/plain, Size: 152 bytes --]


-- 
Jan Nieuwenhuizen <janneke@gnu.org> | GNU LilyPond http://lilypond.org
Freelance IT http://JoyofSource.com | Avatar® http://AvatarAcademy.com