From mboxrd@z Thu Jan 1 00:00:00 1970 From: ludo@gnu.org (Ludovic =?utf-8?Q?Court=C3=A8s?=) Subject: Building Guix with Guile 2.1 Date: Wed, 21 Sep 2016 17:23:18 +0900 Message-ID: <87fuotl367.fsf@gnu.org> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" Return-path: Received: from eggs.gnu.org ([2001:4830:134:3::10]:58262) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1bmipp-00072H-7C for guix-devel@gnu.org; Wed, 21 Sep 2016 10:49:23 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1bmiph-00052T-0n for guix-devel@gnu.org; Wed, 21 Sep 2016 10:49:16 -0400 Received: from fencepost.gnu.org ([2001:4830:134:3::e]:37784) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1bmipg-000527-Rv for guix-devel@gnu.org; Wed, 21 Sep 2016 10:49:08 -0400 Received: from p348175-ipngn200606kyoto.kyoto.ocn.ne.jp ([125.170.228.175]:62805 helo=pluto) by fencepost.gnu.org with esmtpsa (TLS1.2:RSA_AES_256_CBC_SHA1:256) (Exim 4.82) (envelope-from ) id 1bmipH-0007Ox-7Z for guix-devel@gnu.org; Wed, 21 Sep 2016 10:49:06 -0400 List-Id: "Development of GNU Guix and the GNU System distribution." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-devel-bounces+gcggd-guix-devel=m.gmane.org@gnu.org Sender: "Guix-devel" To: guix-devel --=-=-= Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable Hello! Nalaginrut reported that Guix fails to build with Guile 2.2, which was a bit of a shame, hence commits e465d9e19087ab150f7e31f21c09e4a147b93b36 and 9d126aa2b504bb9fad536eac186805ff623e96be. Now, the hack in build-aux/compile-all.scm doesn=E2=80=99t quite work with = 2.2, and it was already quite fragile. So I think we need a different approach. A more robust approach that comes to mind would be to look at our module dependency graph (with help from (guix modules)), run N Guile processes (instead of threads), and have each of them build a subset of the graph. It would essentially require us to reimplement a subset of Make (or guix-daemon). I=E2=80=99ve tried generating makefile rules from module dependencies. Tha= t=E2=80=99s pretty cool, but compilation of the package modules remains damn slow (2nd patch attached). What do people think? Anyone has a better idea? The attached patch allows us to build the modules sequentially but in topologically-sorted order, which means we can avoid the ugly load-modules-first workaround. Still, it=E2=80=99s a bit too slow. Ludo=E2=80=99. --=-=-= Content-Type: text/x-patch Content-Disposition: inline diff --git a/build-aux/compile-all.scm b/build-aux/compile-all.scm index 7c937a0..3703187 100644 --- a/build-aux/compile-all.scm +++ b/build-aux/compile-all.scm @@ -21,6 +21,7 @@ (system base message) (ice-9 match) (ice-9 threads) + (guix modules) (guix build utils)) (define warnings @@ -55,19 +56,15 @@ (map string->symbol (string-split module-path #\/)))) -;;; To work around (FIXME), we want to load all -;;; files to be compiled first. We do this via resolve-interface so that the -;;; top-level of each file (module) is only executed once. -(define (load-module-file file) - (let ((module (file->module file))) - (format #t " LOAD ~a~%" module) - (resolve-interface module))) +(define (module->file module) + (string-append srcdir "/" + (string-join (map symbol->string module) "/") + ".scm")) -(define (compile-file* file output-mutex) +(define (compile-file* file) (let ((go (scm->go file))) - (with-mutex output-mutex - (format #t " GUILEC ~a~%" go) - (force-output)) + (format #t " GUILEC ~a~%" go) + (force-output) (mkdir-p (dirname go)) (with-fluids ((*current-warning-prefix* "")) (with-target host @@ -78,12 +75,12 @@ (match (command-line) ((_ . files) - (let ((files (filter file-needs-compilation? files))) - (for-each load-module-file files) - (let ((mutex (make-mutex))) - (par-for-each (lambda (file) - (compile-file* file mutex)) - files))))) + (for-each (lambda (module) + (let ((file (module->file module))) + (when (file-needs-compilation? file) + (compile-file* file)))) + (source-module-closure + (map file->module files))))) ;;; Local Variables: ;;; eval: (put 'with-target 'scheme-indent-function 1) diff --git a/guix/modules.scm b/guix/modules.scm index 24f613f..837d150 100644 --- a/guix/modules.scm +++ b/guix/modules.scm @@ -118,14 +118,19 @@ module names. Only modules that match SELECT? are considered." (visited (set))) (match modules (() - (reverse result)) + (values result visited)) ((module rest ...) (cond ((set-contains? visited module) (loop rest result visited)) ((select? module) - (loop (append (dependencies module) rest) - (cons module result) - (set-insert module visited))) + (call-with-values + (lambda () + (loop rest result + (set-insert module visited))) + (lambda (result visited) + (loop (dependencies module) + (cons module result) + visited)))) (else (loop rest result visited))))))) --=-=-= Content-Type: text/x-patch; charset=utf-8 Content-Disposition: inline; filename=0001-build-Generate-makefile-rules-from-.scm-files.patch Content-Transfer-Encoding: quoted-printable Content-Description: the 2nd patch >From 9621aa00b4378036e906f58dc40e431748ba9790 Mon Sep 17 00:00:00 2001 From: =3D?UTF-8?q?Ludovic=3D20Court=3DC3=3DA8s?=3D Date: Wed, 21 Sep 2016 23:45:07 +0900 Subject: [PATCH] build: Generate makefile rules from .scm files. * guix/modules.scm (source-module-dependencies): Export. * build-aux/generate-make-rules.scm: New file. * Makefile.am (EXTRA_DIST): Add it. (AM_V_GUILEC, AM_V_GUILEC_, AM_V_GUILEC_0) (GUILD_COMPILE_FLAGS): New variables. (.scm.go): New rule. (%.go, make-go): Remove. (build-aux/module-rules.mk): New rule. Add "-include" statement. (CLEANFILES): Add this file. --- Makefile.am | 45 +++++++++++++++++++------ build-aux/generate-make-rules.scm | 71 +++++++++++++++++++++++++++++++++++= ++++ guix/modules.scm | 3 +- 3 files changed, 108 insertions(+), 11 deletions(-) create mode 100644 build-aux/generate-make-rules.scm diff --git a/Makefile.am b/Makefile.am index f9fe141..6720773 100644 --- a/Makefile.am +++ b/Makefile.am @@ -355,6 +355,7 @@ EXTRA_DIST =3D \ .dir-locals.el \ build-aux/build-self.scm \ build-aux/compile-all.scm \ + build-aux/generate-make-rules.scm \ build-aux/hydra/evaluate.scm \ build-aux/hydra/gnu-system.scm \ build-aux/hydra/demo-os.scm \ @@ -390,26 +391,50 @@ CLEANFILES =3D \ $(GOBJECTS) \ $(SCM_TESTS:tests/%.scm=3D%.log) =20 +AM_V_GUILEC =3D $(AM_V_GUILEC_$(V)) +AM_V_GUILEC_ =3D $(AM_V_GUILEC_$(AM_DEFAULT_VERBOSITY)) +AM_V_GUILEC_0 =3D @echo " GUILEC" $@; + +# Flags passed to 'guild compile'. +GUILD_COMPILE_FLAGS =3D \ + -Wformat -Wunbound-variable -Warity-mismatch + # Unset 'GUILE_LOAD_COMPILED_PATH' altogether while compiling. Otherwise,= if # $GUILE_LOAD_COMPILED_PATH contains $(moduledir), we may find .go files in # there that are newer than the local .scm files (for instance because the # user ran 'make install' recently). When that happens, we end up loading # those previously-installed .go files, which may be stale, thereby breaki= ng -# the whole thing. Likewise, set 'XDG_CACHE_HOME' to avoid loading possib= ly -# stale files from ~/.cache/guile/ccache. -%.go: make-go ; @: -make-go: $(MODULES) guix/config.scm guix/tests.scm - $(AM_V_at)echo "Compiling Scheme modules..." ; \ +# the whole thing. +# +# XXX: Use the C locale for when Guile lacks +# . +.scm.go: + $(AM_V_GUILEC)$(MKDIR_P) `dirname "$@"` ; \ + $(AM_V_P) && out=3D1 || out=3D- ; \ unset GUILE_LOAD_COMPILED_PATH ; \ - XDG_CACHE_HOME=3D/nowhere \ - host=3D$(host) srcdir=3D"$(top_srcdir)" \ + LC_ALL=3DC \ $(top_builddir)/pre-inst-env \ - $(GUILE) -L "$(top_builddir)" -L "$(top_srcdir)" \ - --no-auto-compile \ - -s "$(top_srcdir)"/build-aux/compile-all.scm $^ + $(GUILD) compile -L "$(top_builddir)" -L "$(top_srcdir)" \ + $(GUILD_COMPILE_FLAGS) --target=3D"$(host)" \ + -o "$@" "$<" >&$$out + =20 SUFFIXES =3D .go =20 +# Dependency among Scheme modules. +-include build-aux/module-rules.mk + +build-aux/module-rules.mk: $(MODULES) + $(AM_V_GEN)$(MKDIR_P) `dirname "$@"` ; \ + unset GUILE_LOAD_COMPILED_PATH ; \ + LC_ALL=3DC srcdir=3D"$(srcdir)" \ + $(top_builddir)/pre-inst-env \ + $(GUILE) --no-auto-compile \ + build-aux/generate-make-rules.scm $^ > "$@.tmp" + mv "$@.tmp" "$@" + +CLEANFILES +=3D build-aux/module-rules.mk + # Make sure source files are installed first, so that the mtime of # installed compiled files is greater than that of installed source # files. See diff --git a/build-aux/generate-make-rules.scm b/build-aux/generate-make-ru= les.scm new file mode 100644 index 0000000..c38b364 --- /dev/null +++ b/build-aux/generate-make-rules.scm @@ -0,0 +1,71 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright =C2=A9 2016 Ludovic Court=C3=A8s +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(use-modules (guix modules) + (guix utils) + (srfi srfi-26) + (ice-9 match)) + +(define (relevant-module? module) + (match module + (('guix _ ...) #t) + (('gnu 'packages _ ...) #f) + (('gnu _ ...) #t) + (_ #f))) + +(define srcdir (or (getenv "srcdir")".")) + +(define (relative-file file) + (if (string-prefix? (string-append srcdir "/") file) + (string-drop file (+ 1 (string-length srcdir))) + file)) + +(define (file->module file) + (let* ((relative (relative-file file)) + (module-path (string-drop-right relative 4))) + (map string->symbol + (string-split module-path #\/)))) + +(define (module->file module) + (string-append srcdir "/" + (string-join (map symbol->string module) "/"))) + +(define (display-module-rule module dependencies port) + (format port "~a.go: ~a.scm~{ ~a~}~%" + (module->file module) + (module->file module) + (map (compose (cut string-append <> ".go") + module->file) dependencies))) + +(match (command-line) + ((command . files) + (format #t "# Automatically generated. -*- read-only -*-\n") + (for-each (lambda (file) + (match (file->module file) + (('gnu 'packages _ ...) + ;; We don't want to rebuild package modules too often, + ;; especially since it's rarely useful. Thus, generate a + ;; simple static rule. + (format #t "~a.go: ~a guix/packages.go~%" + (file-sans-extension file) file)) + (module + (display-module-rule module + (filter relevant-module? + (source-module-dependencies= module)) + (current-output-port))))) + files))) diff --git a/guix/modules.scm b/guix/modules.scm index 24f613f..d78faf2 100644 --- a/guix/modules.scm +++ b/guix/modules.scm @@ -21,7 +21,8 @@ #:use-module (guix sets) #:use-module (srfi srfi-26) #:use-module (ice-9 match) - #:export (source-module-closure + #:export (source-module-dependencies + source-module-closure live-module-closure guix-module-name?)) =20 --=20 2.10.0 --=-=-=--