From mboxrd@z Thu Jan 1 00:00:00 1970 From: taylanbayirli@gmail.com (Taylan Ulrich =?utf-8?Q?Bay=C4=B1rl=C4=B1?= =?utf-8?Q?=2FKammer?=) Subject: Re: [PATCH] build: Speed up .go compilation. Date: Mon, 11 Jan 2016 22:47:49 +0100 Message-ID: <874mejwzy2.fsf@T420.taylan> References: <87lha3rx04.fsf@T420.taylan> <87mvsgxpef.fsf@T420.taylan> <87ziwgf1b4.fsf@gnu.org> <87egdqy24m.fsf@T420.taylan> <87bn8u76tl.fsf@gnu.org> <87pox9wjsz.fsf@T420.taylan> <87pox9s5uh.fsf@gnu.org> <87d1t9w419.fsf@T420.taylan> <87d1t9rv5d.fsf@gnu.org> <8737u3g72u.fsf@gnu.org> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" Return-path: Received: from eggs.gnu.org ([2001:4830:134:3::10]:54343) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1aIkJe-0004yQ-K2 for guix-devel@gnu.org; Mon, 11 Jan 2016 16:47:56 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1aIkJc-00080J-JT for guix-devel@gnu.org; Mon, 11 Jan 2016 16:47:54 -0500 In-Reply-To: <8737u3g72u.fsf@gnu.org> ("Ludovic \=\?utf-8\?Q\?Court\=C3\=A8s\=22'\?\= \=\?utf-8\?Q\?s\?\= message of "Mon, 11 Jan 2016 22:05:45 +0100") List-Id: "Development of GNU Guix and the GNU System distribution." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-devel-bounces+gcggd-guix-devel=m.gmane.org@gnu.org Sender: guix-devel-bounces+gcggd-guix-devel=m.gmane.org@gnu.org To: Ludovic =?utf-8?Q?Court=C3=A8s?= Cc: guix-devel@gnu.org --=-=-= Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable ludo@gnu.org (Ludovic Court=C3=A8s) writes: > Mathieu Lirzin skribis: > >>>>> # Unset 'GUILE_LOAD_COMPILED_PATH' altogether while compiling. Othe= rwise, if >>>>> # $GUILE_LOAD_COMPILED_PATH contains $(moduledir), we may find .go f= iles in >>>>> # there that are newer than the local .scm files (for instance becau= se the >>>>> @@ -358,14 +346,16 @@ GUILD_COMPILE_FLAGS =3D \ >>>>> # >>>>> # XXX: Use the C locale for when Guile lacks >>>>> # . >>>> ^^^ >>>> >>>>> -.scm.go: >>>>> - $(AM_V_GUILEC)$(MKDIR_P) `dirname "$@"` ; \ >>>>> +%.go: make-go ; @: >>>>> +make-go: $(MODULES) guix/config.scm guix/tests.scm >>>>> + @echo "Compiling Scheme modules..." ; \ >>>>> unset GUILE_LOAD_COMPILED_PATH ; \ >>>>> LC_ALL=3DC \ >>>> ^^^ >>>> >>>> This is present because (scripts compile) from "old" Guile doesn't do = it >>>> automatically. What about copying the code from the link above in >>>> compile-all.scm and removing this from Makefile.am ? >>> >>> I should be using the whole (catch ...) expression, right? Done, thanks >>> for the heads up. >> >> Yes I suppose. Maybe Ludo can confirm? > > It=E2=80=99s unnecessary to even call =E2=80=98setlocale=E2=80=99 in comp= ile-all.scm because we > don=E2=80=99t rely on anything locale-specific. So there=E2=80=99s no pr= oblem. > > The LC_ALL=3DC line can also be removed from Makefile.am. In light of this and the other changes, here's a new patch: --=-=-= Content-Type: text/x-diff; charset=utf-8 Content-Disposition: inline; filename=0001-build-Speed-up-.go-compilation.patch Content-Transfer-Encoding: quoted-printable >From 21cb57ce693467faee3ed2dfd48d7676f7d58fd5 Mon Sep 17 00:00:00 2001 From: =3D?UTF-8?q?Taylan=3D20Ulrich=3D20Bay=3DC4=3DB1rl=3DC4=3DB1/Kammer?=3D Date: Thu, 5 Nov 2015 23:42:45 +0100 Subject: [PATCH] build: Speed up .go compilation. * build-aux/compile-all.scm: New file. * Makefile.am (EXTRA_DIST): Add it. (%.go, make-go): New rules. --- Makefile.am | 32 ++++++------------ build-aux/compile-all.scm | 82 +++++++++++++++++++++++++++++++++++++++++++= ++++ 2 files changed, 91 insertions(+), 23 deletions(-) create mode 100644 build-aux/compile-all.scm diff --git a/Makefile.am b/Makefile.am index 760caed..dd98564 100644 --- a/Makefile.am +++ b/Makefile.am @@ -152,11 +152,6 @@ endif BUILD_DAEMON_OFFLOAD # Internal module with test suite support. dist_noinst_DATA =3D guix/tests.scm =20 -# Because of the autoload hack in (guix build download), we must build it -# first to avoid errors on systems where (gnutls) is unavailable. -guix/scripts/download.go: guix/build/download.go -guix/download.go: guix/build/download.go - # Linux-Libre configurations. KCONFIGS =3D \ gnu/packages/linux-libre-i686.conf \ @@ -302,6 +297,7 @@ EXTRA_DIST =3D \ CODE-OF-CONDUCT \ .dir-locals.el \ build-aux/build-self.scm \ + build-aux/compile-all.scm \ build-aux/hydra/gnu-system.scm \ build-aux/hydra/demo-os.scm \ build-aux/hydra/guix.scm \ @@ -341,31 +337,21 @@ CLEANFILES =3D \ $(GOBJECTS) \ $(SCM_TESTS:tests/%.scm=3D%.log) =20 -AM_V_GUILEC =3D $(AM_V_GUILEC_$(V)) -AM_V_GUILEC_ =3D $(AM_V_GUILEC_$(AM_DEFAULT_VERBOSITY)) -AM_V_GUILEC_0 =3D @echo " GUILEC" $@; - -# Flags passed to 'guild compile'. -GUILD_COMPILE_FLAGS =3D \ - -Wformat -Wunbound-variable -Warity-mismatch - # Unset 'GUILE_LOAD_COMPILED_PATH' altogether while compiling. Otherwise,= if # $GUILE_LOAD_COMPILED_PATH contains $(moduledir), we may find .go files in # there that are newer than the local .scm files (for instance because the # user ran 'make install' recently). When that happens, we end up loading # those previously-installed .go files, which may be stale, thereby breaki= ng # the whole thing. -# -# XXX: Use the C locale for when Guile lacks -# . -.scm.go: - $(AM_V_GUILEC)$(MKDIR_P) `dirname "$@"` ; \ +%.go: make-go ; @: +make-go: $(MODULES) guix/config.scm guix/tests.scm + $(AM_V_at)echo "Compiling Scheme modules..." ; \ unset GUILE_LOAD_COMPILED_PATH ; \ - LC_ALL=3DC \ + host=3D$(host) srcdir=3D"$(top_srcdir)" \ $(top_builddir)/pre-inst-env \ - $(GUILD) compile -L "$(top_builddir)" -L "$(top_srcdir)" \ - $(GUILD_COMPILE_FLAGS) --target=3D"$(host)" \ - -o "$@" "$<" + $(GUILE) -L "$(top_builddir)" -L "$(top_srcdir)" \ + --no-auto-compile \ + -s "$(top_srcdir)"/build-aux/compile-all.scm $^ =20 SUFFIXES =3D .go =20 @@ -457,6 +443,6 @@ assert-final-inputs-self-contained: $(top_builddir)/pre-inst-env "$(GUILE)" \ "$(top_srcdir)/build-aux/check-final-inputs-self-contained.scm" =20 -.PHONY: sync-descriptions gen-ChangeLog gen-AUTHORS clean-go +.PHONY: sync-descriptions gen-ChangeLog gen-AUTHORS clean-go make-go .PHONY: assert-no-store-file-names assert-binaries-available .PHONY: assert-final-inputs-self-contained diff --git a/build-aux/compile-all.scm b/build-aux/compile-all.scm new file mode 100644 index 0000000..e0877db --- /dev/null +++ b/build-aux/compile-all.scm @@ -0,0 +1,82 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright =C2=A9 2016 Taylan Ulrich Bay=C4=B1rl=C4=B1/Kammer +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(use-modules (system base target) + (ice-9 match) + (ice-9 threads) + (guix build utils)) + +(define compile-options '(format unbound-variable arity-mismatch)) + +(define host (getenv "host")) + +(define srcdir (getenv "srcdir")) + +(define (relative-file file) + (if (string-prefix? (string-append srcdir "/") file) + (string-drop file (+ 1 (string-length srcdir))) + file)) + +(define (file-mtimego file) + (let* ((relative (relative-file file)) + (without-extension (string-drop-right relative 4))) + (string-append without-extension ".go"))) + +(define (file-needs-compilation? file) + (let ((go (scm->go file))) + (or (not (file-exists? go)) + (file-mtimemodule file) + (let* ((relative (relative-file file)) + (module-path (string-drop-right relative 4))) + (map string->symbol + (string-split module-path #\/)))) + +;;; To work around (FIXME), we want to load all +;;; files to be compiled first. We do this via resolve-interface so that = the +;;; top-level of each file (module) is only executed once. +(define (load-module-file file) + (let ((module (file->module file))) + (format #t " LOAD ~a~%" module) + (resolve-interface module))) + +(define (compile-file* file output-mutex) + (let ((go (scm->go file))) + (with-mutex output-mutex + (format #t " GUILEC ~a~%" go) + (force-output)) + (mkdir-p (dirname go)) + (with-target host + (lambda () + (compile-file file + #:output-file go + #:opts compile-options))))) + +(match (command-line) + ((_ . files) + (let ((files (filter file-needs-compilation? files))) + (for-each load-module-file files) + (let ((mutex (make-mutex))) + (par-for-each (lambda (file) + (compile-file* file mutex)) + files))))) --=20 2.6.3 --=-=-=--