* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests. --- gnu/packages/julia.scm | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm index aff7e1a66f..0d4dd2b7f2 100644 --- a/gnu/packages/julia.scm +++ b/gnu/packages/julia.scm @@ -382,7 +382,18 @@ (define-public julia ;(setenv "LC_ALL" "en_US.utf8") (substitute* "test/cmdlineargs.jl" (("test v\\[3") "test_broken v[3") - (("test isempty\\(v\\[3") "test_broken isempty(v[3")))) + (("test isempty\\(v\\[3") "test_broken isempty(v[3")) + ;; These tests randomly fails because they depend on CPU. + (substitute* "stdlib/LinearAlgebra/test/matmul.jl" + ;; Fixed in v1.6.4 + ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155 + (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0") + "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0")) + (substitute* "test/math.jl" + ;; @test_broken cannot be used because if the test randomly + ;; passes, then it also raises an error. + (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)") + " ")))) (add-before 'install 'symlink-libraries (lambda* (#:key inputs outputs #:allow-other-keys) (let ((link base-commit: 6605d9ba356f7975e75c0664d9ef02bb7bc9cc1e -- 2.32.0
* gnu/packages/julia.scm (julia)[origin]: Patch. * gnu/packages/patches/julia-Allow-parallel-build.patch: New file. --- gnu/packages/julia.scm | 3 ++- .../patches/julia-Allow-parallel-build.patch | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 gnu/packages/patches/julia-Allow-parallel-build.patch diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm index 0d4dd2b7f2..2394cb3d5a 100644 --- a/gnu/packages/julia.scm +++ b/gnu/packages/julia.scm @@ -145,7 +145,8 @@ (define-public julia (base32 "1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5")) (patches - (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch")))) + (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch" + "julia-Allow-parallel-build.patch")))) (build-system gnu-build-system) (arguments `(#:test-target "test" diff --git a/gnu/packages/patches/julia-Allow-parallel-build.patch b/gnu/packages/patches/julia-Allow-parallel-build.patch new file mode 100644 index 0000000000..1eb6a4f1f1 --- /dev/null +++ b/gnu/packages/patches/julia-Allow-parallel-build.patch @@ -0,0 +1,18 @@ +diff --git a/test/runtests.jl b/test/runtests.jl +index 2f9cd058bb..d8d6d15b8c 100644 +--- a/test/runtests.jl ++++ b/test/runtests.jl +@@ -83,8 +83,11 @@ prepend!(tests, linalg_tests) + import LinearAlgebra + cd(@__DIR__) do + n = 1 +- if net_on +- n = min(Sys.CPU_THREADS, length(tests)) ++ # Allow parallel tests with isolated environment ++ # https://github.com/JuliaLang/julia/issues/43205 ++ if net_on || haskey(ENV, "JULIA_CPU_THREADS") ++ x = haskey(ENV, "JULIA_CPU_THREADS") ? parse(Int, ENV["JULIA_CPU_THREADS"]) : Sys.CPU_THREADS ++ n = min(x, length(tests)) + n > 1 && addprocs_with_testenv(n) + LinearAlgebra.BLAS.set_num_threads(1) + end -- 2.32.0
* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests. --- gnu/packages/julia.scm | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm index aff7e1a66f..649ffb8ee4 100644 --- a/gnu/packages/julia.scm +++ b/gnu/packages/julia.scm @@ -358,8 +358,8 @@ (define-public julia ;; Dates/io tests fail on master when networking is unavailable ;; https://github.com/JuliaLang/julia/issues/34655 (substitute* "stdlib/Dates/test/io.jl" - (("\"Dates.Date") "\"Date") - (("\"Dates.Time") "\"Time")) + (("using Dates") "import Dates +using Dates: @dateformat_str, Date, DateTime, DateFormat, Time")) ;; Upstream bug I found when packaging ;; https://github.com/JuliaLang/julia/issues/35785 (substitute* "test/file.jl" @@ -382,7 +382,18 @@ (define-public julia ;(setenv "LC_ALL" "en_US.utf8") (substitute* "test/cmdlineargs.jl" (("test v\\[3") "test_broken v[3") - (("test isempty\\(v\\[3") "test_broken isempty(v[3")))) + (("test isempty\\(v\\[3") "test_broken isempty(v[3")) + ;; These tests randomly fails because they depend on CPU. + (substitute* "stdlib/LinearAlgebra/test/matmul.jl" + ;; Fixed in v1.6.4 + ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155 + (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0") + "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0")) + (substitute* "test/math.jl" + ;; @test_broken cannot be used because if the test randomly + ;; passes, then it also raises an error. + (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)") + " ")))) (add-before 'install 'symlink-libraries (lambda* (#:key inputs outputs #:allow-other-keys) (let ((link base-commit: c70eadeaed9367e45be3797a17d3a34e5f8f67cd -- 2.32.0
* gnu/packages/julia.scm (julia)[origin]<patches>: New patch. * gnu/packages/patches/julia-Allow-parallel-build.patch: New file. * gnu/local.mk (dist_patch_DATA): Register it. --- gnu/local.mk | 2 ++ gnu/packages/julia.scm | 3 +- .../patches/julia-allow-parallel-build.patch | 30 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 gnu/packages/patches/julia-allow-parallel-build.patch diff --git a/gnu/local.mk b/gnu/local.mk index ea8a203ac9..706725965b 100644 --- a/gnu/local.mk +++ b/gnu/local.mk @@ -46,6 +46,7 @@ # Copyright © 2021 Sharlatan Hellseher <sharlatanus@gmail.com> # Copyright © 2021 Dmitry Polyakov <polyakov@liltechdude.xyz> # Copyright © 2021 Andrew Tropin <andrew@trop.in> +# Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> # # This file is part of GNU Guix. # @@ -1296,6 +1297,7 @@ dist_patch_DATA = \ %D%/packages/patches/jfsutils-include-systypes.patch \ %D%/packages/patches/julia-SOURCE_DATE_EPOCH-mtime.patch \ %D%/packages/patches/julia-tracker-16-compat.patch \ + %D%/packages/patches/julia-allow-parallel-build.patch \ %D%/packages/patches/kdbusaddons-kinit-file-name.patch \ %D%/packages/patches/libffi-3.3-powerpc-fixes.patch \ %D%/packages/patches/libffi-float128-powerpc64le.patch \ diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm index 649ffb8ee4..cf76cfa23c 100644 --- a/gnu/packages/julia.scm +++ b/gnu/packages/julia.scm @@ -145,7 +145,8 @@ (define-public julia (base32 "1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5")) (patches - (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch")))) + (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch" + "julia-allow-parallel-build.patch")))) (build-system gnu-build-system) (arguments `(#:test-target "test" diff --git a/gnu/packages/patches/julia-allow-parallel-build.patch b/gnu/packages/patches/julia-allow-parallel-build.patch new file mode 100644 index 0000000000..5a382e7c89 --- /dev/null +++ b/gnu/packages/patches/julia-allow-parallel-build.patch @@ -0,0 +1,30 @@ +diff --git a/test/runtests.jl b/test/runtests.jl +index 2f9cd058bb..150395e78c 100644 +--- a/test/runtests.jl ++++ b/test/runtests.jl +@@ -4,7 +4,7 @@ using Test + using Distributed + using Dates + import REPL +-using Printf: @sprintf ++using Printf: @sprintf, @printf + using Base: Experimental + + include("choosetests.jl") +@@ -83,11 +83,15 @@ prepend!(tests, linalg_tests) + import LinearAlgebra + cd(@__DIR__) do + n = 1 +- if net_on ++ # Allow parallel tests with isolated environment ++ # https://github.com/JuliaLang/julia/issues/43205 ++ # https://github.com/JuliaLang/julia/pull/43211 ++ if net_on || haskey(ENV, "JULIA_CPU_THREADS") + n = min(Sys.CPU_THREADS, length(tests)) + n > 1 && addprocs_with_testenv(n) + LinearAlgebra.BLAS.set_num_threads(1) + end ++ @printf("Number of threads: %i\n", n) + skipped = 0 + + @everywhere include("testdefs.jl") -- 2.32.0
The tests are defined by the script 'test/runtests.jl' from packages and the parallelism depends on the implementation of this script. Therefore, 'julia' is launched using local worker processes accordingly with 'parallel?'. * guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add options '--procs'. * guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'. --- guix/build-system/julia.scm | 3 +++ guix/build/julia-build-system.scm | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm index 2998663df1..6261f8a55a 100644 --- a/guix/build-system/julia.scm +++ b/guix/build-system/julia.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2019 Nicolò Balzarotti <nicolo@nixo.xyz> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -79,6 +80,7 @@ (define private-keywords (define* (julia-build name inputs #:key source (tests? #t) + (parallel-tests? #t) (phases '%standard-phases) (outputs '("out")) (search-paths '()) @@ -98,6 +100,7 @@ (define builder #:source #+source #:system #$system #:tests? #$tests? + #:parallel-tests? #$parallel-tests? #:phases #$phases #:outputs #$(outputs->gexp outputs) #:search-paths '#$(sexp->gexp diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm index 41c69665c6..f0dc419c17 100644 --- a/guix/build/julia-build-system.scm +++ b/guix/build/julia-build-system.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo@nixo.xyz> ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name #t) (define* (check #:key tests? source inputs outputs julia-package-name + parallel-tests? #:allow-other-keys) (when tests? (let* ((out (assoc-ref outputs "out")) (package (or julia-package-name (project.toml->name "Project.toml"))) - (builddir (string-append out "/share/julia/"))) + (builddir (string-append out "/share/julia/")) + (jobs (if parallel-tests? + (number->string (parallel-job-count)) + "1"))) ;; With a patch, SOURCE_DATE_EPOCH is honored (setenv "SOURCE_DATE_EPOCH" "1") (setenv "JULIA_DEPOT_PATH" builddir) @@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name (string-append builddir "loadpath/" ":" (or (getenv "JULIA_LOAD_PATH") ""))) + (setenv "JULIA_CPU_THREADS" jobs) (setenv "HOME" "/tmp") (invoke "julia" "--depwarn=yes" + (string-append "--procs=" jobs) (string-append builddir "loadpath/" package "/test/runtests.jl")))) #t) -- 2.32.0
Hello Simon,
zimoun <zimon.toutoune@gmail.com> writes:
> The tests are defined by the script 'test/runtests.jl' from packages and the
> parallelism depends on the implementation of this script. Therefore, 'julia'
> is launched using local worker processes accordingly with 'parallel?'.
>
> * guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add
> options '--procs'.
> * guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'.
> ---
> guix/build-system/julia.scm | 3 +++
> guix/build/julia-build-system.scm | 9 ++++++++-
> 2 files changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm
> index 2998663df1..6261f8a55a 100644
> --- a/guix/build-system/julia.scm
> +++ b/guix/build-system/julia.scm
> @@ -2,6 +2,7 @@
> ;;; Copyright © 2019 Nicolò Balzarotti <nicolo@nixo.xyz>
> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org>
> ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
> ;;;
> ;;; This file is part of GNU Guix.
> ;;;
> @@ -79,6 +80,7 @@ (define private-keywords
> (define* (julia-build name inputs
> #:key source
> (tests? #t)
> + (parallel-tests? #t)
> (phases '%standard-phases)
> (outputs '("out"))
> (search-paths '())
> @@ -98,6 +100,7 @@ (define builder
> #:source #+source
> #:system #$system
> #:tests? #$tests?
> + #:parallel-tests? #$parallel-tests?
> #:phases #$phases
> #:outputs #$(outputs->gexp outputs)
> #:search-paths '#$(sexp->gexp
> diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm
> index 41c69665c6..f0dc419c17 100644
> --- a/guix/build/julia-build-system.scm
> +++ b/guix/build/julia-build-system.scm
> @@ -1,6 +1,7 @@
> ;;; GNU Guix --- Functional package management for GNU
> ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo@nixo.xyz>
> ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
> ;;;
> ;;; This file is part of GNU Guix.
> ;;;
> @@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name
> #t)
>
> (define* (check #:key tests? source inputs outputs julia-package-name
> + parallel-tests?
> #:allow-other-keys)
> (when tests?
> (let* ((out (assoc-ref outputs "out"))
> (package (or julia-package-name (project.toml->name "Project.toml")))
> - (builddir (string-append out "/share/julia/")))
> + (builddir (string-append out "/share/julia/"))
> + (jobs (if parallel-tests?
> + (number->string (parallel-job-count))
> + "1")))
> ;; With a patch, SOURCE_DATE_EPOCH is honored
> (setenv "SOURCE_DATE_EPOCH" "1")
> (setenv "JULIA_DEPOT_PATH" builddir)
> @@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name
> (string-append builddir "loadpath/" ":"
> (or (getenv "JULIA_LOAD_PATH")
> "")))
> + (setenv "JULIA_CPU_THREADS" jobs)
> (setenv "HOME" "/tmp")
> (invoke "julia" "--depwarn=yes"
> + (string-append "--procs=" jobs)
> (string-append builddir "loadpath/"
> package "/test/runtests.jl"))))
> #t)
I've applied this series as of commit
905b090582e5c03ef4fb183fda0a1ababf5cb284. Thanks for the great hacking
session (and result :-)).
Closing.
Maxim