[bug#52078] [PATCH 1/2] gnu: julia: Disable broken tests.

[bug#52078] [PATCH 1/2] gnu: julia: Disable broken tests.

[zimoun]

* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests.
---
 gnu/packages/julia.scm | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index aff7e1a66f..0d4dd2b7f2 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -382,7 +382,18 @@ (define-public julia
              ;(setenv "LC_ALL" "en_US.utf8")
              (substitute* "test/cmdlineargs.jl"
                (("test v\\[3") "test_broken v[3")
-               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))))
+               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))
+             ;; These tests randomly fails because they depend on CPU.
+             (substitute* "stdlib/LinearAlgebra/test/matmul.jl"
+               ;; Fixed in v1.6.4
+               ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155
+               (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0")
+                "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0"))
+             (substitute* "test/math.jl"
+               ;; @test_broken cannot be used because if the test randomly
+               ;; passes, then it also raises an error.
+               (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)")
+                " "))))
          (add-before 'install 'symlink-libraries
            (lambda* (#:key inputs outputs #:allow-other-keys)
              (let ((link

base-commit: 6605d9ba356f7975e75c0664d9ef02bb7bc9cc1e
-- 
2.32.0

[bug#52078] [PATCH 2/2] gnu: julia: Allow parallel tests.

[zimoun]

* gnu/packages/julia.scm (julia)[origin]: Patch.
* gnu/packages/patches/julia-Allow-parallel-build.patch: New file.
---
 gnu/packages/julia.scm                         |  3 ++-
 .../patches/julia-Allow-parallel-build.patch   | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 gnu/packages/patches/julia-Allow-parallel-build.patch

diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index 0d4dd2b7f2..2394cb3d5a 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -145,7 +145,8 @@ (define-public julia
                (base32
                 "1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5"))
               (patches
-               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"))))
+               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"
+                               "julia-Allow-parallel-build.patch"))))
     (build-system gnu-build-system)
     (arguments
      `(#:test-target "test"
diff --git a/gnu/packages/patches/julia-Allow-parallel-build.patch b/gnu/packages/patches/julia-Allow-parallel-build.patch
new file mode 100644
index 0000000000..1eb6a4f1f1
--- /dev/null
+++ b/gnu/packages/patches/julia-Allow-parallel-build.patch
@@ -0,0 +1,18 @@
+diff --git a/test/runtests.jl b/test/runtests.jl
+index 2f9cd058bb..d8d6d15b8c 100644
+--- a/test/runtests.jl
++++ b/test/runtests.jl
+@@ -83,8 +83,11 @@ prepend!(tests, linalg_tests)
+ import LinearAlgebra
+ cd(@__DIR__) do
+     n = 1
+-    if net_on
+-        n = min(Sys.CPU_THREADS, length(tests))
++    # Allow parallel tests with isolated environment
++    # https://github.com/JuliaLang/julia/issues/43205
++    if net_on || haskey(ENV, "JULIA_CPU_THREADS")
++        x = haskey(ENV, "JULIA_CPU_THREADS") ? parse(Int, ENV["JULIA_CPU_THREADS"]) : Sys.CPU_THREADS
++        n = min(x, length(tests))
+         n > 1 && addprocs_with_testenv(n)
+         LinearAlgebra.BLAS.set_num_threads(1)
+     end
-- 
2.32.0

[bug#52078] [PATCH v2 1/3] gnu: julia: Disable broken tests.

[zimoun]

* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests.
---
 gnu/packages/julia.scm | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index aff7e1a66f..649ffb8ee4 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -358,8 +358,8 @@ (define-public julia
              ;; Dates/io tests fail on master when networking is unavailable
              ;; https://github.com/JuliaLang/julia/issues/34655
              (substitute* "stdlib/Dates/test/io.jl"
-               (("\"Dates.Date") "\"Date")
-               (("\"Dates.Time") "\"Time"))
+               (("using Dates") "import Dates
+using Dates: @dateformat_str, Date, DateTime, DateFormat, Time"))
              ;; Upstream bug I found when packaging
              ;; https://github.com/JuliaLang/julia/issues/35785
              (substitute* "test/file.jl"
@@ -382,7 +382,18 @@ (define-public julia
              ;(setenv "LC_ALL" "en_US.utf8")
              (substitute* "test/cmdlineargs.jl"
                (("test v\\[3") "test_broken v[3")
-               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))))
+               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))
+             ;; These tests randomly fails because they depend on CPU.
+             (substitute* "stdlib/LinearAlgebra/test/matmul.jl"
+               ;; Fixed in v1.6.4
+               ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155
+               (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0")
+                "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0"))
+             (substitute* "test/math.jl"
+               ;; @test_broken cannot be used because if the test randomly
+               ;; passes, then it also raises an error.
+               (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)")
+                " "))))
          (add-before 'install 'symlink-libraries
            (lambda* (#:key inputs outputs #:allow-other-keys)
              (let ((link

base-commit: c70eadeaed9367e45be3797a17d3a34e5f8f67cd
-- 
2.32.0

[bug#52078] [PATCH v2 2/3] gnu: julia: Allow parallel tests.

[zimoun]

* gnu/packages/julia.scm (julia)[origin]<patches>: New patch.
* gnu/packages/patches/julia-Allow-parallel-build.patch: New file.
* gnu/local.mk (dist_patch_DATA): Register it.
---
 gnu/local.mk                                  |  2 ++
 gnu/packages/julia.scm                        |  3 +-
 .../patches/julia-allow-parallel-build.patch  | 30 +++++++++++++++++++
 3 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 gnu/packages/patches/julia-allow-parallel-build.patch

diff --git a/gnu/local.mk b/gnu/local.mk
index ea8a203ac9..706725965b 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -46,6 +46,7 @@
 # Copyright © 2021 Sharlatan Hellseher <sharlatanus@gmail.com>
 # Copyright © 2021 Dmitry Polyakov <polyakov@liltechdude.xyz>
 # Copyright © 2021 Andrew Tropin <andrew@trop.in>
+# Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
 #
 # This file is part of GNU Guix.
 #
@@ -1296,6 +1297,7 @@ dist_patch_DATA =						\
   %D%/packages/patches/jfsutils-include-systypes.patch		\
   %D%/packages/patches/julia-SOURCE_DATE_EPOCH-mtime.patch	\
   %D%/packages/patches/julia-tracker-16-compat.patch		\
+  %D%/packages/patches/julia-allow-parallel-build.patch		\
   %D%/packages/patches/kdbusaddons-kinit-file-name.patch	\
   %D%/packages/patches/libffi-3.3-powerpc-fixes.patch		\
   %D%/packages/patches/libffi-float128-powerpc64le.patch	\
diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index 649ffb8ee4..cf76cfa23c 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -145,7 +145,8 @@ (define-public julia
                (base32
                 "1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5"))
               (patches
-               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"))))
+               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"
+                               "julia-allow-parallel-build.patch"))))
     (build-system gnu-build-system)
     (arguments
      `(#:test-target "test"
diff --git a/gnu/packages/patches/julia-allow-parallel-build.patch b/gnu/packages/patches/julia-allow-parallel-build.patch
new file mode 100644
index 0000000000..5a382e7c89
--- /dev/null
+++ b/gnu/packages/patches/julia-allow-parallel-build.patch
@@ -0,0 +1,30 @@
+diff --git a/test/runtests.jl b/test/runtests.jl
+index 2f9cd058bb..150395e78c 100644
+--- a/test/runtests.jl
++++ b/test/runtests.jl
+@@ -4,7 +4,7 @@ using Test
+ using Distributed
+ using Dates
+ import REPL
+-using Printf: @sprintf
++using Printf: @sprintf, @printf
+ using Base: Experimental
+ 
+ include("choosetests.jl")
+@@ -83,11 +83,15 @@ prepend!(tests, linalg_tests)
+ import LinearAlgebra
+ cd(@__DIR__) do
+     n = 1
+-    if net_on
++    # Allow parallel tests with isolated environment
++    # https://github.com/JuliaLang/julia/issues/43205
++    # https://github.com/JuliaLang/julia/pull/43211
++    if net_on || haskey(ENV, "JULIA_CPU_THREADS")
+         n = min(Sys.CPU_THREADS, length(tests))
+         n > 1 && addprocs_with_testenv(n)
+         LinearAlgebra.BLAS.set_num_threads(1)
+     end
++    @printf("Number of threads: %i\n", n)
+     skipped = 0
+ 
+     @everywhere include("testdefs.jl")
-- 
2.32.0

[bug#52078] [PATCH v2 3/3] build: julia-build-system: Allow parallel tests for package.

[zimoun]

The tests are defined by the script 'test/runtests.jl' from packages and the
parallelism depends on the implementation of this script.  Therefore, 'julia'
is launched using local worker processes accordingly with 'parallel?'.

* guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add
options '--procs'.
* guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'.
---
 guix/build-system/julia.scm       | 3 +++
 guix/build/julia-build-system.scm | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm
index 2998663df1..6261f8a55a 100644
--- a/guix/build-system/julia.scm
+++ b/guix/build-system/julia.scm
@@ -2,6 +2,7 @@
 ;;; Copyright © 2019 Nicolò Balzarotti <nicolo@nixo.xyz>
 ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org>
 ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
+;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -79,6 +80,7 @@ (define private-keywords
 (define* (julia-build name inputs
                       #:key source
                       (tests? #t)
+                      (parallel-tests? #t)
                       (phases '%standard-phases)
                       (outputs '("out"))
                       (search-paths '())
@@ -98,6 +100,7 @@ (define builder
                        #:source #+source
                        #:system #$system
                        #:tests? #$tests?
+                       #:parallel-tests? #$parallel-tests?
                        #:phases #$phases
                        #:outputs #$(outputs->gexp outputs)
                        #:search-paths '#$(sexp->gexp
diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm
index 41c69665c6..f0dc419c17 100644
--- a/guix/build/julia-build-system.scm
+++ b/guix/build/julia-build-system.scm
@@ -1,6 +1,7 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo@nixo.xyz>
 ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
+;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name
   #t)
 
 (define* (check #:key tests? source inputs outputs julia-package-name
+                parallel-tests?
                 #:allow-other-keys)
   (when tests?
     (let* ((out (assoc-ref outputs "out"))
            (package (or julia-package-name (project.toml->name "Project.toml")))
-           (builddir (string-append out "/share/julia/")))
+           (builddir (string-append out "/share/julia/"))
+           (jobs (if parallel-tests?
+                     (number->string (parallel-job-count))
+                     "1")))
       ;; With a patch, SOURCE_DATE_EPOCH is honored
       (setenv "SOURCE_DATE_EPOCH" "1")
       (setenv "JULIA_DEPOT_PATH" builddir)
@@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name
               (string-append builddir "loadpath/" ":"
                              (or (getenv "JULIA_LOAD_PATH")
                                  "")))
+      (setenv "JULIA_CPU_THREADS" jobs)
       (setenv "HOME" "/tmp")
       (invoke "julia" "--depwarn=yes"
+              (string-append "--procs=" jobs)
               (string-append builddir "loadpath/"
                              package "/test/runtests.jl"))))
   #t)
-- 
2.32.0

bug#52078: [PATCH 1/2] gnu: julia: Disable broken tests.

[Maxim Cournoyer]

Hello Simon,

zimoun <zimon.toutoune@gmail.com> writes:

> The tests are defined by the script 'test/runtests.jl' from packages and the
> parallelism depends on the implementation of this script.  Therefore, 'julia'
> is launched using local worker processes accordingly with 'parallel?'.
>
> * guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add
> options '--procs'.
> * guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'.
> ---
>  guix/build-system/julia.scm       | 3 +++
>  guix/build/julia-build-system.scm | 9 ++++++++-
>  2 files changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm
> index 2998663df1..6261f8a55a 100644
> --- a/guix/build-system/julia.scm
> +++ b/guix/build-system/julia.scm
> @@ -2,6 +2,7 @@
>  ;;; Copyright © 2019 Nicolò Balzarotti <nicolo@nixo.xyz>
>  ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org>
>  ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
>  ;;;
>  ;;; This file is part of GNU Guix.
>  ;;;
> @@ -79,6 +80,7 @@ (define private-keywords
>  (define* (julia-build name inputs
>                        #:key source
>                        (tests? #t)
> +                      (parallel-tests? #t)
>                        (phases '%standard-phases)
>                        (outputs '("out"))
>                        (search-paths '())
> @@ -98,6 +100,7 @@ (define builder
>                         #:source #+source
>                         #:system #$system
>                         #:tests? #$tests?
> +                       #:parallel-tests? #$parallel-tests?
>                         #:phases #$phases
>                         #:outputs #$(outputs->gexp outputs)
>                         #:search-paths '#$(sexp->gexp
> diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm
> index 41c69665c6..f0dc419c17 100644
> --- a/guix/build/julia-build-system.scm
> +++ b/guix/build/julia-build-system.scm
> @@ -1,6 +1,7 @@
>  ;;; GNU Guix --- Functional package management for GNU
>  ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo@nixo.xyz>
>  ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
>  ;;;
>  ;;; This file is part of GNU Guix.
>  ;;;
> @@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name
>    #t)
>  
>  (define* (check #:key tests? source inputs outputs julia-package-name
> +                parallel-tests?
>                  #:allow-other-keys)
>    (when tests?
>      (let* ((out (assoc-ref outputs "out"))
>             (package (or julia-package-name (project.toml->name "Project.toml")))
> -           (builddir (string-append out "/share/julia/")))
> +           (builddir (string-append out "/share/julia/"))
> +           (jobs (if parallel-tests?
> +                     (number->string (parallel-job-count))
> +                     "1")))
>        ;; With a patch, SOURCE_DATE_EPOCH is honored
>        (setenv "SOURCE_DATE_EPOCH" "1")
>        (setenv "JULIA_DEPOT_PATH" builddir)
> @@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name
>                (string-append builddir "loadpath/" ":"
>                               (or (getenv "JULIA_LOAD_PATH")
>                                   "")))
> +      (setenv "JULIA_CPU_THREADS" jobs)
>        (setenv "HOME" "/tmp")
>        (invoke "julia" "--depwarn=yes"
> +              (string-append "--procs=" jobs)
>                (string-append builddir "loadpath/"
>                               package "/test/runtests.jl"))))
>    #t)

I've applied this series as of commit
905b090582e5c03ef4fb183fda0a1ababf5cb284.  Thanks for the great hacking
session (and result :-)).

Closing.

Maxim