all messages for Guix-related lists mirrored at yhetil.org
 help / color / mirror / Atom feed
* [bug#52078] [PATCH 1/2] gnu: julia: Disable broken tests.
@ 2021-11-24 14:19 zimoun
  2021-11-24 14:20 ` [bug#52078] [PATCH 2/2] gnu: julia: Allow parallel tests zimoun
  2021-11-25  1:24 ` [bug#52078] [PATCH v2 1/3] gnu: julia: Disable broken tests zimoun
  0 siblings, 2 replies; 6+ messages in thread
From: zimoun @ 2021-11-24 14:19 UTC (permalink / raw)
  To: 52078; +Cc: zimoun

* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests.
---
 gnu/packages/julia.scm | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index aff7e1a66f..0d4dd2b7f2 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -382,7 +382,18 @@ (define-public julia
              ;(setenv "LC_ALL" "en_US.utf8")
              (substitute* "test/cmdlineargs.jl"
                (("test v\\[3") "test_broken v[3")
-               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))))
+               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))
+             ;; These tests randomly fails because they depend on CPU.
+             (substitute* "stdlib/LinearAlgebra/test/matmul.jl"
+               ;; Fixed in v1.6.4
+               ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155
+               (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0")
+                "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0"))
+             (substitute* "test/math.jl"
+               ;; @test_broken cannot be used because if the test randomly
+               ;; passes, then it also raises an error.
+               (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)")
+                " "))))
          (add-before 'install 'symlink-libraries
            (lambda* (#:key inputs outputs #:allow-other-keys)
              (let ((link

base-commit: 6605d9ba356f7975e75c0664d9ef02bb7bc9cc1e
-- 
2.32.0





^ permalink raw reply	[flat|nested] 6+ messages in thread

* [bug#52078] [PATCH 2/2] gnu: julia: Allow parallel tests.
  2021-11-24 14:19 [bug#52078] [PATCH 1/2] gnu: julia: Disable broken tests zimoun
@ 2021-11-24 14:20 ` zimoun
  2021-11-25  1:24 ` [bug#52078] [PATCH v2 1/3] gnu: julia: Disable broken tests zimoun
  1 sibling, 0 replies; 6+ messages in thread
From: zimoun @ 2021-11-24 14:20 UTC (permalink / raw)
  To: 52078; +Cc: zimoun

* gnu/packages/julia.scm (julia)[origin]: Patch.
* gnu/packages/patches/julia-Allow-parallel-build.patch: New file.
---
 gnu/packages/julia.scm                         |  3 ++-
 .../patches/julia-Allow-parallel-build.patch   | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 gnu/packages/patches/julia-Allow-parallel-build.patch

diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index 0d4dd2b7f2..2394cb3d5a 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -145,7 +145,8 @@ (define-public julia
                (base32
                 "1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5"))
               (patches
-               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"))))
+               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"
+                               "julia-Allow-parallel-build.patch"))))
     (build-system gnu-build-system)
     (arguments
      `(#:test-target "test"
diff --git a/gnu/packages/patches/julia-Allow-parallel-build.patch b/gnu/packages/patches/julia-Allow-parallel-build.patch
new file mode 100644
index 0000000000..1eb6a4f1f1
--- /dev/null
+++ b/gnu/packages/patches/julia-Allow-parallel-build.patch
@@ -0,0 +1,18 @@
+diff --git a/test/runtests.jl b/test/runtests.jl
+index 2f9cd058bb..d8d6d15b8c 100644
+--- a/test/runtests.jl
++++ b/test/runtests.jl
+@@ -83,8 +83,11 @@ prepend!(tests, linalg_tests)
+ import LinearAlgebra
+ cd(@__DIR__) do
+     n = 1
+-    if net_on
+-        n = min(Sys.CPU_THREADS, length(tests))
++    # Allow parallel tests with isolated environment
++    # https://github.com/JuliaLang/julia/issues/43205
++    if net_on || haskey(ENV, "JULIA_CPU_THREADS")
++        x = haskey(ENV, "JULIA_CPU_THREADS") ? parse(Int, ENV["JULIA_CPU_THREADS"]) : Sys.CPU_THREADS
++        n = min(x, length(tests))
+         n > 1 && addprocs_with_testenv(n)
+         LinearAlgebra.BLAS.set_num_threads(1)
+     end
-- 
2.32.0





^ permalink raw reply	[flat|nested] 6+ messages in thread

* [bug#52078] [PATCH v2 1/3] gnu: julia: Disable broken tests.
  2021-11-24 14:19 [bug#52078] [PATCH 1/2] gnu: julia: Disable broken tests zimoun
  2021-11-24 14:20 ` [bug#52078] [PATCH 2/2] gnu: julia: Allow parallel tests zimoun
@ 2021-11-25  1:24 ` zimoun
  2021-11-25  1:24   ` [bug#52078] [PATCH v2 2/3] gnu: julia: Allow parallel tests zimoun
  2021-11-25  1:24   ` [bug#52078] [PATCH v2 3/3] build: julia-build-system: Allow parallel tests for package zimoun
  1 sibling, 2 replies; 6+ messages in thread
From: zimoun @ 2021-11-25  1:24 UTC (permalink / raw)
  To: 52078; +Cc: maxim.cournoyer, zimoun

* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests.
---
 gnu/packages/julia.scm | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index aff7e1a66f..649ffb8ee4 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -358,8 +358,8 @@ (define-public julia
              ;; Dates/io tests fail on master when networking is unavailable
              ;; https://github.com/JuliaLang/julia/issues/34655
              (substitute* "stdlib/Dates/test/io.jl"
-               (("\"Dates.Date") "\"Date")
-               (("\"Dates.Time") "\"Time"))
+               (("using Dates") "import Dates
+using Dates: @dateformat_str, Date, DateTime, DateFormat, Time"))
              ;; Upstream bug I found when packaging
              ;; https://github.com/JuliaLang/julia/issues/35785
              (substitute* "test/file.jl"
@@ -382,7 +382,18 @@ (define-public julia
              ;(setenv "LC_ALL" "en_US.utf8")
              (substitute* "test/cmdlineargs.jl"
                (("test v\\[3") "test_broken v[3")
-               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))))
+               (("test isempty\\(v\\[3") "test_broken isempty(v[3"))
+             ;; These tests randomly fails because they depend on CPU.
+             (substitute* "stdlib/LinearAlgebra/test/matmul.jl"
+               ;; Fixed in v1.6.4
+               ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155
+               (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0")
+                "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0"))
+             (substitute* "test/math.jl"
+               ;; @test_broken cannot be used because if the test randomly
+               ;; passes, then it also raises an error.
+               (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)")
+                " "))))
          (add-before 'install 'symlink-libraries
            (lambda* (#:key inputs outputs #:allow-other-keys)
              (let ((link

base-commit: c70eadeaed9367e45be3797a17d3a34e5f8f67cd
-- 
2.32.0





^ permalink raw reply	[flat|nested] 6+ messages in thread

* [bug#52078] [PATCH v2 2/3] gnu: julia: Allow parallel tests.
  2021-11-25  1:24 ` [bug#52078] [PATCH v2 1/3] gnu: julia: Disable broken tests zimoun
@ 2021-11-25  1:24   ` zimoun
  2021-11-25  1:24   ` [bug#52078] [PATCH v2 3/3] build: julia-build-system: Allow parallel tests for package zimoun
  1 sibling, 0 replies; 6+ messages in thread
From: zimoun @ 2021-11-25  1:24 UTC (permalink / raw)
  To: 52078; +Cc: maxim.cournoyer, zimoun

* gnu/packages/julia.scm (julia)[origin]<patches>: New patch.
* gnu/packages/patches/julia-Allow-parallel-build.patch: New file.
* gnu/local.mk (dist_patch_DATA): Register it.
---
 gnu/local.mk                                  |  2 ++
 gnu/packages/julia.scm                        |  3 +-
 .../patches/julia-allow-parallel-build.patch  | 30 +++++++++++++++++++
 3 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 gnu/packages/patches/julia-allow-parallel-build.patch

diff --git a/gnu/local.mk b/gnu/local.mk
index ea8a203ac9..706725965b 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -46,6 +46,7 @@
 # Copyright © 2021 Sharlatan Hellseher <sharlatanus@gmail.com>
 # Copyright © 2021 Dmitry Polyakov <polyakov@liltechdude.xyz>
 # Copyright © 2021 Andrew Tropin <andrew@trop.in>
+# Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
 #
 # This file is part of GNU Guix.
 #
@@ -1296,6 +1297,7 @@ dist_patch_DATA =						\
   %D%/packages/patches/jfsutils-include-systypes.patch		\
   %D%/packages/patches/julia-SOURCE_DATE_EPOCH-mtime.patch	\
   %D%/packages/patches/julia-tracker-16-compat.patch		\
+  %D%/packages/patches/julia-allow-parallel-build.patch		\
   %D%/packages/patches/kdbusaddons-kinit-file-name.patch	\
   %D%/packages/patches/libffi-3.3-powerpc-fixes.patch		\
   %D%/packages/patches/libffi-float128-powerpc64le.patch	\
diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index 649ffb8ee4..cf76cfa23c 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -145,7 +145,8 @@ (define-public julia
                (base32
                 "1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5"))
               (patches
-               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"))))
+               (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"
+                               "julia-allow-parallel-build.patch"))))
     (build-system gnu-build-system)
     (arguments
      `(#:test-target "test"
diff --git a/gnu/packages/patches/julia-allow-parallel-build.patch b/gnu/packages/patches/julia-allow-parallel-build.patch
new file mode 100644
index 0000000000..5a382e7c89
--- /dev/null
+++ b/gnu/packages/patches/julia-allow-parallel-build.patch
@@ -0,0 +1,30 @@
+diff --git a/test/runtests.jl b/test/runtests.jl
+index 2f9cd058bb..150395e78c 100644
+--- a/test/runtests.jl
++++ b/test/runtests.jl
+@@ -4,7 +4,7 @@ using Test
+ using Distributed
+ using Dates
+ import REPL
+-using Printf: @sprintf
++using Printf: @sprintf, @printf
+ using Base: Experimental
+ 
+ include("choosetests.jl")
+@@ -83,11 +83,15 @@ prepend!(tests, linalg_tests)
+ import LinearAlgebra
+ cd(@__DIR__) do
+     n = 1
+-    if net_on
++    # Allow parallel tests with isolated environment
++    # https://github.com/JuliaLang/julia/issues/43205
++    # https://github.com/JuliaLang/julia/pull/43211
++    if net_on || haskey(ENV, "JULIA_CPU_THREADS")
+         n = min(Sys.CPU_THREADS, length(tests))
+         n > 1 && addprocs_with_testenv(n)
+         LinearAlgebra.BLAS.set_num_threads(1)
+     end
++    @printf("Number of threads: %i\n", n)
+     skipped = 0
+ 
+     @everywhere include("testdefs.jl")
-- 
2.32.0





^ permalink raw reply	[flat|nested] 6+ messages in thread

* [bug#52078] [PATCH v2 3/3] build: julia-build-system: Allow parallel tests for package.
  2021-11-25  1:24 ` [bug#52078] [PATCH v2 1/3] gnu: julia: Disable broken tests zimoun
  2021-11-25  1:24   ` [bug#52078] [PATCH v2 2/3] gnu: julia: Allow parallel tests zimoun
@ 2021-11-25  1:24   ` zimoun
  2021-11-25  6:43     ` bug#52078: [PATCH 1/2] gnu: julia: Disable broken tests Maxim Cournoyer
  1 sibling, 1 reply; 6+ messages in thread
From: zimoun @ 2021-11-25  1:24 UTC (permalink / raw)
  To: 52078; +Cc: maxim.cournoyer, zimoun

The tests are defined by the script 'test/runtests.jl' from packages and the
parallelism depends on the implementation of this script.  Therefore, 'julia'
is launched using local worker processes accordingly with 'parallel?'.

* guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add
options '--procs'.
* guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'.
---
 guix/build-system/julia.scm       | 3 +++
 guix/build/julia-build-system.scm | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm
index 2998663df1..6261f8a55a 100644
--- a/guix/build-system/julia.scm
+++ b/guix/build-system/julia.scm
@@ -2,6 +2,7 @@
 ;;; Copyright © 2019 Nicolò Balzarotti <nicolo@nixo.xyz>
 ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org>
 ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
+;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -79,6 +80,7 @@ (define private-keywords
 (define* (julia-build name inputs
                       #:key source
                       (tests? #t)
+                      (parallel-tests? #t)
                       (phases '%standard-phases)
                       (outputs '("out"))
                       (search-paths '())
@@ -98,6 +100,7 @@ (define builder
                        #:source #+source
                        #:system #$system
                        #:tests? #$tests?
+                       #:parallel-tests? #$parallel-tests?
                        #:phases #$phases
                        #:outputs #$(outputs->gexp outputs)
                        #:search-paths '#$(sexp->gexp
diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm
index 41c69665c6..f0dc419c17 100644
--- a/guix/build/julia-build-system.scm
+++ b/guix/build/julia-build-system.scm
@@ -1,6 +1,7 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo@nixo.xyz>
 ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
+;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name
   #t)
 
 (define* (check #:key tests? source inputs outputs julia-package-name
+                parallel-tests?
                 #:allow-other-keys)
   (when tests?
     (let* ((out (assoc-ref outputs "out"))
            (package (or julia-package-name (project.toml->name "Project.toml")))
-           (builddir (string-append out "/share/julia/")))
+           (builddir (string-append out "/share/julia/"))
+           (jobs (if parallel-tests?
+                     (number->string (parallel-job-count))
+                     "1")))
       ;; With a patch, SOURCE_DATE_EPOCH is honored
       (setenv "SOURCE_DATE_EPOCH" "1")
       (setenv "JULIA_DEPOT_PATH" builddir)
@@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name
               (string-append builddir "loadpath/" ":"
                              (or (getenv "JULIA_LOAD_PATH")
                                  "")))
+      (setenv "JULIA_CPU_THREADS" jobs)
       (setenv "HOME" "/tmp")
       (invoke "julia" "--depwarn=yes"
+              (string-append "--procs=" jobs)
               (string-append builddir "loadpath/"
                              package "/test/runtests.jl"))))
   #t)
-- 
2.32.0





^ permalink raw reply	[flat|nested] 6+ messages in thread

* bug#52078: [PATCH 1/2] gnu: julia: Disable broken tests.
  2021-11-25  1:24   ` [bug#52078] [PATCH v2 3/3] build: julia-build-system: Allow parallel tests for package zimoun
@ 2021-11-25  6:43     ` Maxim Cournoyer
  0 siblings, 0 replies; 6+ messages in thread
From: Maxim Cournoyer @ 2021-11-25  6:43 UTC (permalink / raw)
  To: zimoun; +Cc: 52078-done

Hello Simon,

zimoun <zimon.toutoune@gmail.com> writes:

> The tests are defined by the script 'test/runtests.jl' from packages and the
> parallelism depends on the implementation of this script.  Therefore, 'julia'
> is launched using local worker processes accordingly with 'parallel?'.
>
> * guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add
> options '--procs'.
> * guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'.
> ---
>  guix/build-system/julia.scm       | 3 +++
>  guix/build/julia-build-system.scm | 9 ++++++++-
>  2 files changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm
> index 2998663df1..6261f8a55a 100644
> --- a/guix/build-system/julia.scm
> +++ b/guix/build-system/julia.scm
> @@ -2,6 +2,7 @@
>  ;;; Copyright © 2019 Nicolò Balzarotti <nicolo@nixo.xyz>
>  ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org>
>  ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
>  ;;;
>  ;;; This file is part of GNU Guix.
>  ;;;
> @@ -79,6 +80,7 @@ (define private-keywords
>  (define* (julia-build name inputs
>                        #:key source
>                        (tests? #t)
> +                      (parallel-tests? #t)
>                        (phases '%standard-phases)
>                        (outputs '("out"))
>                        (search-paths '())
> @@ -98,6 +100,7 @@ (define builder
>                         #:source #+source
>                         #:system #$system
>                         #:tests? #$tests?
> +                       #:parallel-tests? #$parallel-tests?
>                         #:phases #$phases
>                         #:outputs #$(outputs->gexp outputs)
>                         #:search-paths '#$(sexp->gexp
> diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm
> index 41c69665c6..f0dc419c17 100644
> --- a/guix/build/julia-build-system.scm
> +++ b/guix/build/julia-build-system.scm
> @@ -1,6 +1,7 @@
>  ;;; GNU Guix --- Functional package management for GNU
>  ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo@nixo.xyz>
>  ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv@pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com>
>  ;;;
>  ;;; This file is part of GNU Guix.
>  ;;;
> @@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name
>    #t)
>  
>  (define* (check #:key tests? source inputs outputs julia-package-name
> +                parallel-tests?
>                  #:allow-other-keys)
>    (when tests?
>      (let* ((out (assoc-ref outputs "out"))
>             (package (or julia-package-name (project.toml->name "Project.toml")))
> -           (builddir (string-append out "/share/julia/")))
> +           (builddir (string-append out "/share/julia/"))
> +           (jobs (if parallel-tests?
> +                     (number->string (parallel-job-count))
> +                     "1")))
>        ;; With a patch, SOURCE_DATE_EPOCH is honored
>        (setenv "SOURCE_DATE_EPOCH" "1")
>        (setenv "JULIA_DEPOT_PATH" builddir)
> @@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name
>                (string-append builddir "loadpath/" ":"
>                               (or (getenv "JULIA_LOAD_PATH")
>                                   "")))
> +      (setenv "JULIA_CPU_THREADS" jobs)
>        (setenv "HOME" "/tmp")
>        (invoke "julia" "--depwarn=yes"
> +              (string-append "--procs=" jobs)
>                (string-append builddir "loadpath/"
>                               package "/test/runtests.jl"))))
>    #t)

I've applied this series as of commit
905b090582e5c03ef4fb183fda0a1ababf5cb284.  Thanks for the great hacking
session (and result :-)).

Closing.

Maxim




^ permalink raw reply	[flat|nested] 6+ messages in thread

end of thread, other threads:[~2021-11-25  6:44 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-11-24 14:19 [bug#52078] [PATCH 1/2] gnu: julia: Disable broken tests zimoun
2021-11-24 14:20 ` [bug#52078] [PATCH 2/2] gnu: julia: Allow parallel tests zimoun
2021-11-25  1:24 ` [bug#52078] [PATCH v2 1/3] gnu: julia: Disable broken tests zimoun
2021-11-25  1:24   ` [bug#52078] [PATCH v2 2/3] gnu: julia: Allow parallel tests zimoun
2021-11-25  1:24   ` [bug#52078] [PATCH v2 3/3] build: julia-build-system: Allow parallel tests for package zimoun
2021-11-25  6:43     ` bug#52078: [PATCH 1/2] gnu: julia: Disable broken tests Maxim Cournoyer

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.