[bug#72766] [PATCH 0/2] slurm: Enable REST API.

[bug#72766] [PATCH 0/2] slurm: Enable REST API.

[Arun Isaac]

This short patchset enables the REST API support in our slurm package.

Arun Isaac (2):
  gnu: Add libjwt.
  gnu: slurm: Enable REST API.

 gnu/packages/parallel.scm | 26 ++++++++++++++++++++++----
 gnu/packages/web.scm      | 26 +++++++++++++++++++++++++-
 2 files changed, 47 insertions(+), 5 deletions(-)


base-commit: 6bec6c20b6b18dc173383a6828f1bb84314f986b
-- 
2.45.2

[bug#72766] [PATCH 1/2] gnu: Add libjwt.

[Arun Isaac]

* gnu/packages/web.scm (libjwt): New variable.

Change-Id: Ied516fb41a90de316588622b999b27f0f7ee8de3
---
 gnu/packages/web.scm | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/web.scm b/gnu/packages/web.scm
index 34922911a7..dd1b9438a2 100644
--- a/gnu/packages/web.scm
+++ b/gnu/packages/web.scm
@@ -15,7 +15,7 @@
 ;;; Copyright © 2016 Ben Woodcroft <donttrustben@gmail.com>
 ;;; Copyright © 2016, 2023 Clément Lassieur <clement@lassieur.org>
 ;;; Copyright © 2016, 2017 Nikita <nikita@n0.is>
-;;; Copyright © 2016–2023 Arun Isaac <arunisaac@systemreboot.net>
+;;; Copyright © 2016–2024 Arun Isaac <arunisaac@systemreboot.net>
 ;;; Copyright © 2016–2022 Tobias Geerinckx-Rice <me@tobias.gr>
 ;;; Copyright © 2016 Bake Timmons <b3timmons@speedymail.org>
 ;;; Copyright © 2017 Thomas Danckaert <post@thomasdanckaert.be>
@@ -1604,6 +1604,30 @@ (define-public rapidjson
 style API.")
       (license license:expat))))
 
+(define-public libjwt
+  (package
+    (name "libjwt")
+    (version "1.17.2")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (string-append "https://github.com/benmcollins/libjwt/releases/download/v"
+                           version "/libjwt-"
+                           version ".tar.bz2"))
+       (sha256
+        (base32
+         "1bpfaa0y8bccz5hr677lkrprs07akx02k0qbf82z2c8syr24a77i"))))
+    (build-system gnu-build-system)
+    (inputs
+     (list jansson openssl))
+    (native-inputs
+     (list check pkg-config))
+    (home-page "https://github.com/benmcollins/libjwt")
+    (synopsis "C @acronym{JWT, JSON Web Token} library")
+    (description "@code{libjwt} is a @acronym{JWT, JSON Web Token} library for
+C.")
+    (license license:mpl2.0)))
+
 (define-public yajl
   (package
     (name "yajl")
-- 
2.45.2

[bug#72766] [PATCH 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

* gnu/packages/parallel.scm (slurm)[inputs]: Add dbus, http-parser, libjwt,
librdkafka, libtaml, linux-libre-headers and (mariadb "dev").
[arguments]: Add --enable-slurmrestd, --with-bpf, --with-http-parser and
--with-rdkafka to configure flags.

Change-Id: Icddaa8f883754129d975e82fccd732a47a467517
---
 gnu/packages/parallel.scm | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/gnu/packages/parallel.scm b/gnu/packages/parallel.scm
index e75e902a68..21c945382f 100644
--- a/gnu/packages/parallel.scm
+++ b/gnu/packages/parallel.scm
@@ -15,6 +15,7 @@
 ;;; Copyright © 2024 Zheng Junjie <873216071@qq.com>
 ;;; Copyright © 2024 David Elsing <david.elsing@posteo.net>
 ;;; Copyright © 2024 Romain Garbage <romain.garbage@inria.fr>
+;;; Copyright © 2024 Arun Isaac <arunisaac@systemreboot.net>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -49,13 +50,16 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages base)
   #:use-module (gnu packages bash)
   #:use-module (gnu packages check)
+  #:use-module (gnu packages databases)
   #:use-module (gnu packages documentation)
   #:use-module (gnu packages flex)
   #:use-module (gnu packages freeipmi)
+  #:use-module (gnu packages glib)
   #:use-module (gnu packages libevent)
   #:use-module (gnu packages linux)
   #:use-module (gnu packages maths)
   #:use-module (gnu packages mpi)
+  #:use-module (gnu packages networking)
   #:use-module (gnu packages perl)
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages python)
@@ -63,6 +67,7 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages python-science)
   #:use-module (gnu packages python-xyz)
   #:use-module (gnu packages readline)
+  #:use-module (gnu packages serialization)
   #:use-module (gnu packages tcl)
   #:use-module (gnu packages tls)
   #:use-module (gnu packages web))
@@ -223,12 +228,18 @@ (define-public slurm
                   (mkdir "contribs")
                   (rename-file "tmp-pmi" "contribs/pmi")
                   (rename-file "tmp-pmi2" "contribs/pmi2")))))
-    ;; FIXME: More optional inputs could be added,
-    ;; in particular mysql and gtk+.
-    (inputs (list freeipmi
+    ;; FIXME: More optional inputs (for example, gtk+) could be added.
+    (inputs (list dbus
+                  freeipmi
+                  http-parser
                   `(,hwloc-2 "lib")
                   json-c
+                  libjwt
+                  librdkafka
+                  libyaml
+                  linux-libre-headers
                   linux-pam
+                  `(,mariadb "dev")
                   openpmix
                   munge
                   numactl
@@ -238,13 +249,20 @@ (define-public slurm
     (build-system gnu-build-system)
     (arguments
      (list #:configure-flags
-           #~(list "--enable-pam" "--sysconfdir=/etc/slurm"
+           #~(list "--enable-pam"
+                   "--enable-slurmrestd"
+                   "--sysconfdir=/etc/slurm"
                    "--disable-static"
+                   (string-append "--with-bpf="
+                                  #$(this-package-input "linux-libre-headers"))
                    (string-append "--with-freeipmi=" #$(this-package-input "freeipmi"))
+                   (string-append "--with-http-parser="
+                                  #$(this-package-input "http-parser"))
                    (string-append "--with-hwloc="
                                   (ungexp (this-package-input "hwloc") "lib"))
                    (string-append "--with-json=" #$(this-package-input "json-c"))
                    (string-append "--with-munge=" #$(this-package-input "munge"))
+                   (string-append "--with-rdkafka=" #$(this-package-input "librdkafka"))
 
                    ;; Use PMIx bundled with Open MPI (this is required for Open MPI 5.x).
                    ;; Note: Older versions that inherit from this package lack the
-- 
2.45.2

[bug#72766] [PATCH 2/2] gnu: slurm: Enable REST API.

[Ludovic Courtès]

Hi Arun,

Arun Isaac <arunisaac@systemreboot.net> skribis:

> * gnu/packages/parallel.scm (slurm)[inputs]: Add dbus, http-parser, libjwt,
> librdkafka, libtaml, linux-libre-headers and (mariadb "dev").
> [arguments]: Add --enable-slurmrestd, --with-bpf, --with-http-parser and
> --with-rdkafka to configure flags.
>
> Change-Id: Icddaa8f883754129d975e82fccd732a47a467517

I’m curious what the REST interface has to offer compared to DRMAA.

> +    (inputs (list dbus
> +                  freeipmi
> +                  http-parser
>                    `(,hwloc-2 "lib")
>                    json-c
> +                  libjwt
> +                  librdkafka
> +                  libyaml
> +                  linux-libre-headers
>                    linux-pam
> +                  `(,mariadb "dev")

That’s quite a few heavyweight dependencies (‘linux-libre-headers’ is
not necessary; it’s included by default).

It’s a bit of Inheritance Game in this file (ah ha!) so we should make
sure the other slurm variants still work fine.  qa.guix hasn’t tested it
yet.

Thanks,
Ludo’.

[bug#72766] [PATCH 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

Hi Ludo,

> I’m curious what the REST interface has to offer compared to DRMAA.

I don't know, to be honest.

I've started working on ravanan, a new implementation of the Common
Workflow Language that takes inspiration from Guix and integrates with
Guix for strong reproducibility.

https://git.systemreboot.net/ravanan/about/

ravanan uses the slurm REST API to talk to slurm, and that's why I need
this patch. I chose the slurm REST API over DRMAA rather arbitrarily. In
the future, I could support both in ravanan. So, it's still good to have
both.

>> +    (inputs (list dbus
>> +                  freeipmi
>> +                  http-parser
>>                    `(,hwloc-2 "lib")
>>                    json-c
>> +                  libjwt
>> +                  librdkafka
>> +                  libyaml
>> +                  linux-libre-headers
>>                    linux-pam
>> +                  `(,mariadb "dev")
>
> That’s quite a few heavyweight dependencies (‘linux-libre-headers’ is
> not necessary; it’s included by default).

Good catch! I'll remove it before pushing.

> It’s a bit of Inheritance Game in this file (ah ha!) so we should make
> sure the other slurm variants still work fine.  qa.guix hasn’t tested it
> yet.

I have built the other slurm variants, and they seem to build fine. I
also used these packages to migrate a cluster from ancient slurm 18.08
to modern slurm 23.02 in increments of two versions, and tested
functionality at each step. They all seem to work. So, I am relatively
confident these packages work.

Regards,
Arun

[bug#72766] [PATCH 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

>> That’s quite a few heavyweight dependencies (‘linux-libre-headers’ is
>> not necessary; it’s included by default).
>
> Good catch! I'll remove it before pushing.

But, how do I refer to an implicit input (linux-libre-headers, in this
case) using this-package-input? Is there some other way to do this?

Thanks!

[bug#72766] [PATCH 2/2] gnu: slurm: Enable REST API.

[Ludovic Courtès]

Hey Arun,

Arun Isaac <arunisaac@systemreboot.net> skribis:

>>> That’s quite a few heavyweight dependencies (‘linux-libre-headers’ is
>>> not necessary; it’s included by default).
>>
>> Good catch! I'll remove it before pushing.
>
> But, how do I refer to an implicit input (linux-libre-headers, in this
> case) using this-package-input? Is there some other way to do this?

It depends on the details, but perhaps you can use ‘search-input-file’?
‘this-package-input’ doesn’t let you access implicit inputs; it just
accesses the regular inputs fields.

HTH,
Ludo’.

[bug#72766] [PATCH 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

>> But, how do I refer to an implicit input (linux-libre-headers, in this
>> case) using this-package-input? Is there some other way to do this?
>
> It depends on the details, but perhaps you can use ‘search-input-file’?
> ‘this-package-input’ doesn’t let you access implicit inputs; it just
> accesses the regular inputs fields.

Fixed with search-input-directory. Patchset v2 follows.

Thanks!

[bug#72766] [PATCH v2 1/2] gnu: Add libjwt.

[Arun Isaac]

* gnu/packages/web.scm (libjwt): New variable.

Change-Id: Ied516fb41a90de316588622b999b27f0f7ee8de3
---
 gnu/packages/web.scm | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/web.scm b/gnu/packages/web.scm
index c768874a4a..735cc5ce98 100644
--- a/gnu/packages/web.scm
+++ b/gnu/packages/web.scm
@@ -15,7 +15,7 @@
 ;;; Copyright © 2016 Ben Woodcroft <donttrustben@gmail.com>
 ;;; Copyright © 2016, 2023 Clément Lassieur <clement@lassieur.org>
 ;;; Copyright © 2016, 2017 Nikita <nikita@n0.is>
-;;; Copyright © 2016–2023 Arun Isaac <arunisaac@systemreboot.net>
+;;; Copyright © 2016–2024 Arun Isaac <arunisaac@systemreboot.net>
 ;;; Copyright © 2016–2022 Tobias Geerinckx-Rice <me@tobias.gr>
 ;;; Copyright © 2016 Bake Timmons <b3timmons@speedymail.org>
 ;;; Copyright © 2017 Thomas Danckaert <post@thomasdanckaert.be>
@@ -1610,6 +1610,30 @@ (define-public rapidjson
 style API.")
       (license license:expat))))
 
+(define-public libjwt
+  (package
+    (name "libjwt")
+    (version "1.17.2")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (string-append "https://github.com/benmcollins/libjwt/releases/download/v"
+                           version "/libjwt-"
+                           version ".tar.bz2"))
+       (sha256
+        (base32
+         "1bpfaa0y8bccz5hr677lkrprs07akx02k0qbf82z2c8syr24a77i"))))
+    (build-system gnu-build-system)
+    (inputs
+     (list jansson openssl))
+    (native-inputs
+     (list check pkg-config))
+    (home-page "https://github.com/benmcollins/libjwt")
+    (synopsis "C @acronym{JWT, JSON Web Token} library")
+    (description "@code{libjwt} is a @acronym{JWT, JSON Web Token} library for
+C.")
+    (license license:mpl2.0)))
+
 (define-public yajl
   (package
     (name "yajl")

base-commit: cc2619a7bd7542d834762de2430937a59d1f3a36
-- 
2.45.2

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

* gnu/packages/parallel.scm (slurm)[inputs]: Add dbus, http-parser, libjwt,
librdkafka, libtaml and (mariadb "dev").
[arguments]: Add --enable-slurmrestd, --with-bpf, --with-http-parser,
--with-rdkafka and --with-yaml to configure flags.

Change-Id: Icddaa8f883754129d975e82fccd732a47a467517
---
 gnu/packages/parallel.scm | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/gnu/packages/parallel.scm b/gnu/packages/parallel.scm
index d5a55a8641..5d7544783a 100644
--- a/gnu/packages/parallel.scm
+++ b/gnu/packages/parallel.scm
@@ -15,6 +15,7 @@
 ;;; Copyright © 2024 Zheng Junjie <873216071@qq.com>
 ;;; Copyright © 2024 David Elsing <david.elsing@posteo.net>
 ;;; Copyright © 2024 Romain Garbage <romain.garbage@inria.fr>
+;;; Copyright © 2024 Arun Isaac <arunisaac@systemreboot.net>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -49,13 +50,16 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages base)
   #:use-module (gnu packages bash)
   #:use-module (gnu packages check)
+  #:use-module (gnu packages databases)
   #:use-module (gnu packages documentation)
   #:use-module (gnu packages flex)
   #:use-module (gnu packages freeipmi)
+  #:use-module (gnu packages glib)
   #:use-module (gnu packages libevent)
   #:use-module (gnu packages linux)
   #:use-module (gnu packages maths)
   #:use-module (gnu packages mpi)
+  #:use-module (gnu packages networking)
   #:use-module (gnu packages perl)
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages python)
@@ -63,6 +67,7 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages python-science)
   #:use-module (gnu packages python-xyz)
   #:use-module (gnu packages readline)
+  #:use-module (gnu packages serialization)
   #:use-module (gnu packages tcl)
   #:use-module (gnu packages tls)
   #:use-module (gnu packages web))
@@ -224,12 +229,17 @@ (define-public slurm
                   (mkdir "contribs")
                   (rename-file "tmp-pmi" "contribs/pmi")
                   (rename-file "tmp-pmi2" "contribs/pmi2")))))
-    ;; FIXME: More optional inputs could be added,
-    ;; in particular mysql and gtk+.
-    (inputs (list freeipmi
+    ;; FIXME: More optional inputs (for example, gtk+) could be added.
+    (inputs (list dbus
+                  freeipmi
+                  http-parser
                   `(,hwloc-2 "lib")
                   json-c
+                  libjwt
+                  librdkafka
+                  libyaml
                   linux-pam
+                  `(,mariadb "dev")
                   openpmix
                   munge
                   numactl
@@ -239,13 +249,22 @@ (define-public slurm
     (build-system gnu-build-system)
     (arguments
      (list #:configure-flags
-           #~(list "--enable-pam" "--sysconfdir=/etc/slurm"
+           #~(list "--enable-pam"
+                   "--enable-slurmrestd"
+                   "--sysconfdir=/etc/slurm"
                    "--disable-static"
+                   (string-append "--with-bpf="
+                                  (dirname (dirname (search-input-directory
+                                                     %build-inputs "include/linux"))))
                    (string-append "--with-freeipmi=" #$(this-package-input "freeipmi"))
+                   (string-append "--with-http-parser="
+                                  #$(this-package-input "http-parser"))
                    (string-append "--with-hwloc="
                                   (ungexp (this-package-input "hwloc") "lib"))
                    (string-append "--with-json=" #$(this-package-input "json-c"))
                    (string-append "--with-munge=" #$(this-package-input "munge"))
+                   (string-append "--with-rdkafka=" #$(this-package-input "librdkafka"))
+                   (string-append "--with-yaml=" #$(this-package-input "libyaml"))
 
                    ;; Use PMIx bundled with Open MPI (this is required for Open MPI 5.x).
                    ;; Note: Older versions that inherit from this package lack the
-- 
2.45.2

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Ludovic Courtès]

Hello Arun!

Arun Isaac <arunisaac@systemreboot.net> skribis:

> * gnu/packages/parallel.scm (slurm)[inputs]: Add dbus, http-parser, libjwt,
> librdkafka, libtaml and (mariadb "dev").
> [arguments]: Add --enable-slurmrestd, --with-bpf, --with-http-parser,
> --with-rdkafka and --with-yaml to configure flags.
>
> Change-Id: Icddaa8f883754129d975e82fccd732a47a467517

[...]

> +    (inputs (list dbus
> +                  freeipmi
> +                  http-parser
>                    `(,hwloc-2 "lib")
>                    json-c
> +                  libjwt
> +                  librdkafka
> +                  libyaml
>                    linux-pam
> +                  `(,mariadb "dev")

Could you check how this impacts ‘guix size slurm’?

The reason I’m asking is that we at work often end up with slurm in
packs that we carry to supercomputers, so it would be best if it wasn’t
too big:

--8<---------------cut here---------------start------------->8---
$ guix size slurm |tail -1
total: 359.8 MiB
$ guix size slurm mariadb |tail -1
total: 663.4 MiB
--8<---------------cut here---------------end--------------->8---

I wonder if we should have a ‘slurm-minimal’ variant or something?

(That could probably come as a patch after this series.)

Apart from that, both patches look good to me!

Thanks,
Ludo’.

[bug#72766] [PATCH 2/2] gnu: slurm: Enable REST API.

[Ludovic Courtès]

Hey,

Arun Isaac <arunisaac@systemreboot.net> skribis:

>> I’m curious what the REST interface has to offer compared to DRMAA.
>
> I don't know, to be honest.
>
> I've started working on ravanan, a new implementation of the Common
> Workflow Language that takes inspiration from Guix and integrates with
> Guix for strong reproducibility.
>
> https://git.systemreboot.net/ravanan/about/

Nice!

> ravanan uses the slurm REST API to talk to slurm, and that's why I need
> this patch. I chose the slurm REST API over DRMAA rather arbitrarily. In
> the future, I could support both in ravanan. So, it's still good to have
> both.

I’m mentioning DRMAA because Ricardo wrote guile-drmaa for similar
purposes, and I think DRMAA is implemented by several batch schedulers.
But I don’t know whether it would be a good fit for what you need.

Ludo’.

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

Hi Ludo,

> Could you check how this impacts ‘guix size slurm’?

The size goes up from 141 MiB to 407 MiB.

> The reason I’m asking is that we at work often end up with slurm in
> packs that we carry to supercomputers, so it would be best if it wasn’t
> too big:
>
> --8<---------------cut here---------------start------------->8---
> $ guix size slurm |tail -1
> total: 359.8 MiB
> $ guix size slurm mariadb |tail -1
> total: 663.4 MiB
> --8<---------------cut here---------------end--------------->8---
>
> I wonder if we should have a ‘slurm-minimal’ variant or something?
>
> (That could probably come as a patch after this series.)

Yes, I could add a slurm-minimal. Would you like to have a slurm-minimal
for every version of slurm or just the latest?

Regards,
Arun

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Ludovic Courtès]

Hello!

Arun Isaac <arunisaac@systemreboot.net> skribis:

>> Could you check how this impacts ‘guix size slurm’?
>
> The size goes up from 141 MiB to 407 MiB.

D’oh!

>> I wonder if we should have a ‘slurm-minimal’ variant or something?
>>
>> (That could probably come as a patch after this series.)
>
> Yes, I could add a slurm-minimal. Would you like to have a slurm-minimal
> for every version of slurm or just the latest?

Hmm!  The reason we keep different versions of slurm is because one
always needs a slurm client compatible with the version of slurmd
running on the target cluster.  And so yes, we’d need ‘slurm-minimal’
for each version.

But that doesn’t sound great so I’m not sure.

Another possibility would be to do the opposite: keep ‘slurm’ without
MariaDB & co., and have a ‘slurm-rest’ (?) variant with all the bells
and whistles?

Thanks,
Ludo’.

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

Hi Ludo,

> Another possibility would be to do the opposite: keep ‘slurm’ without
> MariaDB & co., and have a ‘slurm-rest’ (?) variant with all the bells
> and whistles?

The problem with that approach is that I'm sure there will be more
optional features to add to slurm. Then, we would have a combinatorial
explosion of packages---slurm-rest, slurm-rest+feature1,
slurm-rest+feature2, slurm-feature1+feature2, etc. I am ok with having a
slurm-minimal for all versions. That's the lesser evil.

Regards,
Arun

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Ludovic Courtès]

Hi,

Arun Isaac <arunisaac@systemreboot.net> skribis:

>> Another possibility would be to do the opposite: keep ‘slurm’ without
>> MariaDB & co., and have a ‘slurm-rest’ (?) variant with all the bells
>> and whistles?
>
> The problem with that approach is that I'm sure there will be more
> optional features to add to slurm. Then, we would have a combinatorial
> explosion of packages---slurm-rest, slurm-rest+feature1,
> slurm-rest+feature2, slurm-feature1+feature2, etc.

Not necessarily: that second variant could be maximal, where additional
features would be added.

> I am ok with having a slurm-minimal for all versions. That's the
> lesser evil.

OK.

There might be a third option: moving server-side code to a separate
output.  That way, perhaps client-side code (things like ‘openmpi’)
would not have MariaDB & co. in their closure?

Ludo’.

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

>> I am ok with having a slurm-minimal for all versions. That's the
>> lesser evil.
>
> OK.

I'll send a v3 patchset with the minimal variants soon..

> There might be a third option: moving server-side code to a separate
> output.  That way, perhaps client-side code (things like ‘openmpi’)
> would not have MariaDB & co. in their closure?

I presume you might need to carry both the server-side and the
client-side code in a guix pack. So, how would this option help?

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Ludovic Courtès]

Morning!

Arun Isaac <arunisaac@systemreboot.net> skribis:

>> There might be a third option: moving server-side code to a separate
>> output.  That way, perhaps client-side code (things like ‘openmpi’)
>> would not have MariaDB & co. in their closure?
>
> I presume you might need to carry both the server-side and the
> client-side code in a guix pack. So, how would this option help?

No, to run code on a supercomputer, you only need MPI and things like
‘salloc’, ‘sbatch’, etc.  There’s definitely no need for ‘slurmd’ or
anything like that in that case because it’s already running on the
machine.

Ludo’.

[bug#72766] [PATCH v2 2/2] gnu: slurm: Enable REST API.

[Arun Isaac]

>> I presume you might need to carry both the server-side and the
>> client-side code in a guix pack. So, how would this option help?
>
> No, to run code on a supercomputer, you only need MPI and things like
> ‘salloc’, ‘sbatch’, etc.  There’s definitely no need for ‘slurmd’ or
> anything like that in that case because it’s already running on the
> machine.

Ah, ok, the server-side is maintained by a different admin team. I was
thinking you were maintaining the server-side as well. That makes sense,
thank you!

In that case, I'll split slurm into two outputs---a server and a
client-side. I'll send a patch in the coming days.

[bug#72766] [PATCH v3 0/2] slurm: Enable REST API

[Arun Isaac]

Hi Ludo,

I pushed the patch adding libjwt. And, I have added a patch moving slurm
client executables into a separate "client" output. `guix size' reports that
the "out" and "client" outputs are respectively 192.6 MiB and 201.3 MiB. For
comparison, before this patch, the size of slurm was 141.3 MiB.

Everything LGTM. I'll push if there are no objections.

Thanks!
Arun

Arun Isaac (2):
  gnu: slurm: Enable REST API.
  gnu: slurm: Move client executables into separate output.

 gnu/packages/parallel.scm | 40 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 5 deletions(-)


base-commit: ba0340eec2e50439cfd94e85b40bf41c2d488a74
prerequisite-patch-id: 0a3dc095f7f4c2fc32778dfc299d7bfa447a8ec4
-- 
2.45.2

[bug#72766] [PATCH v3 1/2] gnu: slurm: Enable REST API.

[Arun Isaac]

* gnu/packages/parallel.scm (slurm)[inputs]: Add dbus, http-parser, libjwt,
librdkafka, libtaml and (mariadb "dev").
[arguments]: Add --enable-slurmrestd, --with-bpf, --with-http-parser,
--with-rdkafka and --with-yaml to configure flags.

Change-Id: Icddaa8f883754129d975e82fccd732a47a467517
---
 gnu/packages/parallel.scm | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/gnu/packages/parallel.scm b/gnu/packages/parallel.scm
index fc96136065..522d326ef2 100644
--- a/gnu/packages/parallel.scm
+++ b/gnu/packages/parallel.scm
@@ -15,6 +15,7 @@
 ;;; Copyright © 2024 Zheng Junjie <873216071@qq.com>
 ;;; Copyright © 2024 David Elsing <david.elsing@posteo.net>
 ;;; Copyright © 2024 Romain Garbage <romain.garbage@inria.fr>
+;;; Copyright © 2024 Arun Isaac <arunisaac@systemreboot.net>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -49,14 +50,17 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages base)
   #:use-module (gnu packages bash)
   #:use-module (gnu packages check)
+  #:use-module (gnu packages databases)
   #:use-module (gnu packages documentation)
   #:use-module (gnu packages flex)
   #:use-module (gnu packages freeipmi)
   #:use-module (gnu packages gcc)
+  #:use-module (gnu packages glib)
   #:use-module (gnu packages libevent)
   #:use-module (gnu packages linux)
   #:use-module (gnu packages maths)
   #:use-module (gnu packages mpi)
+  #:use-module (gnu packages networking)
   #:use-module (gnu packages perl)
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages python)
@@ -64,6 +68,7 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages python-science)
   #:use-module (gnu packages python-xyz)
   #:use-module (gnu packages readline)
+  #:use-module (gnu packages serialization)
   #:use-module (gnu packages tcl)
   #:use-module (gnu packages tls)
   #:use-module (gnu packages web))
@@ -223,12 +228,17 @@ (define-public slurm
                   (mkdir "contribs")
                   (rename-file "tmp-pmi" "contribs/pmi")
                   (rename-file "tmp-pmi2" "contribs/pmi2")))))
-    ;; FIXME: More optional inputs could be added,
-    ;; in particular mysql and gtk+.
-    (inputs (list freeipmi
+    ;; FIXME: More optional inputs (for example, gtk+) could be added.
+    (inputs (list dbus
+                  freeipmi
+                  http-parser
                   `(,hwloc-2 "lib")
                   json-c
+                  libjwt
+                  librdkafka
+                  libyaml
                   linux-pam
+                  `(,mariadb "dev")
                   openpmix
                   munge
                   numactl
@@ -238,13 +248,22 @@ (define-public slurm
     (build-system gnu-build-system)
     (arguments
      (list #:configure-flags
-           #~(list "--enable-pam" "--sysconfdir=/etc/slurm"
+           #~(list "--enable-pam"
+                   "--enable-slurmrestd"
+                   "--sysconfdir=/etc/slurm"
                    "--disable-static"
+                   (string-append "--with-bpf="
+                                  (dirname (dirname (search-input-directory
+                                                     %build-inputs "include/linux"))))
                    (string-append "--with-freeipmi=" #$(this-package-input "freeipmi"))
+                   (string-append "--with-http-parser="
+                                  #$(this-package-input "http-parser"))
                    (string-append "--with-hwloc="
                                   (ungexp (this-package-input "hwloc") "lib"))
                    (string-append "--with-json=" #$(this-package-input "json-c"))
                    (string-append "--with-munge=" #$(this-package-input "munge"))
+                   (string-append "--with-rdkafka=" #$(this-package-input "librdkafka"))
+                   (string-append "--with-yaml=" #$(this-package-input "libyaml"))
 
                    ;; Use PMIx bundled with Open MPI (this is required for Open MPI 5.x).
                    ;; Note: Older versions that inherit from this package lack the
-- 
2.45.2

[bug#72766] [PATCH v3 2/2] gnu: slurm: Move client executables into separate output.

[Arun Isaac]

* gnu/packages/parallel.scm (slurm)[outputs]: Add client.
[arguments]: Add move-client-executables phase.

Change-Id: Id7441b0b34a5b8433e4c38bd5c56e4ca1fec587c
---
 gnu/packages/parallel.scm | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/gnu/packages/parallel.scm b/gnu/packages/parallel.scm
index 522d326ef2..f27f07ca1e 100644
--- a/gnu/packages/parallel.scm
+++ b/gnu/packages/parallel.scm
@@ -246,6 +246,7 @@ (define-public slurm
     (native-inputs
      (list autoconf expect perl pkg-config python-wrapper))
     (build-system gnu-build-system)
+    (outputs '("out" "client"))
     (arguments
      (list #:configure-flags
            #~(list "--enable-pam"
@@ -291,7 +292,17 @@ (define-public slurm
                    (invoke "make" "install" "-C" "contribs/pmi")
 
                    ;; Others expect pmi2.
-                   (invoke "make" "install" "-C" "contribs/pmi2"))))))
+                   (invoke "make" "install" "-C" "contribs/pmi2")))
+               (add-after 'install 'move-client-executables
+                 (lambda _
+                   (let ((client-bin (string-append #$output:client "/bin")))
+                     (mkdir-p client-bin)
+                     (for-each (lambda (executable)
+                                 (rename-file executable
+                                              (string-append client-bin
+                                                             "/"
+                                                             (basename executable))))
+                               (find-files (string-append #$output "/bin")))))))))
     (home-page "https://slurm.schedmd.com/")
     (synopsis "Workload manager for cluster computing")
     (description
-- 
2.45.2

[bug#72766] [PATCH v3 2/2] gnu: slurm: Move client executables into separate output.

[Ludovic Courtès]

Hi Arun,

Arun Isaac <arunisaac@systemreboot.net> skribis:

> * gnu/packages/parallel.scm (slurm)[outputs]: Add client.
> [arguments]: Add move-client-executables phase.
>
> Change-Id: Id7441b0b34a5b8433e4c38bd5c56e4ca1fec587c

[...]

> +    (outputs '("out" "client"))
>      (arguments
>       (list #:configure-flags
>             #~(list "--enable-pam"
> @@ -291,7 +292,17 @@ (define-public slurm
>                     (invoke "make" "install" "-C" "contribs/pmi")
>  
>                     ;; Others expect pmi2.
> -                   (invoke "make" "install" "-C" "contribs/pmi2"))))))
> +                   (invoke "make" "install" "-C" "contribs/pmi2")))
> +               (add-after 'install 'move-client-executables
> +                 (lambda _
> +                   (let ((client-bin (string-append #$output:client "/bin")))
> +                     (mkdir-p client-bin)
> +                     (for-each (lambda (executable)
> +                                 (rename-file executable
> +                                              (string-append client-bin
> +                                                             "/"
> +                                                             (basename executable))))
> +                               (find-files (string-append #$output "/bin")))))))))

I think “bin” would be a more conventional name for the output; this is
used in a couple of packages and recognized by ‘package->recutils’.

However, what does ‘guix size slurm:client’ report?  I suspect it
depends on slurm:out, in which case moving to a separate output makes no
difference from that perspective.

Thanks,
Ludo’.

[bug#72766] [PATCH v3 2/2] gnu: slurm: Move client executables into separate output.

[Arun Isaac]

Hi Ludo,

> I think “bin” would be a more conventional name for the output; this is
> used in a couple of packages and recognized by ‘package->recutils’.

Sure, fixed now.

> However, what does ‘guix size slurm:client’ report?  I suspect it
> depends on slurm:out, in which case moving to a separate output makes no
> difference from that perspective.

Yep, that's right.

$ ./pre-inst-env guix size slurm:out | tail -n1
total: 192.6 MiB
$ ./pre-inst-env guix size slurm:bin | tail -n1
total: 201.3 MiB
$ ./pre-inst-env guix size slurm:bin | grep slurm
/gnu/store/2qhgkq79ahw64n4kmr3znl8v66z0n87j-slurm-23.11.10         192.6    17.5   8.7%
/gnu/store/6f3d93i0zdr48v6bp471dfsk7r89xdqh-slurm-23.11.10-bin     201.3     1.3   0.6%

Should I then go back to the idea of creating a slurm-minimal variant?

Regards,
Arun

[bug#72766] [PATCH v3 2/2] gnu: slurm: Move client executables into separate output.

[Ludovic Courtès]

Hello,

Arun Isaac <arunisaac@systemreboot.net> skribis:

>> I think “bin” would be a more conventional name for the output; this is
>> used in a couple of packages and recognized by ‘package->recutils’.
>
> Sure, fixed now.
>
>> However, what does ‘guix size slurm:client’ report?  I suspect it
>> depends on slurm:out, in which case moving to a separate output makes no
>> difference from that perspective.
>
> Yep, that's right.
>
> $ ./pre-inst-env guix size slurm:out | tail -n1
> total: 192.6 MiB
> $ ./pre-inst-env guix size slurm:bin | tail -n1
> total: 201.3 MiB
> $ ./pre-inst-env guix size slurm:bin | grep slurm
> /gnu/store/2qhgkq79ahw64n4kmr3znl8v66z0n87j-slurm-23.11.10         192.6    17.5   8.7%
> /gnu/store/6f3d93i0zdr48v6bp471dfsk7r89xdqh-slurm-23.11.10-bin     201.3     1.3   0.6%

OK.

Hmm I just realized that the introduction of this extra output would at
least deserve a news entry too, because I can already see myself as the
Guix support guy at my workplace having to reply to confused colleagues
who realize that ‘guix shell slurm‘ or similar no longer gives them
‘salloc’ & co.  :-)

> Should I then go back to the idea of creating a slurm-minimal variant?

Yes, probably.

Apologies for all the back and forth and hesitations; it’s tricky!

Ludo’.

[bug#72766] [PATCH v3 2/2] gnu: slurm: Move client executables into separate output.

[Arun Isaac]

> Hmm I just realized that the introduction of this extra output would at
> least deserve a news entry too, because I can already see myself as the
> Guix support guy at my workplace having to reply to confused colleagues
> who realize that ‘guix shell slurm‘ or similar no longer gives them
> ‘salloc’ & co.  :-)
>
>> Should I then go back to the idea of creating a slurm-minimal variant?
>
> Yes, probably.
>
> Apologies for all the back and forth and hesitations; it’s tricky!

No worries! A v4 patch follows. Do I need to add a news entry for this
v4 patch? I guess not, but just checking.

[bug#72766] [PATCH v4] gnu: slurm: Enable REST API.

[Arun Isaac]

Rename slurm and slurm-* to slurm-minimal and slurm-minimal-*. Add new slurm
packages that inherit from slurm-minimal and have the REST API enabled.

* gnu/packages/parallel.scm (make-slurm): New function.
(slurm-minimal, slurm-minimal-23.02, slurm-minimal-22.05, slurm-minimal-21.08,
slurm-minimal-20.11, slurm-minimal-20.02, slurm-minimal-19.05,
slurm-minimal-18.08): New variables.
(slurm, slurm-23.02, slurm-22.05, slurm-21.08, slurm-20.11, slurm-20.02,
slurm-19.05, slurm-18.08): Inherit from slurm-minimal, slurm-minimal-23.02,
slurm-minimal-22.05, slurm-minimal-21.08, slurm-minimal-20.11,
slurm-minimal-20.02, slurm-minimal-19.05 and slurm-minimal-18.08 respectively.

Change-Id: I6410ac385d8876f4b6a908ddb1272172475a87bb
---
 gnu/packages/parallel.scm | 98 ++++++++++++++++++++++++++++-----------
 1 file changed, 72 insertions(+), 26 deletions(-)

diff --git a/gnu/packages/parallel.scm b/gnu/packages/parallel.scm
index 89d82fb8da..66997951c3 100644
--- a/gnu/packages/parallel.scm
+++ b/gnu/packages/parallel.scm
@@ -15,6 +15,7 @@
 ;;; Copyright © 2024 Zheng Junjie <873216071@qq.com>
 ;;; Copyright © 2024 David Elsing <david.elsing@posteo.net>
 ;;; Copyright © 2024 Romain Garbage <romain.garbage@inria.fr>
+;;; Copyright © 2024 Arun Isaac <arunisaac@systemreboot.net>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -49,14 +50,17 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages base)
   #:use-module (gnu packages bash)
   #:use-module (gnu packages check)
+  #:use-module (gnu packages databases)
   #:use-module (gnu packages documentation)
   #:use-module (gnu packages flex)
   #:use-module (gnu packages freeipmi)
   #:use-module (gnu packages gcc)
+  #:use-module (gnu packages glib)
   #:use-module (gnu packages libevent)
   #:use-module (gnu packages linux)
   #:use-module (gnu packages maths)
   #:use-module (gnu packages mpi)
+  #:use-module (gnu packages networking)
   #:use-module (gnu packages perl)
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages python)
@@ -64,6 +68,7 @@ (define-module (gnu packages parallel)
   #:use-module (gnu packages python-science)
   #:use-module (gnu packages python-xyz)
   #:use-module (gnu packages readline)
+  #:use-module (gnu packages serialization)
   #:use-module (gnu packages tcl)
   #:use-module (gnu packages tls)
   #:use-module (gnu packages web))
@@ -190,9 +195,9 @@ (define-public xjobs
 when jobs finish.")
     (license license:gpl2+)))
 
-(define-public slurm
+(define-public slurm-minimal
   (package
-    (name "slurm")
+    (name "slurm-minimal")
     (version "23.11.10")
     (source (origin
               (method url-fetch)
@@ -223,8 +228,6 @@ (define-public slurm
                   (mkdir "contribs")
                   (rename-file "tmp-pmi" "contribs/pmi")
                   (rename-file "tmp-pmi2" "contribs/pmi2")))))
-    ;; FIXME: More optional inputs could be added,
-    ;; in particular mysql and gtk+.
     (inputs (list freeipmi
                   `(,hwloc-2 "lib")
                   json-c
@@ -291,6 +294,35 @@ (define-public slurm
                     "https://github.com/SchedMD/slurm/blob/master/contribs/pmi2/COPYRIGHT")
                    license:gpl2+))))   ; the rest, often with OpenSSL exception
 
+(define (make-slurm base-slurm)
+  "Make a slurm package with all optional features enabled. Base it off of the
+minimal slurm package BASE-SLURM."
+  (package
+    (inherit base-slurm)
+    (name "slurm")
+    (arguments
+     (substitute-keyword-arguments (package-arguments base-slurm)
+       ((#:configure-flags flags #~'())
+        #~(cons* "--enable-slurmrestd"
+                 (string-append "--with-bpf="
+                                (dirname
+                                 (dirname (search-input-directory
+                                           %build-inputs "include/linux"))))
+                 (string-append "--with-http-parser="
+                                #$(this-package-input "http-parser"))
+                 (string-append "--with-rdkafka="
+                                #$(this-package-input "librdkafka"))
+                 (string-append "--with-yaml="
+                                #$(this-package-input "libyaml"))
+                 #$flags))))
+    ;; FIXME: More optional inputs could be added.
+    (inputs
+     (modify-inputs (package-inputs base-slurm)
+       (prepend dbus freeipmi http-parser
+                libjwt librdkafka libyaml (list mariadb "dev"))))))
+
+(define-public slurm (make-slurm slurm-minimal))
+
 ;; The SLURM client/daemon protocol and file format changes from time to time
 ;; in incompatible ways, as noted in
 ;; <https://slurm.schedmd.com/troubleshoot.html#network>.  Thus, keep older
@@ -298,9 +330,9 @@ (define-public slurm
 ;; As noted in the link, YY.MM is the release scheme, and the 'maintenance'
 ;; digit does not introduce incompatibilities.
 
-(define-public slurm-23.02
+(define-public slurm-minimal-23.02
   (package
-   (inherit slurm)
+   (inherit slurm-minimal)
    (version "23.02.6")
     (source (origin
              (inherit (package-source slurm))
@@ -314,12 +346,14 @@ (define-public slurm-23.02
               (base32
                "08rz3r1rlnb3pmfdnbh542gm44ja0fdy8rkj4vm4lclc48cvqp2a"))))))
 
-(define-public slurm-22.05
+(define-public slurm-23.02 (make-slurm slurm-minimal-23.02))
+
+(define-public slurm-minimal-22.05
   (package
-    (inherit slurm-23.02)
+    (inherit slurm-minimal-23.02)
     (version "22.05.1")
     (source (origin
-              (inherit (package-source slurm-23.02))
+              (inherit (package-source slurm-minimal-23.02))
               (method url-fetch)
               (uri (string-append
                     "https://download.schedmd.com/slurm/slurm-"
@@ -329,12 +363,14 @@ (define-public slurm-22.05
                (base32
                 "0f3hhlki8g7slllsnyj1qikbsvr62i0hig85lcdcfnmsagzlhbyi"))))))
 
-(define-public slurm-21.08
+(define-public slurm-22.05 (make-slurm slurm-minimal-22.05))
+
+(define-public slurm-minimal-21.08
   (package
-    (inherit slurm-22.05)
+    (inherit slurm-minimal-22.05)
     (version "21.08.8")
     (source (origin
-              (inherit (package-source slurm-22.05))
+              (inherit (package-source slurm-minimal-22.05))
               (method url-fetch)
               (uri (string-append
                     "https://download.schedmd.com/slurm/slurm-"
@@ -346,15 +382,17 @@ (define-public slurm-21.08
 
     ;; This and older versions of slurm have PMIx support but they seem to
     ;; require an older version of openpmix.  Disable PMIx support.
-    (inputs (modify-inputs (package-inputs slurm-22.05)
+    (inputs (modify-inputs (package-inputs slurm-minimal-22.05)
               (delete "openpmix")))))
 
-(define-public slurm-20.11
+(define-public slurm-21.08 (make-slurm slurm-minimal-21.08))
+
+(define-public slurm-minimal-20.11
   (package
-    (inherit slurm-21.08)
+    (inherit slurm-minimal-21.08)
     (version "20.11.9")
     (source (origin
-              (inherit (package-source slurm-21.08))
+              (inherit (package-source slurm-minimal-21.08))
               (method url-fetch)
               (uri (string-append
                     "https://download.schedmd.com/slurm/slurm-"
@@ -364,12 +402,14 @@ (define-public slurm-20.11
                (base32
                 "0xq2d6dm285y541dyg1h66z7svsisrq8c81ag0f601xz1cn3mq9m"))))))
 
-(define-public slurm-20.02
+(define-public slurm-20.11 (make-slurm slurm-minimal-20.11))
+
+(define-public slurm-minimal-20.02
   (package
-    (inherit slurm-20.11)
+    (inherit slurm-minimal-20.11)
     (version "20.02.6-1")
     (source (origin
-              (inherit (package-source slurm-20.11))
+              (inherit (package-source slurm-minimal-20.11))
               (method url-fetch)
               (uri (string-append
                     "https://download.schedmd.com/slurm/slurm-"
@@ -379,17 +419,19 @@ (define-public slurm-20.02
                (base32
                 "0qj4blfymrd2ry2qmb58l3jbr4jwygc3adcfw7my27rippcijlyc"))))
     (arguments
-     (substitute-keyword-arguments (package-arguments slurm-20.11)
+     (substitute-keyword-arguments (package-arguments slurm-minimal-20.11)
        ((#:configure-flags flags ''())
         #~(append '("CFLAGS=-O2 -g -fcommon" "LDFLAGS=-fcommon")
                   #$flags))))))
 
-(define-public slurm-19.05
+(define-public slurm-20.02 (make-slurm slurm-minimal-20.02))
+
+(define-public slurm-minimal-19.05
   (package
-    (inherit slurm-20.02)
+    (inherit slurm-minimal-20.02)
     (version "19.05.8")
     (source (origin
-              (inherit (package-source slurm-20.02))
+              (inherit (package-source slurm-minimal-20.02))
               (method url-fetch)
               (uri (string-append
                     "https://download.schedmd.com/slurm/slurm-"
@@ -399,14 +441,16 @@ (define-public slurm-19.05
                (base32
                 "10c9j4a9a6d4ibpf75006mn03p8xgpaprc247x2idakysjf2fw43"))))))
 
+(define-public slurm-19.05 (make-slurm slurm-minimal-19.05))
+
 ;; Same as Debian 10
-(define-public slurm-18.08
+(define-public slurm-minimal-18.08
   (package
-    (inherit slurm-19.05)
+    (inherit slurm-minimal-19.05)
     (version "18.08.9")
     (source
       (origin
-        (inherit (package-source slurm-20.02))
+        (inherit (package-source slurm-minimal-20.02))
         (uri (string-append
                "https://download.schedmd.com/slurm/slurm-"
                version ".tar.bz2"))
@@ -415,6 +459,8 @@ (define-public slurm-18.08
          (base32
           "1bgrpz75m7l4xhirsd0fvnkzlkrl8v2qpmjcz60barc5qm2kn457"))))))
 
+(define-public slurm-18.08 (make-slurm slurm-minimal-18.08))
+
 (define-public slurm-drmaa
   (package
     (name "slurm-drmaa")

base-commit: 2394a7f5fbf60dd6adc0a870366adb57166b6d8b
-- 
2.45.2