[bug#67613] Introduce unit tests for oci-container-service-type.

[bug#67613] Introduce unit tests for oci-container-service-type.

[paul via Guix-patches via]

Hi,

as discussed in issue #66160 and #67574 I'm sending a follow up with 
some unit tests for most of the internals of oci-container-service-type. 
These tests depend on the hotfix from #67574 since #66160 was merged 
with a blocking bug due to a last minute feature I added during the 
review process :( Hence if this gets merged before #67574 tests will fail .


Thank you for your help  and apologies for the noise,


giacomo

[bug#67613] [PATCH] tests: Add oci-container-service-type unit tests.

[Giacomo Leidi via Guix-patches via]

This patch is a followup to issue #66160 and issue #67574. It introduces
unit tests for the oci-container-service-type. 8 out 11 tests depend on
issue #67574 being merged since issue #66160 was merged with a blocking
bug from the beginning.

* gnu/services/docker.scm: Export
oci-container-configuration-container-user and
oci-container-configuration-workdir.
* tests/services/docker.scm: New file.
* Makefile.am (SCM_TESTS): Register it.

Change-Id: I47ed0fe36060ba84dd50b548a66f36e3df8a3710
---
 Makefile.am               |   1 +
 gnu/services/docker.scm   |   2 +
 tests/services/docker.scm | 187 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 190 insertions(+)
 create mode 100644 tests/services/docker.scm

diff --git a/Makefile.am b/Makefile.am
index cbc3191dfc..91f7a77a94 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -564,6 +564,7 @@ SCM_TESTS =					\
   tests/services.scm				\
   tests/services/file-sharing.scm		\
   tests/services/configuration.scm		\
+  tests/services/docker.scm			\
   tests/services/lightdm.scm			\
   tests/services/linux.scm			\
   tests/services/pam-mount.scm			\
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index ebea0a473a..263cb41df3 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -58,6 +58,8 @@ (define-module (gnu services docker)
             oci-container-configuration-network
             oci-container-configuration-ports
             oci-container-configuration-volumes
+            oci-container-configuration-container-user
+            oci-container-configuration-workdir
             oci-container-service-type
             oci-container-shepherd-service))
 
diff --git a/tests/services/docker.scm b/tests/services/docker.scm
new file mode 100644
index 0000000000..fad28a228c
--- /dev/null
+++ b/tests/services/docker.scm
@@ -0,0 +1,187 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2023 Giacomo Leidi <goodoldpaul@autistici.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (tests services docker)
+  #:use-module (gnu packages docker)
+  #:use-module (gnu services docker)
+  #:use-module (guix derivations)
+  #:use-module (guix gexp)
+  #:use-module (guix monads)
+  #:use-module (guix packages)
+  #:use-module (guix store)
+  #:use-module (guix tests)
+  #:use-module (ice-9 match)
+  #:use-module (srfi srfi-1)
+  #:use-module (srfi srfi-64))
+
+
+;;; Commentary:
+;;;
+;;; Unit tests for the (gnu services docker) module.
+;;;
+;;; Code:
+
+
+;;;
+;;; Unit tests for the oci-container-service-type.
+;;;
+
+
+;;; Access some internals for whitebox testing.
+(define %store
+  (open-connection-for-tests))
+(define (gexp->sexp . x)
+  (apply (@@ (guix gexp) gexp->sexp) x))
+(define* (gexp->sexp* exp #:optional target)
+  (run-with-store %store (gexp->sexp exp (%current-system) target)
+                  #:guile-for-build (%guile-for-build)))
+(define (list->sexp-list* lst)
+  (map (lambda (el)
+         (if (gexp? el)
+             (gexp->sexp* el)
+             el))
+       lst))
+(define oci-sanitize-mixed-list
+  (@@ (gnu services docker) oci-sanitize-mixed-list))
+(define (oci-container-configuration->options config)
+  (list->sexp-list*
+   ((@@ (gnu services docker) oci-container-configuration->options) config)))
+
+(test-begin "oci-containers-service")
+
+(test-group "oci-sanitize-mixed-list"
+  (define delimiter "=")
+  (define file-like-key
+    (plain-file "oci-tests-file-like-key" "some-content"))
+  (define mixed-list
+    `("any kind of string"
+      ("KEY" . "VALUE")
+      (,#~(string-append "COMPUTED" "_KEY") . "VALUE")
+      (,file-like-key . "VALUE")))
+
+  (test-assertm "successfully lower mixed values"
+    (mlet* %store-monad ((ml ->             (oci-sanitize-mixed-list "field-name" mixed-list delimiter))
+                         (actual ->         (list->sexp-list* ml))
+                         (file-like-item    (lower-object file-like-key))
+                         (expected ->       `("any kind of string"
+                                              (string-append "KEY" "=" "VALUE")
+                                              (string-append (string-append "COMPUTED" "_KEY") "=" "VALUE")
+                                              (string-append ,file-like-item "=" "VALUE"))))
+      (mbegin %store-monad
+        (return
+         (every (lambda (pair)
+                  (apply (if (string? (first pair))
+                             string=?
+                             equal?)
+                         pair))
+                (zip expected actual))))))
+
+  (test-error
+   "illegal list values" #t
+   (oci-sanitize-mixed-list "field-name" '(("KEY" . "VALUE") #f) delimiter))
+
+  (test-error
+   "illegal pair member values" #t
+   (oci-sanitize-mixed-list "field-name" '(("KEY" . 1)) delimiter)))
+
+(test-group "oci-container-configuration->options"
+  (define config
+    (oci-container-configuration
+     (image "guix/guix:latest")))
+
+  (test-equal "entrypoint"
+    (list "--entrypoint" "entrypoint")
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (entrypoint "entrypoint"))))
+
+  (test-equal "environment"
+    (list "--env" '(string-append "key" "=" "value")
+          "--env" '(string-append "environment" "=" "variable"))
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (environment
+       '(("key" . "value")
+         ("environment" . "variable"))))))
+
+  (test-equal "network"
+    (list "--network" "host")
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (network "host"))))
+
+  (test-equal "container-user"
+    (list "--user" "service-account")
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (container-user "service-account"))))
+
+  (test-equal "workdir"
+    (list "--workdir" "/srv/http")
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (workdir "/srv/http"))))
+
+  (test-equal "ports"
+    (list "-p" '(string-append "10443" ":" "443")
+          "-p" '(string-append "9022" ":" "22"))
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (ports
+       '(("10443" . "443")
+         ("9022" . "22"))))))
+
+  (test-equal "volumes"
+    (list "-v" '(string-append "/gnu/store" ":" "/gnu/store")
+          "-v" '(string-append "/var/lib/guix" ":" "/var/lib/guix"))
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (volumes
+       '(("/gnu/store" . "/gnu/store")
+         ("/var/lib/guix" . "/var/lib/guix"))))))
+
+  (test-equal "complete configuration"
+    (list "--entrypoint" "entrypoint"
+          "--env" '(string-append "key" "=" "value")
+          "--network" "host"
+          "--user" "service-account"
+          "--workdir" "/srv/http"
+          "-p" '(string-append "10443" ":" "443")
+          "-v" '(string-append "/gnu/store" ":" "/gnu/store"))
+    (oci-container-configuration->options
+     (oci-container-configuration
+      (inherit config)
+      (entrypoint "entrypoint")
+      (environment
+       '(("key" . "value")))
+      (network "host")
+      (container-user "service-account")
+      (workdir "/srv/http")
+      (ports
+       '(("10443" . "443")))
+      (volumes
+       '(("/gnu/store" . "/gnu/store")))))))
+
+(test-end "oci-containers-service")

base-commit: 2c9ac9ab20c76abe570ff83f8746fa089fea3047
-- 
2.41.0

[bug#67613] Introduce unit tests for oci-container-service-type.

[Ludovic Courtès]

Hello,

Giacomo Leidi <goodoldpaul@autistici.org> skribis:

> This patch is a followup to issue #66160 and issue #67574. It introduces
> unit tests for the oci-container-service-type. 8 out 11 tests depend on
> issue #67574 being merged since issue #66160 was merged with a blocking
> bug from the beginning.
>
> * gnu/services/docker.scm: Export
> oci-container-configuration-container-user and
> oci-container-configuration-workdir.
> * tests/services/docker.scm: New file.
> * Makefile.am (SCM_TESTS): Register it.
>
> Change-Id: I47ed0fe36060ba84dd50b548a66f36e3df8a3710

Thanks for working on this!

To me, what’s really helpful is a system test: a test that spins up a VM
running an OCI service and makes sure said service is functional.
Apologies if I wasn’t clear!

Unit tests can be interesting too, but only if their “bug-finding
performance” is good.  The tests below, for instance, are likely to be
mirroring the implementation too closely to be really able to find bugs:

> +  (test-equal "environment"
> +    (list "--env" '(string-append "key" "=" "value")
> +          "--env" '(string-append "environment" "=" "variable"))
> +    (oci-container-configuration->options
> +     (oci-container-configuration
> +      (inherit config)
> +      (environment
> +       '(("key" . "value")
> +         ("environment" . "variable"))))))
> +
> +  (test-equal "network"
> +    (list "--network" "host")
> +    (oci-container-configuration->options
> +     (oci-container-configuration
> +      (inherit config)
> +      (network "host"))))
> +
> +  (test-equal "container-user"
> +    (list "--user" "service-account")
> +    (oci-container-configuration->options
> +     (oci-container-configuration
> +      (inherit config)
> +      (container-user "service-account"))))

Thus my suggestion would be to instead focus on a system test, like
those in (gnu tests docker).

Does that make sense?  WDYT?

Ludo’.

[bug#67613] Introduce unit tests for oci-container-service-type.

[paul via Guix-patches via]

[-- Attachment #1: Type: text/plain, Size: 811 bytes --]

Hi Ludo’,

On 12/10/23 22:47, Ludovic Courtès wrote:
> Thus my suggestion would be to instead focus on a system test, like
> those in (gnu tests docker).
>
> Does that make sense?  WDYT?

I definitely misunderstood, I'll work also on system tests like those 
you pointed out. Thank you, I was not aware of them, I was wondering how 
do I run them?

guix shell --pure -D guix -- make check TESTS=gnu/tests/docker.scm

gives me

============================================================================
Testsuite summary for GNU Guix 1.3.0.50882-34e1c
============================================================================
# TOTAL: 0
# PASS:  0
# SKIP:  0
# XFAIL: 0
# FAIL:  0
# XPASS: 0
# ERROR: 0
============================================================================

Thank you,

giacomo

[-- Attachment #2: Type: text/html, Size: 1253 bytes --]

[bug#67613] Introduce unit tests for oci-container-service-type.

[Ludovic Courtès]

Hi,

paul <goodoldpaul@autistici.org> skribis:

> I definitely misunderstood, I'll work also on system tests like those
> you pointed out. Thank you, I was not aware of them, I was wondering
> how do I run them?

With ‘make check-system TESTS=…’:

  https://guix.gnu.org/manual/devel/en/html_node/Running-the-Test-Suite.html

Apologies for the miscommunication!

Ludo’.

[bug#67613] Introduce unit tests for oci-container-service-type.

[paul via Guix-patches via]

Hi Ludo’ ,

I should have created a suitable system test for the 
oci-container-service-type. Thanks to a nice input from 
@graywolf@emacs.ch on mastodon, and actually to be able to run the test 
since the vm doesn't have internet access and can't pull OCI images, I 
implemented a new oci-image record that can be given some lowerable 
value that can be lowered to an OCI tarballed image and passed to the 
image field of the oci-container-configuration record. I'd like to point 
out two things:

- It's the first time I use Guix internal API to build derivations, I 
took most of my implementation from other places around Guix and I hope 
is sound but I may have missed something. I'd like your feedback about it.

- I was tempted to make the image field of the 
oci-container-configuration record directly only accept oci-image 
records (hence making the value field of oci-image optional) but that 
would break existing configurations. I'm not sure about the contract we 
have for configuration records API, should I wait 1.5.0 for this change?


I'm sending an updated patchset, thank you for all your help and efforts.


giacomo

[bug#67613] Introduce unit tests for oci-container-service-type.

[paul via Guix-patches via]

Hi Ludo’ ,

I'm sending a patchset rebased on current master. I hope patch 1-3 are 
non-controversial enough to be directly merged as they add features 
without breaking existing configurations. About patch 4 and 5 I stand by 
my request for help in my last email.

Thank you for your work,


giacomo

[bug#67613] [PATCH v2 1/5] gnu: docker: Provide escape hatch in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

* gnu/services/docker.scm (oci-container-configuration)
[extra-arguments]: New field;
(oci-sanitize-extra-arguments): sanitize it;
(oci-container-shepherd-service): use it;
* doc/guix.texi: document it.

Change-Id: I54c74ac2fe0f5ca65ca5a1d0d7f3fb55ff428063
---
 doc/guix.texi           | 13 ++++++++++---
 gnu/services/docker.scm | 42 ++++++++++++++++++++++++++++++++++-------
 2 files changed, 45 insertions(+), 10 deletions(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index 395545bed7..ce239c603d 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -39844,7 +39844,8 @@ Set environment variables. This can be a list of pairs or strings, even mixed:
       "JAVA_HOME=/opt/java")
 @end lisp
 
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI.  You can refer to the
 @uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
 documentation for semantics.
 
@@ -39868,7 +39869,8 @@ list of pairs or strings, even mixed:
       "10443:443")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI.  You can refer to the
 @uref{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
 documentation for semantics.
 
@@ -39881,7 +39883,8 @@ list of pairs or strings, even mixed:
       "/gnu/store:/gnu/store")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI.  You can refer to the
 @uref{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
 documentation for semantics.
 
@@ -39896,6 +39899,10 @@ You can refer to the
 @url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
 documentation for semantics.
 
+@item @code{extra-arguments} (default: @code{()}) (type: list)
+A list of strings, gexps or file-like objects that will be directly
+passed to the @command{docker run} invokation.
+
 @end table
 
 @end deftp
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 4d32b96847..b4fd94d1fd 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -58,6 +58,9 @@ (define-module (gnu services docker)
             oci-container-configuration-network
             oci-container-configuration-ports
             oci-container-configuration-volumes
+            oci-container-configuration-container-user
+            oci-container-configuration-workdir
+            oci-container-configuration-extra-arguments
             oci-container-service-type
             oci-container-shepherd-service))
 
@@ -297,6 +300,21 @@ (define (oci-sanitize-volumes value)
   ;; '(("/mnt/dir" . "/dir") "/run/current-system/profile:/java")
   (oci-sanitize-mixed-list "volumes" value ":"))
 
+(define (oci-sanitize-extra-arguments value)
+  (define (valid? member)
+    (or (string? member)
+        (gexp? member)
+        (file-like? member)))
+  (map
+   (lambda (el)
+     (if (valid? el)
+         el
+         (raise
+          (formatted-message
+           (G_ "extra arguments may only be strings, gexps or file-like objects
+but ~a was found") el))))
+   value))
+
 (define-maybe/no-serialization string)
 
 (define-configuration/no-serialization oci-container-configuration
@@ -322,7 +340,8 @@ (define-configuration/no-serialization oci-container-configuration
       \"JAVA_HOME=/opt/java\")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI.  You can refer to the
 @url{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
 documentation for semantics."
    (sanitizer oci-sanitize-environment))
@@ -347,7 +366,8 @@ (define-configuration/no-serialization oci-container-configuration
       \"10443:443\")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI.  You can refer to the
 @url{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
 documentation for semantics."
    (sanitizer oci-sanitize-ports))
@@ -361,7 +381,8 @@ (define-configuration/no-serialization oci-container-configuration
       \"/gnu/store:/gnu/store\")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI.  You can refer to the
 @url{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
 documentation for semantics."
    (sanitizer oci-sanitize-volumes))
@@ -375,7 +396,12 @@ (define-configuration/no-serialization oci-container-configuration
    "Set the current working for the spawned Shepherd service.
 You can refer to the
 @url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
-documentation for semantics."))
+documentation for semantics.")
+  (extra-arguments
+   (list '())
+   "A list of strings, gexps or file-like objects that will be directly passed
+to the @command{docker run} invokation."
+   (sanitizer oci-sanitize-extra-arguments)))
 
 (define oci-container-configuration->options
   (lambda (config)
@@ -428,7 +454,9 @@ (define (guess-name name image)
          (provision (oci-container-configuration-provision config))
          (image (oci-container-configuration-image config))
          (options (oci-container-configuration->options config))
-         (name (guess-name provision image)))
+         (name (guess-name provision image))
+         (extra-arguments
+          (oci-container-configuration-extra-arguments config)))
 
     (shepherd-service (provision `(,(string->symbol name)))
                       (requirement '(dockerd user-processes))
@@ -441,7 +469,7 @@ (define (guess-name name image)
                           ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
                           (list #$docker-command "run" "--rm"
                                 "--name" #$name
-                                #$@options #$image #$@command)
+                                #$@options #$@extra-arguments #$image #$@command)
                           #:user #$user
                           #:group #$group))
                       (stop
@@ -482,5 +510,5 @@ (define oci-container-service-type
                 (extend append)
                 (compose concatenate)
                 (description
-                 "This service allows the management of Docker and OCI
+                 "This service allows the management of OCI
 containers as Shepherd services.")))

base-commit: 637b72e2b83a6332849218ef1f193124fa8239eb
-- 
2.41.0

[bug#67613] [PATCH v2 2/5] gnu: docker: Allow setting host environment variables in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

* gnu/services/docker.scm (oci-container-configuration)
[host-environment]: New field;
(oci-sanitize-host-environment): sanitize it;
(oci-container-shepherd-service): use it.

Change-Id: I4d54d37736cf09f042a71cb0b6e673abc0948d9c
---
 gnu/services/docker.scm | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index b4fd94d1fd..7706b4a29a 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -5,7 +5,7 @@
 ;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il>
 ;;; Copyright © 2020 Jesse Dowell <jessedowell@gmail.com>
 ;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re>
-;;; Copyright © 2023 Giacomo Leidi <goodoldpaul@autistici.org>
+;;; Copyright © 2023, 2024 Giacomo Leidi <goodoldpaul@autistici.org>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -285,6 +285,11 @@ (define (oci-sanitize-mixed-list name value delimiter)
               name el)))))
    value))
 
+(define (oci-sanitize-host-environment value)
+  ;; Expected spec format:
+  ;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
+  (oci-sanitize-mixed-list "host-environment" value "="))
+
 (define (oci-sanitize-environment value)
   ;; Expected spec format:
   ;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
@@ -330,6 +335,24 @@ (define-configuration/no-serialization oci-container-configuration
   (entrypoint
    (maybe-string)
    "Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.")
+  (host-environment
+   (list '())
+   "Set environment variables in the host environment where @command{docker run}
+is invoked.  This is especially useful to pass secrets from the host to the
+container without having them on the @command{docker run}'s command line: by
+setting the @{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment.  This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+      \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}."
+   (sanitizer oci-sanitize-host-environment))
   (environment
    (list '())
    "Set environment variables.  This can be a list of pairs or strings, even
@@ -450,6 +473,8 @@ (define (guess-name name image)
   (let* ((docker-command (file-append docker-cli "/bin/docker"))
          (user (oci-container-configuration-user config))
          (group (oci-container-configuration-group config))
+         (host-environment
+          (oci-container-configuration-host-environment config))
          (command (oci-container-configuration-command config))
          (provision (oci-container-configuration-provision config))
          (image (oci-container-configuration-image config))
@@ -471,7 +496,9 @@ (define (guess-name name image)
                                 "--name" #$name
                                 #$@options #$@extra-arguments #$image #$@command)
                           #:user #$user
-                          #:group #$group))
+                          #:group #$group
+                          #:environment-variables
+                          (list #$@host-environment)))
                       (stop
                        #~(lambda _
                            (invoke #$docker-command "rm" "-f" #$name)))
-- 
2.41.0

[bug#67613] [PATCH v2 3/5] gnu: docker: Allow setting Shepherd dependencies in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

* gnu/services/docker.scm (oci-container-configuration)
[requirement]: New field;
(list-of-symbols): sanitize it;
(oci-container-shepherd-service): use it.

Change-Id: Ic0ba336a2257d6ef7c658cfc6cd630116661f581
---
 gnu/services/docker.scm | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 7706b4a29a..43ffb71901 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -320,6 +320,9 @@ (define (valid? member)
 but ~a was found") el))))
    value))
 
+(define list-of-symbols?
+  (list-of symbol?))
+
 (define-maybe/no-serialization string)
 
 (define-configuration/no-serialization oci-container-configuration
@@ -376,6 +379,10 @@ (define-configuration/no-serialization oci-container-configuration
   (provision
    (maybe-string)
    "Set the name of the provisioned Shepherd service.")
+  (requirement
+   (list-of-symbols '())
+   "Set additional Shepherd services dependencies to the provisioned Shepherd
+service.")
   (network
    (maybe-string)
    "Set a Docker network for the spawned container.")
@@ -477,6 +484,7 @@ (define (guess-name name image)
           (oci-container-configuration-host-environment config))
          (command (oci-container-configuration-command config))
          (provision (oci-container-configuration-provision config))
+         (requirement (oci-container-configuration-requirement config))
          (image (oci-container-configuration-image config))
          (options (oci-container-configuration->options config))
          (name (guess-name provision image))
@@ -484,7 +492,7 @@ (define (guess-name name image)
           (oci-container-configuration-extra-arguments config)))
 
     (shepherd-service (provision `(,(string->symbol name)))
-                      (requirement '(dockerd user-processes))
+                      (requirement `(dockerd user-processes ,@requirement))
                       (respawn? #f)
                       (documentation
                        (string-append
-- 
2.41.0

[bug#67613] [PATCH v2 4/5] gnu: docker: Allow passing tarballs for images in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

This commit allows for loading an OCI image tarball before running an
OCI backed Shepherd service. It does so by adding a one shot Shepherd
service to the dependencies of the OCI backed service that at boot runs
docker load on the tarball.

* gnu/services/docker.scm (oci-image): New record;
(lower-oci-image): new variable, lower it;
(string-or-oci-image?): sanitize it;
(oci-container-configuration)[image]: allow also for oci-image records;
(oci-container-shepherd-service): use it;
(%oci-image-loader): new variable.

Change-Id: Ie504f479ea0d47f74b0ec5df9085673ffd3f639d
---
 gnu/services/docker.scm | 244 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 219 insertions(+), 25 deletions(-)

diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 43ffb71901..58a725737c 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -23,11 +23,14 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (gnu services docker)
+  #:use-module (gnu image)
   #:use-module (gnu services)
   #:use-module (gnu services configuration)
   #:use-module (gnu services base)
   #:use-module (gnu services dbus)
   #:use-module (gnu services shepherd)
+  #:use-module (gnu system)
+  #:use-module (gnu system image)
   #:use-module (gnu system setuid)
   #:use-module (gnu system shadow)
   #:use-module (gnu packages admin)               ;shadow
@@ -37,7 +40,11 @@ (define-module (gnu services docker)
   #:use-module (guix diagnostics)
   #:use-module (guix gexp)
   #:use-module (guix i18n)
+  #:use-module (guix monads)
   #:use-module (guix packages)
+  #:use-module (guix profiles)
+  #:use-module ((guix scripts pack) #:prefix pack:)
+  #:use-module (guix store)
   #:use-module (srfi srfi-1)
   #:use-module (ice-9 format)
   #:use-module (ice-9 match)
@@ -45,6 +52,16 @@ (define-module (gnu services docker)
   #:export (docker-configuration
             docker-service-type
             singularity-service-type
+            oci-image
+            oci-image?
+            oci-image-fields
+            oci-image-repository
+            oci-image-tag
+            oci-image-value
+            oci-image-pack-options
+            oci-image-target
+            oci-image-system
+            oci-image-grafts?
             oci-container-configuration
             oci-container-configuration?
             oci-container-configuration-fields
@@ -52,9 +69,11 @@ (define-module (gnu services docker)
             oci-container-configuration-group
             oci-container-configuration-command
             oci-container-configuration-entrypoint
+            oci-container-configuration-host-environment
             oci-container-configuration-environment
             oci-container-configuration-image
             oci-container-configuration-provision
+            oci-container-configuration-requirement
             oci-container-configuration-network
             oci-container-configuration-ports
             oci-container-configuration-volumes
@@ -62,7 +81,8 @@ (define-module (gnu services docker)
             oci-container-configuration-workdir
             oci-container-configuration-extra-arguments
             oci-container-service-type
-            oci-container-shepherd-service))
+            oci-container-shepherd-service
+            %oci-container-accounts))
 
 (define-maybe file-like)
 
@@ -320,11 +340,68 @@ (define (valid? member)
 but ~a was found") el))))
    value))
 
+(define (oci-image-reference image)
+  (if (string? image)
+      image
+      (string-append (oci-image-repository image)
+                     ":" (oci-image-tag image))))
+
+(define (oci-lowerable-image? image)
+  (or (manifest? image)
+      (operating-system? image)
+      (gexp? image)
+      (file-like? image)))
+
+(define (string-or-oci-image? image)
+  (or (string? image)
+      (oci-image? image)))
+
 (define list-of-symbols?
   (list-of symbol?))
 
 (define-maybe/no-serialization string)
 
+(define-configuration/no-serialization oci-image
+  (repository
+   (string)
+   "A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.")
+  (tag
+   (string "latest")
+   "A string representing the OCI image tag. Defaults to @code{latest}.")
+  (value
+   (oci-lowerable-image)
+   "A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball.  Otherwise this field's value can be a gexp
+or a file-like object that evaluates to an OCI compatible tarball.")
+  (pack-options
+   (list '())
+   "An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}.  They can be used
+to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+  (repository \"guile\")
+  (tag \"3\")
+  (manifest (specifications->manifest '(\"guile\")))
+  (pack-options
+    '(#:symlinks ((\"/bin/guile\" -> \"bin/guile\"))
+      #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.")
+  (system
+   (maybe-string)
+   "Attempt to build for a given system, e.g. \"i686-linux\"")
+  (target
+   (maybe-string)
+   "Attempt to cross-build for a given triple, e.g. \"aarch64-linux-gnu\"")
+  (grafts?
+   (boolean #f)
+   "Whether to allow grafting or not in the pack build."))
+
 (define-configuration/no-serialization oci-container-configuration
   (user
    (string "oci-container")
@@ -372,8 +449,9 @@ (define-configuration/no-serialization oci-container-configuration
 documentation for semantics."
    (sanitizer oci-sanitize-environment))
   (image
-   (string)
-   "The image used to build the container.  Images are resolved by the Docker
+   (string-or-oci-image)
+   "The image used to build the container.  It can be a string or an
+@code{oci-image} record.  Strings are resolved by the Docker
 Engine, and follow the usual format
 @code{myregistry.local:5000/testing/test-image:tag}.")
   (provision
@@ -470,14 +548,122 @@ (define oci-container-configuration->options
                            (list "-v" spec))
                          (oci-container-configuration-volumes config))))))))
 
+(define* (get-keyword-value args keyword #:key (default #f))
+  (let ((kv (memq keyword args)))
+    (if (and kv (>= (length kv) 2))
+        (cadr kv)
+        default)))
+
+(define (lower-operating-system os target system)
+  (mlet* %store-monad
+      ((tarball
+        (lower-object
+         (system-image (os->image os #:type docker-image-type))
+         system
+         #:target target)))
+    (return tarball)))
+
+(define (lower-manifest name image target system)
+  (define value (oci-image-value image))
+  (define options (oci-image-pack-options image))
+  (define image-reference
+    (oci-image-reference image))
+  (define image-tag
+    (let* ((extra-options
+            (get-keyword-value options #:extra-options))
+           (image-tag-option
+            (and extra-options
+                 (get-keyword-value extra-options #:image-tag))))
+      (if image-tag-option
+          '()
+          `(#:extra-options (#:image-tag ,image-reference)))))
+
+  (mlet* %store-monad
+      ((_ (set-grafting
+           (oci-image-grafts? image)))
+       (guile (set-guile-for-build (default-guile)))
+       (profile
+        (profile-derivation value
+                            #:target target
+                            #:system system
+                            #:hooks '()
+                            #:locales? #f))
+       (tarball (apply pack:docker-image
+                       `(,name ,profile
+                         ,@options
+                         ,@image-tag
+                         #:localstatedir? #t))))
+    (return tarball)))
+
+(define (lower-oci-image name image)
+  (define value (oci-image-value image))
+  (define image-target (oci-image-target image))
+  (define image-system (oci-image-system image))
+  (define target
+    (if (maybe-value-set? image-target)
+        image-target
+        (%current-target-system)))
+  (define system
+    (if (maybe-value-set? image-system)
+        image-system
+        (%current-system)))
+  (with-store store
+   (run-with-store store
+     (match value
+       ((? manifest? value)
+        (lower-manifest name image target system))
+       ((? operating-system? value)
+        (lower-operating-system value target system))
+       ((or (? gexp? value)
+            (? file-like? value))
+        value)
+       (_
+        (raise
+         (formatted-message
+          (G_ "oci-image value must contain only manifest,
+operating-system, gexp or file-like records but ~a was found")
+          value))))
+     #:target target
+     #:system system)))
+
+(define (%oci-image-loader name image tag)
+  (let ((docker (file-append docker-cli "/bin/docker"))
+        (tarball (lower-oci-image name image)))
+    (with-imported-modules '((guix build utils))
+      (program-file (format #f "~a-image-loader" name)
+       #~(begin
+           (use-modules (guix build utils)
+                        (ice-9 popen)
+                        (ice-9 rdelim))
+
+           (format #t "Loading image for ~a from ~a...~%" #$name #$tarball)
+           (define line
+             (read-line
+              (open-input-pipe
+               (string-append #$docker " load -i " #$tarball))))
+
+           (unless (or (eof-object? line)
+                       (string-null? line))
+             (format #t "~a~%" line)
+             (let ((repository&tag
+                    (string-drop line
+                                 (string-length
+                                   "Loaded image: "))))
+
+               (invoke #$docker "tag" repository&tag #$tag)
+               (format #t "Tagged ~a with ~a...~%" #$tarball #$tag))))))))
+
 (define (oci-container-shepherd-service config)
   (define (guess-name name image)
     (if (maybe-value-set? name)
         name
         (string-append "docker-"
-                       (basename (car (string-split image #\:))))))
+                       (basename
+                        (if (string? image)
+                            (first (string-split image #\:))
+                            (oci-image-repository image))))))
 
-  (let* ((docker-command (file-append docker-cli "/bin/docker"))
+  (let* ((docker (file-append docker-cli "/bin/docker"))
          (user (oci-container-configuration-user config))
          (group (oci-container-configuration-group config))
          (host-environment
@@ -486,6 +672,7 @@ (define (guess-name name image)
          (provision (oci-container-configuration-provision config))
          (requirement (oci-container-configuration-requirement config))
          (image (oci-container-configuration-image config))
+         (image-reference (oci-image-reference image))
          (options (oci-container-configuration->options config))
          (name (guess-name provision image))
          (extra-arguments
@@ -496,30 +683,37 @@ (define (guess-name name image)
                       (respawn? #f)
                       (documentation
                        (string-append
-                        "Docker backed Shepherd service for image: " image))
+                        "Docker backed Shepherd service for "
+                        (if (oci-image? image) name image) "."))
                       (start
-                       #~(make-forkexec-constructor
-                          ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
-                          (list #$docker-command "run" "--rm"
-                                "--name" #$name
-                                #$@options #$@extra-arguments #$image #$@command)
-                          #:user #$user
-                          #:group #$group
-                          #:environment-variables
-                          (list #$@host-environment)))
+                       #~(lambda ()
+                          (when #$(oci-image? image)
+                            (invoke #$(%oci-image-loader
+                                       name image image-reference)))
+                          (fork+exec-command
+                           ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
+                           (list #$docker "run" "--rm" "--name" #$name
+                                 #$@options #$@extra-arguments
+                                 #$image-reference #$@command)
+                           #:user #$user
+                           #:group #$group
+                           #:environment-variables
+                           (list #$@host-environment))))
                       (stop
                        #~(lambda _
-                           (invoke #$docker-command "rm" "-f" #$name)))
+                           (invoke #$docker "rm" "-f" #$name)))
                       (actions
-                       (list
-                        (shepherd-action
-                         (name 'pull)
-                         (documentation
-                          (format #f "Pull ~a's image (~a)."
-                                  name image))
-                         (procedure
-                          #~(lambda _
-                              (invoke #$docker-command "pull" #$image)))))))))
+                       (if (oci-image? image)
+                           '()
+                           (list
+                            (shepherd-action
+                             (name 'pull)
+                             (documentation
+                              (format #f "Pull ~a's image (~a)."
+                                      name image))
+                             (procedure
+                              #~(lambda _
+                                  (invoke #$docker "pull" #$image))))))))))
 
 (define %oci-container-accounts
   (list (user-account
-- 
2.41.0

[bug#67613] [PATCH v2 5/5] gnu: Add tests and documentation for oci-container-service-type.

[Giacomo Leidi via Guix-patches via]

* doc/guix.texi: Add documentation for the oci-image record and update
the oci-container-configuration documentation.
* gnu/tests/docker.scm (run-oci-container-test): New variable;
(%test-oci-container): new variable.

Change-Id: Id8f4f5454aa3b88d8aa3fa47de823e921acece05
---
 doc/guix.texi           |  91 +++++++++++++++++++++++++++-
 gnu/services/docker.scm |   6 +-
 gnu/tests/docker.scm    | 131 +++++++++++++++++++++++++++++++++++++++-
 3 files changed, 221 insertions(+), 7 deletions(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index ce239c603d..1916a00412 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -39790,6 +39790,17 @@ processes as Shepherd Services.
 @lisp
 (service oci-container-service-type
          (list
+          (oci-container-configuration
+           (image
+            (oci-image
+             (repository "guile")
+             (tag "3")
+             (value (specifications->manifest '("guile")))
+             (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+                             #:max-layers 2))))
+           (entrypoint "/bin/guile")
+           (command
+            '("-c" "(display \"hello!\n\")")))
           (oci-container-configuration
            (image "prom/prometheus")
            (network "host")
@@ -39836,6 +39847,23 @@ Overwrite the default command (@code{CMD}) of the image.
 @item @code{entrypoint} (default: @code{""}) (type: string)
 Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.
 
+@item @code{host-environment} (default: @code{()}) (type: list)
+Set environment variables in the host environment where @command{docker
+run} is invoked.  This is especially useful to pass secrets from the
+host to the container without having them on the @command{docker run}'s
+command line: by setting the @code{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment.  This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+      \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}.
+
 @item @code{environment} (default: @code{()}) (type: list)
 Set environment variables. This can be a list of pairs or strings, even mixed:
 
@@ -39849,14 +39877,19 @@ Strings are passed directly to the Docker CLI.  You can refer to the
 @uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
 documentation for semantics.
 
-@item @code{image} (type: string)
-The image used to build the container.  Images are resolved by the
-Docker Engine, and follow the usual format
+@item @code{image} (type: string-or-oci-image)
+The image used to build the container.  It can be a string or an
+@code{oci-image} record.  Strings are resolved by the Docker Engine, and
+follow the usual format
 @code{myregistry.local:5000/testing/test-image:tag}.
 
 @item @code{provision} (default: @code{""}) (type: string)
 Set the name of the provisioned Shepherd service.
 
+@item @code{requirement} (default: @code{()}) (type: list-of-symbols)
+Set additional Shepherd services dependencies to the provisioned
+Shepherd service.
+
 @item @code{network} (default: @code{""}) (type: string)
 Set a Docker network for the spawned container.
 
@@ -39908,6 +39941,58 @@ passed to the @command{docker run} invokation.
 @end deftp
 
 
+@c %end of fragment
+
+@c %start of fragment
+
+@deftp {Data Type} oci-image
+Available @code{oci-image} fields are:
+
+@table @asis
+@item @code{repository} (type: string)
+A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.
+
+@item @code{tag} (default: @code{"latest"}) (type: string)
+A string representing the OCI image tag.  Defaults to @code{latest}.
+
+@item @code{value} (type: oci-lowerable-image)
+A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball.  Otherwise this field's value can be a
+gexp or a file-like object that evaluates to an OCI compatible tarball.
+
+@item @code{pack-options} (default: @code{()}) (type: list)
+An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}.  They can
+be used to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+  (repository "guile")
+  (tag "3")
+  (value
+    (specifications->manifest '("guile")))
+  (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+                  #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.
+
+@item @code{system} (default: @code{""}) (type: string)
+Attempt to build for a given system, e.g. "i686-linux"
+
+@item @code{target} (default: @code{""}) (type: string)
+Attempt to cross-build for a given triple, e.g. "aarch64-linux-gnu"
+
+@item @code{grafts?} (default: @code{#f}) (type: boolean)
+Whether to allow grafting or not in the pack build.
+
+@end table
+
+@end deftp
+
+
 @c %end of fragment
 
 @cindex Audit
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 58a725737c..7aff8dcc5f 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -420,7 +420,7 @@ (define-configuration/no-serialization oci-container-configuration
    "Set environment variables in the host environment where @command{docker run}
 is invoked.  This is especially useful to pass secrets from the host to the
 container without having them on the @command{docker run}'s command line: by
-setting the @{MYSQL_PASSWORD} on the host and by passing
+setting the @code{MYSQL_PASSWORD} on the host and by passing
 @code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
 possible to securely set values in the container environment.  This field's
 value can be a list of pairs or strings, even mixed:
@@ -435,8 +435,8 @@ (define-configuration/no-serialization oci-container-configuration
    (sanitizer oci-sanitize-host-environment))
   (environment
    (list '())
-   "Set environment variables.  This can be a list of pairs or strings, even
-mixed:
+   "Set environment variables inside the container.  This can be a list of pairs
+or strings, even mixed:
 
 @lisp
 (list '(\"LANGUAGE\" . \"eo:ca:eu\")
diff --git a/gnu/tests/docker.scm b/gnu/tests/docker.scm
index 9e9d2e2d07..d550136b4a 100644
--- a/gnu/tests/docker.scm
+++ b/gnu/tests/docker.scm
@@ -1,6 +1,7 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2019 Danny Milosavljevic <dannym@scratchpost.org>
 ;;; Copyright © 2019-2023 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2024 Giacomo Leidi <goodoldpaul@autistici.org>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -29,6 +30,7 @@ (define-module (gnu tests docker)
   #:use-module (gnu services networking)
   #:use-module (gnu services docker)
   #:use-module (gnu services desktop)
+  #:use-module (gnu packages)
   #:use-module ((gnu packages base) #:select (glibc))
   #:use-module (gnu packages guile)
   #:use-module (gnu packages docker)
@@ -43,7 +45,8 @@ (define-module (gnu tests docker)
   #:use-module (guix build-system trivial)
   #:use-module ((guix licenses) #:prefix license:)
   #:export (%test-docker
-            %test-docker-system))
+            %test-docker-system
+            %test-oci-container))
 
 (define %docker-os
   (simple-operating-system
@@ -316,3 +319,129 @@ (define %test-docker-system
                                    (locale-libcs (list glibc)))
                                  #:type docker-image-type)))
                  run-docker-system-test)))))
+
+\f
+(define %oci-os
+  (simple-operating-system
+   (service dhcp-client-service-type)
+   (service dbus-root-service-type)
+   (service polkit-service-type)
+   (service elogind-service-type)
+   (service docker-service-type)
+   (extra-special-file "/shared.txt"
+                       (plain-file "shared.txt" "hello"))
+   (service oci-container-service-type
+            (list
+             (oci-container-configuration
+              (image
+               (oci-image
+                (repository "guile")
+                (value
+                 (specifications->manifest '("guile")))
+                (pack-options
+                 '(#:symlinks (("/bin" -> "bin"))))))
+              (entrypoint
+               "/bin/guile")
+              (command
+               '("-c" "(let l ((c 300))(display c)(sleep 1)(when(positive? c)(l (- c 1))))"))
+              (host-environment
+               '(("VARIABLE" . "value")))
+              (volumes
+               '(("/shared.txt" . "/shared.txt:ro")))
+              (extra-arguments
+               '("--env" "VARIABLE")))))))
+
+(define (run-oci-container-test)
+  "Run IMAGE as an OCI backed Shepherd service, inside OS."
+
+  (define os
+    (marionette-operating-system
+     (operating-system-with-gc-roots
+      %oci-os
+      (list))
+     #:imported-modules '((gnu services herd)
+                          (guix combinators))))
+
+  (define vm
+    (virtual-machine
+     (operating-system os)
+     (volatile? #f)
+     (memory-size 1024)
+     (disk-image-size (* 3000 (expt 2 20)))
+     (port-forwardings '())))
+
+  (define test
+    (with-imported-modules '((gnu build marionette))
+      #~(begin
+          (use-modules (srfi srfi-11) (srfi srfi-64)
+                       (gnu build marionette))
+
+          (define marionette
+            ;; Relax timeout to accommodate older systems and
+            ;; allow for pulling the image.
+            (make-marionette (list #$vm) #:timeout 60))
+
+          (test-runner-current (system-test-runner #$output))
+          (test-begin "oci-container")
+
+          (test-assert "dockerd running"
+            (marionette-eval
+             '(begin
+                (use-modules (gnu services herd))
+                (match (start-service 'dockerd)
+                  (#f #f)
+                  (('service response-parts ...)
+                   (match (assq-ref response-parts 'running)
+                     ((pid) (number? pid))))))
+             marionette))
+
+          (sleep 10) ; let service start
+
+          (test-assert "docker-guile running"
+            (marionette-eval
+             '(begin
+                (use-modules (gnu services herd))
+                (match (start-service 'docker-guile)
+                  (#f #f)
+                  (('service response-parts ...)
+                   (match (assq-ref response-parts 'running)
+                     ((pid) (number? pid))))))
+             marionette))
+
+          (test-equal "passing host environment variables and volumes"
+            '("value" "hello")
+            (marionette-eval
+             `(begin
+                (use-modules (ice-9 popen)
+                             (ice-9 rdelim))
+
+                (define slurp
+                  (lambda args
+                    (let* ((port (apply open-pipe* OPEN_READ args))
+                           (output (let ((line (read-line port)))
+                                     (if (eof-object? line)
+                                         ""
+                                         line)))
+                           (status (close-pipe port)))
+                      output)))
+                (let* ((response1 (slurp
+                                   ,(string-append #$docker-cli "/bin/docker")
+                                   "exec" "docker-guile"
+                                   "/bin/guile" "-c" "(display (getenv \"VARIABLE\"))"))
+                       (response2 (slurp
+                                   ,(string-append #$docker-cli "/bin/docker")
+                                   "exec" "docker-guile"
+                                   "/bin/guile" "-c" "(begin (use-modules (ice-9 popen) (ice-9 rdelim))
+(display (call-with-input-file \"/shared.txt\" read-line)))")))
+                  (list response1 response2)))
+             marionette))
+
+          (test-end))))
+
+  (gexp->derivation "oci-container-test" test))
+
+(define %test-oci-container
+  (system-test
+   (name "oci-container")
+   (description "Test OCI backed Shepherd service.")
+   (value (run-oci-container-test))))
-- 
2.41.0

[bug#67613] [PATCH v3 1/5] gnu: docker: Provide escape hatch in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

* gnu/services/docker.scm (exports): Add missing procedures;
(oci-container-service-type)[description]: Docker and OCI images should
mean the same thing;
(oci-container-configuration): clarify field types;
[extra-arguments]: new field;
(oci-sanitize-extra-arguments): sanitize it;
(oci-container-shepherd-service): use it.
* doc/guix.texi: Document it.

Change-Id: I64e9d82c8ae538d59d1c482f23070a880156ddf7
---
 doc/guix.texi           | 21 ++++++++++++-------
 gnu/services/docker.scm | 46 +++++++++++++++++++++++++++++++++--------
 2 files changed, 51 insertions(+), 16 deletions(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index 3f5d4e7f0d..19b7563916 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40430,13 +40430,13 @@ Miscellaneous Services
 @item @code{group} (default: @code{"docker"}) (type: string)
 The group under whose authority docker commands will be run.
 
-@item @code{command} (default: @code{()}) (type: list-of-strings)
+@item @code{command} (default: @code{'()}) (type: list-of-strings)
 Overwrite the default command (@code{CMD}) of the image.
 
 @item @code{entrypoint} (default: @code{""}) (type: string)
 Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.
 
-@item @code{environment} (default: @code{()}) (type: list)
+@item @code{environment} (default: @code{'()}) (type: list)
 Set environment variables. This can be a list of pairs or strings, even mixed:
 
 @lisp
@@ -40444,7 +40444,8 @@ Miscellaneous Services
       "JAVA_HOME=/opt/java")
 @end lisp
 
-String are passed directly to the Docker CLI. You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI.  You can refer to the
 @uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
 documentation for semantics.
 
@@ -40459,7 +40460,7 @@ Miscellaneous Services
 @item @code{network} (default: @code{""}) (type: string)
 Set a Docker network for the spawned container.
 
-@item @code{ports} (default: @code{()}) (type: list)
+@item @code{ports} (default: @code{'()}) (type: list)
 Set the port or port ranges to expose from the spawned container.  This can be a
 list of pairs or strings, even mixed:
 
@@ -40468,11 +40469,12 @@ Miscellaneous Services
       "10443:443")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI.  You can refer to the
 @uref{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
 documentation for semantics.
 
-@item @code{volumes} (default: @code{()}) (type: list)
+@item @code{volumes} (default: @code{'()}) (type: list)
 Set volume mappings for the spawned container.  This can be a
 list of pairs or strings, even mixed:
 
@@ -40481,7 +40483,8 @@ Miscellaneous Services
       "/gnu/store:/gnu/store")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects.
+Strings are passed directly to the Docker CLI.  You can refer to the
 @uref{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
 documentation for semantics.
 
@@ -40496,6 +40499,10 @@ Miscellaneous Services
 @url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
 documentation for semantics.
 
+@item @code{extra-arguments} (default: @code{'()}) (type: list)
+A list of strings, gexps or file-like objects that will be directly
+passed to the @command{docker run} invokation.
+
 @end table
 
 @end deftp
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 4d32b96847..824c4ecbe6 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -58,6 +58,9 @@ (define-module (gnu services docker)
             oci-container-configuration-network
             oci-container-configuration-ports
             oci-container-configuration-volumes
+            oci-container-configuration-container-user
+            oci-container-configuration-workdir
+            oci-container-configuration-extra-arguments
             oci-container-service-type
             oci-container-shepherd-service))
 
@@ -297,6 +300,21 @@ (define (oci-sanitize-volumes value)
   ;; '(("/mnt/dir" . "/dir") "/run/current-system/profile:/java")
   (oci-sanitize-mixed-list "volumes" value ":"))
 
+(define (oci-sanitize-extra-arguments value)
+  (define (valid? member)
+    (or (string? member)
+        (gexp? member)
+        (file-like? member)))
+  (map
+   (lambda (el)
+     (if (valid? el)
+         el
+         (raise
+          (formatted-message
+           (G_ "extra arguments may only be strings, gexps or file-like objects
+but ~a was found") el))))
+   value))
+
 (define-maybe/no-serialization string)
 
 (define-configuration/no-serialization oci-container-configuration
@@ -314,15 +332,16 @@ (define-configuration/no-serialization oci-container-configuration
    "Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.")
   (environment
    (list '())
-   "Set environment variables.  This can be a list of pairs or strings, even
-mixed:
+   "Set environment variables inside the container.  This can be a list of pairs
+or strings, even mixed:
 
 @lisp
 (list '(\"LANGUAGE\" . \"eo:ca:eu\")
       \"JAVA_HOME=/opt/java\")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI.  You can refer to the
 @url{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
 documentation for semantics."
    (sanitizer oci-sanitize-environment))
@@ -347,7 +366,8 @@ (define-configuration/no-serialization oci-container-configuration
       \"10443:443\")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI.  You can refer to the
 @url{https://docs.docker.com/engine/reference/commandline/run/#publish,upstream}
 documentation for semantics."
    (sanitizer oci-sanitize-ports))
@@ -361,7 +381,8 @@ (define-configuration/no-serialization oci-container-configuration
       \"/gnu/store:/gnu/store\")
 @end lisp
 
-String are passed directly to the Docker CLI.  You can refer to the
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to the Docker CLI.  You can refer to the
 @url{https://docs.docker.com/engine/reference/commandline/run/#volume,upstream}
 documentation for semantics."
    (sanitizer oci-sanitize-volumes))
@@ -375,7 +396,12 @@ (define-configuration/no-serialization oci-container-configuration
    "Set the current working for the spawned Shepherd service.
 You can refer to the
 @url{https://docs.docker.com/engine/reference/run/#workdir,upstream}
-documentation for semantics."))
+documentation for semantics.")
+  (extra-arguments
+   (list '())
+   "A list of strings, gexps or file-like objects that will be directly passed
+to the @command{docker run} invokation."
+   (sanitizer oci-sanitize-extra-arguments)))
 
 (define oci-container-configuration->options
   (lambda (config)
@@ -428,7 +454,9 @@ (define (oci-container-shepherd-service config)
          (provision (oci-container-configuration-provision config))
          (image (oci-container-configuration-image config))
          (options (oci-container-configuration->options config))
-         (name (guess-name provision image)))
+         (name (guess-name provision image))
+         (extra-arguments
+          (oci-container-configuration-extra-arguments config)))
 
     (shepherd-service (provision `(,(string->symbol name)))
                       (requirement '(dockerd user-processes))
@@ -441,7 +469,7 @@ (define (oci-container-shepherd-service config)
                           ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
                           (list #$docker-command "run" "--rm"
                                 "--name" #$name
-                                #$@options #$image #$@command)
+                                #$@options #$@extra-arguments #$image #$@command)
                           #:user #$user
                           #:group #$group))
                       (stop
@@ -482,5 +510,5 @@ (define oci-container-service-type
                 (extend append)
                 (compose concatenate)
                 (description
-                 "This service allows the management of Docker and OCI
+                 "This service allows the management of OCI
 containers as Shepherd services.")))

base-commit: 7d4ae2fca723114fb1df56de33b82177fbc4d0a6
-- 
2.41.0

[bug#67613] [PATCH v3 2/5] gnu: docker: Allow setting host environment variables in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

* gnu/services/docker.scm (oci-container-configuration)
[host-environment]: New field;
(oci-sanitize-host-environment): sanitize it;
(oci-container-shepherd-service): use it.
* doc/guix.texi: Document it.

Change-Id: I4d54d37736cf09f042a71cb0b6e673abc0948d9c
---
 doc/guix.texi           | 17 +++++++++++++++++
 gnu/services/docker.scm | 31 +++++++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index 19b7563916..fad0bf8c7c 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40436,6 +40436,23 @@ Miscellaneous Services
 @item @code{entrypoint} (default: @code{""}) (type: string)
 Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.
 
+@item @code{host-environment} (default: @code{'()}) (type: list)
+Set environment variables in the host environment where @command{docker
+run} is invoked.  This is especially useful to pass secrets from the
+host to the container without having them on the @command{docker run}'s
+command line: by setting the @code{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment.  This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+      \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}.
+
 @item @code{environment} (default: @code{'()}) (type: list)
 Set environment variables. This can be a list of pairs or strings, even mixed:
 
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index 824c4ecbe6..df5884aca1 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -5,7 +5,7 @@
 ;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il>
 ;;; Copyright © 2020 Jesse Dowell <jessedowell@gmail.com>
 ;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re>
-;;; Copyright © 2023 Giacomo Leidi <goodoldpaul@autistici.org>
+;;; Copyright © 2023, 2024 Giacomo Leidi <goodoldpaul@autistici.org>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -285,6 +285,11 @@ (define (oci-sanitize-mixed-list name value delimiter)
               name el)))))
    value))
 
+(define (oci-sanitize-host-environment value)
+  ;; Expected spec format:
+  ;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
+  (oci-sanitize-mixed-list "host-environment" value "="))
+
 (define (oci-sanitize-environment value)
   ;; Expected spec format:
   ;; '(("HOME" . "/home/nobody") "JAVA_HOME=/java")
@@ -330,6 +335,24 @@ (define-configuration/no-serialization oci-container-configuration
   (entrypoint
    (maybe-string)
    "Overwrite the default entrypoint (@code{ENTRYPOINT}) of the image.")
+  (host-environment
+   (list '())
+   "Set environment variables in the host environment where @command{docker run}
+is invoked.  This is especially useful to pass secrets from the host to the
+container without having them on the @command{docker run}'s command line: by
+setting the @code{MYSQL_PASSWORD} on the host and by passing
+@code{--env MYSQL_PASSWORD} through the @code{extra-arguments} field, it is
+possible to securely set values in the container environment.  This field's
+value can be a list of pairs or strings, even mixed:
+
+@lisp
+(list '(\"LANGUAGE\" . \"eo:ca:eu\")
+      \"JAVA_HOME=/opt/java\")
+@end lisp
+
+Pair members can be strings, gexps or file-like objects. Strings are passed
+directly to @code{make-forkexec-constructor}."
+   (sanitizer oci-sanitize-host-environment))
   (environment
    (list '())
    "Set environment variables inside the container.  This can be a list of pairs
@@ -450,6 +473,8 @@ (define (oci-container-shepherd-service config)
   (let* ((docker-command (file-append docker-cli "/bin/docker"))
          (user (oci-container-configuration-user config))
          (group (oci-container-configuration-group config))
+         (host-environment
+          (oci-container-configuration-host-environment config))
          (command (oci-container-configuration-command config))
          (provision (oci-container-configuration-provision config))
          (image (oci-container-configuration-image config))
@@ -471,7 +496,9 @@ (define (oci-container-shepherd-service config)
                                 "--name" #$name
                                 #$@options #$@extra-arguments #$image #$@command)
                           #:user #$user
-                          #:group #$group))
+                          #:group #$group
+                          #:environment-variables
+                          (list #$@host-environment)))
                       (stop
                        #~(lambda _
                            (invoke #$docker-command "rm" "-f" #$name)))
-- 
2.41.0

[bug#67613] [PATCH v3 3/5] gnu: docker: Allow setting Shepherd dependencies in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

* gnu/services/docker.scm (oci-container-configuration)
[requirement]: New field;
(list-of-symbols): sanitize it;
(oci-container-shepherd-service): use it.
* doc/guix.texi: Document it.

Change-Id: Ic0ba336a2257d6ef7c658cfc6cd630116661f581
---
 doc/guix.texi           |  4 ++++
 gnu/services/docker.scm | 10 +++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index fad0bf8c7c..2b9cc5602c 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40474,6 +40474,10 @@ Miscellaneous Services
 @item @code{provision} (default: @code{""}) (type: string)
 Set the name of the provisioned Shepherd service.
 
+@item @code{requirement} (default: @code{'()}) (type: list-of-symbols)
+Set additional Shepherd services dependencies to the provisioned
+Shepherd service.
+
 @item @code{network} (default: @code{""}) (type: string)
 Set a Docker network for the spawned container.
 
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index df5884aca1..a5b1614fa9 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -320,6 +320,9 @@ (define (oci-sanitize-extra-arguments value)
 but ~a was found") el))))
    value))
 
+(define list-of-symbols?
+  (list-of symbol?))
+
 (define-maybe/no-serialization string)
 
 (define-configuration/no-serialization oci-container-configuration
@@ -376,6 +379,10 @@ (define-configuration/no-serialization oci-container-configuration
   (provision
    (maybe-string)
    "Set the name of the provisioned Shepherd service.")
+  (requirement
+   (list-of-symbols '())
+   "Set additional Shepherd services dependencies to the provisioned Shepherd
+service.")
   (network
    (maybe-string)
    "Set a Docker network for the spawned container.")
@@ -477,6 +484,7 @@ (define (oci-container-shepherd-service config)
           (oci-container-configuration-host-environment config))
          (command (oci-container-configuration-command config))
          (provision (oci-container-configuration-provision config))
+         (requirement (oci-container-configuration-requirement config))
          (image (oci-container-configuration-image config))
          (options (oci-container-configuration->options config))
          (name (guess-name provision image))
@@ -484,7 +492,7 @@ (define (oci-container-shepherd-service config)
           (oci-container-configuration-extra-arguments config)))
 
     (shepherd-service (provision `(,(string->symbol name)))
-                      (requirement '(dockerd user-processes))
+                      (requirement `(dockerd user-processes ,@requirement))
                       (respawn? #f)
                       (documentation
                        (string-append
-- 
2.41.0

[bug#67613] [PATCH v3 4/5] gnu: docker: Allow passing tarballs for images in oci-container-configuration.

[Giacomo Leidi via Guix-patches via]

This commit allows for loading an OCI image tarball before running an
OCI backed Shepherd service. It does so by adding a one shot Shepherd
service to the dependencies of the OCI backed service that at boot runs
docker load on the tarball.

* gnu/services/docker.scm (oci-image): New record;
(lower-oci-image): new variable, lower it;
(string-or-oci-image?): sanitize it;
(oci-container-configuration)[image]: allow also for oci-image records;
(oci-container-shepherd-service): use it;
(%oci-image-loader): new variable.

Change-Id: Ie504f479ea0d47f74b0ec5df9085673ffd3f639d
---
 doc/guix.texi           |  70 +++++++++++-
 gnu/services/docker.scm | 244 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 286 insertions(+), 28 deletions(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index 2b9cc5602c..451bee5615 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -40390,6 +40390,17 @@ Miscellaneous Services
 @lisp
 (service oci-container-service-type
          (list
+          (oci-container-configuration
+           (image
+            (oci-image
+             (repository "guile")
+             (tag "3")
+             (value (specifications->manifest '("guile")))
+             (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+                             #:max-layers 2))))
+           (entrypoint "/bin/guile")
+           (command
+            '("-c" "(display \"hello!\n\")")))
           (oci-container-configuration
            (image "prom/prometheus")
            (network "host")
@@ -40466,9 +40477,10 @@ Miscellaneous Services
 @uref{https://docs.docker.com/engine/reference/commandline/run/#env,upstream}
 documentation for semantics.
 
-@item @code{image} (type: string)
-The image used to build the container.  Images are resolved by the
-Docker Engine, and follow the usual format
+@item @code{image} (type: string-or-oci-image)
+The image used to build the container.  It can be a string or an
+@code{oci-image} record.  Strings are resolved by the Docker Engine, and
+follow the usual format
 @code{myregistry.local:5000/testing/test-image:tag}.
 
 @item @code{provision} (default: @code{""}) (type: string)
@@ -40529,6 +40541,58 @@ Miscellaneous Services
 @end deftp
 
 
+@c %end of fragment
+
+@c %start of fragment
+
+@deftp {Data Type} oci-image
+Available @code{oci-image} fields are:
+
+@table @asis
+@item @code{repository} (type: string)
+A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.
+
+@item @code{tag} (default: @code{"latest"}) (type: string)
+A string representing the OCI image tag.  Defaults to @code{latest}.
+
+@item @code{value} (type: oci-lowerable-image)
+A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball.  Otherwise this field's value can be a
+gexp or a file-like object that evaluates to an OCI compatible tarball.
+
+@item @code{pack-options} (default: @code{'()}) (type: list)
+An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}.  They can
+be used to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+  (repository "guile")
+  (tag "3")
+  (value
+    (specifications->manifest '("guile")))
+  (pack-options '(#:symlinks (("/bin/guile" -> "bin/guile"))
+                  #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.
+
+@item @code{system} (default: @code{""}) (type: string)
+Attempt to build for a given system, e.g. "i686-linux"
+
+@item @code{target} (default: @code{""}) (type: string)
+Attempt to cross-build for a given triple, e.g. "aarch64-linux-gnu"
+
+@item @code{grafts?} (default: @code{#f}) (type: boolean)
+Whether to allow grafting or not in the pack build.
+
+@end table
+
+@end deftp
+
+
 @c %end of fragment
 
 @cindex Audit
diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm
index a5b1614fa9..7aff8dcc5f 100644
--- a/gnu/services/docker.scm
+++ b/gnu/services/docker.scm
@@ -23,11 +23,14 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (gnu services docker)
+  #:use-module (gnu image)
   #:use-module (gnu services)
   #:use-module (gnu services configuration)
   #:use-module (gnu services base)
   #:use-module (gnu services dbus)
   #:use-module (gnu services shepherd)
+  #:use-module (gnu system)
+  #:use-module (gnu system image)
   #:use-module (gnu system setuid)
   #:use-module (gnu system shadow)
   #:use-module (gnu packages admin)               ;shadow
@@ -37,7 +40,11 @@ (define-module (gnu services docker)
   #:use-module (guix diagnostics)
   #:use-module (guix gexp)
   #:use-module (guix i18n)
+  #:use-module (guix monads)
   #:use-module (guix packages)
+  #:use-module (guix profiles)
+  #:use-module ((guix scripts pack) #:prefix pack:)
+  #:use-module (guix store)
   #:use-module (srfi srfi-1)
   #:use-module (ice-9 format)
   #:use-module (ice-9 match)
@@ -45,6 +52,16 @@ (define-module (gnu services docker)
   #:export (docker-configuration
             docker-service-type
             singularity-service-type
+            oci-image
+            oci-image?
+            oci-image-fields
+            oci-image-repository
+            oci-image-tag
+            oci-image-value
+            oci-image-pack-options
+            oci-image-target
+            oci-image-system
+            oci-image-grafts?
             oci-container-configuration
             oci-container-configuration?
             oci-container-configuration-fields
@@ -52,9 +69,11 @@ (define-module (gnu services docker)
             oci-container-configuration-group
             oci-container-configuration-command
             oci-container-configuration-entrypoint
+            oci-container-configuration-host-environment
             oci-container-configuration-environment
             oci-container-configuration-image
             oci-container-configuration-provision
+            oci-container-configuration-requirement
             oci-container-configuration-network
             oci-container-configuration-ports
             oci-container-configuration-volumes
@@ -62,7 +81,8 @@ (define-module (gnu services docker)
             oci-container-configuration-workdir
             oci-container-configuration-extra-arguments
             oci-container-service-type
-            oci-container-shepherd-service))
+            oci-container-shepherd-service
+            %oci-container-accounts))
 
 (define-maybe file-like)
 
@@ -320,11 +340,68 @@ (define (oci-sanitize-extra-arguments value)
 but ~a was found") el))))
    value))
 
+(define (oci-image-reference image)
+  (if (string? image)
+      image
+      (string-append (oci-image-repository image)
+                     ":" (oci-image-tag image))))
+
+(define (oci-lowerable-image? image)
+  (or (manifest? image)
+      (operating-system? image)
+      (gexp? image)
+      (file-like? image)))
+
+(define (string-or-oci-image? image)
+  (or (string? image)
+      (oci-image? image)))
+
 (define list-of-symbols?
   (list-of symbol?))
 
 (define-maybe/no-serialization string)
 
+(define-configuration/no-serialization oci-image
+  (repository
+   (string)
+   "A string like @code{myregistry.local:5000/testing/test-image} that names
+the OCI image.")
+  (tag
+   (string "latest")
+   "A string representing the OCI image tag. Defaults to @code{latest}.")
+  (value
+   (oci-lowerable-image)
+   "A @code{manifest} or @code{operating-system} record that will be lowered
+into an OCI compatible tarball.  Otherwise this field's value can be a gexp
+or a file-like object that evaluates to an OCI compatible tarball.")
+  (pack-options
+   (list '())
+   "An optional set of keyword arguments that will be passed to the
+@code{docker-image} procedure from @code{guix scripts pack}.  They can be used
+to replicate @command{guix pack} behavior:
+
+@lisp
+(oci-image
+  (repository \"guile\")
+  (tag \"3\")
+  (manifest (specifications->manifest '(\"guile\")))
+  (pack-options
+    '(#:symlinks ((\"/bin/guile\" -> \"bin/guile\"))
+      #:max-layers 2)))
+@end lisp
+
+If the @code{value} field is an @code{operating-system} record, this field's
+value will be ignored.")
+  (system
+   (maybe-string)
+   "Attempt to build for a given system, e.g. \"i686-linux\"")
+  (target
+   (maybe-string)
+   "Attempt to cross-build for a given triple, e.g. \"aarch64-linux-gnu\"")
+  (grafts?
+   (boolean #f)
+   "Whether to allow grafting or not in the pack build."))
+
 (define-configuration/no-serialization oci-container-configuration
   (user
    (string "oci-container")
@@ -372,8 +449,9 @@ (define-configuration/no-serialization oci-container-configuration
 documentation for semantics."
    (sanitizer oci-sanitize-environment))
   (image
-   (string)
-   "The image used to build the container.  Images are resolved by the Docker
+   (string-or-oci-image)
+   "The image used to build the container.  It can be a string or an
+@code{oci-image} record.  Strings are resolved by the Docker
 Engine, and follow the usual format
 @code{myregistry.local:5000/testing/test-image:tag}.")
   (provision
@@ -470,14 +548,122 @@ (define oci-container-configuration->options
                            (list "-v" spec))
                          (oci-container-configuration-volumes config))))))))
 
+(define* (get-keyword-value args keyword #:key (default #f))
+  (let ((kv (memq keyword args)))
+    (if (and kv (>= (length kv) 2))
+        (cadr kv)
+        default)))
+
+(define (lower-operating-system os target system)
+  (mlet* %store-monad
+      ((tarball
+        (lower-object
+         (system-image (os->image os #:type docker-image-type))
+         system
+         #:target target)))
+    (return tarball)))
+
+(define (lower-manifest name image target system)
+  (define value (oci-image-value image))
+  (define options (oci-image-pack-options image))
+  (define image-reference
+    (oci-image-reference image))
+  (define image-tag
+    (let* ((extra-options
+            (get-keyword-value options #:extra-options))
+           (image-tag-option
+            (and extra-options
+                 (get-keyword-value extra-options #:image-tag))))
+      (if image-tag-option
+          '()
+          `(#:extra-options (#:image-tag ,image-reference)))))
+
+  (mlet* %store-monad
+      ((_ (set-grafting
+           (oci-image-grafts? image)))
+       (guile (set-guile-for-build (default-guile)))
+       (profile
+        (profile-derivation value
+                            #:target target
+                            #:system system
+                            #:hooks '()
+                            #:locales? #f))
+       (tarball (apply pack:docker-image
+                       `(,name ,profile
+                         ,@options
+                         ,@image-tag
+                         #:localstatedir? #t))))
+    (return tarball)))
+
+(define (lower-oci-image name image)
+  (define value (oci-image-value image))
+  (define image-target (oci-image-target image))
+  (define image-system (oci-image-system image))
+  (define target
+    (if (maybe-value-set? image-target)
+        image-target
+        (%current-target-system)))
+  (define system
+    (if (maybe-value-set? image-system)
+        image-system
+        (%current-system)))
+  (with-store store
+   (run-with-store store
+     (match value
+       ((? manifest? value)
+        (lower-manifest name image target system))
+       ((? operating-system? value)
+        (lower-operating-system value target system))
+       ((or (? gexp? value)
+            (? file-like? value))
+        value)
+       (_
+        (raise
+         (formatted-message
+          (G_ "oci-image value must contain only manifest,
+operating-system, gexp or file-like records but ~a was found")
+          value))))
+     #:target target
+     #:system system)))
+
+(define (%oci-image-loader name image tag)
+  (let ((docker (file-append docker-cli "/bin/docker"))
+        (tarball (lower-oci-image name image)))
+    (with-imported-modules '((guix build utils))
+      (program-file (format #f "~a-image-loader" name)
+       #~(begin
+           (use-modules (guix build utils)
+                        (ice-9 popen)
+                        (ice-9 rdelim))
+
+           (format #t "Loading image for ~a from ~a...~%" #$name #$tarball)
+           (define line
+             (read-line
+              (open-input-pipe
+               (string-append #$docker " load -i " #$tarball))))
+
+           (unless (or (eof-object? line)
+                       (string-null? line))
+             (format #t "~a~%" line)
+             (let ((repository&tag
+                    (string-drop line
+                                 (string-length
+                                   "Loaded image: "))))
+
+               (invoke #$docker "tag" repository&tag #$tag)
+               (format #t "Tagged ~a with ~a...~%" #$tarball #$tag))))))))
+
 (define (oci-container-shepherd-service config)
   (define (guess-name name image)
     (if (maybe-value-set? name)
         name
         (string-append "docker-"
-                       (basename (car (string-split image #\:))))))
+                       (basename
+                        (if (string? image)
+                            (first (string-split image #\:))
+                            (oci-image-repository image))))))
 
-  (let* ((docker-command (file-append docker-cli "/bin/docker"))
+  (let* ((docker (file-append docker-cli "/bin/docker"))
          (user (oci-container-configuration-user config))
          (group (oci-container-configuration-group config))
          (host-environment
@@ -486,6 +672,7 @@ (define (oci-container-shepherd-service config)
          (provision (oci-container-configuration-provision config))
          (requirement (oci-container-configuration-requirement config))
          (image (oci-container-configuration-image config))
+         (image-reference (oci-image-reference image))
          (options (oci-container-configuration->options config))
          (name (guess-name provision image))
          (extra-arguments
@@ -496,30 +683,37 @@ (define (oci-container-shepherd-service config)
                       (respawn? #f)
                       (documentation
                        (string-append
-                        "Docker backed Shepherd service for image: " image))
+                        "Docker backed Shepherd service for "
+                        (if (oci-image? image) name image) "."))
                       (start
-                       #~(make-forkexec-constructor
-                          ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
-                          (list #$docker-command "run" "--rm"
-                                "--name" #$name
-                                #$@options #$@extra-arguments #$image #$@command)
-                          #:user #$user
-                          #:group #$group
-                          #:environment-variables
-                          (list #$@host-environment)))
+                       #~(lambda ()
+                          (when #$(oci-image? image)
+                            (invoke #$(%oci-image-loader
+                                       name image image-reference)))
+                          (fork+exec-command
+                           ;; docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
+                           (list #$docker "run" "--rm" "--name" #$name
+                                 #$@options #$@extra-arguments
+                                 #$image-reference #$@command)
+                           #:user #$user
+                           #:group #$group
+                           #:environment-variables
+                           (list #$@host-environment))))
                       (stop
                        #~(lambda _
-                           (invoke #$docker-command "rm" "-f" #$name)))
+                           (invoke #$docker "rm" "-f" #$name)))
                       (actions
-                       (list
-                        (shepherd-action
-                         (name 'pull)
-                         (documentation
-                          (format #f "Pull ~a's image (~a)."
-                                  name image))
-                         (procedure
-                          #~(lambda _
-                              (invoke #$docker-command "pull" #$image)))))))))
+                       (if (oci-image? image)
+                           '()
+                           (list
+                            (shepherd-action
+                             (name 'pull)
+                             (documentation
+                              (format #f "Pull ~a's image (~a)."
+                                      name image))
+                             (procedure
+                              #~(lambda _
+                                  (invoke #$docker "pull" #$image))))))))))
 
 (define %oci-container-accounts
   (list (user-account
-- 
2.41.0

[bug#67613] [PATCH v3 5/5] gnu: Add tests for oci-container-service-type.

[Giacomo Leidi via Guix-patches via]

* gnu/tests/docker.scm (run-oci-container-test): New variable;
(%test-oci-container): new variable.

Change-Id: Idefc3840bdc6e0ed4264e8f27373cd9a670f87a0
---
 gnu/tests/docker.scm | 131 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 130 insertions(+), 1 deletion(-)

diff --git a/gnu/tests/docker.scm b/gnu/tests/docker.scm
index 9e9d2e2d07..d550136b4a 100644
--- a/gnu/tests/docker.scm
+++ b/gnu/tests/docker.scm
@@ -1,6 +1,7 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2019 Danny Milosavljevic <dannym@scratchpost.org>
 ;;; Copyright © 2019-2023 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2024 Giacomo Leidi <goodoldpaul@autistici.org>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -29,6 +30,7 @@ (define-module (gnu tests docker)
   #:use-module (gnu services networking)
   #:use-module (gnu services docker)
   #:use-module (gnu services desktop)
+  #:use-module (gnu packages)
   #:use-module ((gnu packages base) #:select (glibc))
   #:use-module (gnu packages guile)
   #:use-module (gnu packages docker)
@@ -43,7 +45,8 @@ (define-module (gnu tests docker)
   #:use-module (guix build-system trivial)
   #:use-module ((guix licenses) #:prefix license:)
   #:export (%test-docker
-            %test-docker-system))
+            %test-docker-system
+            %test-oci-container))
 
 (define %docker-os
   (simple-operating-system
@@ -316,3 +319,129 @@ (define %test-docker-system
                                    (locale-libcs (list glibc)))
                                  #:type docker-image-type)))
                  run-docker-system-test)))))
+
+\f
+(define %oci-os
+  (simple-operating-system
+   (service dhcp-client-service-type)
+   (service dbus-root-service-type)
+   (service polkit-service-type)
+   (service elogind-service-type)
+   (service docker-service-type)
+   (extra-special-file "/shared.txt"
+                       (plain-file "shared.txt" "hello"))
+   (service oci-container-service-type
+            (list
+             (oci-container-configuration
+              (image
+               (oci-image
+                (repository "guile")
+                (value
+                 (specifications->manifest '("guile")))
+                (pack-options
+                 '(#:symlinks (("/bin" -> "bin"))))))
+              (entrypoint
+               "/bin/guile")
+              (command
+               '("-c" "(let l ((c 300))(display c)(sleep 1)(when(positive? c)(l (- c 1))))"))
+              (host-environment
+               '(("VARIABLE" . "value")))
+              (volumes
+               '(("/shared.txt" . "/shared.txt:ro")))
+              (extra-arguments
+               '("--env" "VARIABLE")))))))
+
+(define (run-oci-container-test)
+  "Run IMAGE as an OCI backed Shepherd service, inside OS."
+
+  (define os
+    (marionette-operating-system
+     (operating-system-with-gc-roots
+      %oci-os
+      (list))
+     #:imported-modules '((gnu services herd)
+                          (guix combinators))))
+
+  (define vm
+    (virtual-machine
+     (operating-system os)
+     (volatile? #f)
+     (memory-size 1024)
+     (disk-image-size (* 3000 (expt 2 20)))
+     (port-forwardings '())))
+
+  (define test
+    (with-imported-modules '((gnu build marionette))
+      #~(begin
+          (use-modules (srfi srfi-11) (srfi srfi-64)
+                       (gnu build marionette))
+
+          (define marionette
+            ;; Relax timeout to accommodate older systems and
+            ;; allow for pulling the image.
+            (make-marionette (list #$vm) #:timeout 60))
+
+          (test-runner-current (system-test-runner #$output))
+          (test-begin "oci-container")
+
+          (test-assert "dockerd running"
+            (marionette-eval
+             '(begin
+                (use-modules (gnu services herd))
+                (match (start-service 'dockerd)
+                  (#f #f)
+                  (('service response-parts ...)
+                   (match (assq-ref response-parts 'running)
+                     ((pid) (number? pid))))))
+             marionette))
+
+          (sleep 10) ; let service start
+
+          (test-assert "docker-guile running"
+            (marionette-eval
+             '(begin
+                (use-modules (gnu services herd))
+                (match (start-service 'docker-guile)
+                  (#f #f)
+                  (('service response-parts ...)
+                   (match (assq-ref response-parts 'running)
+                     ((pid) (number? pid))))))
+             marionette))
+
+          (test-equal "passing host environment variables and volumes"
+            '("value" "hello")
+            (marionette-eval
+             `(begin
+                (use-modules (ice-9 popen)
+                             (ice-9 rdelim))
+
+                (define slurp
+                  (lambda args
+                    (let* ((port (apply open-pipe* OPEN_READ args))
+                           (output (let ((line (read-line port)))
+                                     (if (eof-object? line)
+                                         ""
+                                         line)))
+                           (status (close-pipe port)))
+                      output)))
+                (let* ((response1 (slurp
+                                   ,(string-append #$docker-cli "/bin/docker")
+                                   "exec" "docker-guile"
+                                   "/bin/guile" "-c" "(display (getenv \"VARIABLE\"))"))
+                       (response2 (slurp
+                                   ,(string-append #$docker-cli "/bin/docker")
+                                   "exec" "docker-guile"
+                                   "/bin/guile" "-c" "(begin (use-modules (ice-9 popen) (ice-9 rdelim))
+(display (call-with-input-file \"/shared.txt\" read-line)))")))
+                  (list response1 response2)))
+             marionette))
+
+          (test-end))))
+
+  (gexp->derivation "oci-container-test" test))
+
+(define %test-oci-container
+  (system-test
+   (name "oci-container")
+   (description "Test OCI backed Shepherd service.")
+   (value (run-oci-container-test))))
-- 
2.41.0

bug#67613: Introduce unit tests for oci-container-service-type.

[Ludovic Courtès]

Hi,

I went ahead and finally applied this series, thanks!

(I’m still waiting for the system test to complete on my laptop: it
requires quite a lot of disk space so I had to GC and try again a couple
of times already…)

Thanks,
Ludo’.