From: pukkamustard <pukkamustard@posteo.net>
To: 52555@debbugs.gnu.org
Cc: pukkamustard <pukkamustard@posteo.net>,
ludo@gnu.org, maximedevos@telenet.be,
"Christopher Baines" <guix@cbaines.net>,
"Josselin Poiret" <dev@jpoiret.xyz>,
"Ludovic Courtès" <ludo@gnu.org>,
"Mathieu Othacehe" <othacehe@gnu.org>,
"Ricardo Wurmus" <rekado@elephly.net>,
"Simon Tournier" <zimon.toutoune@gmail.com>,
"Tobias Geerinckx-Rice" <me@tobias.gr>
Subject: [bug#52555] [PATCH v4 6/7] substitute: Decode substitutes using ERIS.
Date: Thu, 28 Dec 2023 09:40:09 +0000 [thread overview]
Message-ID: <deebb9389ad0f32f9bcfb0d7ca6ed9cc04a980d2.1703316055.git.pukkamustard@posteo.net> (raw)
In-Reply-To: <cover.1703316055.git.pukkamustard@posteo.net>
* guix/scripts/substitute.scm: Decode substitutes using ERIS.
* guix/eris.scm (eris-decode-store-item): New function.
* nix/nix-daemon/guix-daemon.cc (options): Add eris-store-url option.
---
guix/eris.scm | 45 +++++++++++++++++----
guix/scripts/substitute.scm | 76 +++++++++++++++++++++++++++--------
nix/nix-daemon/guix-daemon.cc | 5 +++
3 files changed, 102 insertions(+), 24 deletions(-)
diff --git a/guix/eris.scm b/guix/eris.scm
index 3fbedd0cb7..1aa52e69dd 100644
--- a/guix/eris.scm
+++ b/guix/eris.scm
@@ -21,6 +21,7 @@ (define-module (guix eris)
#:use-module (eris)
#:use-module (eris fs)
#:use-module (eris sqlite)
+ #:use-module (sqlite3)
#:use-module (eris coap)
#:use-module (eris read-capability)
@@ -31,7 +32,8 @@ (define-module (guix eris)
#:use-module (srfi srfi-171)
#:export (%eris-store-url
- eris-encode-store-item))
+ eris-encode-store-item
+ eris-decode-store-item))
(define %eris-store-url
(make-parameter
@@ -83,8 +85,7 @@ (define (guix-eris-block-reducer)
(() (ecbr))
((conn ref-block) (ecbr conn ref-block))
((conn)
- (ecbr conn)
- (close-port conn)))))
+ (ecbr conn)))))
;; TODO
;; ('coap+tcp #f)
@@ -97,10 +98,40 @@ (define (guix-eris-block-reducer)
;; SRFI-171 that counts the number of blocks.
rcount)))
+(define (call-with-guix-eris-block-ref proc)
+ (let ((store-url (%eris-store-url)))
+ (if (uri? store-url)
+ (match (uri-scheme store-url)
+
+ ('sqlite
+ (let ((db (eris-sqlite-open (uri-path store-url))))
+ (proc (lambda (ref) (eris-sqlite-ref db ref)))
+ (sqlite-close db)))
+
+ ('coap+unix
+ (let ((conn (open-coap-unix-socket (uri-path store-url)))
+ (req-uri (build-uri 'coap #:path ".well-known/eris")))
+ (proc
+ (lambda (ref)
+ (eris-coap-block-ref req-uri ref #:connection conn)))
+ (close-port conn)))
+
+ (_ (error "Don't know how to handle ERIS store URL "
+ (uri->string (%eris-store-url)))))
+
+ (error "No ERIS store to get blocks."))))
+
(define* (eris-encode-store-item item)
"Encodes the store item ITEM using ERIS and returns the read capability as
string."
- (eris-read-capability->string
- (eris-fs-encode item
- #:convergence-secret (%guix-eris-convergence-secret)
- #:block-reducer (guix-eris-block-reducer))))
+ (eris-fs-encode item
+ #:convergence-secret (%guix-eris-convergence-secret)
+ #:block-reducer (guix-eris-block-reducer)))
+
+(define* (eris-decode-store-item eris-urn destination)
+ "Decode a store item with read-capability ERIS-URN to DESTINATION."
+ (call-with-guix-eris-block-ref
+ (lambda (block-ref)
+ (eris-fs-decode eris-urn destination
+ #:block-ref block-ref)
+ #t)))
diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm
index 37cd08e289..3c060f1c89 100755
--- a/guix/scripts/substitute.scm
+++ b/guix/scripts/substitute.scm
@@ -45,6 +45,8 @@ (define-module (guix scripts substitute)
. guix:open-connection-for-uri)))
#:autoload (gnutls) (error/invalid-session error/again error/interrupted)
#:use-module (guix progress)
+ #:use-module (guix eris)
+ #:use-module (fibers)
#:use-module ((guix build syscalls)
#:select (set-thread-name))
#:use-module (ice-9 rdelim)
@@ -656,9 +658,11 @@ (define* (process-substitution/fallback port narinfo destination
(()
(loop rest)))))))
+
(define* (process-substitution port store-item destination
#:key cache-urls acl
- deduplicate? print-build-trace?)
+ deduplicate? print-build-trace?
+ (eris? #f))
"Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to
DESTINATION as a nar file. Verify the substitute against ACL, and verify its
hash against what appears in the narinfo. When DEDUPLICATE? is true, and if
@@ -674,20 +678,56 @@ (define* (process-substitution port store-item destination
(leave (G_ "no valid substitute for '~a'~%")
store-item))
- (guard (c ((network-error? c)
- (format (current-error-port)
- (G_ "retrying download of '~a' with other substitute URLs...~%")
- store-item)
- (process-substitution/fallback port narinfo destination
- #:cache-urls cache-urls
- #:acl acl
- #:deduplicate? deduplicate?
- #:print-build-trace?
- print-build-trace?)))
- (download-nar narinfo destination
- #:status-port port
- #:deduplicate? deduplicate?
- #:print-build-trace? print-build-trace?)))
+ (if (and eris?
+ (%eris-store-url)
+ (narinfo-eris-urn narinfo))
+
+ (unless
+
+ ;; Attempt to fetch substitute via ERIS
+ (let ((eris-urn (narinfo-eris-urn narinfo)))
+ (format (current-error-port)
+ (G_ "Downloading ~a...~%") (uri->string eris-urn))
+ (run-fibers
+ (lambda ()
+ (guard
+ (c (else
+ (format (current-error-port)
+ (G_ "failed to decode substitute from ERIS URN ~a: ~a"
+ (uri->string eris-urn)
+ c))
+ #f))
+ (eris-decode-store-item eris-urn destination)
+ ;; Tell the daemon that we're done.
+ (format port "success ~a ~a~%"
+ (narinfo-hash narinfo) (narinfo-size narinfo))))))
+
+ ;; Retry without ERIS on failure.
+ (process-substitution port store-item destination
+ #:cache-urls cache-urls
+ #:acl acl
+ #:deduplicate? deduplicate?
+ #:print-build-trace? print-build-trace?
+ #:eris? #f))
+
+ (guard (c ((network-error? c)
+ (format (current-error-port)
+ (G_ "retrying download of '~a' with other substitute URLs...~%")
+ store-item)
+ (process-substitution/fallback port narinfo destination
+ #:cache-urls cache-urls
+ #:acl acl
+ #:deduplicate? deduplicate?
+ #:print-build-trace?
+ print-build-trace?)))
+ (download-nar narinfo destination
+ #:status-port port
+ #:deduplicate? deduplicate?
+ #:print-build-trace? print-build-trace?))))
+
+
+
+
\f
;;;
@@ -876,7 +916,8 @@ (define-command (guix-substitute . args)
;; Download STORE-PATH and store it as a Nar in file DESTINATION.
;; Specify the number of columns of the terminal so the progress
;; report displays nicely.
- (parameterize ((current-terminal-columns (client-terminal-columns)))
+ (parameterize ((current-terminal-columns (client-terminal-columns))
+ (%eris-store-url (find-daemon-option "eris-store-url")))
(let loop ()
(match (read-line)
((? eof-object?)
@@ -887,7 +928,8 @@ (define-command (guix-substitute . args)
#:acl (current-acl)
#:deduplicate? deduplicate?
#:print-build-trace?
- print-build-trace?)
+ print-build-trace?
+ #:eris? #t)
(loop))))))
(opts
(leave (G_ "~a: unrecognized options~%") opts))))))
diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc
index d7ab9c5e64..d6b054bc6c 100644
--- a/nix/nix-daemon/guix-daemon.cc
+++ b/nix/nix-daemon/guix-daemon.cc
@@ -90,6 +90,7 @@ builds derivations on behalf of its clients.");
#define GUIX_OPT_MAX_SILENT_TIME 19
#define GUIX_OPT_LOG_COMPRESSION 20
#define GUIX_OPT_DISCOVER 21
+#define GUIX_OPT_ERIS_STORE_URL 22
static const struct argp_option options[] =
{
@@ -132,6 +133,8 @@ static const struct argp_option options[] =
n_("use the specified compression type for build logs") },
{ "discover", GUIX_OPT_DISCOVER, "yes/no", OPTION_ARG_OPTIONAL,
n_("use substitute servers discovered on the local network") },
+ { "eris-store-url", GUIX_OPT_ERIS_STORE_URL, n_("URL"), 0,
+ n_("use URL to retrieve blocks of ERIS encoded substitutes") },
/* '--disable-deduplication' was known as '--disable-store-optimization'
up to Guix 0.7 included, so keep the alias around. */
@@ -270,6 +273,8 @@ parse_opt (int key, char *arg, struct argp_state *state)
useDiscover = string_to_bool (arg);
settings.set ("discover", useDiscover ? "true" : "false");
break;
+ case GUIX_OPT_ERIS_STORE_URL:
+ settings.set ("eris-store-url", arg);
case GUIX_OPT_DEBUG:
verbosity = lvlDebug;
break;
--
2.41.0
next prev parent reply other threads:[~2023-12-28 9:41 UTC|newest]
Thread overview: 65+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-12-16 16:17 [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS pukkamustard
2021-12-16 16:20 ` [bug#52555] [RFC PATCH 1/3] publish: Add ERIS URN to narinfo pukkamustard
2021-12-16 16:20 ` [bug#52555] [RFC PATCH 2/3] WIP: gnu: guile-eris: Update to unreleased git version pukkamustard
2021-12-16 16:20 ` [bug#52555] [RFC PATCH 3/3] publish: Add IPFS support pukkamustard
2021-12-20 16:25 ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2021-12-23 11:42 ` pukkamustard
2021-12-24 14:48 ` Ludovic Courtès
2022-01-25 19:21 ` [bug#52555] [RFC PATCH v2 0/5] " pukkamustard
2022-01-25 19:21 ` [bug#52555] [RFC PATCH v2 1/5] WIP: gnu: guile-eris: Update to unreleased git version pukkamustard
2022-01-25 19:21 ` [bug#52555] [RFC PATCH v2 2/5] publish: Add ERIS URN to narinfo pukkamustard
2022-01-29 21:09 ` Maxime Devos
2022-01-29 21:15 ` Maxime Devos
2022-02-02 10:16 ` pukkamustard
2022-01-25 19:21 ` [bug#52555] [RFC PATCH v2 3/5] Add (guix eris) pukkamustard
2022-01-29 21:23 ` Maxime Devos
2022-02-02 10:28 ` pukkamustard
2022-02-02 15:36 ` Maxime Devos
2022-01-29 21:24 ` Maxime Devos
2022-01-25 19:22 ` [bug#52555] [RFC PATCH v2 4/5] publish: Add support for storing ERIS encoded blocks to IPFS pukkamustard
2022-01-29 21:28 ` Maxime Devos
2022-02-02 10:24 ` pukkamustard
2022-01-25 19:22 ` [bug#52555] [RFC PATCH v2 5/5] substitute: Fetch substitutes using ERIS pukkamustard
2022-01-29 21:29 ` Maxime Devos
2022-02-02 10:11 ` pukkamustard
2022-01-29 21:33 ` Maxime Devos
2022-01-29 21:38 ` Maxime Devos
2022-01-29 21:40 ` Maxime Devos
2022-01-29 21:40 ` Maxime Devos
2022-02-02 10:38 ` pukkamustard
2022-01-29 21:00 ` [bug#52555] [RFC PATCH v2 0/5] Decentralized substitute distribution with ERIS Maxime Devos
2022-02-02 9:50 ` pukkamustard
2022-01-29 21:08 ` Maxime Devos
2022-02-02 9:56 ` pukkamustard
2022-02-02 11:09 ` Maxime Devos
2022-01-29 21:52 ` Maxime Devos
2022-02-02 11:10 ` pukkamustard
2022-02-03 20:36 ` Maxime Devos
2022-02-04 10:20 ` pukkamustard
2022-01-30 11:46 ` Maxime Devos
2022-02-02 10:51 ` pukkamustard
2022-02-02 11:27 ` Maxime Devos
2022-02-02 12:42 ` pukkamustard
2022-02-02 15:07 ` Maxime Devos
2022-02-02 15:27 ` Maxime Devos
2022-02-04 16:16 ` Maxime Devos
2022-12-29 18:13 ` [bug#52555] [PATCH v3 0/8] " pukkamustard
2022-12-29 18:13 ` [bug#52555] [PATCH v3 1/8] publish: Add ERIS URN to narinfo pukkamustard
2023-01-14 18:34 ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2022-12-29 18:13 ` [bug#52555] [PATCH v3 2/8] publish: Store ERIS encoded blocks to a local block store pukkamustard
2023-01-14 18:42 ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2022-12-29 18:13 ` [bug#52555] [PATCH v3 3/8] publish: Add HTTP endpoint for resolving ERIS blocks pukkamustard
2022-12-29 18:13 ` [bug#52555] [PATCH v3 4/8] WIP: substitute: Fetch substitutes using ERIS pukkamustard
2022-12-29 18:13 ` [bug#52555] [PATCH v3 5/8] eris/http: Add HTTP block de-referencer pukkamustard
2022-12-29 18:13 ` [bug#52555] [PATCH v3 6/8] WIP: eris: Use HTTP to get ERIS blocks pukkamustard
2022-12-29 18:13 ` [bug#52555] [PATCH v3 7/8] eris: Use parameterized %eris-peers when getting blocks pukkamustard
2022-12-29 18:13 ` [bug#52555] [PATCH v3 8/8] eris: Use IPFS to get ERIS blocks pukkamustard
2023-01-14 18:25 ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2023-12-28 9:40 ` [bug#52555] [PATCH v4 0/7] " pukkamustard
2023-12-28 9:40 ` [bug#52555] [PATCH v4 1/7] narinfo: Add ERIS field pukkamustard
2023-12-28 9:40 ` [bug#52555] [PATCH v4 2/7] gnu: Add guile-coap pukkamustard
2023-12-28 9:40 ` [bug#52555] [PATCH v4 3/7] gnu: guile-eris: Update to 1.2.0-dev pukkamustard
2023-12-28 9:40 ` [bug#52555] [PATCH v4 4/7] publish: Add ERIS URN to narinfo pukkamustard
2023-12-28 9:40 ` [bug#52555] [PATCH v4 5/7] eris: Connect with an ERIS Store over CoAP+Unix pukkamustard
2023-12-28 9:40 ` pukkamustard [this message]
2023-12-28 9:40 ` [bug#52555] [PATCH v4 7/7] gnu: Add kapla pukkamustard
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=deebb9389ad0f32f9bcfb0d7ca6ed9cc04a980d2.1703316055.git.pukkamustard@posteo.net \
--to=pukkamustard@posteo.net \
--cc=52555@debbugs.gnu.org \
--cc=dev@jpoiret.xyz \
--cc=guix@cbaines.net \
--cc=ludo@gnu.org \
--cc=maximedevos@telenet.be \
--cc=me@tobias.gr \
--cc=othacehe@gnu.org \
--cc=rekado@elephly.net \
--cc=zimon.toutoune@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this external index
https://git.savannah.gnu.org/cgit/guix.git
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.