unofficial mirror of guix-patches@gnu.org 
 help / color / mirror / code / Atom feed
From: pukkamustard <pukkamustard@posteo.net>
To: 52555@debbugs.gnu.org
Cc: pukkamustard <pukkamustard@posteo.net>, ~pukkamustard/eris@lists.sr.ht
Subject: [bug#52555] [RFC PATCH v2 2/5] publish: Add ERIS URN to narinfo
Date: Tue, 25 Jan 2022 19:21:58 +0000	[thread overview]
Message-ID: <20220125192201.7582-3-pukkamustard@posteo.net> (raw)
In-Reply-To: <20220125192201.7582-1-pukkamustard@posteo.net>

* guix/scripts/publish.scm: (bake-narinfo+nar): Compute ERIS URN of compressed nars.
(narinfo-string): Add #:eris-urn parameter and honor it.
* guix/scripts/narinfo.scm: (<narinfo>)[eris-format,eris-urn]: New fields.
(narinfo-maker): Handle ERIS URN and ERIS format.
* configure.ac: (HAVE_GUILE_ERIS): New conditional.
* gnu/packages/package-management.scm: (guix)[native-inputs]: Add guile-eris.
---
 configure.ac                        |  5 +++++
 gnu/packages/package-management.scm |  1 +
 guix/narinfo.scm                    | 14 ++++++++++----
 guix/scripts/publish.scm            | 24 ++++++++++++++++++++----
 4 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/configure.ac b/configure.ac
index 341cff8fbd..72396be8aa 100644
--- a/configure.ac
+++ b/configure.ac
@@ -170,6 +170,11 @@ GUILE_MODULE_AVAILABLE([have_guile_avahi], [(avahi)])
 AM_CONDITIONAL([HAVE_GUILE_AVAHI],
   [test "x$have_guile_avahi" = "xyes"])
 
+dnl Check for Guile-eris.
+GUILE_MODULE_AVAILABLE([have_guile_eris], [(eris)])
+AM_CONDITIONAL([HAVE_GUILE_ERIS],
+  [test "x$have_guile_eris" = "xyes"])
+
 dnl Guile-newt is used by the graphical installer.
 GUILE_MODULE_AVAILABLE([have_guile_newt], [(newt)])
 
diff --git a/gnu/packages/package-management.scm b/gnu/packages/package-management.scm
index 05795824b5..a9094b8b7f 100644
--- a/gnu/packages/package-management.scm
+++ b/gnu/packages/package-management.scm
@@ -404,6 +404,7 @@ (define code
                        ("guile-zstd" ,guile-zstd)
                        ("guile-ssh" ,guile-ssh)
                        ("guile-git" ,guile-git)
+                       ("guile-eris" ,guile-eris)
 
                        ;; XXX: Keep the development inputs here even though
                        ;; they're unnecessary, just so that 'guix environment
diff --git a/guix/narinfo.scm b/guix/narinfo.scm
index 4fc550aa6c..a6a5d3b84b 100644
--- a/guix/narinfo.scm
+++ b/guix/narinfo.scm
@@ -45,6 +45,8 @@ (define-module (guix narinfo)
             narinfo-file-sizes
             narinfo-hash
             narinfo-size
+            narinfo-eris-format
+            narinfo-eris-urn
             narinfo-references
             narinfo-deriver
             narinfo-system
@@ -68,8 +70,8 @@ (define-module (guix narinfo)
 
 (define-record-type <narinfo>
   (%make-narinfo path uri-base uris compressions file-sizes file-hashes
-                 nar-hash nar-size references deriver system
-                 signature contents)
+                 nar-hash nar-size eris-format eris-urn references deriver
+                 system signature contents)
   narinfo?
   (path         narinfo-path)
   (uri-base     narinfo-uri-base)        ;URI of the cache it originates from
@@ -79,6 +81,8 @@ (define-record-type <narinfo>
   (file-hashes  narinfo-file-hashes)
   (nar-hash     narinfo-hash)
   (nar-size     narinfo-size)
+  (eris-format  narinfo-eris-format)
+  (eris-urn     narinfo-eris-urn)
   (references   narinfo-references)
   (deriver      narinfo-deriver)
   (system       narinfo-system)
@@ -135,7 +139,7 @@ (define (narinfo-maker str cache-url)
   "Return a narinfo constructor for narinfos originating from CACHE-URL.  STR
 must contain the original contents of a narinfo file."
   (lambda (path urls compressions file-hashes file-sizes
-                nar-hash nar-size references deriver system
+                nar-hash nar-size eris-format eris-urn references deriver system
                 signature)
     "Return a new <narinfo> object."
     (define len (length urls))
@@ -157,6 +161,8 @@ (define len (length urls))
                      ((lst ...) (map string->number lst)))
                    nar-hash
                    (and=> nar-size string->number)
+                   eris-format
+                   (if eris-urn (string->uri eris-urn) #f)
                    (string-tokenize references)
                    (match deriver
                      ((or #f "") #f)
@@ -184,7 +190,7 @@ (define* (read-narinfo port #:optional url
                    (narinfo-maker str url)
                    '("StorePath" "URL" "Compression"
                      "FileHash" "FileSize" "NarHash" "NarSize"
-                     "References" "Deriver" "System"
+                     "ERISFormat" "ERIS" "References" "Deriver" "System"
                      "Signature")
                    '("URL" "Compression" "FileSize" "FileHash"))))
 
diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm
index 6e2b4368da..9c83f5183d 100644
--- a/guix/scripts/publish.scm
+++ b/guix/scripts/publish.scm
@@ -58,6 +58,7 @@ (define-module (guix scripts publish)
   #:use-module (guix workers)
   #:use-module (guix store)
   #:use-module ((guix serialization) #:select (write-file))
+  #:use-module (eris)
   #:use-module (zlib)
   #:autoload   (lzlib) (call-with-lzip-output-port
                         make-lzip-output-port)
@@ -146,6 +147,9 @@ (define %default-gzip-compression
   ;; Since we compress on the fly, default to fast compression.
   (compression 'gzip 3))
 
+(define %eris-zstd-compression
+  (compression 'zstd 19))
+
 (define (default-compression type)
   (compression type 3))
 
@@ -324,7 +328,8 @@ (define* (store-item->recutils store-item
 
 (define* (narinfo-string store store-path
                          #:key (compressions (list %no-compression))
-                         (nar-path "nar") (file-sizes '()))
+                         (nar-path "nar") (file-sizes '())
+                         eris-urn)
   "Generate a narinfo key/value string for STORE-PATH; an exception is raised
 if STORE-PATH is invalid.  Produce a URL that corresponds to COMPRESSION.  The
 narinfo is signed with KEY.  NAR-PATH specifies the prefix for nar URLs.
@@ -347,7 +352,7 @@ (define* (narinfo-string store store-path
 StorePath: ~a
 ~{~a~}\
 NarHash: sha256:~a
-NarSize: ~d
+NarSize: ~d~@[~%ERISFormat: application/x-nix-archive+zstd-19~%ERIS: ~a~]
 References: ~a~%"
                              store-path
                              (map (lambda (compression)
@@ -359,7 +364,7 @@ (define* (narinfo-string store store-path
                                                             #:compression
                                                             compression)))
                                   compressions)
-                             hash size references))
+                             hash size eris-urn references))
          ;; Do not render a "Deriver" line if we are rendering info for a
          ;; derivation.  Also do not render a "System" line that would be
          ;; expensive to compute and is currently unused.
@@ -632,6 +637,16 @@ (define (compressed-nar-size compression)
       (and stat
            (cons compression (stat:size stat)))))
 
+  (define (eris-encode-nar compressions)
+    (and (member %eris-zstd-compression compressions)
+         (let* ((nar (nar-cache-file cache item
+                                     #:compression %eris-zstd-compression))
+                (stat (stat nar #f)))
+           (and stat
+                (call-with-input-file nar
+                  (cut eris-encode->string <>
+                       #:block-size %eris-block-size-large))))))
+
   (let ((compression (actual-compressions item compressions)))
 
     (for-each (cut compress-nar cache item <>) compressions)
@@ -650,7 +665,8 @@ (define (compressed-nar-size compression)
                  (display (narinfo-string store item
                                           #:nar-path nar-path
                                           #:compressions compressions
-                                          #:file-sizes sizes)
+                                          #:file-sizes sizes
+                                          #:eris-urn (eris-encode-nar compression))
                           port)))
 
              ;; Make the cached narinfo world-readable, contrary to what
-- 
2.34.0





  parent reply	other threads:[~2022-01-25 19:23 UTC|newest]

Thread overview: 65+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-12-16 16:17 [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS pukkamustard
2021-12-16 16:20 ` [bug#52555] [RFC PATCH 1/3] publish: Add ERIS URN to narinfo pukkamustard
2021-12-16 16:20   ` [bug#52555] [RFC PATCH 2/3] WIP: gnu: guile-eris: Update to unreleased git version pukkamustard
2021-12-16 16:20   ` [bug#52555] [RFC PATCH 3/3] publish: Add IPFS support pukkamustard
2021-12-20 16:25 ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2021-12-23 11:42   ` pukkamustard
2021-12-24 14:48     ` Ludovic Courtès
2022-01-25 19:21 ` [bug#52555] [RFC PATCH v2 0/5] " pukkamustard
2022-01-25 19:21   ` [bug#52555] [RFC PATCH v2 1/5] WIP: gnu: guile-eris: Update to unreleased git version pukkamustard
2022-01-25 19:21   ` pukkamustard [this message]
2022-01-29 21:09     ` [bug#52555] [RFC PATCH v2 2/5] publish: Add ERIS URN to narinfo Maxime Devos
2022-01-29 21:15     ` Maxime Devos
2022-02-02 10:16       ` pukkamustard
2022-01-25 19:21   ` [bug#52555] [RFC PATCH v2 3/5] Add (guix eris) pukkamustard
2022-01-29 21:23     ` Maxime Devos
2022-02-02 10:28       ` pukkamustard
2022-02-02 15:36         ` Maxime Devos
2022-01-29 21:24     ` Maxime Devos
2022-01-25 19:22   ` [bug#52555] [RFC PATCH v2 4/5] publish: Add support for storing ERIS encoded blocks to IPFS pukkamustard
2022-01-29 21:28     ` Maxime Devos
2022-02-02 10:24       ` pukkamustard
2022-01-25 19:22   ` [bug#52555] [RFC PATCH v2 5/5] substitute: Fetch substitutes using ERIS pukkamustard
2022-01-29 21:29     ` Maxime Devos
2022-02-02 10:11       ` pukkamustard
2022-01-29 21:33     ` Maxime Devos
2022-01-29 21:38     ` Maxime Devos
2022-01-29 21:40       ` Maxime Devos
2022-01-29 21:40     ` Maxime Devos
2022-02-02 10:38       ` pukkamustard
2022-01-29 21:00   ` [bug#52555] [RFC PATCH v2 0/5] Decentralized substitute distribution with ERIS Maxime Devos
2022-02-02  9:50     ` pukkamustard
2022-01-29 21:08   ` Maxime Devos
2022-02-02  9:56     ` pukkamustard
2022-02-02 11:09       ` Maxime Devos
2022-01-29 21:52   ` Maxime Devos
2022-02-02 11:10     ` pukkamustard
2022-02-03 20:36       ` Maxime Devos
2022-02-04 10:20         ` pukkamustard
2022-01-30 11:46   ` Maxime Devos
2022-02-02 10:51     ` pukkamustard
2022-02-02 11:27       ` Maxime Devos
2022-02-02 12:42         ` pukkamustard
2022-02-02 15:07           ` Maxime Devos
2022-02-02 15:27           ` Maxime Devos
2022-02-04 16:16       ` Maxime Devos
2022-12-29 18:13 ` [bug#52555] [PATCH v3 0/8] " pukkamustard
2022-12-29 18:13   ` [bug#52555] [PATCH v3 1/8] publish: Add ERIS URN to narinfo pukkamustard
2023-01-14 18:34     ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2022-12-29 18:13   ` [bug#52555] [PATCH v3 2/8] publish: Store ERIS encoded blocks to a local block store pukkamustard
2023-01-14 18:42     ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2022-12-29 18:13   ` [bug#52555] [PATCH v3 3/8] publish: Add HTTP endpoint for resolving ERIS blocks pukkamustard
2022-12-29 18:13   ` [bug#52555] [PATCH v3 4/8] WIP: substitute: Fetch substitutes using ERIS pukkamustard
2022-12-29 18:13   ` [bug#52555] [PATCH v3 5/8] eris/http: Add HTTP block de-referencer pukkamustard
2022-12-29 18:13   ` [bug#52555] [PATCH v3 6/8] WIP: eris: Use HTTP to get ERIS blocks pukkamustard
2022-12-29 18:13   ` [bug#52555] [PATCH v3 7/8] eris: Use parameterized %eris-peers when getting blocks pukkamustard
2022-12-29 18:13   ` [bug#52555] [PATCH v3 8/8] eris: Use IPFS to get ERIS blocks pukkamustard
2023-01-14 18:25   ` [bug#52555] [RFC PATCH 0/3] Decentralized substitute distribution with ERIS Ludovic Courtès
2023-12-28  9:40 ` [bug#52555] [PATCH v4 0/7] " pukkamustard
2023-12-28  9:40   ` [bug#52555] [PATCH v4 1/7] narinfo: Add ERIS field pukkamustard
2023-12-28  9:40   ` [bug#52555] [PATCH v4 2/7] gnu: Add guile-coap pukkamustard
2023-12-28  9:40   ` [bug#52555] [PATCH v4 3/7] gnu: guile-eris: Update to 1.2.0-dev pukkamustard
2023-12-28  9:40   ` [bug#52555] [PATCH v4 4/7] publish: Add ERIS URN to narinfo pukkamustard
2023-12-28  9:40   ` [bug#52555] [PATCH v4 5/7] eris: Connect with an ERIS Store over CoAP+Unix pukkamustard
2023-12-28  9:40   ` [bug#52555] [PATCH v4 6/7] substitute: Decode substitutes using ERIS pukkamustard
2023-12-28  9:40   ` [bug#52555] [PATCH v4 7/7] gnu: Add kapla pukkamustard

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://guix.gnu.org/

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20220125192201.7582-3-pukkamustard@posteo.net \
    --to=pukkamustard@posteo.net \
    --cc=52555@debbugs.gnu.org \
    --cc=~pukkamustard/eris@lists.sr.ht \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://git.savannah.gnu.org/cgit/guix.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).