From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([208.118.235.92]:37288) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gd1eF-0003QJ-Pe for guix-patches@gnu.org; Fri, 28 Dec 2018 18:34:39 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gd1NI-0003pQ-Vo for guix-patches@gnu.org; Fri, 28 Dec 2018 18:17:06 -0500 Received: from debbugs.gnu.org ([208.118.235.43]:50059) by eggs.gnu.org with esmtps (TLS1.0:RSA_AES_128_CBC_SHA1:16) (Exim 4.71) (envelope-from ) id 1gd1NH-0003oR-Id for guix-patches@gnu.org; Fri, 28 Dec 2018 18:17:04 -0500 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1gd1NH-0001Rs-Bb for guix-patches@gnu.org; Fri, 28 Dec 2018 18:17:03 -0500 Subject: [bug#33899] [PATCH 5/5] DRAFT substitute: Add IPFS support. Resent-Message-ID: From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Date: Sat, 29 Dec 2018 00:15:54 +0100 Message-Id: <20181228231554.8220-5-ludo@gnu.org> In-Reply-To: <20181228231554.8220-1-ludo@gnu.org> References: <20181228231554.8220-1-ludo@gnu.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-patches-bounces+kyle=kyleam.com@gnu.org Sender: "Guix-patches" To: 33899@debbugs.gnu.org Missing: - documentation - command-line options - progress report when downloading over IPFS - fallback when we fail to fetch from IPFS * guix/scripts/substitute.scm ()[ipfs]: New field. (read-narinfo): Read "IPFS". (process-substitution/http): New procedure, with code formerly in 'process-substitution'. (process-substitution): Check for IPFS and call 'ipfs:restore-file-tree' when IPFS is true. --- guix/scripts/substitute.scm | 106 +++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 45 deletions(-) diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 53b1777241..8be15e4f13 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -42,6 +42,7 @@ #:use-module (guix progress) #:use-module ((guix build syscalls) #:select (set-thread-name)) + #:use-module ((guix ipfs) #:prefix ipfs:) #:use-module (ice-9 rdelim) #:use-module (ice-9 regex) #:use-module (ice-9 match) @@ -281,7 +282,7 @@ failure, return #f and #f." (define-record-type (%make-narinfo path uri uri-base compression file-hash file-size nar-hash nar-size - references deriver system signature contents) + references deriver system ipfs signature contents) narinfo? (path narinfo-path) (uri narinfo-uri) @@ -294,6 +295,7 @@ failure, return #f and #f." (references narinfo-references) (deriver narinfo-deriver) (system narinfo-system) + (ipfs narinfo-ipfs) (signature narinfo-signature) ; canonical sexp ;; The original contents of a narinfo file. This field is needed because we ;; want to preserve the exact textual representation for verification purposes. @@ -335,7 +337,7 @@ s-expression: ~s~%") "Return a narinfo constructor for narinfos originating from CACHE-URL. STR must contain the original contents of a narinfo file." (lambda (path url compression file-hash file-size nar-hash nar-size - references deriver system signature) + references deriver system ipfs signature) "Return a new object." (%make-narinfo path ;; Handle the case where URL is a relative URL. @@ -352,6 +354,7 @@ must contain the original contents of a narinfo file." ((or #f "") #f) (_ deriver)) system + ipfs (false-if-exception (and=> signature narinfo-signature->canonical-sexp)) str))) @@ -386,7 +389,7 @@ No authentication and authorization checks are performed here!" (narinfo-maker str url) '("StorePath" "URL" "Compression" "FileHash" "FileSize" "NarHash" "NarSize" - "References" "Deriver" "System" + "References" "Deriver" "System" "IPFS" "Signature")))) (define (narinfo-sha256 narinfo) @@ -947,13 +950,58 @@ authorized substitutes." (wtf (error "unknown `--query' command" wtf)))) +(define* (process-substitution/http narinfo destination uri + #:key print-build-trace?) + (unless print-build-trace? + (format (current-error-port) + (G_ "Downloading ~a...~%") (uri->string uri))) + + (let*-values (((raw download-size) + ;; Note that Hydra currently generates Nars on the fly + ;; and doesn't specify a Content-Length, so + ;; DOWNLOAD-SIZE is #f in practice. + (fetch uri #:buffered? #f #:timeout? #f)) + ((progress) + (let* ((comp (narinfo-compression narinfo)) + (dl-size (or download-size + (and (equal? comp "none") + (narinfo-size narinfo)))) + (reporter (if print-build-trace? + (progress-reporter/trace + destination + (uri->string uri) dl-size + (current-error-port)) + (progress-reporter/file + (uri->string uri) dl-size + (current-error-port) + #:abbreviation nar-uri-abbreviation)))) + (progress-report-port reporter raw))) + ((input pids) + ;; NOTE: This 'progress' port of current process will be + ;; closed here, while the child process doing the + ;; reporting will close it upon exit. + (decompressed-port (and=> (narinfo-compression narinfo) + string->symbol) + progress))) + ;; Unpack the Nar at INPUT into DESTINATION. + (restore-file input destination) + (close-port input) + + ;; Wait for the reporter to finish. + (every (compose zero? cdr waitpid) pids) + + ;; Skip a line after what 'progress-reporter/file' printed, and another + ;; one to visually separate substitutions. + (display "\n\n" (current-error-port)))) + (define* (process-substitution store-item destination #:key cache-urls acl print-build-trace?) "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to DESTINATION as a nar file. Verify the substitute against ACL." (let* ((narinfo (lookup-narinfo cache-urls store-item (cut valid-narinfo? <> acl))) - (uri (and=> narinfo narinfo-uri))) + (uri (and=> narinfo narinfo-uri)) + (ipfs (and=> narinfo narinfo-ipfs))) (unless uri (leave (G_ "no valid substitute for '~a'~%") store-item)) @@ -961,47 +1009,15 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; Tell the daemon what the expected hash of the Nar itself is. (format #t "~a~%" (narinfo-hash narinfo)) - (unless print-build-trace? - (format (current-error-port) - (G_ "Downloading ~a...~%") (uri->string uri))) - - (let*-values (((raw download-size) - ;; Note that Hydra currently generates Nars on the fly - ;; and doesn't specify a Content-Length, so - ;; DOWNLOAD-SIZE is #f in practice. - (fetch uri #:buffered? #f #:timeout? #f)) - ((progress) - (let* ((comp (narinfo-compression narinfo)) - (dl-size (or download-size - (and (equal? comp "none") - (narinfo-size narinfo)))) - (reporter (if print-build-trace? - (progress-reporter/trace - destination - (uri->string uri) dl-size - (current-error-port)) - (progress-reporter/file - (uri->string uri) dl-size - (current-error-port) - #:abbreviation nar-uri-abbreviation)))) - (progress-report-port reporter raw))) - ((input pids) - ;; NOTE: This 'progress' port of current process will be - ;; closed here, while the child process doing the - ;; reporting will close it upon exit. - (decompressed-port (and=> (narinfo-compression narinfo) - string->symbol) - progress))) - ;; Unpack the Nar at INPUT into DESTINATION. - (restore-file input destination) - (close-port input) - - ;; Wait for the reporter to finish. - (every (compose zero? cdr waitpid) pids) - - ;; Skip a line after what 'progress-reporter/file' printed, and another - ;; one to visually separate substitutions. - (display "\n\n" (current-error-port))))) + (if ipfs + (begin + (unless print-build-trace? + (format (current-error-port) + (G_ "Downloading from IPFS ~s...~%") ipfs)) + (ipfs:restore-file-tree ipfs destination)) + (process-substitution/http narinfo destination uri + #:print-build-trace? + print-build-trace?)))) ;;; -- 2.20.1