unofficial mirror of guix-patches@gnu.org 
 help / color / mirror / code / Atom feed
From: Hartmut Goebel <h.goebel@crazy-compilers.com>
To: 42180@debbugs.gnu.org
Subject: [bug#42180] [PATCH 01/22] guix: Add extracting-download.
Date: Fri,  3 Jul 2020 19:43:26 +0200	[thread overview]
Message-ID: <dec6cd7aac012c076896b7793bec1d51d34a245d.1593797694.git.h.goebel@crazy-compilers.com> (raw)
In-Reply-To: <cover.1593797694.git.h.goebel@crazy-compilers.com>

* guix/extracting-download.scm: New file
* Makefile.am (MODULES): Add it.
---
 Makefile.am                  |   1 +
 guix/extracting-download.scm | 172 +++++++++++++++++++++++++++++++++++
 2 files changed, 173 insertions(+)
 create mode 100644 guix/extracting-download.scm

diff --git a/Makefile.am b/Makefile.am
index 8988cdfa12..71c90e0c27 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -81,6 +81,7 @@ MODULES =					\
   guix/download.scm				\
   guix/discovery.scm				\
   guix/bzr-download.scm            		\
+  guix/extracting-download.scm			\
   guix/git-download.scm				\
   guix/hg-download.scm				\
   guix/swh.scm					\
diff --git a/guix/extracting-download.scm b/guix/extracting-download.scm
new file mode 100644
index 0000000000..e2dc5643e2
--- /dev/null
+++ b/guix/extracting-download.scm
@@ -0,0 +1,172 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2017 Mathieu Lirzin <mthl@gnu.org>
+;;; Copyright © 2017 Christopher Baines <mail@cbaines.net>
+;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net>
+;;; Copyright © 2020 Hartmut Goebel <h.goebel@crazy-compilers.com>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix extracting-download)
+  #:use-module (ice-9 match)
+  #:use-module (ice-9 popen)
+  #:use-module ((guix build download) #:prefix build:)
+  #:use-module (guix build utils)
+  #:use-module (guix gexp)
+  #:use-module (guix modules)
+  #:use-module (guix monads)
+  #:use-module (guix packages) ;; for %current-system
+  #:use-module (guix store)
+  #:use-module (guix utils)
+  #:use-module (srfi srfi-26)
+  #:export (http-fetch/extract
+            download-to-store/extract))
+
+;;;
+;;; Produce fixed-output derivations with data extracted from n archive
+;;; fetched over HTTP or FTP.
+;;;
+;;; This is meant to be used for package repositories where the actual source
+;;; archive is packed into another archive, eventually carrying meta-data.
+;;; Using this derivation saves both storing the outer archive and extracting
+;;; the actual one at build time.  The hash is calculated on the actual
+;;; archive to ease validating the stored file.
+;;;
+
+(define* (http-fetch/extract url filename-to-extract hash-algo hash
+                    #:optional name
+                    #:key (system (%current-system)) (guile (default-guile)))
+  "Return a fixed-output derivation that fetches an archive at URL, and
+extracts FILE_TO_EXTRACT from the archive.  The FILE_TO_EXTRACT is expected to
+have hash HASH of type HASH-ALGO (a symbol).  By default, the file name is the
+base name of URL; optionally, NAME can specify a different file name."
+  (define file-name
+    (match url
+      ((head _ ...)
+       (basename head))
+      (_
+       (basename url))))
+
+  (define gnutls
+    (module-ref (resolve-interface '(gnu packages tls)) 'gnutls))
+
+  (define inputs
+    `(("tar" ,(module-ref (resolve-interface '(gnu packages base))
+                          'tar))))
+
+  (define config.scm
+    (scheme-file "config.scm"
+                 #~(begin
+                     (define-module (guix config)
+                       #:export (%system))
+
+                     (define %system
+                       #$(%current-system)))))
+
+  (define modules
+    (cons `((guix config) => ,config.scm)
+          (delete '(guix config)
+                  (source-module-closure '((guix build download)
+                                           (guix build utils)
+                                           (guix utils)
+                                           (web uri))))))
+
+  (define build
+    (with-imported-modules modules
+      (with-extensions (list gnutls)
+        #~(begin
+            (use-modules (guix build download)
+                         (guix build utils)
+                         (guix utils)
+                         (web uri)
+                         (ice-9 match)
+                         (ice-9 popen))
+            ;; The code below expects tar to be in $PATH.
+            (set-path-environment-variable "PATH" '("bin")
+                                           (match '#+inputs
+                                             (((names dirs outputs ...) ...)
+                                              dirs)))
+
+            (setvbuf (current-output-port) 'line)
+            (setvbuf (current-error-port) 'line)
+
+            (call-with-temporary-directory
+             (lambda (directory)
+               ;; TODO: Support different archive types, based on content-type
+               ;; or archive name extention.
+               (let* ((file-to-extract (getenv "extract filename"))
+                      (port (http-fetch (string->uri (getenv "download url"))
+                                        #:verify-certificate? #f))
+                      (tar (open-pipe* OPEN_WRITE "tar" "-C" directory
+                                       "-xf" "-" file-to-extract)))
+                 (dump-port port tar)
+                 (close-port port)
+                 (let ((status (close-pipe tar)))
+                   (unless (zero? status)
+                     (error "tar extraction failure" status)))
+                 (copy-file (string-append directory "/"
+                                           (getenv "extract filename"))
+                            #$output))))))))
+
+  (mlet %store-monad ((guile (package->derivation guile system)))
+    (gexp->derivation (or name file-name) build
+
+                      ;; Use environment variables and a fixed script name so
+                      ;; there's only one script in store for all the
+                      ;; downloads.
+                      #:script-name "extract-download"
+                      #:env-vars
+                      `(("download url" . ,url)
+                        ("extract filename" . ,filename-to-extract))
+                      #:leaked-env-vars '("http_proxy" "https_proxy"
+                                          "LC_ALL" "LC_MESSAGES" "LANG"
+                                          "COLUMNS")
+                      #:system system
+                      #:local-build? #t           ; don't offload download
+                      #:hash-algo hash-algo
+                      #:hash hash
+                      #:guile-for-build guile)))
+
+
+(define* (download-to-store/extract store url filename-to-extract
+                                    #:optional (name (basename url))
+                                    #:key (log (current-error-port))
+                                    (verify-certificate? #t))
+  "Download an archive from URL, and extracts FILE_TO_EXTRACT from the archive
+to STORE, either under NAME or URL's basename if omitted.  Write progress
+reports to LOG.  VERIFY-CERTIFICATE? determines whether or not to validate
+HTTPS server certificates."
+  (call-with-temporary-output-file
+   (lambda (temp port)
+     (let ((result
+            (parameterize ((current-output-port log))
+              (build:url-fetch url temp
+                               ;;#:mirrors %mirrors
+                               #:verify-certificate?
+                               verify-certificate?))))
+       (close port)
+       (and result
+            (call-with-temporary-output-file
+             (lambda (contents port)
+               (let ((tar (open-pipe* OPEN_READ
+                                      "tar"  ;"--auto-compress"
+                                      "-xf" temp "--to-stdout" filename-to-extract)))
+                 (dump-port tar port)
+                 (close-port port)
+                 (let ((status (close-pipe tar)))
+                   (unless (zero? status)
+                     (error "tar extraction failure" status)))
+                 (add-to-store store name #f "sha256" contents)))))))))
-- 
2.21.3





  reply	other threads:[~2020-07-03 17:44 UTC|newest]

Thread overview: 78+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-07-03 17:40 [bug#42180] [PATCH 00/22] Add extracting download, importer for hex.pm and rebar3 build-system for Erlang Hartmut Goebel
2020-07-03 17:43 ` Hartmut Goebel [this message]
2020-07-06  8:08   ` [bug#42180] [PATCH 01/22] guix: Add extracting-download zimoun
2020-07-06  8:15     ` Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 02/22] guix: Add importer for hex.pm Hartmut Goebel
2020-11-02 14:18   ` [bug#42180] [PATCH 0/1] " pukkamustard
2020-11-02 14:18     ` [bug#42180] [PATCH 1/1] " pukkamustard
2020-07-03 17:43 ` [bug#42180] [PATCH 03/22] guix: Add rebar3 build-system Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 04/22] gnu: Add erlang-cf Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 05/22] gnu: Add erlang-certifi Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 06/22] gnu: Add erlang-erlware-commons Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 07/22] gnu: Add erlang-cth-readable Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 08/22] gnu: Add erlang-bbmustache Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 09/22] gnu: Add erlang-getopt Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 10/22] gnu: Add erlang-eunit-formatters Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 11/22] gnu: Add erlang-providers Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 12/22] gnu: Add erlang-parse-trans Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 13/22] gnu: Add erlang-hex-core Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 14/22] gnu: Add erlang-ssl-verify-fun Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 15/22] gnu: Add erlang-relx Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 16/22] gnu: Add rebar3 Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 17/22] gnu: Add erlang-edown Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 18/22] gnu: Add erlang-jsone Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 19/22] gnu: Add erlang-proper Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 20/22] gnu: Add erlang-rebar3-raw-deps Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 21/22] gnu: Add erlang-rebar3-git-vsn Hartmut Goebel
2020-07-03 17:43 ` [bug#42180] [PATCH 22/22] gnu: Add erlang-rebar3-proper Hartmut Goebel
     [not found] ` <handler.42180.B.159379803013215.ack@debbugs.gnu.org>
2021-01-22 20:20   ` [bug#42180] Acknowledgement ([PATCH 00/22] Add extracting download, importer for hex.pm and rebar3 build-system for Erlang) Hartmut Goebel
2021-10-06 15:20 ` [bug#42180] [PATCH v2 01/23] guix: Add extracting-download Hartmut Goebel
2021-10-06 15:20   ` [bug#42180] [PATCH v2 02/23] guix: Add importer for hex.pm Hartmut Goebel
2021-10-06 19:37     ` Maxime Devos
2021-10-06 20:23       ` Hartmut Goebel
2021-10-06 19:38     ` [bug#42180] [bug#51061] " Maxime Devos
2021-10-06 20:25       ` Hartmut Goebel
2021-10-07 22:01     ` [bug#42180] bug#51061: [PATCH v2 01/23] guix: Add extracting-download Ludovic Courtès
2021-10-06 15:20   ` [bug#42180] [PATCH v2 03/23] guix: Add rebar3 build-system Hartmut Goebel
2021-10-06 18:56     ` [bug#51061] " Maxime Devos
2021-10-06 20:27       ` [bug#42180] " Hartmut Goebel
2021-10-06 21:25         ` Maxime Devos
2021-10-06 21:36           ` Hartmut Goebel
2021-10-06 21:47             ` [bug#42180] " Maxime Devos
2021-10-07 20:57               ` Hartmut Goebel
2021-10-07 22:20                 ` [bug#51061] [PATCH v2 01/23] guix: Add extracting-download Ludovic Courtès
2021-10-08  9:49                   ` Hartmut Goebel
2021-10-09 13:16                     ` [bug#42180] bug#51061: " Ludovic Courtès
2021-10-08 20:25                   ` [bug#51061] " Hartmut Goebel
2021-10-07 22:09     ` [bug#42180] bug#51061: " Ludovic Courtès
2021-10-06 15:20   ` [bug#42180] [PATCH v2 04/23] gnu: Add erlang-cf Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 05/23] gnu: Add erlang-certifi Hartmut Goebel
2021-10-06 15:20   ` [bug#42180] [PATCH v2 06/23] gnu: Add erlang-erlware-commons Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 07/23] gnu: Add erlang-cth-readable Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 08/23] gnu: Add erlang-bbmustache Hartmut Goebel
2021-10-06 15:20   ` [bug#42180] [PATCH v2 09/23] gnu: Add erlang-getopt Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 10/23] gnu: Add erlang-eunit-formatters Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 11/23] gnu: Add erlang-providers Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 12/23] gnu: Add erlang-parse-trans Hartmut Goebel
2021-10-06 15:20   ` [bug#42180] [PATCH v2 13/23] gnu: Add erlang-hex-core Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 14/23] gnu: Add erlang-ssl-verify-fun Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 15/23] gnu: Add erlang-relx Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 16/23] gnu: Add rebar3 Hartmut Goebel
2021-10-06 15:20   ` [bug#42180] [PATCH v2 17/23] gnu: Add erlang-edown Hartmut Goebel
2021-10-06 15:20   ` [bug#42180] [PATCH v2 18/23] gnu: Add erlang-jsone Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 19/23] gnu: Add erlang-proper Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 20/23] gnu: Add erlang-rebar3-raw-deps Hartmut Goebel
2021-10-06 15:20   ` [bug#42180] [PATCH v2 21/23] gnu: Add erlang-rebar3-git-vsn Hartmut Goebel
2021-10-06 18:43     ` Maxime Devos
2021-10-06 21:09       ` Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 22/23] gnu: Add erlang-rebar3-proper Hartmut Goebel
2021-10-06 15:20   ` [bug#51061] [PATCH v2 23/23] gnu: Add erlang-covertool Hartmut Goebel
2021-10-07 21:55   ` [bug#42180] bug#51061: [PATCH v2 01/23] guix: Add extracting-download Ludovic Courtès
2021-10-07 22:25     ` [bug#51061] " Tobias Geerinckx-Rice via Guix-patches via
2021-10-07 22:34       ` Tobias Geerinckx-Rice via Guix-patches via
2021-10-08  9:10         ` Hartmut Goebel
2021-10-08  9:39           ` Hartmut Goebel
2021-10-08 21:00             ` Tobias Geerinckx-Rice via Guix-patches via
2021-10-08  5:49     ` [bug#51061] " Maxime Devos
2021-10-08  7:05       ` [bug#51061] " Ludovic Courtès
2021-10-07 20:58 ` bug#42180: (no subject) Hartmut Goebel

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://guix.gnu.org/

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=dec6cd7aac012c076896b7793bec1d51d34a245d.1593797694.git.h.goebel@crazy-compilers.com \
    --to=h.goebel@crazy-compilers.com \
    --cc=42180@debbugs.gnu.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://git.savannah.gnu.org/cgit/guix.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).