From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([209.51.188.92]:36880) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1hUATQ-0008Oj-08 for guix-patches@gnu.org; Fri, 24 May 2019 09:43:05 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1hUATO-0008Mi-OX for guix-patches@gnu.org; Fri, 24 May 2019 09:43:03 -0400 Received: from debbugs.gnu.org ([209.51.188.43]:60990) by eggs.gnu.org with esmtps (TLS1.0:RSA_AES_128_CBC_SHA1:16) (Exim 4.71) (envelope-from ) id 1hUATO-0008Mc-LV for guix-patches@gnu.org; Fri, 24 May 2019 09:43:02 -0400 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1hUATO-0002PR-CS for guix-patches@gnu.org; Fri, 24 May 2019 09:43:02 -0400 Subject: [bug#35880] [PATCH 1/7] lzlib: Add 'make-lzip-input-port/compressed'. References: <20190524133159.22568-1-ludo@gnu.org> In-Reply-To: <20190524133159.22568-1-ludo@gnu.org> Resent-Message-ID: From: Ludovic =?UTF-8?Q?Court=C3=A8s?= Date: Fri, 24 May 2019 15:42:32 +0200 Message-Id: <20190524134238.22802-1-ludo@gnu.org> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-patches-bounces+kyle=kyleam.com@gnu.org Sender: "Guix-patches" To: 35880@debbugs.gnu.org Cc: Pierre Neidhardt * guix/lzlib.scm (lzwrite!, make-lzip-input-port/compressed): New procedures. * tests/lzlib.scm ("make-lzip-input-port/compressed"): New test. * guix/tests.scm (%seed): Export. --- guix/lzlib.scm | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ guix/tests.scm | 1 + tests/lzlib.scm | 10 ++++++++ 3 files changed, 73 insertions(+) diff --git a/guix/lzlib.scm b/guix/lzlib.scm index a6dac46049..48927c6262 100644 --- a/guix/lzlib.scm +++ b/guix/lzlib.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019 Pierre Neidhardt +;;; Copyright © 2019 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -23,9 +24,11 @@ #:use-module (ice-9 match) #:use-module (system foreign) #:use-module (guix config) + #:use-module (srfi srfi-11) #:export (lzlib-available? make-lzip-input-port make-lzip-output-port + make-lzip-input-port/compressed call-with-lzip-input-port call-with-lzip-output-port %default-member-length-limit @@ -515,6 +518,23 @@ the end-of-stream has been reached." (loop rd))) read)) +(define (lzwrite! encoder source source-offset source-count + target target-offset target-count) + "Write up to SOURCE-COUNT bytes from SOURCE to ENCODER, and read up to +TARGET-COUNT bytes into TARGET at TARGET-OFFSET. Return two values: the +number of bytes read from SOURCE, and the number of bytes written to TARGET." + (define read + (if (< 0 (lz-compress-write-size encoder)) + (match (lz-compress-write encoder source source-offset source-count) + (0 (lz-compress-finish encoder) 0) + (n n)) + 0)) + + (let loop () + (match (lz-compress-read encoder target target-offset target-count) + (0 (loop)) + (written (values read written))))) + (define* (lzwrite encoder bv lz-port #:optional (start 0) (count (bytevector-length bv))) "Write up to COUNT bytes from BV at offset START into LZ-PORT. Return @@ -597,6 +617,48 @@ port is closed." (lz-compress-close encoder) (close-port port)))) +(define* (make-lzip-input-port/compressed port + #:key + (level %default-compression-level)) + "Return an input port that compresses data read from PORT, with the given LEVEL. +PORT is automatically closed when the resulting port is closed." + (define encoder (apply lz-compress-open + (car (assoc-ref %compression-levels level)))) + + (define input-buffer (make-bytevector 8192)) + (define input-len 0) + (define input-offset 0) + + (define input-eof? #f) + + (define (read! bv start count) + (cond + (input-eof? + (lz-compress-read encoder bv start count)) + ((= input-offset input-len) + (match (get-bytevector-n! port input-buffer 0 + (bytevector-length input-buffer)) + ((? eof-object?) + (set! input-eof? #t) + (lz-compress-finish encoder)) + (count + (set! input-offset 0) + (set! input-len count))) + (read! bv start count)) + (else + (let-values (((read written) + (lzwrite! encoder + input-buffer input-offset + (- input-len input-offset) + bv start count))) + (set! input-offset (+ input-offset read)) + written)))) + + (make-custom-binary-input-port "lzip-input/compressed" + read! #f #f + (lambda () + (close-port port)))) + (define* (call-with-lzip-input-port port proc) "Call PROC with a port that wraps PORT and decompresses data read from it. PORT is closed upon completion." diff --git a/guix/tests.scm b/guix/tests.scm index 35ebf8464d..66d60e964e 100644 --- a/guix/tests.scm +++ b/guix/tests.scm @@ -33,6 +33,7 @@ #:use-module (web uri) #:export (open-connection-for-tests with-external-store + %seed random-text random-bytevector file=? diff --git a/tests/lzlib.scm b/tests/lzlib.scm index cf53a9417d..543622bb45 100644 --- a/tests/lzlib.scm +++ b/tests/lzlib.scm @@ -108,4 +108,14 @@ (test-assert* "Bytevector of size relative to Lzip internal buffers (1MiB+1)" (compress-and-decompress (random-bytevector (1+ (* 1024 1024))))) +(test-assert "make-lzip-input-port/compressed" + (let* ((len (pk 'len (+ 10 (random 4000 %seed)))) + (data (random-bytevector len)) + (compressed (make-lzip-input-port/compressed + (open-bytevector-input-port data))) + (result (call-with-lzip-input-port compressed + get-bytevector-all))) + (pk (bytevector-length result) (bytevector-length data)) + (bytevector=? result data))) + (test-end) -- 2.21.0