From: "Ludovic Courtès" <ludo@gnu.org>
To: 44760@debbugs.gnu.org
Subject: bug#44760: [PATCH 12/15] system: 'init' does not recompute the hash of each store item.
Date: Fri, 11 Dec 2020 16:09:48 +0100 [thread overview]
Message-ID: <20201211150951.18508-3-ludo@gnu.org> (raw)
In-Reply-To: <20201211150951.18508-1-ludo@gnu.org>
Fixes <https://bugs.gnu.org/44760>.
Previously, the 'register-path' call would re-traverse ITEM to compute
its nar hash, even though that hash is already known in the initial
store. This patch also avoids repeated opening/closing of the
database.
* guix/store/database.scm (call-with-database): Export.
* guix/scripts/system.scm (copy-item): Add 'db' parameter. Call
'sqlite-register' instead of 'register-path'.
(copy-closure): Remove redundant call to 'references*'. Call
'call-with-database' and pass the database to 'copy-item'.
---
.dir-locals.el | 1 +
guix/scripts/system.scm | 59 ++++++++++++++++++++++-------------------
guix/store/database.scm | 1 +
3 files changed, 34 insertions(+), 27 deletions(-)
diff --git a/.dir-locals.el b/.dir-locals.el
index 4eb27d8b1b..8f07a08eb5 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -121,6 +121,7 @@
(eval . (put 'let-system 'scheme-indent-function 1))
(eval . (put 'with-database 'scheme-indent-function 2))
+ (eval . (put 'call-with-database 'scheme-indent-function 1))
(eval . (put 'call-with-transaction 'scheme-indent-function 1))
(eval . (put 'with-statement 'scheme-indent-function 3))
(eval . (put 'call-with-retrying-transaction 'scheme-indent-function 1))
diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm
index 0e543d9460..5427f875ec 100644
--- a/guix/scripts/system.scm
+++ b/guix/scripts/system.scm
@@ -29,7 +29,9 @@
#:use-module (guix ui)
#:use-module ((guix status) #:select (with-status-verbosity))
#:use-module (guix store)
- #:autoload (guix store database) (register-path)
+ #:autoload (guix base16) (bytevector->base16-string)
+ #:autoload (guix store database)
+ (sqlite-register store-database-file call-with-database)
#:autoload (guix build store-copy) (copy-store-item)
#:use-module (guix describe)
#:use-module (guix grafts)
@@ -130,12 +132,11 @@ BODY..., and restore them."
(store-lift topologically-sorted))
-(define* (copy-item item references target
+(define* (copy-item item info target db
#:key (log-port (current-error-port)))
- "Copy ITEM to the store under root directory TARGET and register it with
-REFERENCES as its set of references."
- (let ((dest (string-append target item))
- (state (string-append target "/var/guix")))
+ "Copy ITEM to the store under root directory TARGET and populate DB with the
+given INFO, a <path-info> record."
+ (let ((dest (string-append target item)))
(format log-port "copying '~a'...~%" item)
;; Remove DEST if it exists to make sure that (1) we do not fail badly
@@ -151,41 +152,45 @@ REFERENCES as its set of references."
(copy-store-item item target
#:deduplicate? #t)
- ;; Register ITEM; as a side-effect, it resets timestamps, etc.
- ;; Explicitly use "TARGET/var/guix" as the state directory, to avoid
- ;; reproducing the user's current settings; see
- ;; <http://bugs.gnu.org/18049>.
- (unless (register-path item
- #:prefix target
- #:state-directory state
- #:references references)
- (leave (G_ "failed to register '~a' under '~a'~%")
- item target))))
+ (sqlite-register db
+ #:path item
+ #:references (path-info-references info)
+ #:deriver (path-info-deriver info)
+ #:hash (string-append
+ "sha256:"
+ (bytevector->base16-string (path-info-hash info)))
+ #:nar-size (path-info-nar-size info))))
(define* (copy-closure item target
#:key (log-port (current-error-port)))
"Copy ITEM and all its dependencies to the store under root directory
TARGET, and register them."
(mlet* %store-monad ((to-copy (topologically-sorted* (list item)))
- (refs (mapm %store-monad references* to-copy))
- (info (mapm %store-monad query-path-info*
- (delete-duplicates
- (append to-copy (concatenate refs)))))
+ (info (mapm %store-monad query-path-info* to-copy))
(size -> (reduce + 0 (map path-info-nar-size info))))
(define progress-bar
(progress-reporter/bar (length to-copy)
(format #f (G_ "copying to '~a'...")
target)))
+ (define state
+ (string-append target "/var/guix"))
+
(check-available-space size target)
- (call-with-progress-reporter progress-bar
- (lambda (report)
- (let ((void (%make-void-port "w")))
- (for-each (lambda (item refs)
- (copy-item item refs target #:log-port void)
- (report))
- to-copy refs))))
+ ;; Explicitly use "TARGET/var/guix" as the state directory to avoid
+ ;; reproducing the user's current settings; see
+ ;; <http://bugs.gnu.org/18049>.
+ (call-with-database (store-database-file #:prefix target
+ #:state-directory state)
+ (lambda (db)
+ (call-with-progress-reporter progress-bar
+ (lambda (report)
+ (let ((void (%make-void-port "w")))
+ (for-each (lambda (item info)
+ (copy-item item info target db #:log-port void)
+ (report))
+ to-copy info))))))
(return *unspecified*)))
diff --git a/guix/store/database.scm b/guix/store/database.scm
index c0010b72b9..9d5bc531bb 100644
--- a/guix/store/database.scm
+++ b/guix/store/database.scm
@@ -39,6 +39,7 @@
#:export (sql-schema
%default-database-file
store-database-file
+ call-with-database
with-database
path-id
sqlite-register
--
2.29.2
next prev parent reply other threads:[~2020-12-11 15:12 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-11-20 11:02 bug#44760: Closure copy in ‘guix system init’ is inefficient Ludovic Courtès
2020-11-22 19:46 ` raingloom
2020-11-22 21:10 ` Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 00/15] Speed up 'guix system init' & co Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 01/15] serialization: 'fold-archive' notifies about directory processing completion Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 02/15] serialization: 'restore-file' sets canonical timestamp and permissions Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 03/15] nar: Deduplicate files right as they are restored Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 04/15] store-copy: 'populate-store' resets timestamps Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 05/15] image: 'register-closure' assumes already-reset timestamps Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 06/15] database: Remove #:reset-timestamps? from 'register-items' Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 07/15] store-copy: 'populate-store' can optionally deduplicate files Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 08/15] image: 'register-closure' leaves it up to the caller to deduplicate Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 09/15] database: Remove #:deduplicate? from 'register-items' Ludovic Courtès
2020-12-15 16:33 ` bug#44760: [PATCH 00/15] Speed up 'guix system init' & co Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 10/15] guix system: 'init' copies, resets timestamps, and deduplicates at once Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 11/15] database: Remove #:deduplicate? and #:reset-timestamps? from 'register-path' Ludovic Courtès
2020-12-11 15:09 ` Ludovic Courtès [this message]
2020-12-11 15:09 ` bug#44760: [PATCH 13/15] database: Remove 'register-path' Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 14/15] database: Honor 'SOURCE_DATE_EPOCH' Ludovic Courtès
2020-12-11 15:09 ` bug#44760: [PATCH 15/15] deduplicate: Create the '.links' directory lazily Ludovic Courtès
2020-12-15 16:38 ` bug#44760: Closure copy in ‘guix system init’ is inefficient Ludovic Courtès
2020-12-16 21:53 ` Jonathan Brielmaier
2020-12-17 13:24 ` Ludovic Courtès
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://guix.gnu.org/
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201211150951.18508-3-ludo@gnu.org \
--to=ludo@gnu.org \
--cc=44760@debbugs.gnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://git.savannah.gnu.org/cgit/guix.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).