From mboxrd@z Thu Jan 1 00:00:00 1970 From: ludo@gnu.org (Ludovic =?utf-8?Q?Court=C3=A8s?=) Subject: Estimating build time Date: Fri, 12 Oct 2018 17:58:12 +0200 Message-ID: <87a7njgnsb.fsf_-_@gnu.org> References: <87pnxqkbmg.fsf@gnu.org> <87zhvv5c4n.fsf@lassieur.org> <875zyi191w.fsf@inria.fr> <87sh1l62ge.fsf@lassieur.org> <87zhvs69gd.fsf@inria.fr> <87ftxksk0v.fsf@elephly.net> <87bm885pwt.fsf@inria.fr> <87pnwgnvd8.fsf@elephly.net> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" Return-path: Received: from eggs.gnu.org ([2001:4830:134:3::10]:44548) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gAzpk-0004p8-Da for guix-devel@gnu.org; Fri, 12 Oct 2018 11:58:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gAzpi-00038h-Re for guix-devel@gnu.org; Fri, 12 Oct 2018 11:58:36 -0400 In-Reply-To: <87pnwgnvd8.fsf@elephly.net> (Ricardo Wurmus's message of "Thu, 11 Oct 2018 21:20:19 +0200") List-Id: "Development of GNU Guix and the GNU System distribution." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-devel-bounces+gcggd-guix-devel=m.gmane.org@gnu.org Sender: "Guix-devel" To: Ricardo Wurmus Cc: guix-devel , =?utf-8?Q?Cl=C3=A9ment?= Lassieur --=-=-= Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable Hello! Ricardo Wurmus skribis: > Ludovic Court=C3=A8s writes: > >> Ricardo Wurmus skribis: [...] >>> Currently, there=E2=80=99s no way to tell if the derivations listed und= er =E2=80=9CThese >>> derivations will be built=E2=80=9D are expensive package builds or just= simple >>> graft derivations. >> >> Indeed. A simple trick would be to (ab)use the environment variable >> part of derivations as a property list, the way Nix has traditionally >> done it (see =E2=80=98user+system-env-vars=E2=80=99 in (guix derivations= )). >> >> So we could have, say, a =E2=80=98hint=E2=80=99 environment variable, an= d the UI would >> use that to determine if it=E2=80=99s a graft. > > This sounds like a good trick to me. I think it would be great to give > more hints to the UI and make it clearer to users what work they can > expect Guix to perform. In a similar vein, the attached module provides code to estimate package build time based on locally-available build logs. It can be used to show hints like this: --8<---------------cut here---------------start------------->8--- The following derivations would be built (estimated time: 54 mn): /gnu/store/3627svyhih9cfss8gnxllp9nmxqp23cq-gcc-4.9.4.drv /gnu/store/3didvp9c3sfqwmb9kkdr211vg5myygsf-gcc-4.9.4.tar.xz.drv --8<---------------cut here---------------end--------------->8--- or: --8<---------------cut here---------------start------------->8--- The following derivations would be built (estimated time: 22 hr): /gnu/store/safjgjqhxlf59rknygqdfq175cl5wvks-rust-1.27.2.drv /gnu/store/v154ah7f8wqcga104df9ldb25bjk2pm8-rustc-1.27.2-src.tar.gz.drv /gnu/store/nz2xzl1yizcwcxhnw2w5mdqnp3q1gggx-rustc-1.25.0-src.tar.gz.drv /gnu/store/wn422brymn6ysxq07090ijb4wx78dc1l-rustc-1.24.1-src.tar.gz.drv /gnu/store/3cj5083j5aq7az7n5dmkds77g84xqc33-rust-bootstrap-1.22.1.drv /gnu/store/rm1nghqrvw43wlbwws49rw921qh58m35-rustc-1.23.0-src.tar.xz.drv /gnu/store/a9gzrq0bsc3ynfj4cnrsxxd2jwjgf4zj-rust-1.23.0.drv /gnu/store/bmbpbhgby84yrj0mvkv279cy2ira4xqf-rustc-1.24.1-src.tar.xz.drv /gnu/store/bxxyzp6kzq88ii4aindgnggbb2m193rk-rust-1.24.1.drv /gnu/store/r26s0y5fi055pclhnivvf63llbrj54yw-rustc-1.25.0-src.tar.xz.drv /gnu/store/x4l0rsqni9lglfzazkjyxqjp432yr33s-rustc-1.26.2-src.tar.gz.drv /gnu/store/03y9zf5imbm0ni1llcmxixb8c78nmxdd-rustc-1.26.2-src.tar.xz.drv /gnu/store/ici0m0bia0f6f4wh0ykn12x6wg1ckck0-rust-1.25.0.drv /gnu/store/2l6fn1nxs2sfl93khki5jzz6dh7gfqpr-rust-1.26.2.drv /gnu/store/9bn1gxnsc59zi8bdpvfgqcjpczmk3ch0-rustc-1.27.2-src.tar.xz.drv --8<---------------cut here---------------end--------------->8--- (That=E2=80=99s from my x86_64 laptop.) The obvious downside is that it works by first retrieving the names of the files under /var/log/guix/drvs, and then opening, decompressing, and parsing the candidate log files. That typically takes a few seconds on a recent SSD laptop, but clearly we don=E2=80=99t want to do that every tim= e. We could maintain a cache, but even then, it might still be too expensive. Perhaps we should keep build times in the database somehow; the daemon can keep it up-to-date. Thoughts? There=E2=80=99s the obvious downside that both approaches rely on having previously built the package, but I think that=E2=80=99s a necessary limita= tion, unless we are to resort to external services (which could hardly provide estimates that make sense for the user=E2=80=99s machine anyway.) Ludo=E2=80=99. --=-=-= Content-Type: text/plain; charset=utf-8 Content-Disposition: inline; filename=build-logs.scm Content-Transfer-Encoding: quoted-printable Content-Description: the code ;;; GNU Guix --- Functional package management for GNU ;;; Copyright =C2=A9 2018 Ludovic Court=C3=A8s ;;; ;;; This file is part of GNU Guix. ;;; ;;; GNU Guix is free software; you can redistribute it and/or modify it ;;; under the terms of the GNU General Public License as published by ;;; the Free Software Foundation; either version 3 of the License, or (at ;;; your option) any later version. ;;; ;;; GNU Guix is distributed in the hope that it will be useful, but ;;; WITHOUT ANY WARRANTY; without even the implied warranty of ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;;; GNU General Public License for more details. ;;; ;;; You should have received a copy of the GNU General Public License ;;; along with GNU Guix. If not, see . (define-module (guix build-logs) #:use-module (guix config) #:use-module (guix store) #:use-module (srfi srfi-1) #:use-module (guix utils) #:use-module (ice-9 match) #:use-module (ice-9 ftw) #:use-module (ice-9 regex) #:use-module (ice-9 rdelim) #:export (%log-directory log-file-build-phases log-file-build-time estimated-build-time)) (define %log-directory (string-append (dirname %state-directory) ; XXX "/log/guix/drvs")) (define %end-of-phase-rx (make-regexp "^phase [`']([[:graph:]]+)' succeeded after ([0-9.]+) second= s$")) (define (log-file-build-phases file) "Interpret the build log in FILE and return an alist of name/duration pai= rs for each build phase, such as: ((unpack . 1.3) (configure . 4.2) (build . 383.8) =E2=80=A6) Duration is expressed in seconds. Return the empty list if no build phase information could be extracted from FILE." (define compression (cond ((string-suffix? ".gz" file) 'gzip) ((string-suffix? ".bz2" file) 'bzip2) ((string-suffix? ".xz" file) 'xz) (else 'none))) (call-with-input-file file (lambda (input) (call-with-decompressed-port compression input (lambda (port) (set-port-conversion-strategy! port 'substitute) (let loop ((result '())) (match (read-line port) ((? eof-object?) (reverse result)) (line (match (regexp-exec %end-of-phase-rx line) (#f (loop result)) (hit (loop (alist-cons (string->symbol (match:substring hit 1)) (string->number (match:substring hit 2)) result)))))))))))) (define (log-file-build-time file) "Return the total build time described by FILE, a build log, or zero if build phase information was not found." (match (log-file-build-phases file) (((names . durations) ...) (if (memq 'install names) (reduce + 0 durations) 0)))) (define (matching-log-files package) (define noop (lambda (file stat result) result)) (file-system-fold (const #t) (lambda (file stat result) ;leaf (let* ((base (basename (file-sans-extension (file-sans-extension file)))) (dash (string-index base #\-)) (full (string-drop base (+ dash 1)))) (call-with-values (lambda () (package-name->name+version full #\-)) (lambda (p v) (if (and (string=3D? p package) (not (string-suffix? ".gz" v)) (not (string-suffix? ".bz2" v)) (not (string-suffix? ".xz" v)) (not (string-suffix? ".lz" v)) (not (string-suffix? ".zip" v))) (cons (list file p v) result) result))))) noop ;down noop ;up noop ;skip (lambda (file stat error result) ;error result) '() %log-directory)) (define %not-dot (char-set-complement (char-set #\.))) (define (version-distance version reference) "Compute a super rough estimate of the distance of VERSION to REFERENCE, both of which being version strings." (let* ((reference (string-tokenize reference %not-dot)) (version (string-tokenize version %not-dot)) (len (length reference))) (let loop ((i len) (reference reference) (version version) (distance 0)) (match version (() distance) ((head . tail) (match reference (() distance) ((ref-head . ref-tail) (loop (- i 1) ref-tail tail (if (string=3D? ref-head head) distance (+ distance i)))))))))) (define (estimated-build-time package version) "Return the estimate time it takes to build PACKAGE at VERSION, or #f if = no such estimate is available." (let ((logs (sort (matching-log-files package) (match-lambda* (((file1 _ version1) (file2 _ version2)) (< (version-distance version1 version) (version-distance version2 version))))))) (any (match-lambda ((log package version) (let ((duration (log-file-build-time log))) (and (not (zero? duration)) duration)))) logs))) --=-=-=--