all messages for Guix-related lists mirrored at yhetil.org
 help / color / mirror / code / Atom feed
* [PATCH] Hotfix (repeat)
@ 2014-11-23  3:07 Deck Pickard
  2014-11-23 20:16 ` Ludovic Courtès
  2014-11-23 20:49 ` Ludovic Courtès
  0 siblings, 2 replies; 5+ messages in thread
From: Deck Pickard @ 2014-11-23  3:07 UTC (permalink / raw)
  To: guix-devel


[-- Attachment #1.1: Type: text/plain, Size: 97 bytes --]

Because google sucks, this time attaching the patches...

finger-crossed,
drp

.sig place holder

[-- Attachment #1.2: Type: text/html, Size: 150 bytes --]

[-- Attachment #2: 0001-store-default-to-serial-scheduler.patch --]
[-- Type: application/octet-stream, Size: 1410 bytes --]

From 5ad37d9ab4febcbec8bbae8f7c14d7a49683c771 Mon Sep 17 00:00:00 2001
From: nebuli <nebu@kipple>
Date: Sat, 22 Nov 2014 19:16:51 +0100
Subject: [PATCH 1/4] store: default to serial "scheduler"
Status: RO
Content-Length: 1172
Lines: 28

* guix/store.scm (set-build-options): exchange default argument values
---
 guix/store.scm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/guix/store.scm b/guix/store.scm
index bc4c641..571cc06 100644
--- a/guix/store.scm
+++ b/guix/store.scm
@@ -435,14 +435,14 @@ encoding conversion errors."
 (define* (set-build-options server
                             #:key keep-failed? keep-going? fallback?
                             (verbosity 0)
-                            (max-build-jobs (current-processor-count))
+                            (max-build-jobs 1)
                             timeout
                             (max-silent-time 3600)
                             (use-build-hook? #t)
                             (build-verbosity 0)
                             (log-type 0)
                             (print-build-trace #t)
-                            (build-cores 1)
+                            (build-cores (current-processor-count))
                             (use-substitutes? #t)
                             (binary-caches '())) ; client "untrusted" cache URLs
   ;; Must be called after `open-connection'.
-- 
2.1.2


[-- Attachment #3: 0002-guix-build-Add-max-jobs-option-without-handling-code.patch --]
[-- Type: application/octet-stream, Size: 2652 bytes --]

From 8e297904d80b39cd510ba0cced37acdb9b1aeb89 Mon Sep 17 00:00:00 2001
From: nebuli <nebu@kipple>
Date: Sat, 22 Nov 2014 19:58:24 +0100
Subject: [PATCH 2/4] guix build: Add --max-jobs option (without handling
 code).

* doc/guix.texi: Mention in the docs.
* guix/scripts/build.scm: Extend (show-build-options-help) and
  (%standard-build-options) functions.
---
 doc/guix.texi          |  5 +++++
 guix/scripts/build.scm | 14 ++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/doc/guix.texi b/doc/guix.texi
index 2a33cb5..02edee0 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -2676,6 +2676,11 @@ may be helpful when debugging setup issues with the build daemon.
 Allow the use of up to @var{n} CPU cores for the build.  The special
 value @code{0} means to use as many CPU cores as available.
 
+@item --max-jobs=@var{n}
+@itemx -M @var{n}
+Allow at most @var{n} build jobs. The special value @code{0} means to
+create as many jobs as the number of available CPU cores.
+
 @end table
 
 Behind the scenes, @command{guix build} is essentially an interface to
diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm
index 7b7f419..d10b95b 100644
--- a/guix/scripts/build.scm
+++ b/guix/scripts/build.scm
@@ -119,7 +119,9 @@ options handled by 'set-build-options-from-command-line', and listed in
   (display (_ "
       --verbosity=LEVEL  use the given verbosity LEVEL"))
   (display (_ "
-  -c, --cores=N          allow the use of up to N CPU cores for the build")))
+  -c, --cores=N          allow the use of up to N CPU cores for the build"))
+  (display (_ "
+  -M, --max-jobs=N       allow at most N build jobs")))
 
 (define (set-build-options-from-command-line store opts)
   "Given OPTS, an alist as returned by 'args-fold' given
@@ -192,7 +194,15 @@ options handled by 'set-build-options-from-command-line', and listed in
                   (let ((c (false-if-exception (string->number arg))))
                     (if c
                         (apply values (alist-cons 'cores c result) rest)
-                        (leave (_ "~a: not a number~%") arg)))))))
+                        (leave (_ "not a number: '~a' option argument: ~a~%")
+                               name arg)))))
+        (option '(#\M "max-jobs") #t #f
+                (lambda (opt name arg result . rest)
+                  (let ((c (false-if-exception (string->number arg))))
+                    (if c
+                        (apply values (alist-cons 'max-jobs c result) rest)
+                        (leave (_ "not a number: '~a' option argument: ~a~%")
+                               name arg)))))))
 
 \f
 ;;;
-- 
2.1.2


[-- Attachment #4: 0003-guix-Add-schedule-module.patch --]
[-- Type: application/octet-stream, Size: 5159 bytes --]

From 49d5d2c5e60e10566b8e87ea7956e8775b1325bc Mon Sep 17 00:00:00 2001
From: nebuli <nebu@kipple>
Date: Sat, 22 Nov 2014 20:13:07 +0100
Subject: [PATCH 3/4] guix: Add schedule module.

* guix/schedule.scm: New file. To handle --cores and --max-jobs
  options in 'guix build'.
* Makefile.am (MODULES): Add *this.
---
 Makefile.am       |   1 +
 guix/schedule.scm | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 guix/schedule.scm

diff --git a/Makefile.am b/Makefile.am
index 5c4ce90..1806a05 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -56,6 +56,7 @@ MODULES =					\
   guix/ftp-client.scm				\
   guix/http-client.scm				\
   guix/gnupg.scm				\
+  guix/schedule.scm				\
   guix/store.scm				\
   guix/svn-download.scm				\
   guix/ui.scm					\
diff --git a/guix/schedule.scm b/guix/schedule.scm
new file mode 100644
index 0000000..26c7b6b
--- /dev/null
+++ b/guix/schedule.scm
@@ -0,0 +1,102 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2014 Nebulieu <nebu@kipple>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (guix schedule)
+  #:use-module (guix records)
+  #:export (schedule?
+            schedule
+            schedule-name
+            schedule-max-cores
+            schedule-max-jobs
+            schedule-override-cores?
+            schedule-override-jobs?
+
+            make-schedule-sane))
+
+(define-record-type* <schedule>
+  schedule make-schedule
+  schedule?
+  (name      schedule-name                    ; symbol
+             (default 'serial))
+  (max-cores schedule-max-cores               ; non-negative integer
+             (default 0))                     ; use all available horse-power
+  (max-jobs  schedule-max-jobs                ; non-negative integer
+             (default 1))                     ; there can be only one
+  ;; unused, for now, rethink "unified" override
+  (override-cores?  schedule-override-cores?  ; boolean
+                    (default #t))
+  (override-jobs?   schedule-override-jobs?   ; boolean
+                    (default #t)))
+
+; will rather need one `make-schedule-with-name` and switch on 'symbol
+(define (make-schedule-serial)
+  (schedule))
+; redundant for now...
+
+; macro?
+(define (real-schedule symname cores jobs
+                       override-c override-j)
+  (schedule
+   (name            symname)
+   (max-cores       cores)
+   (max-jobs        jobs)
+   (override-cores? override-c)
+   (override-jobs?  override-j)))
+
+; better name?
+(define (>1 num)
+  (if (< num 1)
+      1
+      num))
+
+;; TODO: increase number of jobs with spare cores??? perhaps in real-schedule
+(define* (make-schedule-sane #:key max-cores max-jobs)
+  (let ((sym-name       'serial-sane)
+        ;; should overriding one override both (think: yes, e.g.
+        ;; setting cores to max with guix-daemon default [max-jobs = 0]
+        ;; will again lead to the N^2 phenomenon...
+        (override-cores (and max-cores #t))
+        (override-jobs  (and max-jobs #t))
+        ;; scheduling needs be centralized (think: override always)
+        (max-threads    (min (current-processor-count)
+                             (total-processor-count))))
+    (let ((default-max-cores (>1 (- max-threads 1)))
+          (default-max-jobs  1)
+          (validate (lambda (arg default)
+                      (if (or (not arg)
+                              (not (integer? arg)))
+                          default
+                          (or (and (= arg 0) max-threads)
+                              (and (< arg 0) default)
+                              arg)))))
+                           ;; perhaps we shouldn't be so symmetrical?
+      (let ((cores (validate max-cores default-max-cores))
+            (jobs  (validate max-jobs  default-max-jobs)))
+                      ;; cut-off
+        (let loop ((c (min cores max-threads))
+                   (j (min jobs max-threads)))
+          (let ((threads (* c j)))     
+            (if (<= threads max-threads)
+                (real-schedule sym-name c j
+                               override-cores override-jobs)
+                ;; maximize cores at the cost of jobs...
+                (let ((j- (>1 (- j 1))))
+                  (if (<= (* c j-) max-threads)
+                      (real-schedule sym-name c j-
+                                     override-cores override-jobs)
+                      (loop (>1 (- c 1)) j-))))))))))
-- 
2.1.2


[-- Attachment #5: 0004-guix-build-Try-to-handle-cores-and-max-jobs-in-a-san.patch --]
[-- Type: application/octet-stream, Size: 2745 bytes --]

From d22504209d43080d04d798c960bd29f3d4891bd4 Mon Sep 17 00:00:00 2001
From: nebuli <nebu@kipple>
Date: Sat, 22 Nov 2014 20:25:11 +0100
Subject: [PATCH 4/4] guix build: Try to handle --cores and --max-jobs in a
 sane way.

* guix/scripts/build.scm (set-build-options-from-command-line): use
  make-schedule-sane to parse and set "scheduling" options.
---
 guix/scripts/build.scm | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm
index d10b95b..31f17d2 100644
--- a/guix/scripts/build.scm
+++ b/guix/scripts/build.scm
@@ -20,6 +20,7 @@
 (define-module (guix scripts build)
   #:use-module (guix ui)
   #:use-module (guix store)
+  #:use-module (guix schedule)
   #:use-module (guix derivations)
   #:use-module (guix packages)
   #:use-module (guix utils)
@@ -127,16 +128,19 @@ options handled by 'set-build-options-from-command-line', and listed in
   "Given OPTS, an alist as returned by 'args-fold' given
 '%standard-build-options', set the corresponding build options on STORE."
   ;; TODO: Add more options.
-  (set-build-options store
-                     #:keep-failed? (assoc-ref opts 'keep-failed?)
-                     #:build-cores (or (assoc-ref opts 'cores) 0)
-                     #:fallback? (assoc-ref opts 'fallback?)
-                     #:use-substitutes? (assoc-ref opts 'substitutes?)
-                     #:use-build-hook? (assoc-ref opts 'build-hook?)
-                     #:max-silent-time (assoc-ref opts 'max-silent-time)
-                     #:timeout (assoc-ref opts 'timeout)
-                     #:print-build-trace (assoc-ref opts 'print-build-trace?)
-                     #:verbosity (assoc-ref opts 'verbosity)))
+  (let ((sched (make-schedule-sane #:max-cores (assoc-ref opts 'cores)
+                                   #:max-jobs  (assoc-ref opts 'max-jobs))))
+    (set-build-options store
+                       #:keep-failed? (assoc-ref opts 'keep-failed?)
+                       #:build-cores (schedule-max-cores sched)
+                       #:max-build-jobs (schedule-max-jobs sched)
+                       #:fallback? (assoc-ref opts 'fallback?)
+                       #:use-substitutes? (assoc-ref opts 'substitutes?)
+                       #:use-build-hook? (assoc-ref opts 'build-hook?)
+                       #:max-silent-time (assoc-ref opts 'max-silent-time)
+                       #:timeout (assoc-ref opts 'timeout)
+                       #:print-build-trace (assoc-ref opts 'print-build-trace?)
+                       #:verbosity (assoc-ref opts 'verbosity))))
 
 (define %standard-build-options
   ;; List of standard command-line options for tools that build something.
-- 
2.1.2


^ permalink raw reply related	[flat|nested] 5+ messages in thread
end of thread, other threads:[~2014-11-25 21:41 UTC | newest]

Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2014-11-23  3:07 [PATCH] Hotfix (repeat) Deck Pickard
2014-11-23 20:16 ` Ludovic Courtès
2014-11-23 20:49 ` Ludovic Courtès
2014-11-24 15:40   ` Deck Pickard
2014-11-25 21:41     ` Ludovic Courtès

Code repositories for project(s) associated with this external index

	https://git.savannah.gnu.org/cgit/guix.git

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.