From mboxrd@z Thu Jan 1 00:00:00 1970 From: ludo@gnu.org (Ludovic =?UTF-8?Q?Court=C3=A8s?=) Subject: bug#27476: guix pull fails on powerful server Date: Thu, 12 Oct 2017 15:37:12 +0200 Message-ID: <877ew0o5br.fsf__25221.0238636055$1507815825$gmane$org@gnu.org> References: <87h8vvp1q7.fsf@elephly.net> <87377esu1a.fsf@gnu.org> <87k20nz18u.fsf@igalia.com> <87a81jj5gg.fsf@gnu.org> <87bmlyzxj7.fsf@elephly.net> <87shf44ny0.fsf@elephly.net> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" Return-path: Received: from eggs.gnu.org ([2001:4830:134:3::10]:53805) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1e2dge-0006qo-SG for bug-guix@gnu.org; Thu, 12 Oct 2017 09:38:10 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1e2dgY-0003ae-Pq for bug-guix@gnu.org; Thu, 12 Oct 2017 09:38:08 -0400 Received: from debbugs.gnu.org ([208.118.235.43]:54594) by eggs.gnu.org with esmtps (TLS1.0:RSA_AES_128_CBC_SHA1:16) (Exim 4.71) (envelope-from ) id 1e2dgY-0003aL-LO for bug-guix@gnu.org; Thu, 12 Oct 2017 09:38:02 -0400 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1e2dgY-0007hJ-EP for bug-guix@gnu.org; Thu, 12 Oct 2017 09:38:02 -0400 Sender: "Debbugs-submit" Resent-Message-ID: In-Reply-To: <87shf44ny0.fsf@elephly.net> (Ricardo Wurmus's message of "Sat, 30 Sep 2017 09:59:03 +0200") List-Id: Bug reports for GNU Guix List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: bug-guix-bounces+gcggb-bug-guix=m.gmane.org@gnu.org Sender: "bug-Guix" To: Ricardo Wurmus Cc: Andy Wingo , help-guix@gnu.org, 27476@debbugs.gnu.org --=-=-= Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable Hi! Ricardo Wurmus skribis: > The following derivation will be built: > /gnu/store/z5bhk17nxmdhvj0g4cy038p25mzh1gys-guix-latest.drv > copying and compiling to '/gnu/store/s3s7xlqa10mvf8v0ypxz8gzw3lcf1x5z-gui= x-latest' with Guile 2.2.2... > loading... 25.7% of 635 filesrandom seed for tests: 1506720257 > loading... 99.8% of 635 files > compiling... 69.1% of 635 filesice-9/threads.scm:289:22: In procedure= loop: > ice-9/threads.scm:289:22: Syntax error: > guix/scripts/graph.scm:103:10: return: return used outside of 'with-monad= ' in form (return (package-node-edges a)) The program below crashes with completely surreal backtraces in less than a minute on my 4-thread laptop: --8<---------------cut here---------------start------------->8--- (use-modules (ice-9 threads) (srfi srfi-1) (guix monads) (guix store)) (define threads (unfold (lambda (x) (> x 100)) (lambda (x) (call-with-new-thread (lambda () (define monad (symbol-append 'foo-monad (string->symbol (number->string x)))) (while #t (macroexpand `(begin (define-monad ,monad (bind +) (return -)) (with-monad ,monad (return 3)) (mapm ,monad + '(1 2 3)))))))) 1+ 0)) (for-each join-thread threads) --8<---------------cut here---------------end--------------->8--- Can you check if that also happens on your many-core machine? The patch below seems to fix the problem: (guix monads) has shared state (hash tables) used both at expansion-time and run-time, and it wasn=E2=80= =99t protected. My hypothesis is that this was causing random memory corruption. The weird thing, though, is that the errors we were getting were not so random. Also, the load phase of =E2=80=98guix pull=E2=80=99 is sequential. Could you test it and report back? Thanks, Ludo=E2=80=99. --=-=-= Content-Type: text/x-patch Content-Disposition: inline diff --git a/guix/monads.scm b/guix/monads.scm index 6ae616aca..c9c5da3bb 100644 --- a/guix/monads.scm +++ b/guix/monads.scm @@ -20,6 +20,7 @@ #:use-module ((system syntax) #:select (syntax-local-binding)) #:use-module (ice-9 match) + #:use-module (ice-9 threads) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) #:use-module (srfi srfi-26) @@ -117,6 +118,7 @@ ;; the syntax object of the parameter over which it is templated, and (2) ;; the syntax of its body. (define-once %templates (make-hash-table)) + (define-once %template-lock (make-mutex)) (define (register-template! name param body) (hash-set! %templates name (cons param body))) @@ -139,8 +141,9 @@ template instances." (syntax-source s)) (define current-info-port - ;; Port for debugging info. - (const (%make-void-port "w"))) + ;; Port for debugging info. Return a fresh port at each call to make + ;; sure we're thread-safe. + (lambda () (%make-void-port "w"))) (define location-string (format #f "~a:~a:~a" @@ -204,12 +207,14 @@ template instances." ;; Search for an instance of template NAME for this ACTUAL parameter. ;; On success, expand to the identifier of the instance; otherwise ;; expand to #f. - (any (matching-instance? #'name #'actual) %template-instances)) + (with-mutex %template-lock + (any (matching-instance? #'name #'actual) %template-instances))) ((_ exists? name actual) ;; Likewise, but return a Boolean. (let ((result (->bool - (any (matching-instance? #'name #'actual) - %template-instances)))) + (with-mutex %template-lock + (any (matching-instance? #'name #'actual) + %template-instances))))) (unless result (format (current-warning-port) "~a: warning: no specialization of template '~a' for '~a'~%" @@ -220,8 +225,9 @@ template instances." ;; Expand to the definitions of all the existing templates ;; specialized for ACTUAL. #`(begin - #,@(hash-map->list (cut instance-definition <> <> #'actual) - %templates)))))) + #,@(with-mutex %template-lock + (hash-map->list (cut instance-definition <> <> #'actual) + %templates))))))) (define-syntax define-template (lambda (s) --=-=-=--