From fb4bf72be3fbc23bca35ba4b842b7e1517ef0e3a Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 28 Oct 2015 15:20:06 +0100 Subject: store: Use the daemon's substitute URLs by default. Partly fixes . * guix/store.scm (set-build-options): Change #:substitute-urls to default to #f. Send the 'substitute-urls' pair only if SUBSTITUTE-URLS is true. * guix/scripts/build.scm (set-build-options-from-command-line): Do not default to %DEFAULT-SUBSTITUTE-URLS for #:substitute-urls. * guix/scripts/size.scm (%default-options): Remove 'substitute-urls'. --- guix/store.scm | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index c4e3573711..8413d1f452 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -501,11 +501,11 @@ encoding conversion errors." (build-cores (current-processor-count)) (use-substitutes? #t) - ;; Client-provided substitute URLs. For - ;; unprivileged clients, these are considered - ;; "untrusted"; for "trusted" users, they override - ;; the daemon's settings. - (substitute-urls %default-substitute-urls)) + ;; Client-provided substitute URLs. If it is #f, + ;; the daemon's settings are used. Otherwise, it + ;; overrides the daemons settings; see 'guix + ;; substitute'. + (substitute-urls #f)) ;; Must be called after `open-connection'. (define socket @@ -533,7 +533,10 @@ encoding conversion errors." (let ((pairs `(,@(if timeout `(("build-timeout" . ,(number->string timeout))) '()) - ("substitute-urls" . ,(string-join substitute-urls))))) + ,@(if substitute-urls + `(("substitute-urls" + . ,(string-join substitute-urls))) + '())))) (send (string-pairs pairs)))) (let loop ((done? (process-stderr server))) (or done? (process-stderr server))))) -- cgit v1.2.3 From d203d3d4cb3d83beaadaaef6c279c3d84ac142f7 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 3 Dec 2015 18:53:01 +0200 Subject: store: Update to the new daemon protocol. * guix/store.scm (%protocol-version): Set minor to 14. (open-connection): Add 'cpu-affinity' parameter and honor it. --- guix/store.scm | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index 8413d1f452..89f5df052a 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -129,7 +129,7 @@ direct-store-path log-file)) -(define %protocol-version #x10c) +(define %protocol-version #x10e) (define %worker-magic-1 #x6e697863) ; "nixc" (define %worker-magic-2 #x6478696f) ; "dxio" @@ -328,11 +328,13 @@ (status nix-protocol-error-status)) (define* (open-connection #:optional (file (%daemon-socket-file)) - #:key (reserve-space? #t)) + #:key (reserve-space? #t) cpu-affinity) "Connect to the daemon over the Unix-domain socket at FILE. When -RESERVE-SPACE? is true, instruct it to reserve a little bit of extra -space on the file system so that the garbage collector can still -operate, should the disk become full. Return a server object." +RESERVE-SPACE? is true, instruct it to reserve a little bit of extra space on +the file system so that the garbage collector can still operate, should the +disk become full. When CPU-AFFINITY is true, it must be an integer +corresponding to an OS-level CPU number to which the daemon's worker process +for this connection will be pinned. Return a server object." (let ((s (with-fluids ((%default-port-encoding #f)) ;; This trick allows use of the `scm_c_read' optimization. (socket PF_UNIX SOCK_STREAM 0))) @@ -355,8 +357,12 @@ operate, should the disk become full. Return a server object." (protocol-major v)) (begin (write-int %protocol-version s) - (if (>= (protocol-minor v) 11) - (write-int (if reserve-space? 1 0) s)) + (when (>= (protocol-minor v) 14) + (write-int (if cpu-affinity 1 0) s) + (when cpu-affinity + (write-int cpu-affinity s))) + (when (>= (protocol-minor v) 11) + (write-int (if reserve-space? 1 0) s)) (let ((s (%make-nix-server s (protocol-major v) (protocol-minor v) -- cgit v1.2.3 From 07e70f4846521c1fa5319b25f23eea171a03fccd Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 3 Dec 2015 19:08:35 +0200 Subject: store: Add mode parameter to 'build-paths'. * guix/store.scm (%protocol-version): Set minor to 15. (build-mode): New enumerate type. (build-things): Add 'mode' parameter; pass it to the RPC. * tests/store.scm ("build-things, check mode"): New check. --- guix/store.scm | 20 ++++++++++++++++---- tests/store.scm | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index 89f5df052a..1818187155 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -53,6 +53,7 @@ nix-protocol-error-status hash-algo + build-mode open-connection close-connection @@ -129,7 +130,7 @@ direct-store-path log-file)) -(define %protocol-version #x10e) +(define %protocol-version #x10f) (define %worker-magic-1 #x6e697863) ; "nixc" (define %worker-magic-2 #x6478696f) ; "dxio" @@ -188,6 +189,12 @@ (sha1 2) (sha256 3)) +(define-enumerate-type build-mode + ;; store-api.hh + (normal 0) + (repair 1) + (check 2)) + (define-enumerate-type gc-action ;; store-api.hh (return-live 0) @@ -637,12 +644,17 @@ bits are kept. HASH-ALGO must be a string such as \"sha256\"." (hash-set! cache args path) path)))))) -(define-operation (build-things (string-list things)) - "Build THINGS, a list of store items which may be either '.drv' files or +(define build-things + (let ((build (operation (build-things (string-list things) + (integer mode)) + "Do it!" + boolean))) + (lambda* (store things #:optional (mode (build-mode normal))) + "Build THINGS, a list of store items which may be either '.drv' files or outputs, and return when the worker is done building them. Elements of THINGS that are not derivations can only be substituted and not built locally. Return #t on success." - boolean) + (build store things mode)))) (define-operation (add-temp-root (store-path path)) "Make PATH a temporary root for the duration of the current session. diff --git a/tests/store.scm b/tests/store.scm index 60d1085f99..72abf2c694 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -756,6 +756,41 @@ ;; Delete the corrupt item to leave the store in a clean state. (delete-paths s (list file))))))) +(test-assert "build-things, check mode" + (with-store store + (call-with-temporary-output-file + (lambda (entropy entropy-port) + (write (random-text) entropy-port) + (force-output entropy-port) + (let* ((drv (build-expression->derivation + store "non-deterministic" + `(begin + (use-modules (rnrs io ports)) + (let ((out (assoc-ref %outputs "out"))) + (call-with-output-file out + (lambda (port) + (display (call-with-input-file ,entropy + get-string-all) + port))) + #t)) + #:guile-for-build + (package-derivation store %bootstrap-guile (%current-system)))) + (file (derivation->output-path drv))) + (and (build-things store (list (derivation-file-name drv))) + (begin + (write (random-text) entropy-port) + (force-output entropy-port) + (guard (c ((nix-protocol-error? c) + (pk 'determinism-exception c) + (and (not (zero? (nix-protocol-error-status c))) + (string-contains (nix-protocol-error-message c) + "deterministic")))) + ;; This one will produce a different result. Since we're in + ;; 'check' mode, this must fail. + (build-things store (list (derivation-file-name drv)) + (build-mode check)) + #f)))))))) + (test-equal "store-lower" "Lowered." (let* ((add (store-lower text-file)) -- cgit v1.2.3 From 2734cbb89598dbd212d598800bef5a1e649f71f7 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 4 Dec 2015 11:32:50 +0200 Subject: store: 'build-things' now supports older daemon protocols. This is a followup to d203d3d. * guix/store.scm (build-things): Add 'build/old'. Use it when STORE's minor version is less than 15. --- guix/store.scm | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index 1818187155..98ccbd1004 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -648,13 +648,22 @@ bits are kept. HASH-ALGO must be a string such as \"sha256\"." (let ((build (operation (build-things (string-list things) (integer mode)) "Do it!" - boolean))) + boolean)) + (build/old (operation (build-things (string-list things)) + "Do it!" + boolean))) (lambda* (store things #:optional (mode (build-mode normal))) "Build THINGS, a list of store items which may be either '.drv' files or outputs, and return when the worker is done building them. Elements of THINGS that are not derivations can only be substituted and not built locally. Return #t on success." - (build store things mode)))) + (if (>= (nix-server-minor-version store) 15) + (build store things mode) + (if (= mode (build-mode normal)) + (build/old store things) + (raise (condition (&nix-protocol-error + (message "unsupported build mode") + (status 1))))))))) (define-operation (add-temp-root (store-path path)) "Make PATH a temporary root for the duration of the current session. -- cgit v1.2.3 From 2fba87ac7c3e6fc6ca1a6e94131303c37425b2ba Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 8 Dec 2015 22:58:32 +0100 Subject: store: Allow clients to request multiple builds. * guix/store.scm (set-build-options): Add #:rounds parameter and honor it. * tests/store.scm ("build multiple times"): New test. --- guix/store.scm | 5 +++++ tests/store.scm | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index 98ccbd1004..3c4d1c0058 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -504,6 +504,7 @@ encoding conversion errors." (define* (set-build-options server #:key keep-failed? keep-going? fallback? (verbosity 0) + rounds ;number of build rounds (max-build-jobs 1) timeout (max-silent-time 3600) @@ -549,6 +550,10 @@ encoding conversion errors." ,@(if substitute-urls `(("substitute-urls" . ,(string-join substitute-urls))) + '()) + ,@(if rounds + `(("build-repeat" + . ,(number->string (max 0 (1- rounds))))) '())))) (send (string-pairs pairs)))) (let loop ((done? (process-stderr server))) diff --git a/tests/store.scm b/tests/store.scm index 72abf2c694..394c06bc0f 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -769,6 +769,8 @@ (let ((out (assoc-ref %outputs "out"))) (call-with-output-file out (lambda (port) + ;; Rely on the fact that tests do not use the + ;; chroot, and thus ENTROPY is readable. (display (call-with-input-file ,entropy get-string-all) port))) @@ -791,6 +793,44 @@ (build-mode check)) #f)))))))) +(test-assert "build multiple times" + (with-store store + ;; Ask to build twice. + (set-build-options store #:rounds 2 #:use-substitutes? #f) + + (call-with-temporary-output-file + (lambda (entropy entropy-port) + (write (random-text) entropy-port) + (force-output entropy-port) + (let* ((drv (build-expression->derivation + store "non-deterministic" + `(begin + (use-modules (rnrs io ports)) + (let ((out (assoc-ref %outputs "out"))) + (call-with-output-file out + (lambda (port) + ;; Rely on the fact that tests do not use the + ;; chroot, and thus ENTROPY is accessible. + (display (call-with-input-file ,entropy + get-string-all) + port) + (call-with-output-file ,entropy + (lambda (port) + (write 'foobar port))))) + #t)) + #:guile-for-build + (package-derivation store %bootstrap-guile (%current-system)))) + (file (derivation->output-path drv))) + (guard (c ((nix-protocol-error? c) + (pk 'multiple-build c) + (and (not (zero? (nix-protocol-error-status c))) + (string-contains (nix-protocol-error-message c) + "deterministic")))) + ;; This one will produce a different result on the second run. + (current-build-output-port (current-error-port)) + (build-things store (list (derivation-file-name drv))) + #f)))))) + (test-equal "store-lower" "Lowered." (let* ((add (store-lower text-file)) -- cgit v1.2.3