From d9d7b9ec41e280ff18b14dba410f93fd4653e84b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 28 May 2021 17:22:03 +0200 Subject: store: Support dynamic allocation of per-connection caches. * guix/store.scm ()[object-cache]: Remove. [caches]: New field. (open-connection, port->connection): Adjust '%make-store-connection' calls accordingly. (%store-connection-caches, %object-cache-id): New variables. (allocate-store-connection-cache, vector-set) (store-connection-cache, set-store-connection-cache) (set-store-connection-caches!, set-store-connection-cache!): New procedures. (cache-object-mapping): Add #:cache parameter. (set-store-connection-object-cache!): Remove. (lookup-cached-object): Use 'store-connection-cache'. (run-with-store): Use 'store-connection-caches' and 'set-store-connection-caches!'. --- guix/store.scm | 94 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 16 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index cf5d5eeccc..897062efff 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -36,6 +36,7 @@ #:use-module (rnrs bytevectors) #:use-module (ice-9 binary-ports) #:use-module ((ice-9 control) #:select (let/ec)) + #:use-module (ice-9 atomic) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) #:use-module (srfi srfi-9 gnu) @@ -47,7 +48,7 @@ #:use-module (ice-9 match) #:use-module (ice-9 vlist) #:use-module (ice-9 popen) - #:use-module (ice-9 threads) + #:autoload (ice-9 threads) (current-processor-count) #:use-module (ice-9 format) #:use-module (web uri) #:export (%daemon-socket-uri @@ -87,6 +88,11 @@ nix-protocol-error-message nix-protocol-error-status + allocate-store-connection-cache + store-connection-cache + set-store-connection-cache + set-store-connection-cache! + hash-algo build-mode @@ -383,8 +389,8 @@ ;; the session. (ats-cache store-connection-add-to-store-cache) (atts-cache store-connection-add-text-to-store-cache) - (object-cache store-connection-object-cache - (default vlist-null)) ;vhash + (caches store-connection-caches + (default '#())) ;vector (built-in-builders store-connection-built-in-builders (default (delay '())))) ;promise @@ -586,6 +592,10 @@ for this connection will be pinned. Return a server object." (write-int (if reserve-space? 1 0) port)) (letrec* ((built-in-builders (delay (%built-in-builders conn))) + (caches + (make-vector + (atomic-box-ref %store-connection-caches) + vlist-null)) (conn (%make-store-connection port (protocol-major v) @@ -593,7 +603,7 @@ for this connection will be pinned. Return a server object." output flush (make-hash-table 100) (make-hash-table 100) - vlist-null + caches built-in-builders))) (let loop ((done? (process-stderr conn))) (or done? (process-stderr conn))) @@ -616,7 +626,9 @@ connection. Use with care." output flush (make-hash-table 100) (make-hash-table 100) - vlist-null + (make-vector + (atomic-box-ref %store-connection-caches) + vlist-null) (delay (%built-in-builders connection)))) connection)) @@ -1799,6 +1811,57 @@ The result is always the empty list unless the daemon was started with This makes sense only when the daemon was started with '--cache-failures'." boolean) + +;;; +;;; Per-connection caches. +;;; + +;; Number of currently allocated store connection caches--things that go in +;; the 'caches' vector of . +(define %store-connection-caches (make-atomic-box 0)) + +(define (allocate-store-connection-cache name) + "Allocate a new cache for store connections and return its identifier. Said +identifier can be passed as an argument to " + (let loop ((current (atomic-box-ref %store-connection-caches))) + (let ((previous (atomic-box-compare-and-swap! %store-connection-caches + current (+ current 1)))) + (if (= previous current) + current + (loop current))))) + +(define %object-cache-id + ;; The "object cache", mapping lowerable objects such as records + ;; to derivations. + (allocate-store-connection-cache 'object-cache)) + +(define (vector-set vector index value) + (let ((new (vector-copy vector))) + (vector-set! new index value) + new)) + +(define (store-connection-cache store cache) + "Return the cache of STORE identified by CACHE, an identifier as returned by +'allocate-store-connection-cache'." + (vector-ref (store-connection-caches store) cache)) + +(define (set-store-connection-cache store cache value) + "Return a copy of STORE where CACHE has the given VALUE. CACHE must be a +value returned by 'allocate-store-connection-cache'." + (store-connection + (inherit store) + (caches (vector-set (store-connection-caches store) cache value)))) + +(define set-store-connection-caches! ;private + (record-modifier 'caches)) + +(define (set-store-connection-cache! store cache value) + "Set STORE's CACHE to VALUE. + +This is a mutating version that should be avoided. Prefer the functional +'set-store-connection-cache' instead, together with using %STORE-MONAD." + (vector-set! (store-connection-caches store) cache value)) + ;;; ;;; Store monad. @@ -1819,7 +1882,9 @@ This makes sense only when the daemon was started with '--cache-failures'." (template-directory instantiations %store-monad) (define* (cache-object-mapping object keys result - #:key (vhash-cons vhash-consq)) + #:key + (cache %object-cache-id) + (vhash-cons vhash-consq)) "Augment the store's object cache with a mapping from OBJECT/KEYS to RESULT. KEYS is a list of additional keys to match against, for instance a (SYSTEM TARGET) tuple. Use VHASH-CONS to insert OBJECT into the cache. @@ -1828,10 +1893,10 @@ OBJECT is typically a high-level object such as a or an , and RESULT is typically its derivation." (lambda (store) (values result - (store-connection - (inherit store) - (object-cache (vhash-cons object (cons result keys) - (store-connection-object-cache store))))))) + (set-store-connection-cache + store cache + (vhash-cons object (cons result keys) + (store-connection-cache store cache)))))) (define record-cache-lookup! (if (profiled? "object-cache") @@ -1871,7 +1936,7 @@ and KEYS; use VHASH-FOLD* to look for OBJECT in the cache. KEYS is a list of additional keys to match against, and which are compared with 'equal?'. Return #f on failure and the cached result otherwise." (lambda (store) - (let* ((cache (store-connection-object-cache store)) + (let* ((cache (store-connection-cache store %object-cache-id)) ;; Escape as soon as we find the result. This avoids traversing ;; the whole vlist chain and significantly reduces the number of @@ -2048,9 +2113,6 @@ the store." ;; when using 'gexp->derivation' and co. (make-parameter #f)) -(define set-store-connection-object-cache! - (record-modifier 'object-cache)) - (define* (run-with-store store mval #:key (guile-for-build (%guile-for-build)) @@ -2070,8 +2132,8 @@ connection, and return the result." (when (and store new-store) ;; Copy the object cache from NEW-STORE so we don't fully discard ;; the state. - (let ((cache (store-connection-object-cache new-store))) - (set-store-connection-object-cache! store cache))) + (let ((caches (store-connection-caches new-store))) + (set-store-connection-caches! store caches))) result)))) -- cgit v1.2.3 From 0a3c723e0771e488cad79ae7c6f20bc83023eeb0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 28 May 2021 17:45:11 +0200 Subject: store: Generalize cache lookup recording. * guix/store.scm (cache-lookup-recorder): New procedure. (record-cache-lookup!): Define in terms of it. --- guix/store.scm | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index 897062efff..38d12ac5d7 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -69,6 +69,7 @@ nix-server-socket current-store-protocol-version ;for internal use + cache-lookup-recorder ;for internal use mcached &store-error store-error? @@ -1898,21 +1899,24 @@ and RESULT is typically its derivation." (vhash-cons object (cons result keys) (store-connection-cache store cache)))))) -(define record-cache-lookup! - (if (profiled? "object-cache") +(define (cache-lookup-recorder component title) + "Return a procedure of two arguments to record cache lookups, hits, and +misses for COMPONENT. The procedure must be passed a Boolean indicating +whether the cache lookup was a hit, and the actual cache (a vhash)." + (if (profiled? component) (let ((fresh 0) (lookups 0) (hits 0) (size 0)) (register-profiling-hook! - "object-cache" + component (lambda () - (format (current-error-port) "Store object cache: + (format (current-error-port) "~a: fresh caches: ~5@a lookups: ~5@a hits: ~5@a (~,1f%) cache size: ~5@a entries~%" - fresh lookups hits + title fresh lookups hits (if (zero? lookups) 100. (* 100. (/ hits lookups))) @@ -1920,9 +1924,9 @@ and RESULT is typically its derivation." (lambda (hit? cache) (set! fresh - (if (eq? cache vlist-null) - (+ 1 fresh) - fresh)) + (if (eq? cache vlist-null) + (+ 1 fresh) + fresh)) (set! lookups (+ 1 lookups)) (set! hits (if hit? (+ hits 1) hits)) (set! size (+ (if hit? 0 1) @@ -1930,6 +1934,9 @@ and RESULT is typically its derivation." (lambda (x y) #t))) +(define record-cache-lookup! + (cache-lookup-recorder "object-cache" "Store object cache")) + (define-inlinable (lookup-cached-object object keys vhash-fold*) "Return the cached object in the store connection corresponding to OBJECT and KEYS; use VHASH-FOLD* to look for OBJECT in the cache. KEYS is a list of -- cgit v1.2.3 From 2725f0463421dfb446bce393b87a13139922f1cc Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 31 May 2021 22:28:43 +0200 Subject: store: Remove 'references/substitutes'. This procedure lost its only user in commit 710854304b1ab29332edcb76f3de532e0724c197. * guix/store.scm (references/substitutes): Remove. * tests/store.scm ("references/substitutes missing reference info") ("references/substitutes with substitute info"): Remove. --- guix/store.scm | 55 +------------------------------------------------------ tests/store.scm | 36 ------------------------------------ 2 files changed, 1 insertion(+), 90 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index 38d12ac5d7..ea784a33d2 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -148,7 +148,6 @@ built-in-builders references references/cached - references/substitutes references* query-path-info* requisites @@ -1481,7 +1480,7 @@ error if there is no such root." ;; Brute-force cache mapping store items to their list of references. ;; Caching matters because when building a profile in the presence of ;; grafts, we keep calling 'graft-derivation', which in turn calls - ;; 'references/substitutes' many times with the same arguments. Ideally we + ;; 'references/cached' many times with the same arguments. Ideally we ;; would use a cache associated with the daemon connection instead (XXX). (make-hash-table 100)) @@ -1492,58 +1491,6 @@ error if there is no such root." (hash-set! %reference-cache item references) references))) -(define (references/substitutes store items) - "Return the list of list of references of ITEMS; the result has the same -length as ITEMS. Query substitute information for any item missing from the -store at once. Raise a '&store-protocol-error' exception if reference -information for one of ITEMS is missing." - (let* ((requested items) - (local-refs (map (lambda (item) - (or (hash-ref %reference-cache item) - (guard (c ((store-protocol-error? c) #f)) - (references store item)))) - items)) - (missing (fold-right (lambda (item local-ref result) - (if local-ref - result - (cons item result))) - '() - items local-refs)) - - ;; Query all the substitutes at once to minimize the cost of - ;; launching 'guix substitute' and making HTTP requests. - (substs (if (null? missing) - '() - (substitutable-path-info store missing)))) - (when (< (length substs) (length missing)) - (raise (condition (&store-protocol-error - (message "cannot determine \ -the list of references") - (status 1))))) - - ;; Intersperse SUBSTS and LOCAL-REFS. - (let loop ((items items) - (local-refs local-refs) - (result '())) - (match items - (() - (let ((result (reverse result))) - (for-each (cut hash-set! %reference-cache <> <>) - requested result) - result)) - ((item items ...) - (match local-refs - ((#f tail ...) - (loop items tail - (cons (any (lambda (subst) - (and (string=? (substitutable-path subst) item) - (substitutable-references subst))) - substs) - result))) - ((head tail ...) - (loop items tail - (cons head result))))))))) - (define* (fold-path store proc seed paths #:optional (relatives (cut references store <>))) "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the diff --git a/tests/store.scm b/tests/store.scm index 9c25adf5e9..3266fa7a82 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -308,42 +308,6 @@ (null? (references %store t1)) (null? (referrers %store t2))))) -(test-assert "references/substitutes missing reference info" - (with-store s - (set-build-options s #:use-substitutes? #f) - (guard (c ((store-protocol-error? c) #t)) - (let* ((b (add-to-store s "bash" #t "sha256" - (search-bootstrap-binary "bash" - (%current-system)))) - (d (derivation s "the-thing" b '("--help") - #:inputs `((,b))))) - (references/substitutes s (list (derivation->output-path d) b)) - #f)))) - -(test-assert "references/substitutes with substitute info" - (with-store s - (set-build-options s #:use-substitutes? #t) - (let* ((t1 (add-text-to-store s "random1" (random-text))) - (t2 (add-text-to-store s "random2" (random-text) - (list t1))) - (t3 (add-text-to-store s "build" "echo -n $t2 > $out")) - (b (add-to-store s "bash" #t "sha256" - (search-bootstrap-binary "bash" - (%current-system)))) - (d (derivation s "the-thing" b `("-e" ,t3) - #:inputs `((,b) (,t3) (,t2)) - #:env-vars `(("t2" . ,t2)))) - (o (derivation->output-path d))) - (with-derivation-narinfo d - (sha256 => (gcrypt:sha256 (string->utf8 t2))) - (references => (list t2)) - - (equal? (references/substitutes s (list o t3 t2 t1)) - `((,t2) ;refs of O - () ;refs of T3 - (,t1) ;refs of T2 - ())))))) ;refs of T1 - (test-equal "substitutable-path-info when substitutes are turned off" '() (with-store s -- cgit v1.2.3 From fde3c349f511ac1237099511b5bdba5fbf541879 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 31 May 2021 22:38:03 +0200 Subject: store: 'references/cached' now uses a per-session cache. * guix/store.scm (%reference-cache): Remove. (%reference-cache-id): New variable. (references/cached): Rewrite in terms of it. --- guix/store.scm | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index ea784a33d2..b761264ac0 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -1476,21 +1476,6 @@ error if there is no such root." "Return the list of references of PATH." store-path-list)) -(define %reference-cache - ;; Brute-force cache mapping store items to their list of references. - ;; Caching matters because when building a profile in the presence of - ;; grafts, we keep calling 'graft-derivation', which in turn calls - ;; 'references/cached' many times with the same arguments. Ideally we - ;; would use a cache associated with the daemon connection instead (XXX). - (make-hash-table 100)) - -(define (references/cached store item) - "Like 'references', but cache results." - (or (hash-ref %reference-cache item) - (let ((references (references store item))) - (hash-set! %reference-cache item references) - references))) - (define* (fold-path store proc seed paths #:optional (relatives (cut references store <>))) "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the @@ -1810,6 +1795,26 @@ This is a mutating version that should be avoided. Prefer the functional 'set-store-connection-cache' instead, together with using %STORE-MONAD." (vector-set! (store-connection-caches store) cache value)) + +(define %reference-cache-id + ;; Cache mapping store items to their list of references. Caching matters + ;; because when building a profile in the presence of grafts, we keep + ;; calling 'graft-derivation', which in turn calls 'references/cached' many + ;; times with the same arguments. + (allocate-store-connection-cache 'reference-cache)) + +(define (references/cached store item) + "Like 'references', but cache results." + (let ((cache (store-connection-cache store %reference-cache-id))) + (match (vhash-assoc item cache) + ((_ . references) + references) + (#f + (let* ((references (references store item)) + (cache (vhash-cons item references cache))) + (set-store-connection-cache! store %reference-cache-id cache) + references))))) + ;;; ;;; Store monad. -- cgit v1.2.3 From 4985a4272497bf9ba87a2190353d915da9b55906 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sat, 15 May 2021 11:02:36 +0100 Subject: Start enabling substitutes from bordeaux.guix.gnu.org. In addition to substitutes from ci.guix.gnu.org. There are more changes that can be made in the future, but these changes seem like a good start. * config-daemon.ac (guix_substitute_urls): Add https://bordeaux.guix.gnu.org. * guix/scripts/substitute.scm (%default-substitute-urls): Add http://bordeaux.guix.gnu.org. * guix/store.scm (%default-substitute-urls): Add bordeaux.guix.gnu.org. * doc/guix.texi: Adjust accordingly. * doc/contributing.texi: Adjust accordingly. --- config-daemon.ac | 2 +- doc/contributing.texi | 8 +-- doc/guix.texi | 142 ++++++++++++++++++++++++-------------------- guix/scripts/substitute.scm | 3 +- guix/store.scm | 3 +- 5 files changed, 86 insertions(+), 72 deletions(-) (limited to 'guix/store.scm') diff --git a/config-daemon.ac b/config-daemon.ac index 85caee269b..5ddc740600 100644 --- a/config-daemon.ac +++ b/config-daemon.ac @@ -117,7 +117,7 @@ if test "x$guix_build_daemon" = "xyes"; then dnl Determine the appropriate default list of substitute URLs (GnuTLS dnl is required so we can default to 'https'.) - guix_substitute_urls="https://ci.guix.gnu.org" + guix_substitute_urls="https://ci.guix.gnu.org https://bordeaux.guix.gnu.org" AC_MSG_CHECKING([for default substitute URLs]) AC_MSG_RESULT([$guix_substitute_urls]) diff --git a/doc/contributing.texi b/doc/contributing.texi index d67e632520..1086bb9fd4 100644 --- a/doc/contributing.texi +++ b/doc/contributing.texi @@ -376,12 +376,12 @@ Once your package builds correctly, please send us a patch (@pxref{Submitting Patches}). Well, if you need help, we will be happy to help you too. Once the patch is committed in the Guix repository, the new package automatically gets built on the supported platforms by -@url{@value{SUBSTITUTE-URL}, our continuous integration system}. +@url{https://@value{SUBSTITUTE-SERVER-1}, our continuous integration system}. @cindex substituter Users can obtain the new package definition simply by running @command{guix pull} (@pxref{Invoking guix pull}). When -@code{@value{SUBSTITUTE-SERVER}} is done building the package, installing the +@code{@value{SUBSTITUTE-SERVER-1}} is done building the package, installing the package automatically downloads binaries from there (@pxref{Substitutes}). The only place where human intervention is needed is to review and apply the patch. @@ -1107,7 +1107,7 @@ changes). This branch is intended to be merged in @code{master} every until late in its development process. @end table -All these branches are @uref{@value{SUBSTITUTE-URL}, +All these branches are @uref{https://@value{SUBSTITUTE-SERVER-1}, tracked by our build farm} and merged into @code{master} once everything has been successfully built. This allows us to fix issues before they hit users, and to reduce the window during which pre-built @@ -1141,7 +1141,7 @@ as timestamps or randomly-generated output in the build result. Another option is to use @command{guix challenge} (@pxref{Invoking guix challenge}). You may run it once the package has been committed and -built by @code{@value{SUBSTITUTE-SERVER}} to check whether it obtains the same +built by @code{@value{SUBSTITUTE-SERVER-1}} to check whether it obtains the same result as you did. Better yet: Find another machine that can build it and run @command{guix publish}. Since the remote build machine is likely different from yours, this can catch non-determinism issues diff --git a/doc/guix.texi b/doc/guix.texi index 0930a514c7..094d1acd2c 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -17,8 +17,9 @@ @set BASE-URL https://ftp.gnu.org/gnu/guix @c The official substitute server used by default. -@set SUBSTITUTE-SERVER ci.guix.gnu.org -@set SUBSTITUTE-URL https://@value{SUBSTITUTE-SERVER} +@set SUBSTITUTE-SERVER-1 ci.guix.gnu.org +@set SUBSTITUTE-SERVER-2 bordeaux.guix.gnu.org +@set SUBSTITUTE-URLS https://@value{SUBSTITUTE-SERVER-1} https://@value{SUBSTITUTE-SERVER-2} @copying Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès@* @@ -229,7 +230,7 @@ Package Management Substitutes -* Official Substitute Server:: One particular source of substitutes. +* Official Substitute Servers:: One particular source of substitutes. * Substitute Server Authorization:: How to enable or disable substitutes. * Getting Substitutes from Other Servers:: Substitute diversity. * Substitute Authentication:: How Guix verifies substitutes. @@ -780,12 +781,15 @@ Info search path). @item @cindex substitutes, authorization thereof -To use substitutes from @code{@value{SUBSTITUTE-SERVER}} or one of its mirrors -(@pxref{Substitutes}), authorize them: +To use substitutes from @code{@value{SUBSTITUTE-SERVER-1}}, +@code{@value{SUBSTITUTE-SERVER-2}} or a mirror (@pxref{Substitutes}), +authorize them: @example # guix archive --authorize < \ - ~root/.config/guix/current/share/guix/@value{SUBSTITUTE-SERVER}.pub + ~root/.config/guix/current/share/guix/@value{SUBSTITUTE-SERVER-1}.pub +# guix archive --authorize < \ + ~root/.config/guix/current/share/guix/@value{SUBSTITUTE-SERVER-2}.pub @end example @quotation Note @@ -1547,7 +1551,7 @@ remote procedure call (@pxref{The Store}). @item --substitute-urls=@var{urls} Consider @var{urls} the default whitespace-separated list of substitute source URLs. When this option is omitted, -@indicateurl{https://@value{SUBSTITUTE-SERVER}} is used. +@indicateurl{@value{SUBSTITUTE-URLS}} is used. This means that substitutes may be downloaded from @var{urls}, as long as they are signed by a trusted signature (@pxref{Substitutes}). @@ -3685,7 +3689,7 @@ pre-built package binaries, but source tarballs, for instance, which also result from derivation builds, can be available as substitutes. @menu -* Official Substitute Server:: One particular source of substitutes. +* Official Substitute Servers:: One particular source of substitutes. * Substitute Server Authorization:: How to enable or disable substitutes. * Getting Substitutes from Other Servers:: Substitute diversity. * Substitute Authentication:: How Guix verifies substitutes. @@ -3694,14 +3698,15 @@ also result from derivation builds, can be available as substitutes. * On Trusting Binaries:: How can you trust that binary blob? @end menu -@node Official Substitute Server -@subsection Official Substitute Server +@node Official Substitute Servers +@subsection Official Substitute Servers @cindex build farm -The @code{@value{SUBSTITUTE-SERVER}} server is a front-end to an official build farm -that builds packages from Guix continuously for some -architectures, and makes them available as substitutes. This is the -default source of substitutes; it can be overridden by passing the +@code{@value{SUBSTITUTE-SERVER-1}} and +@code{@value{SUBSTITUTE-SERVER-2}} are both front-ends to official build +farms that build packages from Guix continuously for some architectures, +and make them available as substitutes. These are the default source of +substitutes; which can be overridden by passing the @option{--substitute-urls} option either to @command{guix-daemon} (@pxref{daemon-substitute-urls,, @code{guix-daemon --substitute-urls}}) or to client tools such as @command{guix package} @@ -3714,7 +3719,7 @@ using HTTP makes all communications visible to an eavesdropper, who could use the information gathered to determine, for instance, whether your system has unpatched security vulnerabilities. -Substitutes from the official build farm are enabled by default when +Substitutes from the official build farms are enabled by default when using Guix System (@pxref{GNU Distribution}). However, they are disabled by default when using Guix on a foreign distribution, unless you have explicitly enabled them via one of the recommended @@ -3730,27 +3735,28 @@ other substitute server. @cindex substitutes, authorization thereof @cindex access control list (ACL), for substitutes @cindex ACL (access control list), for substitutes -To allow Guix to download substitutes from @code{@value{SUBSTITUTE-SERVER}} or a -mirror thereof, you -must add its public key to the access control list (ACL) of archive +To allow Guix to download substitutes from @code{@value{SUBSTITUTE-SERVER-1}}, @code{@value{SUBSTITUTE-SERVER-2}} or a mirror, you +must add the releavnt public key to the access control list (ACL) of archive imports, using the @command{guix archive} command (@pxref{Invoking guix -archive}). Doing so implies that you trust @code{@value{SUBSTITUTE-SERVER}} to not +archive}). Doing so implies that you trust the substitute server to not be compromised and to serve genuine substitutes. @quotation Note If you are using Guix System, you can skip this section: Guix System -authorizes substitutes from @code{@value{SUBSTITUTE-SERVER}} by default. +authorizes substitutes from @code{@value{SUBSTITUTE-SERVER-1}} and +@code{@value{SUBSTITUTE-SERVER-2}} by default. @end quotation -The public key for @code{@value{SUBSTITUTE-SERVER}} is installed along with Guix, in -@code{@var{prefix}/share/guix/@value{SUBSTITUTE-SERVER}.pub}, where @var{prefix} is -the installation prefix of Guix. If you installed Guix from source, -make sure you checked the GPG signature of +The public keys for each of the project maintained substitute servers +are installed along with Guix, in @code{@var{prefix}/share/guix/}, where +@var{prefix} is the installation prefix of Guix. If you installed Guix +from source, make sure you checked the GPG signature of @file{guix-@value{VERSION}.tar.gz}, which contains this public key file. Then, you can run something like this: @example -# guix archive --authorize < @var{prefix}/share/guix/@value{SUBSTITUTE-SERVER}.pub +# guix archive --authorize < @var{prefix}/share/guix/@value{SUBSTITUTE-SERVER-1}.pub +# guix archive --authorize < @var{prefix}/share/guix/@value{SUBSTITUTE-SERVER-2}.pub @end example Once this is in place, the output of a command like @code{guix build} @@ -3782,8 +3788,8 @@ $ guix build emacs --dry-run @noindent The text changed from ``The following derivations would be built'' to ``112.3 MB would be downloaded''. This indicates that substitutes from -@code{@value{SUBSTITUTE-SERVER}} are usable and will be downloaded, when -possible, for future builds. +the configured substitute servers are usable and will be downloaded, +when possible, for future builds. @cindex substitutes, how to disable The substitute mechanism can be disabled globally by running @@ -3817,8 +3823,9 @@ its configuration and add the URLs and substitute keys that you want As an example, suppose you want to fetch substitutes from @code{guix.example.org} and to authorize the signing key of that server, -in addition to the default @code{@value{SUBSTITUTE-SERVER}}. The -resulting operating system configuration will look something like: +in addition to the default @code{@value{SUBSTITUTE-SERVER-1}} and +@code{@value{SUBSTITUTE-SERVER-2}}. The resulting operating system +configuration will look something like: @lisp (operating-system @@ -3862,7 +3869,7 @@ line and list the URLs of interest (@pxref{daemon-substitute-urls, @code{guix-daemon --substitute-urls}}): @example -@dots{} --substitute-urls='https://guix.example.org https://@value{SUBSTITUTE-SERVER}' +@dots{} --substitute-urls='https://guix.example.org @value{SUBSTITUTE-URLS}' @end example @item @@ -3885,10 +3892,12 @@ Again this assumes @file{key.pub} contains the public key that @end enumerate Now you're all set! Substitutes will be preferably taken from -@code{https://guix.example.org}, using @code{@value{SUBSTITUTE-SERVER}} -as a fallback. Of course you can list as many substitute servers as you -like, with the caveat that substitute lookup can be slowed down if too -many servers need to be contacted. +@code{https://guix.example.org}, using +@code{@value{SUBSTITUTE-SERVER-1}} then +@code{@value{SUBSTITUTE-SERVER-2}} as fallback options. Of course you +can list as many substitute servers as you like, with the caveat that +substitute lookup can be slowed down if too many servers need to be +contacted. Note that there are also situations where one may want to add the URL of a substitute server @emph{without} authorizing its key. @@ -3976,12 +3985,12 @@ by a server. Today, each individual's control over their own computing is at the mercy of institutions, corporations, and groups with enough power and determination to subvert the computing infrastructure and exploit its -weaknesses. While using @code{@value{SUBSTITUTE-SERVER}} substitutes can be -convenient, we encourage users to also build on their own, or even run -their own build farm, such that @code{@value{SUBSTITUTE-SERVER}} is less of an -interesting target. One way to help is by publishing the software you -build using @command{guix publish} so that others have one more choice -of server to download substitutes from (@pxref{Invoking guix publish}). +weaknesses. While using substitutes can be convenient, we encourage +users to also build on their own, or even run their own build farm, such +that the project run substitute servers are less of an interesting +target. One way to help is by publishing the software you build using +@command{guix publish} so that others have one more choice of server to +download substitutes from (@pxref{Invoking guix publish}). Guix has the foundations to maximize build reproducibility (@pxref{Features}). In most cases, independent builds of a given @@ -4945,11 +4954,11 @@ Read a single-item archive as served by substitute servers low-level operation needed in only very narrow use cases; see below. For example, the following command extracts the substitute for Emacs -served by @code{@value{SUBSTITUTE-SERVER}} to @file{/tmp/emacs}: +served by @code{@value{SUBSTITUTE-SERVER-1}} to @file{/tmp/emacs}: @example $ wget -O - \ - https://@value{SUBSTITUTE-SERVER}/nar/gzip/@dots{}-emacs-24.5 \ + https://@value{SUBSTITUTE-SERVER-1}/nar/gzip/@dots{}-emacs-24.5 \ | gunzip | guix archive -x /tmp/emacs @end example @@ -4971,7 +4980,7 @@ this example: @example $ wget -O - \ - https://@value{SUBSTITUTE-SERVER}/nar/lzip/@dots{}-emacs-26.3 \ + https://@value{SUBSTITUTE-SERVER-1}/nar/lzip/@dots{}-emacs-26.3 \ | lzip -d | guix archive -t @end example @@ -10905,7 +10914,7 @@ but you are actually on an @code{x86_64} machine: @example $ guix build --log-file gdb -s aarch64-linux -https://@value{SUBSTITUTE-SERVER}/log/@dots{}-gdb-7.10 +https://@value{SUBSTITUTE-SERVER-1}/log/@dots{}-gdb-7.10 @end example You can freely access a huge library of build logs! @@ -12558,7 +12567,7 @@ When @command{guix publish} runs, it spawns an HTTP server which allows anyone with network access to obtain substitutes from it. This means that any machine running Guix can also act as if it were a build farm, since the HTTP interface is compatible with Cuirass, the software behind -the @code{@value{SUBSTITUTE-SERVER}} build farm. +the @code{@value{SUBSTITUTE-SERVER-1}} build farm. For security, each substitute is signed, allowing recipients to check their authenticity and integrity (@pxref{Substitutes}). Because @@ -12847,12 +12856,12 @@ any given store item. The command output looks like this: @smallexample -$ guix challenge --substitute-urls="https://@value{SUBSTITUTE-SERVER} https://guix.example.org" -updating list of substitutes from 'https://@value{SUBSTITUTE-SERVER}'... 100.0% +$ guix challenge --substitute-urls="https://@value{SUBSTITUTE-SERVER-1} https://guix.example.org" +updating list of substitutes from 'https://@value{SUBSTITUTE-SERVER-1}'... 100.0% updating list of substitutes from 'https://guix.example.org'... 100.0% /gnu/store/@dots{}-openssl-1.0.2d contents differ: local hash: 0725l22r5jnzazaacncwsvp9kgf42266ayyp814v7djxs7nk963q - https://@value{SUBSTITUTE-SERVER}/nar/@dots{}-openssl-1.0.2d: 0725l22r5jnzazaacncwsvp9kgf42266ayyp814v7djxs7nk963q + https://@value{SUBSTITUTE-SERVER-1}/nar/@dots{}-openssl-1.0.2d: 0725l22r5jnzazaacncwsvp9kgf42266ayyp814v7djxs7nk963q https://guix.example.org/nar/@dots{}-openssl-1.0.2d: 1zy4fmaaqcnjrzzajkdn3f5gmjk754b43qkq47llbyak9z0qjyim differing files: /lib/libcrypto.so.1.1 @@ -12860,14 +12869,14 @@ updating list of substitutes from 'https://guix.example.org'... 100.0% /gnu/store/@dots{}-git-2.5.0 contents differ: local hash: 00p3bmryhjxrhpn2gxs2fy0a15lnip05l97205pgbk5ra395hyha - https://@value{SUBSTITUTE-SERVER}/nar/@dots{}-git-2.5.0: 069nb85bv4d4a6slrwjdy8v1cn4cwspm3kdbmyb81d6zckj3nq9f + https://@value{SUBSTITUTE-SERVER-1}/nar/@dots{}-git-2.5.0: 069nb85bv4d4a6slrwjdy8v1cn4cwspm3kdbmyb81d6zckj3nq9f https://guix.example.org/nar/@dots{}-git-2.5.0: 0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73 differing file: /libexec/git-core/git-fsck /gnu/store/@dots{}-pius-2.1.1 contents differ: local hash: 0k4v3m9z1zp8xzzizb7d8kjj72f9172xv078sq4wl73vnq9ig3ax - https://@value{SUBSTITUTE-SERVER}/nar/@dots{}-pius-2.1.1: 0k4v3m9z1zp8xzzizb7d8kjj72f9172xv078sq4wl73vnq9ig3ax + https://@value{SUBSTITUTE-SERVER-1}/nar/@dots{}-pius-2.1.1: 0k4v3m9z1zp8xzzizb7d8kjj72f9172xv078sq4wl73vnq9ig3ax https://guix.example.org/nar/@dots{}-pius-2.1.1: 1cy25x1a4fzq5rk0pmvc8xhwyffnqz95h2bpvqsz2mpvlbccy0gs differing file: /share/man/man1/pius.1.gz @@ -12889,7 +12898,7 @@ the servers obtained a result different from the local build. @cindex non-determinism, in package builds As an example, @code{guix.example.org} always gets a different answer. -Conversely, @code{@value{SUBSTITUTE-SERVER}} agrees with local builds, except in the +Conversely, @code{@value{SUBSTITUTE-SERVER-1}} agrees with local builds, except in the case of Git. This might indicate that the build process of Git is non-deterministic, meaning that its output varies as a function of various things that Guix does not fully control, in spite of building @@ -12905,7 +12914,7 @@ to run: @example guix challenge git \ --diff=diffoscope \ - --substitute-urls="https://@value{SUBSTITUTE-SERVER} https://guix.example.org" + --substitute-urls="https://@value{SUBSTITUTE-SERVER-1} https://guix.example.org" @end example This automatically invokes @command{diffoscope}, which displays detailed @@ -12915,14 +12924,14 @@ Alternatively, we can do something along these lines (@pxref{Invoking guix archive}): @example -$ wget -q -O - https://@value{SUBSTITUTE-SERVER}/nar/lzip/@dots{}-git-2.5.0 \ +$ wget -q -O - https://@value{SUBSTITUTE-SERVER-1}/nar/lzip/@dots{}-git-2.5.0 \ | lzip -d | guix archive -x /tmp/git $ diff -ur --no-dereference /gnu/store/@dots{}-git.2.5.0 /tmp/git @end example This command shows the difference between the files resulting from the local build, and the files resulting from the build on -@code{@value{SUBSTITUTE-SERVER}} (@pxref{Overview, Comparing and Merging Files,, +@code{@value{SUBSTITUTE-SERVER-1}} (@pxref{Overview, Comparing and Merging Files,, diffutils, Comparing and Merging Files}). The @command{diff} command works great for text files. When binary files differ, a better option is @uref{https://diffoscope.org/, Diffoscope}, a tool that helps @@ -12937,7 +12946,7 @@ In the meantime, @command{guix challenge} is one tool to help address the problem. If you are writing packages for Guix, you are encouraged to check -whether @code{@value{SUBSTITUTE-SERVER}} and other substitute servers obtain the +whether @code{@value{SUBSTITUTE-SERVER-1}} and other substitute servers obtain the same build result as you did with: @example @@ -13218,14 +13227,14 @@ on @var{a} and @var{a} has no substitutes, only @var{a} is listed, even though @var{b} usually lacks substitutes as well. The result looks like this: @example -$ guix weather --substitute-urls=@value{SUBSTITUTE-URL} -c 10 +$ guix weather --substitute-urls=@value{SUBSTITUTE-URLS} -c 10 computing 8,983 package derivations for x86_64-linux... -looking for 9,343 store items on @value{SUBSTITUTE-URL}... -updating substitutes from '@value{SUBSTITUTE-URL}'... 100.0% -@value{SUBSTITUTE-URL} +looking for 9,343 store items on @value{SUBSTITUTE-URLS}... +updating substitutes from '@value{SUBSTITUTE-URLS}'... 100.0% +@value{SUBSTITUTE-URLS} 64.7% substitutes available (6,047 out of 9,343) @dots{} -2502 packages are missing from '@value{SUBSTITUTE-URL}' for 'x86_64-linux', among which: +2502 packages are missing from '@value{SUBSTITUTE-URLS}' for 'x86_64-linux', among which: 58 kcoreaddons@@5.49.0 /gnu/store/@dots{}-kcoreaddons-5.49.0 46 qgpgme@@1.11.1 /gnu/store/@dots{}-qgpgme-1.11.1 37 perl-http-cookiejar@@0.008 /gnu/store/@dots{}-perl-http-cookiejar-0.008 @@ -13234,7 +13243,7 @@ updating substitutes from '@value{SUBSTITUTE-URL}'... 100.0% What this example shows is that @code{kcoreaddons} and presumably the 58 packages that depend on it have no substitutes at -@code{@value{SUBSTITUTE-SERVER}}; likewise for @code{qgpgme} and the 46 +@code{@value{SUBSTITUTE-SERVER-1}}; likewise for @code{qgpgme} and the 46 packages that depend on it. If you are a Guix developer, or if you are taking care of this build farm, @@ -15441,7 +15450,9 @@ Number of build user accounts to create. @item @code{authorize-key?} (default: @code{#t}) @cindex substitutes, authorization thereof Whether to authorize the substitute keys listed in -@code{authorized-keys}---by default that of @code{@value{SUBSTITUTE-SERVER}} +@code{authorized-keys}---by default that of +@code{@value{SUBSTITUTE-SERVER-1}} and +@code{@value{SUBSTITUTE-SERVER-2}} (@pxref{Substitutes}). When @code{authorize-key?} is true, @file{/etc/guix/acl} cannot be @@ -15462,8 +15473,9 @@ allowed for in-place modifications to @file{/etc/guix/acl}. @item @code{authorized-keys} (default: @code{%default-authorized-guix-keys}) The list of authorized key files for archive imports, as a list of string-valued gexps (@pxref{Invoking guix archive}). By default, it -contains that of @code{@value{SUBSTITUTE-SERVER}} (@pxref{Substitutes}). -See @code{substitute-urls} below for an example on how to change it. +contains that of @code{@value{SUBSTITUTE-SERVER-1}} and +@code{@value{SUBSTITUTE-SERVER-2}} (@pxref{Substitutes}). See +@code{substitute-urls} below for an example on how to change it. @item @code{use-substitutes?} (default: @code{#t}) Whether to use substitutes. @@ -15472,7 +15484,7 @@ Whether to use substitutes. The list of URLs where to look for substitutes by default. Suppose you would like to fetch substitutes from @code{guix.example.org} -in addition to @code{@value{SUBSTITUTE-SERVER}}. You will need to do +in addition to @code{@value{SUBSTITUTE-SERVER-1}}. You will need to do two things: (1) add @code{guix.example.org} to @code{substitute-urls}, and (2) authorize its signing key, having done appropriate checks (@pxref{Substitute Server Authorization}). The configuration below does diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 44448ff3e9..3ea1c73e10 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -643,7 +643,8 @@ found." (#f ;; This can only happen when this script is not invoked by the ;; daemon. - '("http://ci.guix.gnu.org")))) + '("http://ci.guix.gnu.org" + "http://bordeaux.guix.gnu.org")))) ;; In order to prevent using large number of discovered local substitute ;; servers, limit the local substitute urls list size. diff --git a/guix/store.scm b/guix/store.scm index b761264ac0..1ab2b08b47 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -787,7 +787,8 @@ encoding conversion errors." (map (if (false-if-exception (resolve-interface '(gnutls))) (cut string-append "https://" <>) (cut string-append "http://" <>)) - '("ci.guix.gnu.org"))) + '("ci.guix.gnu.org" + "bordeaux.guix.gnu.org"))) (define (current-user-name) "Return the name of the calling user." -- cgit v1.2.3