From cf7648f882380dd7a4e82760ecc10cc6078498eb Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 27 Oct 2019 15:18:31 +0100 Subject: derivations: Introduce 'imported+compiled-modules'. * guix/derivations.scm (imported+compiled-modules): New procedure. (build-expression->derivation): Use it instead of separate calls to '%imported-modules' and '%compiled-modules'. --- guix/derivations.scm | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'guix') diff --git a/guix/derivations.scm b/guix/derivations.scm index e1073ea39b..8309f845d9 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -1207,6 +1207,14 @@ they can refer to each other." #:guile-for-build guile #:local-build? #t))) +(define* (imported+compiled-modules store modules #:key + (system (%current-system)) + (guile (%guile-for-build))) + "Return a pair containing the derivation to import MODULES and that where +MODULES are compiled." + (cons (%imported-modules store modules #:system system #:guile guile) + (%compiled-modules store modules #:system system #:guile guile))) + (define* (build-expression->derivation store name exp ;deprecated #:key (system (%current-system)) @@ -1330,16 +1338,15 @@ and PROPERTIES." ;; fixed-output. (filter-map source-path inputs))) - (mod-drv (and (pair? modules) - (%imported-modules store modules - #:guile guile-drv - #:system system))) + (mod+go-drv (if (pair? modules) + (imported+compiled-modules store modules + #:guile guile-drv + #:system system) + '(#f . #f))) + (mod-drv (car mod+go-drv)) + (go-drv (cdr mod+go-drv)) (mod-dir (and mod-drv (derivation->output-path mod-drv))) - (go-drv (and (pair? modules) - (%compiled-modules store modules - #:guile guile-drv - #:system system))) (go-dir (and go-drv (derivation->output-path go-drv)))) (derivation store name guile -- cgit v1.2.3 From f726f6f8021e78b6a50ca0dbdb4acc91ed2161c4 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 27 Oct 2019 15:24:41 +0100 Subject: derivations: 'build-expression->derivation' caches its module derivations. This reduces the number of lookups in the 'add-data-to-store' cache from 7505 to 3329 (hit rate from 68% to 27%) when running: GUIX_PROFILING=add-data-to-store-cache guix build libreoffice -nd The execution time of "guix build libreoffice -nd" goes from 2.12s to 1.87s. * guix/derivations.scm (%module-cache): New variable. (imported+compiled-modules)[key]: New variable. Lookup KEY in %MODULE-CACHE and populate %MODULE-CACHE upon cache miss. --- guix/derivations.scm | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'guix') diff --git a/guix/derivations.scm b/guix/derivations.scm index 8309f845d9..140c22b620 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -1207,13 +1207,25 @@ they can refer to each other." #:guile-for-build guile #:local-build? #t))) +(define %module-cache + ;; Map a list of modules to its 'imported+compiled-modules' result. + (make-weak-value-hash-table)) + (define* (imported+compiled-modules store modules #:key (system (%current-system)) (guile (%guile-for-build))) "Return a pair containing the derivation to import MODULES and that where MODULES are compiled." - (cons (%imported-modules store modules #:system system #:guile guile) - (%compiled-modules store modules #:system system #:guile guile))) + (define key + (list modules (derivation-file-name guile) system)) + + (or (hash-ref %module-cache key) + (let ((result (cons (%imported-modules store modules + #:system system #:guile guile) + (%compiled-modules store modules + #:system system #:guile guile)))) + (hash-set! %module-cache key result) + result))) (define* (build-expression->derivation store name exp ;deprecated #:key -- cgit v1.2.3 From f58b45350b0ebfc36a707d9e986f5fe904af3605 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 27 Oct 2019 18:55:44 +0100 Subject: gexp: Add 'imported+compiled-modules'. * guix/gexp.scm (imported+compiled-modules): New procedure. (lower-gexp): Use it instead of separate calls to 'imported-modules' and 'compiled-modules'. --- guix/gexp.scm | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) (limited to 'guix') diff --git a/guix/gexp.scm b/guix/gexp.scm index 7323277511..fa74e80cd6 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -654,6 +654,28 @@ names and file names suitable for the #:allowed-references argument to (load-path lowered-gexp-load-path) ;list of store items (load-compiled-path lowered-gexp-load-compiled-path)) ;list of store items +(define* (imported+compiled-modules modules system + #:key (extensions '()) + deprecation-warnings guile + (module-path %load-path)) + "Return a pair where the first element is the imported MODULES and the +second element is the derivation to compile them." + (mlet %store-monad ((modules (if (pair? modules) + (imported-modules modules + #:system system + #:module-path module-path) + (return #f))) + (compiled (if (pair? modules) + (compiled-modules modules + #:system system + #:module-path module-path + #:extensions extensions + #:guile guile + #:deprecation-warnings + deprecation-warnings) + (return #f)))) + (return (cons modules compiled)))) + (define* (lower-gexp exp #:key (module-path %load-path) @@ -719,20 +741,15 @@ derivations--e.g., code evaluated for its side effects." (lambda (obj) (lower-object obj system)) extensions)) - (modules (if (pair? %modules) - (imported-modules %modules - #:system system - #:module-path module-path) - (return #f))) - (compiled (if (pair? %modules) - (compiled-modules %modules - #:system system - #:module-path module-path - #:extensions extensions - #:guile guile - #:deprecation-warnings - deprecation-warnings) - (return #f)))) + (modules+compiled (imported+compiled-modules + %modules system + #:extensions extensions + #:deprecation-warnings + deprecation-warnings + #:guile guile + #:module-path module-path)) + (modules -> (car modules+compiled)) + (compiled -> (cdr modules+compiled))) (define load-path (search-path modules exts (string-append "/share/guile/site/" effective-version))) -- cgit v1.2.3 From c57e417eff8649fce44041bc8e187a3e0c91b801 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 27 Oct 2019 19:08:15 +0100 Subject: store: Allow objects in the cache to be inserted and search for with 'equal?'. * guix/store.scm (cache-object-mapping): Add #:vhash-cons parameter and honor it. (lookup-cached-object): Add #:vhash-fold* parameter and honor it. (%mcached): Add #:vhash-fold* and #:vhash-cons and honor them. (mcached): Add clauses with 'eq?' and 'equal?' as the first argument. --- guix/store.scm | 67 +++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 24 deletions(-) (limited to 'guix') diff --git a/guix/store.scm b/guix/store.scm index 382aad29d9..a276554a52 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -1612,10 +1612,11 @@ This makes sense only when the daemon was started with '--cache-failures'." ;; from %STATE-MONAD. (template-directory instantiations %store-monad) -(define* (cache-object-mapping object keys result) +(define* (cache-object-mapping object keys result + #:key (vhash-cons vhash-consq)) "Augment the store's object cache with a mapping from OBJECT/KEYS to RESULT. KEYS is a list of additional keys to match against, for instance a (SYSTEM -TARGET) tuple. +TARGET) tuple. Use VHASH-CONS to insert OBJECT into the cache. OBJECT is typically a high-level object such as a or an , and RESULT is typically its derivation." @@ -1623,8 +1624,8 @@ and RESULT is typically its derivation." (values result (store-connection (inherit store) - (object-cache (vhash-consq object (cons result keys) - (store-connection-object-cache store))))))) + (object-cache (vhash-cons object (cons result keys) + (store-connection-object-cache store))))))) (define record-cache-lookup! (if (profiled? "object-cache") @@ -1653,11 +1654,12 @@ and RESULT is typically its derivation." (lambda (x y) #t))) -(define* (lookup-cached-object object #:optional (keys '())) +(define* (lookup-cached-object object #:optional (keys '()) + #:key (vhash-fold* vhash-foldq*)) "Return the cached object in the store connection corresponding to OBJECT -and KEYS. KEYS is a list of additional keys to match against, and which are -compared with 'equal?'. Return #f on failure and the cached result -otherwise." +and KEYS; use VHASH-FOLD* to look for OBJECT in the cache. KEYS is a list of +additional keys to match against, and which are compared with 'equal?'. +Return #f on failure and the cached result otherwise." (lambda (store) (let* ((cache (store-connection-object-cache store)) @@ -1665,33 +1667,50 @@ otherwise." ;; the whole vlist chain and significantly reduces the number of ;; 'hashq' calls. (value (let/ec return - (vhash-foldq* (lambda (item result) - (match item - ((value . keys*) - (if (equal? keys keys*) - (return value) - result)))) - #f object - cache)))) + (vhash-fold* (lambda (item result) + (match item + ((value . keys*) + (if (equal? keys keys*) + (return value) + result)))) + #f object + cache)))) (record-cache-lookup! value cache) (values value store)))) -(define* (%mcached mthunk object #:optional (keys '())) +(define* (%mcached mthunk object #:optional (keys '()) + #:key + (vhash-cons vhash-consq) + (vhash-fold* vhash-foldq*)) "Bind the monadic value returned by MTHUNK, which supposedly corresponds to -OBJECT/KEYS, or return its cached value." - (mlet %store-monad ((cached (lookup-cached-object object keys))) +OBJECT/KEYS, or return its cached value. Use VHASH-CONS to insert OBJECT into +the cache, and VHASH-FOLD* to look it up." + (mlet %store-monad ((cached (lookup-cached-object object keys + #:vhash-fold* vhash-fold*))) (if cached (return cached) (>>= (mthunk) (lambda (result) - (cache-object-mapping object keys result)))))) + (cache-object-mapping object keys result + #:vhash-cons vhash-cons)))))) -(define-syntax-rule (mcached mvalue object keys ...) - "Run MVALUE, which corresponds to OBJECT/KEYS, and cache it; or return the +(define-syntax mcached + (syntax-rules (eq? equal?) + "Run MVALUE, which corresponds to OBJECT/KEYS, and cache it; or return the value associated with OBJECT/KEYS in the store's object cache if there is one." - (%mcached (lambda () mvalue) - object (list keys ...))) + ((_ eq? mvalue object keys ...) + (%mcached (lambda () mvalue) + object (list keys ...) + #:vhash-cons vhash-consq + #:vhash-fold* vhash-foldq*)) + ((_ equal? mvalue object keys ...) + (%mcached (lambda () mvalue) + object (list keys ...) + #:vhash-cons vhash-cons + #:vhash-fold* vhash-fold*)) + ((_ mvalue object keys ...) + (mcached eq? mvalue object keys ...)))) (define (preserve-documentation original proc) "Return PROC with documentation taken from ORIGINAL." -- cgit v1.2.3 From f5fca9a82cec76d2e10b8b6c96be2dd79f638ba0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 27 Oct 2019 19:12:11 +0100 Subject: gexp: Cache the module to derivation mappings. This reduces the number of 'add-data-to-store' cache lookups from 3329 to 2743 (hit rate: 27% to 11%) when running: GUIX_PROFILING=add-data-to-store-cache guix build libreoffice -nd Execution time of "guix build libreoffice -nd" goes from 1.86s to 1.80s. * guix/gexp.scm (imported+compiled-modules): Wrap body in 'mcached'. --- guix/gexp.scm | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) (limited to 'guix') diff --git a/guix/gexp.scm b/guix/gexp.scm index fa74e80cd6..b640c079e4 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -660,21 +660,24 @@ names and file names suitable for the #:allowed-references argument to (module-path %load-path)) "Return a pair where the first element is the imported MODULES and the second element is the derivation to compile them." - (mlet %store-monad ((modules (if (pair? modules) - (imported-modules modules - #:system system - #:module-path module-path) - (return #f))) - (compiled (if (pair? modules) - (compiled-modules modules - #:system system - #:module-path module-path - #:extensions extensions - #:guile guile - #:deprecation-warnings - deprecation-warnings) - (return #f)))) - (return (cons modules compiled)))) + (mcached equal? + (mlet %store-monad ((modules (if (pair? modules) + (imported-modules modules + #:system system + #:module-path module-path) + (return #f))) + (compiled (if (pair? modules) + (compiled-modules modules + #:system system + #:module-path module-path + #:extensions extensions + #:guile guile + #:deprecation-warnings + deprecation-warnings) + (return #f)))) + (return (cons modules compiled))) + modules + system extensions guile deprecation-warnings module-path)) (define* (lower-gexp exp #:key -- cgit v1.2.3 From d727a9343d861cf775645df8be5bfefd43d6c6f0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 27 Oct 2019 19:19:14 +0100 Subject: derivations: Don't memoize 'derivation->bytevector'. Its hit rate was only 8%. Removing it reduces heap size of "guix build libreoffice -nd" from 69MiB to 61MiB and the wall-clock time is unchanged. * guix/derivations.scm (derivation->bytevector): Change from 'mlambda' to 'lambda'. --- guix/derivations.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/derivations.scm b/guix/derivations.scm index 140c22b620..706c650469 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -622,7 +622,7 @@ that form." (display ")" port)))) (define derivation->bytevector - (mlambda (drv) + (lambda (drv) "Return the external representation of DRV as a UTF-8-encoded string." (with-fluids ((%default-port-encoding "UTF-8")) (call-with-values open-bytevector-output-port -- cgit v1.2.3 From b74ed90916dce6239dbe6842548f82e22fc8c249 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 27 Oct 2019 23:00:39 +0100 Subject: channels: Refer to 'guile-json-3'. Fixes a regression introduced in 84af1e74029fd4c43636f7d8d3e6f82ddab9ce82. * guix/channels.scm (whole-package-for-legacy): Refer to GUILE-JSON-3, not GUILE-JSON. --- guix/channels.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/channels.scm b/guix/channels.scm index 2c28dccbcb..826ee729ad 100644 --- a/guix/channels.scm +++ b/guix/channels.scm @@ -505,7 +505,7 @@ modules in the old ~/.config/guix/latest style." ;; In the "old style", %SELF-BUILD-FILE would simply return a ;; derivation that builds modules. We have to infer what the ;; dependencies of these modules were. - (list guile-json guile-git guile-bytestructures + (list guile-json-3 guile-git guile-bytestructures (ssh -> guile-ssh) (tls -> gnutls))))) (define (old-style-guix? drv) -- cgit v1.2.3 From 49af34cfac89d384c46269bfd9388b2c73b1220a Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 22 Oct 2019 18:05:51 +0200 Subject: pull: Honor '/etc/guix/channels.scm'. * guix/scripts/pull.scm (channel-list)[global-file]: New variable. [channels]: Honor it. * doc/guix.texi (Invoking guix pull): Document it. --- doc/guix.texi | 18 +++++++++++++++++- guix/scripts/pull.scm | 5 +++++ 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'guix') diff --git a/doc/guix.texi b/doc/guix.texi index a934626e5a..7cc33c6e22 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -3666,6 +3666,21 @@ descriptions, and deploys it. Source code is downloaded from a @uref{https://git-scm.com, Git} repository, by default the official GNU@tie{}Guix repository, though this can be customized. +Specifically, @command{guix pull} downloads code from the @dfn{channels} +(@pxref{Channels}) specified by one of the followings, in this order: + +@enumerate +@item +the @option{--channels} option; +@item +the user's @file{~/.config/guix/channels.scm} file; +@item +the system-wide @file{/etc/guix/channels.scm} file; +@item +the built-in default channels specified in the @code{%default-channels} +variable. +@end enumerate + On completion, @command{guix package} will use packages and package versions from this just-retrieved copy of Guix. Not only that, but all the Guix commands and Scheme modules will also be taken from that latest @@ -3763,7 +3778,8 @@ configuration in the @file{~/.config/guix/channels.scm} file or using the @item --channels=@var{file} @itemx -C @var{file} Read the list of channels from @var{file} instead of -@file{~/.config/guix/channels.scm}. @var{file} must contain Scheme code that +@file{~/.config/guix/channels.scm} or @file{/etc/guix/channels.scm}. +@var{file} must contain Scheme code that evaluates to a list of channel objects. @xref{Channels}, for more information. diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 7876019eac..80d070652b 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -714,6 +714,9 @@ transformations specified in OPTS (resulting from '--url', '--commit', or (define default-file (string-append (config-directory) "/channels.scm")) + (define global-file + (string-append %sysconfdir "/guix/channels.scm")) + (define (load-channels file) (let ((result (load* file (make-user-module '((guix channels)))))) (if (and (list? result) (every channel? result)) @@ -725,6 +728,8 @@ transformations specified in OPTS (resulting from '--url', '--commit', or (load-channels file)) ((file-exists? default-file) (load-channels default-file)) + ((file-exists? global-file) + (load-channels global-file)) (else %default-channels))) -- cgit v1.2.3 From 8c8d60752e1ad73d5bd87d8497b357f8a8a389ab Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 28 Oct 2019 15:54:47 +0100 Subject: derivation: Remove memoization invalidation for 'derivation->bytevector'. This is a followup to d727a9343d861cf775645df8be5bfefd43d6c6f0, which broke 'hydra-jobs' from (gnu ci). * guix/derivations.scm (invalidate-derivation-caches!): Remove call to 'invalidate-memoization!' for 'derivation->bytevector'. --- guix/derivations.scm | 1 - 1 file changed, 1 deletion(-) (limited to 'guix') diff --git a/guix/derivations.scm b/guix/derivations.scm index 706c650469..bde937044a 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -919,7 +919,6 @@ derivation. It is kept as-is, uninterpreted, in the derivation." long-running processes that know what they're doing. Use with care!" ;; Typically this is meant to be used by Cuirass and Hydra, which can clear ;; caches when they start evaluating packages for another architecture. - (invalidate-memoization! derivation->bytevector) (invalidate-memoization! derivation-base16-hash) ;; FIXME: Comment out to work around . -- cgit v1.2.3