diff options
Diffstat (limited to 'guix/scripts')
| -rw-r--r-- | guix/scripts/archive.scm | 2 | ||||
| -rw-r--r-- | guix/scripts/challenge.scm | 1 | ||||
| -rw-r--r-- | guix/scripts/discover.scm | 142 | ||||
| -rw-r--r-- | guix/scripts/environment.scm | 2 | ||||
| -rw-r--r-- | guix/scripts/hash.scm | 5 | ||||
| -rw-r--r-- | guix/scripts/import/cran.scm | 46 | ||||
| -rw-r--r-- | guix/scripts/import/crate.scm | 15 | ||||
| -rw-r--r-- | guix/scripts/import/elpa.scm | 19 | ||||
| -rw-r--r-- | guix/scripts/import/opam.scm | 10 | ||||
| -rw-r--r-- | guix/scripts/offload.scm | 15 | ||||
| -rw-r--r-- | guix/scripts/pack.scm | 260 | ||||
| -rw-r--r-- | guix/scripts/processes.scm | 152 | ||||
| -rw-r--r-- | guix/scripts/publish.scm | 60 | ||||
| -rw-r--r-- | guix/scripts/pull.scm | 7 | ||||
| -rwxr-xr-x | guix/scripts/substitute.scm | 438 | ||||
| -rw-r--r-- | guix/scripts/system.scm | 109 | ||||
| -rw-r--r-- | guix/scripts/upgrade.scm | 14 |
17 files changed, 904 insertions, 393 deletions
diff --git a/guix/scripts/archive.scm b/guix/scripts/archive.scm index c04baf9784..1f73fff711 100644 --- a/guix/scripts/archive.scm +++ b/guix/scripts/archive.scm @@ -347,6 +347,8 @@ output port." (match type ('directory (format #t "D ~a~%" file)) + ('directory-complete + #t) ('symlink (format #t "S ~a -> ~a~%" file content)) ((or 'regular 'executable) diff --git a/guix/scripts/challenge.scm b/guix/scripts/challenge.scm index 39bd2c1c0f..d0a456ac1d 100644 --- a/guix/scripts/challenge.scm +++ b/guix/scripts/challenge.scm @@ -210,6 +210,7 @@ taken since we do not import the archives." (cons `(,file ,type ,(port-sha256* port size)) result)))) ('directory result) + ('directory-complete result) ('symlink (cons `(,file ,type ,contents) result)))) '() diff --git a/guix/scripts/discover.scm b/guix/scripts/discover.scm new file mode 100644 index 0000000000..6aade81ed1 --- /dev/null +++ b/guix/scripts/discover.scm @@ -0,0 +1,142 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2020 Mathieu Othacehe <othacehe@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts discover) + #:use-module (guix avahi) + #:use-module (guix config) + #:use-module (guix scripts) + #:use-module (guix ui) + #:use-module (guix utils) + #:use-module (guix build syscalls) + #:use-module (guix build utils) + #:use-module (guix scripts publish) + #:use-module (ice-9 rdelim) + #:use-module (srfi srfi-37) + #:export (read-substitute-urls + + guix-discover)) + +(define (show-help) + (format #t (G_ "Usage: guix discover [OPTION]... +Discover Guix related services using Avahi.\n")) + (display (G_ " + -c, --cache=DIRECTORY cache discovery results in DIRECTORY")) + (display (G_ " + -h, --help display this help and exit")) + (display (G_ " + -V, --version display version information and exit")) + (newline) + (show-bug-report-information)) + +(define %options + (list (option '(#\c "cache") #t #f + (lambda (opt name arg result) + (alist-cons 'cache arg result))) + (option '(#\h "help") #f #f + (lambda _ + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda _ + (show-version-and-exit "guix discover"))))) + +(define %default-options + `((cache . ,%state-directory))) + + +;;; +;;; Publish servers. +;;; + +(define %publish-services + ;; Set of discovered publish services. + (make-hash-table)) + +(define (publish-file cache-directory) + "Return the name of the file storing the discovered publish services inside +CACHE-DIRECTORY." + (let ((directory (string-append cache-directory "/discover"))) + (string-append directory "/publish"))) + +(define %publish-file + (make-parameter (publish-file %state-directory))) + +(define* (write-publish-file #:key (file (%publish-file))) + "Dump the content of %PUBLISH-SERVICES hash table into FILE. Use a write +lock on FILE to synchronize with any potential readers." + (with-atomic-file-output file + (lambda (port) + (hash-for-each + (lambda (name service) + (format port "http://~a:~a~%" + (avahi-service-address service) + (avahi-service-port service))) + %publish-services))) + (chmod file #o644)) + +(define* (read-substitute-urls #:key (file (%publish-file))) + "Read substitute urls list from FILE and return it. Use a read lock on FILE +to synchronize with the writer." + (if (file-exists? file) + (call-with-input-file file + (lambda (port) + (let loop ((url (read-line port)) + (urls '())) + (if (eof-object? url) + urls + (loop (read-line port) (cons url urls)))))) + '())) + + +;;; +;;; Entry point. +;;; + +(define %services + ;; List of services we want to discover. + (list publish-service-type)) + +(define (service-proc action service) + (let ((name (avahi-service-name service)) + (type (avahi-service-type service))) + (when (string=? type publish-service-type) + (case action + ((new-service) + (hash-set! %publish-services name service)) + ((remove-service) + (hash-remove! %publish-services name))) + (write-publish-file)))) + +(define-command (guix-discover . args) + (category internal) + (synopsis "discover Guix related services using Avahi") + + (with-error-handling + (let* ((opts (args-fold* args %options + (lambda (opt name arg result) + (leave (G_ "~A: unrecognized option~%") name)) + (lambda (arg result) + (leave (G_ "~A: extraneous argument~%") arg)) + %default-options)) + (cache (assoc-ref opts 'cache)) + (publish-file (publish-file cache))) + (parameterize ((%publish-file publish-file)) + (mkdir-p (dirname publish-file)) + (false-if-exception (delete-file publish-file)) + (avahi-browse-service-thread service-proc + #:types %services))))) diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index e435bf0ce4..fbc202c658 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -42,9 +42,7 @@ #:use-module (gnu packages bash) #:use-module ((gnu packages bootstrap) #:select (bootstrap-executable %bootstrap-guile)) - #:use-module (ice-9 format) #:use-module (ice-9 match) - #:use-module (ice-9 rdelim) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) diff --git a/guix/scripts/hash.scm b/guix/scripts/hash.scm index 797b99f053..b8622373cc 100644 --- a/guix/scripts/hash.scm +++ b/guix/scripts/hash.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2016, 2017, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2016, 2017, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2016 Jan Nieuwenhuizen <janneke@gnu.org> ;;; Copyright © 2018 Tim Gesthuizen <tim.gesthuizen@yahoo.de> @@ -151,7 +151,8 @@ and 'base16' ('hex' and 'hexadecimal' can be used as well).\n")) ;; Catch and gracefully report possible '&nar-error' conditions. (with-error-handling (if (assoc-ref opts 'recursive?) - (let-values (((port get-hash) (open-sha256-port))) + (let-values (((port get-hash) + (open-hash-port (assoc-ref opts 'hash-algorithm)))) (write-file file port #:select? select?) (force-output port) (get-hash)) diff --git a/guix/scripts/import/cran.scm b/guix/scripts/import/cran.scm index d6f371ef3a..4767bc082d 100644 --- a/guix/scripts/import/cran.scm +++ b/guix/scripts/import/cran.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 Eric Bavier <bavier@member.fsf.org> -;;; Copyright © 2015, 2017, 2019 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2015, 2017, 2019, 2020 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -66,6 +67,10 @@ Import and convert the CRAN package for PACKAGE-NAME.\n")) (lambda (opt name arg result) (alist-cons 'repo (string->symbol arg) (alist-delete 'repo result)))) + (option '(#\s "style") #t #f + (lambda (opt name arg result) + (alist-cons 'style (string->symbol arg) + (alist-delete 'style result)))) (option '(#\r "recursive") #f #f (lambda (opt name arg result) (alist-cons 'recursive #t result))) @@ -92,21 +97,24 @@ Import and convert the CRAN package for PACKAGE-NAME.\n")) value) (_ #f)) (reverse opts)))) - (match args - ((package-name) - (if (assoc-ref opts 'recursive) - ;; Recursive import - (map package->definition - (cran-recursive-import package-name - (or (assoc-ref opts 'repo) 'cran))) - ;; Single import - (let ((sexp (cran->guix-package package-name - (or (assoc-ref opts 'repo) 'cran)))) - (unless sexp - (leave (G_ "failed to download description for package '~a'~%") - package-name)) - sexp))) - (() - (leave (G_ "too few arguments~%"))) - ((many ...) - (leave (G_ "too many arguments~%")))))) + (parameterize ((%input-style (assoc-ref opts 'style))) + (match args + ((package-name) + (if (assoc-ref opts 'recursive) + ;; Recursive import + (with-error-handling + (map package->definition + (filter identity + (cran-recursive-import package-name + #:repo (or (assoc-ref opts 'repo) 'cran))))) + ;; Single import + (let ((sexp (cran->guix-package package-name + #:repo (or (assoc-ref opts 'repo) 'cran)))) + (unless sexp + (leave (G_ "failed to download description for package '~a'~%") + package-name)) + sexp))) + (() + (leave (G_ "too few arguments~%"))) + ((many ...) + (leave (G_ "too many arguments~%"))))))) diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm index d834518c18..3a96defb86 100644 --- a/guix/scripts/import/crate.scm +++ b/guix/scripts/import/crate.scm @@ -2,7 +2,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson <davet@gnu.org> ;;; Copyright © 2016 David Craven <david@craven.ch> -;;; Copyright © 2019 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2019, 2020 Martin Becze <mjbecze@riseup.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -42,7 +42,7 @@ (define (show-help) (display (G_ "Usage: guix import crate PACKAGE-NAME -Import and convert the crate.io package for PACKAGE-NAME.\n")) +Import and convert the crates.io package for PACKAGE-NAME.\n")) (display (G_ " -r, --recursive import packages recursively")) (newline) @@ -95,19 +95,14 @@ Import and convert the crate.io package for PACKAGE-NAME.\n")) (package-name->name+version spec)) (if (assoc-ref opts 'recursive) - (map (match-lambda - ((and ('package ('name name) . rest) pkg) - `(define-public ,(string->symbol name) - ,pkg)) - (_ #f)) - (crate-recursive-import name version)) - (let ((sexp (crate->guix-package name version))) + (crate-recursive-import name #:version version) + (let ((sexp (crate->guix-package name #:version version #:include-dev-deps? #t))) (unless sexp (leave (G_ "failed to download meta-data for package '~a'~%") (if version (string-append name "@" version) name))) - sexp))) + (list sexp)))) (() (leave (G_ "too few arguments~%"))) ((many ...) diff --git a/guix/scripts/import/elpa.scm b/guix/scripts/import/elpa.scm index d270d2b4bc..d6b38e5c4b 100644 --- a/guix/scripts/import/elpa.scm +++ b/guix/scripts/import/elpa.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 Federico Beffa <beffa@fbengineering.ch> ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> +;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -95,14 +96,16 @@ Import the latest package named PACKAGE-NAME from an ELPA repository.\n")) (match args ((package-name) (if (assoc-ref opts 'recursive) - (map (match-lambda - ((and ('package ('name name) . rest) pkg) - `(define-public ,(string->symbol name) - ,pkg)) - (_ #f)) - (elpa-recursive-import package-name - (or (assoc-ref opts 'repo) 'gnu))) - (let ((sexp (elpa->guix-package package-name (assoc-ref opts 'repo)))) + (with-error-handling + (map (match-lambda + ((and ('package ('name name) . rest) pkg) + `(define-public ,(string->symbol name) + ,pkg)) + (_ #f)) + (elpa-recursive-import package-name + (or (assoc-ref opts 'repo) 'gnu)))) + (let ((sexp (elpa->guix-package package-name + #:repo (assoc-ref opts 'repo)))) (unless sexp (leave (G_ "failed to download package '~a'~%") package-name)) sexp))) diff --git a/guix/scripts/import/opam.scm b/guix/scripts/import/opam.scm index 20da1437fe..da9392821c 100644 --- a/guix/scripts/import/opam.scm +++ b/guix/scripts/import/opam.scm @@ -45,6 +45,8 @@ Import and convert the opam package for PACKAGE-NAME.\n")) (display (G_ " -r, --recursive import packages recursively")) (display (G_ " + --repo import packages from this opam repository")) + (display (G_ " -V, --version display version information and exit")) (newline) (show-bug-report-information)) @@ -58,6 +60,9 @@ Import and convert the opam package for PACKAGE-NAME.\n")) (option '(#\V "version") #f #f (lambda args (show-version-and-exit "guix import opam"))) + (option '(#f "repo") #t #f + (lambda (opt name arg result) + (alist-cons 'repo arg result))) (option '(#\r "recursive") #f #f (lambda (opt name arg result) (alist-cons 'recursive #t result))) @@ -79,6 +84,7 @@ Import and convert the opam package for PACKAGE-NAME.\n")) %default-options)) (let* ((opts (parse-options)) + (repo (and=> (assoc-ref opts 'repo) string->symbol)) (args (filter-map (match-lambda (('argument . value) value) @@ -93,9 +99,9 @@ Import and convert the opam package for PACKAGE-NAME.\n")) `(define-public ,(string->symbol name) ,pkg)) (_ #f)) - (opam-recursive-import package-name)) + (opam-recursive-import package-name #:repo repo)) ;; Single import - (let ((sexp (opam->guix-package package-name))) + (let ((sexp (opam->guix-package package-name #:repo repo))) (unless sexp (leave (G_ "failed to download meta-data for package '~a'~%") package-name)) diff --git a/guix/scripts/offload.scm b/guix/scripts/offload.scm index 6366556647..835078cb97 100644 --- a/guix/scripts/offload.scm +++ b/guix/scripts/offload.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> ;;; ;;; This file is part of GNU Guix. ;;; @@ -65,6 +66,7 @@ build-machine-overload-threshold build-machine-systems build-machine-features + build-machine-location build-requirements build-requirements? @@ -112,11 +114,17 @@ (speed build-machine-speed ; inexact real (default 1.0)) (features build-machine-features ; list of strings - (default '()))) + (default '())) + (location build-machine-location + (default (and=> (current-source-location) + source-properties->location)) + (innate))) ;;; Deprecated. (define (build-machine-system machine) - (warning (G_ "The 'system' field is deprecated, \ + (warning + (build-machine-location machine) + (G_ "The 'system' field is deprecated, \ please use 'systems' instead.~%")) (%build-machine-system machine)) @@ -626,7 +634,8 @@ daemon is not running." (and add-text-to-store 'alright)) node) ('alright #t) - (_ (report-module-error name))) + (_ (leave (G_ "(guix) module not usable on remote host '~a'") + name))) (match (inferior-eval '(begin (use-modules (guix)) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 6e0a16f033..169cbc2500 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -174,8 +174,6 @@ dependencies are registered." (let ((items (append-map read-closure '#$labels))) (with-database db-file db (register-items db items - #:deduplicate? #f - #:reset-timestamps? #f #:registration-time %epoch))))))) (computed-file "store-database" build @@ -211,12 +209,19 @@ added to the pack." #+(file-append glibc-utf8-locales "/lib/locale")) (setlocale LC_ALL "en_US.utf8")))) + (define (import-module? module) + ;; Since we don't use deduplication support in 'populate-store', don't + ;; import (guix store deduplication) and its dependencies, which includes + ;; Guile-Gcrypt. That way we can run tests with '--bootstrap'. + (and (not-config? module) + (not (equal? '(guix store deduplication) module)))) + (define build (with-imported-modules (source-module-closure `((guix build utils) (guix build union) (gnu build install)) - #:select? not-config?) + #:select? import-module?) #~(begin (use-modules (guix build utils) ((guix build union) #:select (relative-file-name)) @@ -390,138 +395,139 @@ added to the pack." `(("/bin" -> "bin") ,@symlinks))) (define build - (with-imported-modules (source-module-closure - '((guix build utils) - (guix build store-copy) - (guix build union) - (gnu build install)) - #:select? not-config?) - #~(begin - (use-modules (guix build utils) - (guix build store-copy) - ((guix build union) #:select (relative-file-name)) - (gnu build install) - (srfi srfi-1) - (srfi srfi-26) - (ice-9 match)) + (with-extensions (list guile-gcrypt) + (with-imported-modules (source-module-closure + '((guix build utils) + (guix build store-copy) + (guix build union) + (gnu build install)) + #:select? not-config?) + #~(begin + (use-modules (guix build utils) + (guix build store-copy) + ((guix build union) #:select (relative-file-name)) + (gnu build install) + (srfi srfi-1) + (srfi srfi-26) + (ice-9 match)) - (define database #+database) - (define entry-point #$entry-point) + (define database #+database) + (define entry-point #$entry-point) - (define (mksquashfs args) - (apply invoke "mksquashfs" - `(,@args + (define (mksquashfs args) + (apply invoke "mksquashfs" + `(,@args - ;; Do not create a "recovery file" when appending to the - ;; file system since it's useless in this case. - "-no-recovery" + ;; Do not create a "recovery file" when appending to the + ;; file system since it's useless in this case. + "-no-recovery" - ;; Do not attempt to store extended attributes. - ;; See <https://bugs.gnu.org/40043>. - "-no-xattrs" + ;; Do not attempt to store extended attributes. + ;; See <https://bugs.gnu.org/40043>. + "-no-xattrs" - ;; Set file times and the file system creation time to - ;; one second after the Epoch. - "-all-time" "1" "-mkfs-time" "1" + ;; Set file times and the file system creation time to + ;; one second after the Epoch. + "-all-time" "1" "-mkfs-time" "1" - ;; Reset all UIDs and GIDs. - "-force-uid" "0" "-force-gid" "0"))) + ;; Reset all UIDs and GIDs. + "-force-uid" "0" "-force-gid" "0"))) - (setenv "PATH" #+(file-append archiver "/bin")) + (setenv "PATH" #+(file-append archiver "/bin")) - ;; We need an empty file in order to have a valid file argument when - ;; we reparent the root file system. Read on for why that's - ;; necessary. - (with-output-to-file ".empty" (lambda () (display ""))) - - ;; Create the squashfs image in several steps. - ;; Add all store items. Unfortunately mksquashfs throws away all - ;; ancestor directories and only keeps the basename. We fix this - ;; in the following invocations of mksquashfs. - (mksquashfs `(,@(map store-info-item - (call-with-input-file "profile" - read-reference-graph)) - #$environment - ,#$output - - ;; Do not perform duplicate checking because we - ;; don't have any dupes. - "-no-duplicates" - "-comp" - ,#+(compressor-name compressor))) - - ;; Here we reparent the store items. For each sub-directory of - ;; the store prefix we need one invocation of "mksquashfs". - (for-each (lambda (dir) - (mksquashfs `(".empty" - ,#$output - "-root-becomes" ,dir))) - (reverse (string-tokenize (%store-directory) - (char-set-complement (char-set #\/))))) - - ;; Add symlinks and mount points. - (mksquashfs - `(".empty" - ,#$output - ;; Create SYMLINKS via pseudo file definitions. - ,@(append-map - (match-lambda - ((source '-> target) - ;; Create relative symlinks to work around a bug in - ;; Singularity 2.x: - ;; https://bugs.gnu.org/34913 - ;; https://github.com/sylabs/singularity/issues/1487 - (let ((target (string-append #$profile "/" target))) - (list "-p" - (string-join - ;; name s mode uid gid symlink - (list source - "s" "777" "0" "0" - (relative-file-name (dirname source) - target))))))) - '#$symlinks*) - - "-p" "/.singularity.d d 555 0 0" - - ;; Create the environment file. - "-p" "/.singularity.d/env d 555 0 0" - "-p" ,(string-append - "/.singularity.d/env/90-environment.sh s 777 0 0 " - (relative-file-name "/.singularity.d/env" - #$environment)) - - ;; Create /.singularity.d/actions, and optionally the 'run' - ;; script, used by 'singularity run'. - "-p" "/.singularity.d/actions d 555 0 0" - - ,@(if entry-point - `(;; This one if for Singularity 2.x. - "-p" - ,(string-append - "/.singularity.d/actions/run s 777 0 0 " - (relative-file-name "/.singularity.d/actions" - (string-append #$profile "/" - entry-point))) - - ;; This one is for Singularity 3.x. - "-p" - ,(string-append - "/.singularity.d/runscript s 777 0 0 " - (relative-file-name "/.singularity.d" - (string-append #$profile "/" - entry-point)))) - '()) - - ;; Create empty mount points. - "-p" "/proc d 555 0 0" - "-p" "/sys d 555 0 0" - "-p" "/dev d 555 0 0" - "-p" "/home d 555 0 0")) - - (when database - ;; Initialize /var/guix. - (install-database-and-gc-roots "var-etc" database #$profile) - (mksquashfs `("var-etc" ,#$output)))))) + ;; We need an empty file in order to have a valid file argument when + ;; we reparent the root file system. Read on for why that's + ;; necessary. + (with-output-to-file ".empty" (lambda () (display ""))) + + ;; Create the squashfs image in several steps. + ;; Add all store items. Unfortunately mksquashfs throws away all + ;; ancestor directories and only keeps the basename. We fix this + ;; in the following invocations of mksquashfs. + (mksquashfs `(,@(map store-info-item + (call-with-input-file "profile" + read-reference-graph)) + #$environment + ,#$output + + ;; Do not perform duplicate checking because we + ;; don't have any dupes. + "-no-duplicates" + "-comp" + ,#+(compressor-name compressor))) + + ;; Here we reparent the store items. For each sub-directory of + ;; the store prefix we need one invocation of "mksquashfs". + (for-each (lambda (dir) + (mksquashfs `(".empty" + ,#$output + "-root-becomes" ,dir))) + (reverse (string-tokenize (%store-directory) + (char-set-complement (char-set #\/))))) + + ;; Add symlinks and mount points. + (mksquashfs + `(".empty" + ,#$output + ;; Create SYMLINKS via pseudo file definitions. + ,@(append-map + (match-lambda + ((source '-> target) + ;; Create relative symlinks to work around a bug in + ;; Singularity 2.x: + ;; https://bugs.gnu.org/34913 + ;; https://github.com/sylabs/singularity/issues/1487 + (let ((target (string-append #$profile "/" target))) + (list "-p" + (string-join + ;; name s mode uid gid symlink + (list source + "s" "777" "0" "0" + (relative-file-name (dirname source) + target))))))) + '#$symlinks*) + + "-p" "/.singularity.d d 555 0 0" + + ;; Create the environment file. + "-p" "/.singularity.d/env d 555 0 0" + "-p" ,(string-append + "/.singularity.d/env/90-environment.sh s 777 0 0 " + (relative-file-name "/.singularity.d/env" + #$environment)) + + ;; Create /.singularity.d/actions, and optionally the 'run' + ;; script, used by 'singularity run'. + "-p" "/.singularity.d/actions d 555 0 0" + + ,@(if entry-point + `( ;; This one if for Singularity 2.x. + "-p" + ,(string-append + "/.singularity.d/actions/run s 777 0 0 " + (relative-file-name "/.singularity.d/actions" + (string-append #$profile "/" + entry-point))) + + ;; This one is for Singularity 3.x. + "-p" + ,(string-append + "/.singularity.d/runscript s 777 0 0 " + (relative-file-name "/.singularity.d" + (string-append #$profile "/" + entry-point)))) + '()) + + ;; Create empty mount points. + "-p" "/proc d 555 0 0" + "-p" "/sys d 555 0 0" + "-p" "/dev d 555 0 0" + "-p" "/home d 555 0 0")) + + (when database + ;; Initialize /var/guix. + (install-database-and-gc-roots "var-etc" database #$profile) + (mksquashfs `("var-etc" ,#$output))))))) (gexp->derivation (string-append name (compressor-extension compressor) diff --git a/guix/scripts/processes.scm b/guix/scripts/processes.scm index b4ca7b1687..3db5603286 100644 --- a/guix/scripts/processes.scm +++ b/guix/scripts/processes.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2020 John Soo <jsoo1@asu.edu> ;;; ;;; This file is part of GNU Guix. ;;; @@ -176,6 +177,9 @@ active sessions, and the master 'guix-daemon' process." (values (filter-map child-process->session children) master))) +(define (lock->recutils lock port) + (format port "LockHeld: ~a~%" lock)) + (define (daemon-session->recutils session port) "Display SESSION information in recutils format on PORT." (format port "SessionPID: ~a~%" @@ -184,28 +188,111 @@ active sessions, and the master 'guix-daemon' process." (process-id (daemon-session-client session))) (format port "ClientCommand:~{ ~a~}~%" (process-command (daemon-session-client session))) - (for-each (lambda (lock) - (format port "LockHeld: ~a~%" lock)) + (for-each (lambda (lock) (lock->recutils lock port)) (daemon-session-locks-held session)) (for-each (lambda (process) - (format port "ChildProcess: ~a:~{ ~a~}~%" - (process-id process) + (format port "ChildPID: ~a~%" + (process-id process)) + (format port "ChildCommand: :~{ ~a~}~%" (process-command process))) (daemon-session-children session))) +(define (daemon-sessions->recutils port sessions) + "Display denormalized SESSIONS information to PORT." + (for-each (lambda (session) + (daemon-session->recutils session port) + (newline port)) + sessions)) + +(define session-rec-type + "%rec: Session +%type: PID int +%type: ClientPID int +%key: PID +%mandatory: ClientPID ClientCommand") + +(define lock-rec-type + "%rec: Lock +%mandatory: LockHeld +%type: Session rec Session") + +(define child-process-rec-type + "%rec: ChildProcess +%type: PID int +%type: Session rec Session +%key: PID +%mandatory: Command") + +(define (session-key->recutils session port) + "Display SESSION PID as a recutils field on PORT." + (format + port "Session: ~a" + (process-id (daemon-session-process session)))) + +(define (session-scalars->normalized-record session port) + "Display SESSION scalar fields to PORT in normalized form." + (format port "PID: ~a~%" + (process-id (daemon-session-process session))) + (format port "ClientPID: ~a~%" + (process-id (daemon-session-client session))) + (format port "ClientCommand:~{ ~a~}~%" + (process-command (daemon-session-client session)))) + +(define (child-process->normalized-record process port) + "Display PROCESS record on PORT in normalized form" + (format port "PID: ~a" (process-id process)) + (newline port) + (format port "Command:~{ ~a~}" (process-command process))) + +(define (daemon-sessions->normalized-record port sessions) + "Display SESSIONS recutils on PORT in normalized form" + (display session-rec-type port) + (newline port) + (newline port) + (for-each (lambda (session) + (session-scalars->normalized-record session port) + (newline port)) + sessions) + + (display lock-rec-type port) + (newline port) + (newline port) + (for-each (lambda (session) + (for-each (lambda (lock) + (lock->recutils "testing testing" port) + (session-key->recutils session port) + (newline port) + (newline port)) + (daemon-session-locks-held session))) + sessions) + + (display child-process-rec-type port) + (newline port) + (newline port) + (for-each (lambda (session) + (for-each (lambda (process) + (child-process->normalized-record process port) + (newline port) + (session-key->recutils session port) + (newline port) + (newline port)) + (daemon-session-children session))) + sessions)) + ;;; ;;; Options. ;;; -(define %options - (list (option '(#\h "help") #f #f - (lambda args - (show-help) - (exit 0))) - (option '(#\V "version") #f #f - (lambda args - (show-version-and-exit "guix processes"))))) +(define %available-formats + '("recutils" "normalized")) + +(define (list-formats) + (display (G_ "The available formats are:\n")) + (newline) + (for-each (lambda (f) + (format #t " - ~a~%" f)) + %available-formats)) (define (show-help) (display (G_ "Usage: guix processes @@ -216,8 +303,33 @@ List the current Guix sessions and their processes.")) (display (G_ " -V, --version display version information and exit")) (newline) + (display (G_ " + -f, --format=FORMAT display results as normalized record sets")) + (display (G_ " + --list-formats display available formats")) + (newline) (show-bug-report-information)) +(define %options + (list (option '(#\h "help") #f #f + (lambda args + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda args + (show-version-and-exit "guix processes"))) + (option '(#\f "format") #t #f + (lambda (opt name arg result) + (unless (member arg %available-formats) + (leave (G_ "~a: unsupported output format~%") arg)) + (alist-cons 'format (string->symbol arg) result))) + (option '("list-formats") #f #f + (lambda (opt name arg result) + (list-formats) + (exit 0))))) + +(define %default-options '((format . recutils))) + ;;; ;;; Entry point. @@ -226,18 +338,16 @@ List the current Guix sessions and their processes.")) (define-command (guix-processes . args) (category plumbing) (synopsis "list currently running sessions") + (define options - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - cons - '())) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (with-paginated-output-port port - (for-each (lambda (session) - (daemon-session->recutils session port) - (newline port)) - (daemon-sessions)) + (match (assoc-ref options 'format) + ('normalized + (daemon-sessions->normalized-record port (daemon-sessions))) + (_ (daemon-sessions->recutils port (daemon-sessions)))) ;; Pass 'R' (instead of 'r') so 'less' correctly estimates line length. #:less-options "FRX")) diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index 2a2185e2b9..5a865c838d 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -42,6 +42,7 @@ #:use-module (web server) #:use-module (web uri) #:autoload (sxml simple) (sxml->xml) + #:autoload (guix avahi) (avahi-publish-service-thread) #:use-module (guix base32) #:use-module (guix base64) #:use-module (guix config) @@ -70,6 +71,7 @@ signed-string open-server-socket + publish-service-type run-publish-server guix-publish)) @@ -83,6 +85,8 @@ Publish ~a over HTTP.\n") %store-directory) (display (G_ " -u, --user=USER change privileges to USER as soon as possible")) (display (G_ " + -a, --advertise advertise on the local network")) + (display (G_ " -C, --compression[=METHOD:LEVEL] compress archives with METHOD at LEVEL")) (display (G_ " @@ -157,6 +161,9 @@ usage." (option '(#\V "version") #f #f (lambda _ (show-version-and-exit "guix publish"))) + (option '(#\a "advertise") #f #f + (lambda (opt name arg result) + (alist-cons 'advertise? #t result))) (option '(#\u "user") #t #f (lambda (opt name arg result) (alist-cons 'user arg result))) @@ -817,32 +824,6 @@ example: \"/foo/bar\" yields '(\"foo\" \"bar\")." (define %http-write (@@ (web server http) http-write)) -(match (list (major-version) (minor-version) (micro-version)) - (("2" "2" "5") ;Guile 2.2.5 - (let () - (define %read-line (@ (ice-9 rdelim) %read-line)) - (define bad-header (@@ (web http) bad-header)) - - ;; XXX: Work around <https://bugs.gnu.org/36350> by reverting to the - ;; definition of 'read-header-line' as found in 2.2.4 and earlier. - (define (read-header-line port) - "Read an HTTP header line and return it without its final CRLF or LF. -Raise a 'bad-header' exception if the line does not end in CRLF or LF, -or if EOF is reached." - (match (%read-line port) - (((? string? line) . #\newline) - ;; '%read-line' does not consider #\return a delimiter; so if it's - ;; there, remove it. We are more tolerant than the RFC in that we - ;; tolerate LF-only endings. - (if (string-suffix? "\r" line) - (string-drop-right line 1) - line)) - ((line . _) ;EOF or missing delimiter - (bad-header 'read-header-line line)))) - - (set! (@@ (web http) read-header-line) read-header-line))) - (_ #t)) - (define (strip-headers response) "Return RESPONSE's headers minus 'Content-Length' and our internal headers." (fold alist-delete @@ -1069,11 +1050,29 @@ methods, return the applicable compression." (x (not-found request))) (not-found request)))) +(define (service-name) + "Return the Avahi service name of the server." + (string-append "guix-publish-" (gethostname))) + +(define publish-service-type + ;; Return the Avahi service type of the server. + "_guix_publish._tcp") + (define* (run-publish-server socket store #:key + advertise? port (compressions (list %no-compression)) (nar-path "nar") narinfo-ttl cache pool) + (when advertise? + (let ((name (service-name))) + ;; XXX: Use a callback from Guile-Avahi here, as Avahi can pick a + ;; different name to avoid name clashes. + (info (G_ "Advertising ~a~%.") name) + (avahi-publish-service-thread name + #:type publish-service-type + #:port port))) + (run-server (make-request-handler store #:cache cache #:pool pool @@ -1119,9 +1118,10 @@ methods, return the applicable compression." (lambda (arg result) (leave (G_ "~A: extraneous argument~%") arg)) %default-options)) - (user (assoc-ref opts 'user)) - (port (assoc-ref opts 'port)) - (ttl (assoc-ref opts 'narinfo-ttl)) + (advertise? (assoc-ref opts 'advertise?)) + (user (assoc-ref opts 'user)) + (port (assoc-ref opts 'port)) + (ttl (assoc-ref opts 'narinfo-ttl)) (compressions (match (filter-map (match-lambda (('compression . compression) compression) @@ -1179,6 +1179,8 @@ consider using the '--user' option!~%"))) (with-store store (run-publish-server socket store + #:advertise? advertise? + #:port port #:cache cache #:pool (and cache (make-pool workers #:thread-name diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 7fd8b3f1a4..83cdc1d1eb 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2013, 2014, 2015, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com> +;;; Copyright © 2020 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -754,10 +755,14 @@ Use '~/.config/guix/channels.scm' instead.")) (define-command (guix-pull . args) (synopsis "pull the latest revision of Guix") + (define (no-arguments arg _) + (leave (G_ "~A: extraneous argument~%") arg)) + (with-error-handling (with-git-error-handling (let* ((opts (parse-command-line args %options - (list %default-options))) + (list %default-options) + #:argument-handler no-arguments)) (substitutes? (assoc-ref opts 'substitutes?)) (dry-run? (assoc-ref opts 'dry-run?)) (channels (channel-list opts)) diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index ddb885d344..e53de8c304 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -26,7 +26,11 @@ #:use-module (guix combinators) #:use-module (guix config) #:use-module (guix records) - #:use-module ((guix serialization) #:select (restore-file)) + #:use-module (guix diagnostics) + #:use-module (guix i18n) + #:use-module ((guix serialization) #:select (restore-file dump-file)) + #:autoload (guix store deduplication) (dump-file/deduplicate) + #:autoload (guix scripts discover) (read-substitute-urls) #:use-module (gcrypt hash) #:use-module (guix base32) #:use-module (guix base64) @@ -39,6 +43,7 @@ (open-connection-for-uri . guix:open-connection-for-uri) store-path-abbreviation byte-count->string)) + #:autoload (gnutls) (error/invalid-session) #:use-module (guix progress) #:use-module ((guix build syscalls) #:select (set-thread-name)) @@ -87,6 +92,7 @@ write-narinfo %allow-unauthenticated-substitutes? + %error-to-file-descriptor-4? substitute-urls guix-substitute)) @@ -123,11 +129,7 @@ disabled!~%")) ;; purposes, and should be avoided otherwise. (make-parameter (and=> (getenv "GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES") - (cut string-ci=? <> "yes")) - (lambda (value) - (when value - (warn-about-missing-authentication)) - value))) + (cut string-ci=? <> "yes")))) (define %narinfo-ttl ;; Number of seconds during which cached narinfo lookups are considered @@ -190,9 +192,14 @@ again." (sigaction SIGALRM SIG_DFL) (apply values result))))) -(define* (fetch uri #:key (buffered? #t) (timeout? #t)) +(define* (fetch uri #:key (buffered? #t) (timeout? #t) + (keep-alive? #f) (port #f)) "Return a binary input port to URI and the number of bytes it's expected to -provide." +provide. + +When PORT is true, use it as the underlying I/O port for HTTP transfers; when +PORT is false, open a new connection for URI. When KEEP-ALIVE? is true, the +connection (typically PORT) is kept open once data has been fetched from URI." (case (uri-scheme uri) ((file) (let ((port (open-file (uri-path uri) @@ -208,7 +215,7 @@ provide." ;; sudo tc qdisc add dev eth0 root netem delay 1500ms ;; and then cancel with: ;; sudo tc qdisc del dev eth0 root - (let ((port #f)) + (let ((port port)) (with-timeout (if timeout? %fetch-timeout 0) @@ -219,10 +226,11 @@ provide." (begin (when (or (not port) (port-closed? port)) (set! port (guix:open-connection-for-uri - uri #:verify-certificate? #f)) - (unless (or buffered? (not (file-port? port))) - (setvbuf port 'none))) + uri #:verify-certificate? #f))) + (unless (or buffered? (not (file-port? port))) + (setvbuf port 'none)) (http-fetch uri #:text? #f #:port port + #:keep-alive? keep-alive? #:verify-certificate? #f)))))) (else (leave (G_ "unsupported substitute URI scheme: ~a~%") @@ -252,6 +260,18 @@ provide." ;; for more information. (contents narinfo-contents)) +(define (narinfo-hash-algorithm+value narinfo) + "Return two values: the hash algorithm used by NARINFO and its value as a +bytevector." + (match (string-tokenize (narinfo-hash narinfo) + (char-set-complement (char-set #\:))) + ((algorithm base32) + (values (lookup-hash-algorithm (string->symbol algorithm)) + (nix-base32-string->bytevector base32))) + (_ + (raise (formatted-message + (G_ "invalid narinfo hash: ~s") (narinfo-hash narinfo)))))) + (define (narinfo-hash->sha256 hash) "If the string HASH denotes a sha256 hash, return it as a bytevector. Otherwise return #f." @@ -480,27 +500,33 @@ indicates that PATH is unavailable at CACHE-URL." (build-request (string->uri url) #:method 'GET #:headers headers))) (define (at-most max-length lst) - "If LST is shorter than MAX-LENGTH, return it; otherwise return its -MAX-LENGTH first elements." + "If LST is shorter than MAX-LENGTH, return it and the empty list; otherwise +return its MAX-LENGTH first elements and its tail." (let loop ((len 0) (lst lst) (result '())) (match lst (() - (reverse result)) + (values (reverse result) '())) ((head . tail) (if (>= len max-length) - (reverse result) + (values (reverse result) lst) (loop (+ 1 len) tail (cons head result))))))) (define* (http-multiple-get base-uri proc seed requests #:key port (verify-certificate? #t) + (open-connection guix:open-connection-for-uri) + (keep-alive? #t) (batch-size 1000)) "Send all of REQUESTS to the server at BASE-URI. Call PROC for each response, passing it the request object, the response, a port from which to read the response body, and the previous result, starting with SEED, à la -'fold'. Return the final result. When PORT is specified, use it as the -initial connection on which HTTP requests are sent." +'fold'. Return the final result. + +When PORT is specified, use it as the initial connection on which HTTP +requests are sent; otherwise call OPEN-CONNECTION to open a new connection for +a URI. When KEEP-ALIVE? is false, close the connection port before +returning." (let connect ((port port) (requests requests) (result seed)) @@ -509,10 +535,9 @@ initial connection on which HTTP requests are sent." ;; (format (current-error-port) "connecting (~a requests left)..." ;; (length requests)) - (let ((p (or port (guix:open-connection-for-uri - base-uri - #:verify-certificate? - verify-certificate?)))) + (let ((p (or port (open-connection base-uri + #:verify-certificate? + verify-certificate?)))) ;; For HTTPS, P is not a file port and does not support 'setvbuf'. (when (file-port? p) (setvbuf p 'block (expt 2 16))) @@ -537,7 +562,8 @@ initial connection on which HTTP requests are sent." (() (match (drop requests processed) (() - (close-port p) + (unless keep-alive? + (close-port p)) (reverse result)) (remainder (connect p remainder result)))) @@ -579,18 +605,18 @@ if file doesn't exist, and the narinfo otherwise." (define* (open-connection-for-uri/maybe uri #:key - (verify-certificate? #f) + fresh? (time %fetch-timeout)) - "Open a connection to URI and return a port to it, or, if connection failed, -print a warning and return #f." + "Open a connection to URI via 'open-connection-for-uri/cached' and return a +port to it, or, if connection failed, print a warning and return #f. Pass +#:fresh? to 'open-connection-for-uri/cached'." (define host (uri-host uri)) (catch #t (lambda () - (guix:open-connection-for-uri uri - #:verify-certificate? verify-certificate? - #:timeout time)) + (open-connection-for-uri/cached uri #:timeout time + #:fresh? fresh?)) (match-lambda* (('getaddrinfo-error error) (unless (hash-ref %unreachable-hosts host) @@ -664,23 +690,26 @@ print a warning and return #f." (define (do-fetch uri) (case (and=> uri uri-scheme) ((http https) - (let ((requests (map (cut narinfo-request url <>) paths))) - (match (open-connection-for-uri/maybe uri) - (#f - '()) - (port - (update-progress!) - ;; Note: Do not check HTTPS server certificates to avoid depending - ;; on the X.509 PKI. We can do it because we authenticate - ;; narinfos, which provides a much stronger guarantee. - (let ((result (http-multiple-get uri - handle-narinfo-response '() - requests - #:verify-certificate? #f - #:port port))) - (close-port port) - (newline (current-error-port)) - result))))) + ;; Note: Do not check HTTPS server certificates to avoid depending + ;; on the X.509 PKI. We can do it because we authenticate + ;; narinfos, which provides a much stronger guarantee. + (let* ((requests (map (cut narinfo-request url <>) paths)) + (result (call-with-cached-connection uri + (lambda (port) + (if port + (begin + (update-progress!) + (http-multiple-get uri + handle-narinfo-response '() + requests + #:open-connection + open-connection-for-uri/cached + #:verify-certificate? #f + #:port port)) + '())) + open-connection-for-uri/maybe))) + (newline (current-error-port)) + result)) ((file #f) (let* ((base (string-append (uri-path uri) "/")) (files (map (compose (cut string-append base <> ".narinfo") @@ -892,6 +921,9 @@ authorized substitutes." (define (valid? obj) (valid-narinfo? obj acl)) + (when (%allow-unauthenticated-substitutes?) + (warn-about-missing-authentication)) + (match (string-tokenize command) (("have" paths ..1) ;; Return the subset of PATHS available in CACHE-URLS. @@ -961,32 +993,121 @@ the URI, its compression method (a string), and the compressed file size." (((uri compression file-size) _ ...) (values uri compression file-size)))) +(define %max-cached-connections + ;; Maximum number of connections kept in cache by + ;; 'open-connection-for-uri/cached'. + 16) + +(define open-connection-for-uri/cached + (let ((cache '())) + (lambda* (uri #:key fresh? timeout verify-certificate?) + "Return a connection for URI, possibly reusing a cached connection. +When FRESH? is true, delete any cached connections for URI and open a new one. +Return #f if URI's scheme is 'file' or #f. + +When true, TIMEOUT is the maximum number of milliseconds to wait for +connection establishment. When VERIFY-CERTIFICATE? is true, verify HTTPS +server certificates." + (define host (uri-host uri)) + (define scheme (uri-scheme uri)) + (define key (list host scheme (uri-port uri))) + + (and (not (memq scheme '(file #f))) + (match (assoc-ref cache key) + (#f + ;; Open a new connection to URI and evict old entries from + ;; CACHE, if any. + (let-values (((socket) + (guix:open-connection-for-uri + uri + #:verify-certificate? verify-certificate? + #:timeout timeout)) + ((new-cache evicted) + (at-most (- %max-cached-connections 1) cache))) + (for-each (match-lambda + ((_ . port) + (false-if-exception (close-port port)))) + evicted) + (set! cache (alist-cons key socket new-cache)) + socket)) + (socket + (if (or fresh? (port-closed? socket)) + (begin + (false-if-exception (close-port socket)) + (set! cache (alist-delete key cache)) + (open-connection-for-uri/cached uri #:timeout timeout + #:verify-certificate? + verify-certificate?)) + (begin + ;; Drain input left from the previous use. + (drain-input socket) + socket)))))))) + +(define* (call-with-cached-connection uri proc + #:optional + (open-connection + open-connection-for-uri/cached)) + (let ((port (open-connection uri))) + (catch #t + (lambda () + (proc port)) + (lambda (key . args) + ;; If PORT was cached and the server closed the connection in the + ;; meantime, we get EPIPE. In that case, open a fresh connection and + ;; retry. We might also get 'bad-response or a similar exception from + ;; (web response) later on, once we've sent the request, or a + ;; ERROR/INVALID-SESSION from GnuTLS. + (if (or (and (eq? key 'system-error) + (= EPIPE (system-error-errno `(,key ,@args)))) + (and (eq? key 'gnutls-error) + (eq? (first args) error/invalid-session)) + (memq key '(bad-response bad-header bad-header-component))) + (proc (open-connection uri #:fresh? #t)) + (apply throw key args)))))) + +(define-syntax-rule (with-cached-connection uri port exp ...) + "Bind PORT with EXP... to a socket connected to URI." + (call-with-cached-connection uri (lambda (port) exp ...))) + (define* (process-substitution store-item destination - #:key cache-urls acl print-build-trace?) + #:key cache-urls acl + deduplicate? print-build-trace?) "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to -DESTINATION as a nar file. Verify the substitute against ACL." +DESTINATION as a nar file. Verify the substitute against ACL, and verify its +hash against what appears in the narinfo. When DEDUPLICATE? is true, and if +DESTINATION is in the store, deduplicate its files. Print a status line on +the current output port." (define narinfo (lookup-narinfo cache-urls store-item (cut valid-narinfo? <> acl))) + (define destination-in-store? + (string-prefix? (string-append (%store-prefix) "/") + destination)) + + (define (dump-file/deduplicate* . args) + ;; Make sure deduplication looks at the right store (necessary in test + ;; environments). + (apply dump-file/deduplicate + (append args (list #:store (%store-prefix))))) + (unless narinfo (leave (G_ "no valid substitute for '~a'~%") store-item)) (let-values (((uri compression file-size) (narinfo-best-uri narinfo))) - ;; Tell the daemon what the expected hash of the Nar itself is. - (format #t "~a~%" (narinfo-hash narinfo)) - (unless print-build-trace? (format (current-error-port) (G_ "Downloading ~a...~%") (uri->string uri))) (let*-values (((raw download-size) - ;; Note that Hydra currently generates Nars on the fly - ;; and doesn't specify a Content-Length, so - ;; DOWNLOAD-SIZE is #f in practice. - (fetch uri #:buffered? #f #:timeout? #f)) + ;; 'guix publish' without '--cache' doesn't specify a + ;; Content-Length, so DOWNLOAD-SIZE is #f in this case. + (with-cached-connection uri port + (fetch uri #:buffered? #f #:timeout? #f + #:port port + #:keep-alive? #t))) ((progress) (let* ((dl-size (or download-size (and (equal? compression "none") @@ -1000,15 +1121,28 @@ DESTINATION as a nar file. Verify the substitute against ACL." (uri->string uri) dl-size (current-error-port) #:abbreviation nar-uri-abbreviation)))) - (progress-report-port reporter raw))) + ;; Keep RAW open upon completion so we can later reuse + ;; the underlying connection. + (progress-report-port reporter raw #:close? #f))) ((input pids) ;; NOTE: This 'progress' port of current process will be ;; closed here, while the child process doing the ;; reporting will close it upon exit. (decompressed-port (string->symbol compression) - progress))) + progress)) + + ;; Compute the actual nar hash as we read it. + ((algorithm expected) + (narinfo-hash-algorithm+value narinfo)) + ((hashed get-hash) + (open-hash-input-port algorithm input))) ;; Unpack the Nar at INPUT into DESTINATION. - (restore-file input destination) + (restore-file hashed destination + #:dump-file (if (and destination-in-store? + deduplicate?) + dump-file/deduplicate* + dump-file)) + (close-port hashed) (close-port input) ;; Wait for the reporter to finish. @@ -1016,7 +1150,19 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; Skip a line after what 'progress-reporter/file' printed, and another ;; one to visually separate substitutions. - (display "\n\n" (current-error-port))))) + (display "\n\n" (current-error-port)) + + ;; Check whether we got the data announced in NARINFO. + (let ((actual (get-hash))) + (if (bytevector=? actual expected) + ;; Tell the daemon that we're done. + (format (current-output-port) "success ~a ~a~%" + (narinfo-hash narinfo) (narinfo-size narinfo)) + ;; The actual data has a different hash than that in NARINFO. + (format (current-output-port) "hash-mismatch ~a ~a ~a~%" + (hash-algorithm-name algorithm) + (bytevector->nix-base32-string expected) + (bytevector->nix-base32-string actual))))))) ;;; @@ -1078,9 +1224,40 @@ found." ;; daemon. '("http://ci.guix.gnu.org")))) +;; In order to prevent using large number of discovered local substitute +;; servers, limit the local substitute urls list size. +(define %max-substitute-urls 50) + +(define* (randomize-substitute-urls urls + #:key + (max %max-substitute-urls)) + "Return a list containing MAX urls from URLS, picked randomly. If URLS list +is shorter than MAX elements, then it is directly returned." + (define (random-item list) + (list-ref list (random (length list)))) + + (if (<= (length urls) max) + urls + (let loop ((res '()) + (urls urls)) + (if (eq? (length res) max) + res + (let ((url (random-item urls))) + (loop (cons url res) (delete url urls))))))) + +(define %local-substitute-urls + ;; If the following option is passed to the daemon, use the substitutes list + ;; provided by "guix discover" process. + (let* ((option (find-daemon-option "discover")) + (discover? (and option (string=? option "yes")))) + (if discover? + (randomize-substitute-urls (read-substitute-urls)) + '()))) + (define substitute-urls ;; List of substitute URLs. - (make-parameter %default-substitute-urls)) + (make-parameter (append %local-substitute-urls + %default-substitute-urls))) (define (client-terminal-columns) "Return the number of columns in the client's terminal, if it is known, or a @@ -1096,6 +1273,11 @@ default value." (unless (string->uri uri) (leave (G_ "~a: invalid URI~%") uri))) +(define %error-to-file-descriptor-4? + ;; Whether to direct 'current-error-port' to file descriptor 4 like + ;; 'guix-daemon' expects. + (make-parameter #t)) + (define-command (guix-substitute . args) (category internal) (synopsis "implement the build daemon's substituter protocol") @@ -1107,71 +1289,83 @@ default value." ((= string->number number) (> number 0)) (_ #f))) - (mkdir-p %narinfo-cache-directory) - (maybe-remove-expired-cache-entries %narinfo-cache-directory - cached-narinfo-files - #:entry-expiration - cached-narinfo-expiration-time - #:cleanup-period - %narinfo-expired-cache-entry-removal-delay) - (check-acl-initialized) - - ;; Starting from commit 22144afa in Nix, we are allowed to bail out directly - ;; when we know we cannot substitute, but we must emit a newline on stdout - ;; when everything is alright. - (when (null? (substitute-urls)) - (exit 0)) - - ;; Say hello (see above.) - (newline) - (force-output (current-output-port)) - - ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error message. - (for-each validate-uri (substitute-urls)) - - ;; Attempt to install the client's locale so that messages are suitably - ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default so - ;; don't change it. - (match (or (find-daemon-option "untrusted-locale") - (find-daemon-option "locale")) - (#f #f) - (locale (false-if-exception (setlocale LC_MESSAGES locale)))) - - (catch 'system-error - (lambda () - (set-thread-name "guix substitute")) - (const #t)) ;GNU/Hurd lacks 'prctl' - - (with-networking - (with-error-handling ; for signature errors - (match args - (("--query") - (let ((acl (current-acl))) - (let loop ((command (read-line))) - (or (eof-object? command) - (begin - (process-query command - #:cache-urls (substitute-urls) - #:acl acl) - (loop (read-line))))))) - (("--substitute" store-path destination) - ;; Download STORE-PATH and add store it as a Nar in file DESTINATION. - ;; Specify the number of columns of the terminal so the progress - ;; report displays nicely. - (parameterize ((current-terminal-columns (client-terminal-columns))) - (process-substitution store-path destination - #:cache-urls (substitute-urls) - #:acl (current-acl) - #:print-build-trace? print-build-trace?))) - ((or ("-V") ("--version")) - (show-version-and-exit "guix substitute")) - (("--help") - (show-help)) - (opts - (leave (G_ "~a: unrecognized options~%") opts)))))) + (define deduplicate? + (find-daemon-option "deduplicate")) + + ;; The daemon's agent code opens file descriptor 4 for us and this is where + ;; stderr should go. + (parameterize ((current-error-port (if (%error-to-file-descriptor-4?) + (fdopen 4 "wl") + (current-error-port)))) + ;; Redirect diagnostics to file descriptor 4 as well. + (guix-warning-port (current-error-port)) + + (mkdir-p %narinfo-cache-directory) + (maybe-remove-expired-cache-entries %narinfo-cache-directory + cached-narinfo-files + #:entry-expiration + cached-narinfo-expiration-time + #:cleanup-period + %narinfo-expired-cache-entry-removal-delay) + (check-acl-initialized) + + ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error + ;; message. + (for-each validate-uri (substitute-urls)) + + ;; Attempt to install the client's locale so that messages are suitably + ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default + ;; so don't change it. + (match (or (find-daemon-option "untrusted-locale") + (find-daemon-option "locale")) + (#f #f) + (locale (false-if-exception (setlocale LC_MESSAGES locale)))) + + (catch 'system-error + (lambda () + (set-thread-name "guix substitute")) + (const #t)) ;GNU/Hurd lacks 'prctl' + + (with-networking + (with-error-handling ; for signature errors + (match args + (("--query") + (let ((acl (current-acl))) + (let loop ((command (read-line))) + (or (eof-object? command) + (begin + (process-query command + #:cache-urls (substitute-urls) + #:acl acl) + (loop (read-line))))))) + (("--substitute") + ;; Download STORE-PATH and store it as a Nar in file DESTINATION. + ;; Specify the number of columns of the terminal so the progress + ;; report displays nicely. + (parameterize ((current-terminal-columns (client-terminal-columns))) + (let loop () + (match (read-line) + ((? eof-object?) + #t) + ((= string-tokenize ("substitute" store-path destination)) + (process-substitution store-path destination + #:cache-urls (substitute-urls) + #:acl (current-acl) + #:deduplicate? deduplicate? + #:print-build-trace? + print-build-trace?) + (loop)))))) + ((or ("-V") ("--version")) + (show-version-and-exit "guix substitute")) + (("--help") + (show-help)) + (opts + (leave (G_ "~a: unrecognized options~%") opts))))))) ;;; Local Variables: ;;; eval: (put 'with-timeout 'scheme-indent-function 1) +;;; eval: (put 'with-cached-connection 'scheme-indent-function 2) +;;; eval: (put 'call-with-cached-connection 'scheme-indent-function 1) ;;; End: ;;; substitute.scm ends here diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index db80e0be8f..51c8cf2f76 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2016 Alex Kost <alezost@gmail.com> ;;; Copyright © 2016, 2017, 2018 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2017, 2019 Mathieu Othacehe <m.othacehe@gmail.com> @@ -29,7 +29,10 @@ #:use-module (guix ui) #:use-module ((guix status) #:select (with-status-verbosity)) #:use-module (guix store) - #:autoload (guix store database) (register-path) + #:autoload (guix base16) (bytevector->base16-string) + #:autoload (guix store database) + (sqlite-register store-database-file call-with-database) + #:autoload (guix build store-copy) (copy-store-item) #:use-module (guix describe) #:use-module (guix grafts) #:use-module (guix gexp) @@ -45,7 +48,8 @@ #:autoload (guix scripts package) (delete-generations delete-matching-generations) #:autoload (guix scripts pull) (channel-commit-hyperlink) - #:use-module (guix graph) + #:autoload (guix graph) (export-graph node-type + graph-backend-name %graph-backends) #:use-module (guix scripts graph) #:use-module (guix scripts system reconfigure) #:use-module (guix build utils) @@ -129,12 +133,11 @@ BODY..., and restore them." (store-lift topologically-sorted)) -(define* (copy-item item references target +(define* (copy-item item info target db #:key (log-port (current-error-port))) - "Copy ITEM to the store under root directory TARGET and register it with -REFERENCES as its set of references." - (let ((dest (string-append target item)) - (state (string-append target "/var/guix"))) + "Copy ITEM to the store under root directory TARGET and populate DB with the +given INFO, a <path-info> record." + (let ((dest (string-append target item))) (format log-port "copying '~a'...~%" item) ;; Remove DEST if it exists to make sure that (1) we do not fail badly @@ -147,44 +150,48 @@ REFERENCES as its set of references." #:directories? #t)) (delete-file-recursively dest)) - (copy-recursively item dest - #:log (%make-void-port "w")) + (copy-store-item item target + #:deduplicate? #t) - ;; Register ITEM; as a side-effect, it resets timestamps, etc. - ;; Explicitly use "TARGET/var/guix" as the state directory, to avoid - ;; reproducing the user's current settings; see - ;; <http://bugs.gnu.org/18049>. - (unless (register-path item - #:prefix target - #:state-directory state - #:references references) - (leave (G_ "failed to register '~a' under '~a'~%") - item target)))) + (sqlite-register db + #:path item + #:references (path-info-references info) + #:deriver (path-info-deriver info) + #:hash (string-append + "sha256:" + (bytevector->base16-string (path-info-hash info))) + #:nar-size (path-info-nar-size info)))) (define* (copy-closure item target #:key (log-port (current-error-port))) "Copy ITEM and all its dependencies to the store under root directory TARGET, and register them." (mlet* %store-monad ((to-copy (topologically-sorted* (list item))) - (refs (mapm %store-monad references* to-copy)) - (info (mapm %store-monad query-path-info* - (delete-duplicates - (append to-copy (concatenate refs))))) + (info (mapm %store-monad query-path-info* to-copy)) (size -> (reduce + 0 (map path-info-nar-size info)))) (define progress-bar (progress-reporter/bar (length to-copy) (format #f (G_ "copying to '~a'...") target))) + (define state + (string-append target "/var/guix")) + (check-available-space size target) - (call-with-progress-reporter progress-bar - (lambda (report) - (let ((void (%make-void-port "w"))) - (for-each (lambda (item refs) - (copy-item item refs target #:log-port void) - (report)) - to-copy refs)))) + ;; Explicitly use "TARGET/var/guix" as the state directory to avoid + ;; reproducing the user's current settings; see + ;; <http://bugs.gnu.org/18049>. + (call-with-database (store-database-file #:prefix target + #:state-directory state) + (lambda (db) + (call-with-progress-reporter progress-bar + (lambda (report) + (let ((void (%make-void-port "w"))) + (for-each (lambda (item info) + (copy-item item info target db #:log-port void) + (report)) + to-copy info)))))) (return *unspecified*))) @@ -385,6 +392,7 @@ STORE is an open connection to the store." (params (first (profile-boot-parameters %system-profile (list number)))) (locale (boot-parameters-locale params)) + (store-crypto-devices (boot-parameters-store-crypto-devices params)) (store-directory-prefix (boot-parameters-store-directory-prefix params)) (old-generations @@ -400,6 +408,7 @@ STORE is an open connection to the store." ((bootloader-configuration-file-generator bootloader) bootloader-config entries #:locale locale + #:store-crypto-devices store-crypto-devices #:store-directory-prefix store-directory-prefix #:old-entries old-entries))) (drvs -> (list bootcfg))) @@ -879,18 +888,28 @@ Run 'herd status' to view the list of services on your system.\n")))))) (register-root* (list output) gc-root)) (return output))))))))) -(define (export-extension-graph os port) - "Export the service extension graph of OS to PORT." +(define (lookup-backend name) ;TODO: factorize + "Return the graph backend called NAME. Raise an error if it is not found." + (or (find (lambda (backend) + (string=? (graph-backend-name backend) name)) + %graph-backends) + (leave (G_ "~a: unknown backend~%") name))) + +(define* (export-extension-graph os port + #:key (backend (lookup-backend "graphviz"))) + "Export the service extension graph of OS to PORT using BACKEND." (let* ((services (operating-system-services os)) (system (find (lambda (service) (eq? (service-kind service) system-service-type)) services))) (export-graph (list system) (current-output-port) + #:backend backend #:node-type (service-node-type services) #:reverse-edges? #t))) -(define (export-shepherd-graph os port) - "Export the graph of shepherd services of OS to PORT." +(define* (export-shepherd-graph os port + #:key (backend (lookup-backend "graphviz"))) + "Export the graph of shepherd services of OS to PORT using BACKEND." (let* ((services (operating-system-services os)) (pid1 (fold-services services #:target-type shepherd-root-service-type)) @@ -899,6 +918,7 @@ Run 'herd status' to view the list of services on your system.\n")))))) (null? (shepherd-service-requirement service))) shepherds))) (export-graph sinks (current-output-port) + #:backend backend #:node-type (shepherd-service-node-type shepherds) #:reverse-edges? #t))) @@ -1007,6 +1027,10 @@ Some ACTIONS support additional ARGS.\n")) -v, --verbosity=LEVEL use the given verbosity LEVEL")) (newline) (display (G_ " + --graph-backend=BACKEND + use BACKEND for 'extension-graphs' and 'shepherd-graph'")) + (newline) + (display (G_ " -h, --help display this help and exit")) (display (G_ " -V, --version display version information and exit")) @@ -1101,6 +1125,9 @@ Some ACTIONS support additional ARGS.\n")) (option '(#\r "root") #t #f (lambda (opt name arg result) (alist-cons 'gc-root arg result))) + (option '("graph-backend") #t #f + (lambda (opt name arg result) + (alist-cons 'graph-backend arg result))) %standard-build-options)) (define %default-options @@ -1120,7 +1147,8 @@ Some ACTIONS support additional ARGS.\n")) (image-size . guess) (install-bootloader? . #t) (label . #f) - (volatile-root? . #f))) + (volatile-root? . #f) + (graph-backend . "graphviz"))) (define (verbosity-level opts) "Return the verbosity level based on OPTS, the alist of parsed options." @@ -1183,6 +1211,9 @@ resulting from command-line parsing." (bootloader-configuration-target (operating-system-bootloader os))))) + (define (graph-backend) + (lookup-backend (assoc-ref opts 'graph-backend))) + (with-store store (set-build-options-from-command-line store opts) @@ -1197,9 +1228,11 @@ resulting from command-line parsing." (set-guile-for-build (default-guile)) (case action ((extension-graph) - (export-extension-graph os (current-output-port))) + (export-extension-graph os (current-output-port) + #:backend (graph-backend))) ((shepherd-graph) - (export-shepherd-graph os (current-output-port))) + (export-shepherd-graph os (current-output-port) + #:backend (graph-backend))) (else (unless (memq action '(build init)) (warn-about-old-distro #:suggested-command diff --git a/guix/scripts/upgrade.scm b/guix/scripts/upgrade.scm index dcbcb2ab09..beb59cbe6f 100644 --- a/guix/scripts/upgrade.scm +++ b/guix/scripts/upgrade.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net> +;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -74,15 +75,10 @@ This is an alias for 'guix package -u'.\n")) (synopsis "upgrade packages to their latest version") (define (handle-argument arg result arg-handler) - ;; Accept at most one non-option argument, and treat it as an upgrade - ;; regexp. - (match (assq-ref result 'upgrade) - (#f - (values (alist-cons 'upgrade arg - (alist-delete 'upgrade result)) - arg-handler)) - (_ - (leave (G_ "~A: extraneous argument~%") arg)))) + ;; Treat non-option arguments as upgrade regexps. + (values (alist-cons 'upgrade arg + (delete '(upgrade . #f) result)) + arg-handler)) (define opts (parse-command-line args %options |