From bea3b17739fc591b8cf6db1f8d28a6f6c9585577 Mon Sep 17 00:00:00 2001 From: Martin Becze Date: Tue, 4 Feb 2020 07:18:18 -0500 Subject: import: utils: 'recursive-import' accepts an optional version parameter. This adds a key VERSION to 'recursive-import' and moves the parameter REPO to a key. This also changes all the places that rely on 'recursive-import'. * guix/import/utils.scm (recursive-import): Add the VERSION key. Make REPO a key. (package->definition): Add optional 'append-version?'. * guix/scripts/import/crate.scm (guix-import-crate): Add the VERSION key. * guix/import/crate.scm (crate->guix-package): Add the VERSION key. (crate-recursive-import): Pass VERSION to recursive-import, remove now unnecessary code. * guix/import/cran.scm (cran->guix-package, cran-recursive-import): Change the REPO parameter to a key. * guix/import/elpa.scm (elpa->guix-package, elpa-recursive-import): Likewise. * guix/import/gem.scm (gem->guix-package, recursive-import): Likewise. * guix/import/opam.scm (opam-recurive-import): Likewise. * guix/import/pypi.scm (pypi-recursive-import): Likewise. * guix/import/stackage.scm (stackage-recursive-import): Likewise. * guix/scripts/import/cran.scm (guix-import-cran): Likewise. * guix/scripts/import/elpa.scm (guix-import-elpa): Likewise. * tests/elpa.scm (eval-test-with-elpa): Likewise. * tests/import-utils.scm (recursive-import): Likewise. Co-authored-by: Hartmut Goebel --- tests/elpa.scm | 3 ++- tests/import-utils.scm | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/elpa.scm b/tests/elpa.scm index b70539bda6..a008cf993c 100644 --- a/tests/elpa.scm +++ b/tests/elpa.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 Federico Beffa ;;; Copyright © 2020 Ludovic Courtès +;;; Copyright © 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -51,7 +52,7 @@ (200 "This is the description.") (200 "fake tarball contents")) (parameterize ((current-http-proxy (%local-url))) - (match (elpa->guix-package pkg 'gnu/http) + (match (elpa->guix-package pkg #:repo 'gnu/http) (('package ('name "emacs-auctex") ('version "11.88.6") diff --git a/tests/import-utils.scm b/tests/import-utils.scm index 87dda3238f..2357ea5c40 100644 --- a/tests/import-utils.scm +++ b/tests/import-utils.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015, 2017 Ricardo Wurmus ;;; Copyright © 2016 Ben Woodcroft +;;; Copyright © 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -48,15 +49,16 @@ (package (name "foo") (inputs `(("bar" ,bar))))) - (recursive-import "foo" 'repo + (recursive-import "foo" + #:repo 'repo #:repo->guix-package (match-lambda* - (("foo" 'repo) + (("foo" #:version #f #:repo 'repo) (values '(package (name "foo") (inputs `(("bar" ,bar)))) '("bar"))) - (("bar" 'repo) + (("bar" #:version #f #:repo 'repo) (values '(package (name "bar")) '()))) -- cgit v1.2.3 From 269c1db41bd82f93c7ae5c62a4969a423e556183 Mon Sep 17 00:00:00 2001 From: Martin Becze Date: Tue, 4 Feb 2020 03:50:48 -0500 Subject: import: crate: Use guile-semver to resolve module versions. * guix/import/crate.scm: Add guile-semver as a soft dependency. (make-crate-sexp): Don't allow other keys. Add '#:skip-build?' to build system args. Pass a VERSION argument to 'cargo-inputs'. (crate->guix-package): Use guile-semver to resolve the correct module versions. Treat "build" dependencies as normal dependencies. (crate-name->package-name): Reuse the procedure 'guix-name' instead of duplicating its logic. * guix/import/utils.scm (package-names->package-inputs): Implement handling of (name version) pairs. * guix/scripts/import/crate.scm (guix-import-crate): Use crate-recursive-import instead of duplicate code. * tests/crate.scm (recursive-import): Change test packages versions to be distinguishable. Add version data to the test. Check created symbols, too. Co-authored-by: Hartmut Goebel --- guix/import/crate.scm | 91 +++++--- guix/import/utils.scm | 21 +- guix/scripts/import/crate.scm | 11 +- tests/crate.scm | 500 ++++++++++++++++++++++++++---------------- 4 files changed, 391 insertions(+), 232 deletions(-) (limited to 'tests') diff --git a/guix/import/crate.scm b/guix/import/crate.scm index 47bfc16105..5498d1f0ff 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016 David Craven ;;; Copyright © 2019, 2020 Ludovic Courtès -;;; Copyright © 2019 Martin Becze +;;; Copyright © 2019, 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -37,6 +37,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-2) #:use-module (srfi srfi-26) + #:use-module (srfi srfi-71) #:export (crate->guix-package guix-package->crate-name string->license @@ -85,10 +86,15 @@ crate-dependency? json->crate-dependency (id crate-dependency-id "crate_id") ;string - (kind crate-dependency-kind "kind" ;'normal | 'dev + (kind crate-dependency-kind "kind" ;'normal | 'dev | 'build string->symbol) (requirement crate-dependency-requirement "req")) ;string +(module-autoload! (current-module) + '(semver) '(string->semver semversemver-range semver-range-contains?)) + (define (lookup-crate name) "Look up NAME on https://crates.io and return the corresopnding record or #f if it was not found." @@ -142,16 +148,21 @@ record or #f if it was not found." `((arguments (,'quasiquote ,args)))))) (define* (make-crate-sexp #:key name version cargo-inputs cargo-development-inputs - home-page synopsis description license - #:allow-other-keys) + home-page synopsis description license) "Return the `package' s-expression for a rust package with the given NAME, VERSION, CARGO-INPUTS, CARGO-DEVELOPMENT-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." + (define (format-inputs inputs) + (map + (match-lambda + ((name version) + (list (crate-name->package-name name) version))) + inputs)) + (let* ((port (http-fetch (crate-uri name version))) (guix-name (crate-name->package-name name)) - (cargo-inputs (map crate-name->package-name cargo-inputs)) - (cargo-development-inputs (map crate-name->package-name - cargo-development-inputs)) + (cargo-inputs (format-inputs cargo-inputs)) + (cargo-development-inputs (format-inputs cargo-development-inputs)) (pkg `(package (name ,guix-name) (version ,version) @@ -163,7 +174,8 @@ and LICENSE." (base32 ,(bytevector->nix-base32-string (port-sha256 port)))))) (build-system cargo-build-system) - ,@(maybe-arguments (append (maybe-cargo-inputs cargo-inputs) + ,@(maybe-arguments (append '(#:skip-build? #t) + (maybe-cargo-inputs cargo-inputs) (maybe-cargo-development-inputs cargo-development-inputs))) (home-page ,(match home-page @@ -176,7 +188,7 @@ and LICENSE." ((license) license) (_ `(list ,@license))))))) (close-port port) - pkg)) + (package->definition pkg #t))) (define (string->license string) (filter-map (lambda (license) @@ -190,11 +202,17 @@ and LICENSE." (define* (crate->guix-package crate-name #:key version repo) "Fetch the metadata for CRATE-NAME from crates.io, and return the `package' s-expression corresponding to that package, or #f on failure. -When VERSION is specified, attempt to fetch that version; otherwise fetch the -latest version of CRATE-NAME." +When VERSION is specified, convert it into a semver range and attempt to fetch +the latest version matching this semver range; otherwise fetch the latest +version of CRATE-NAME." + + (define (semver-range-contains-string? range version) + (semver-range-contains? (string->semver-range range) + (string->semver version))) (define (normal-dependency? dependency) - (eq? (crate-dependency-kind dependency) 'normal)) + (or (eq? (crate-dependency-kind dependency) 'build) + (eq? (crate-dependency-kind dependency) 'normal))) (define crate (lookup-crate crate-name)) @@ -204,22 +222,45 @@ latest version of CRATE-NAME." (or version (crate-latest-version crate)))) + ;; find the highest version of a crate that fulfills the semver + (define (find-crate-version crate range) + (let* ((semver-range (string->semver-range range)) + (versions + (sort + (filter (lambda (entry) + (semver-range-contains? semver-range (first entry))) + (map (lambda (ver) + (list (string->semver (crate-version-number ver)) + ver)) + (crate-versions crate))) + (match-lambda* (((semver _) ...) + (apply semverpackage-name name) - (string-append "rust-" (string-join (string-split name #\_) "-"))) + (guix-name "rust-" name)) ;;; diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 895fbb11a8..10eb030188 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -229,13 +229,20 @@ into a proper sentence and by using two spaces between sentences." cleaned 'pre ". " 'post))) (define* (package-names->package-inputs names #:optional (output #f)) - "Given a list of PACKAGE-NAMES, and an optional OUTPUT, tries to generate a -quoted list of inputs, as suitable to use in an 'inputs' field of a package -definition." - (map (lambda (input) - (cons* input (list 'unquote (string->symbol input)) - (or (and output (list output)) - '()))) + "Given a list of PACKAGE-NAMES or (PACKAGE-NAME VERSION) pairs, and an +optional OUTPUT, tries to generate a quoted list of inputs, as suitable to +use in an 'inputs' field of a package definition." + (define (make-input input version) + (cons* input (list 'unquote (string->symbol + (if version + (string-append input "-" version) + input))) + (or (and output (list output)) + '()))) + + (map (match-lambda + ((input version) (make-input input version)) + (input (make-input input #f))) names)) (define* (maybe-inputs package-names #:optional (output #f)) diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm index 9c4286f8bd..33dae56561 100644 --- a/guix/scripts/import/crate.scm +++ b/guix/scripts/import/crate.scm @@ -2,7 +2,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson ;;; Copyright © 2016 David Craven -;;; Copyright © 2019 Martin Becze +;;; Copyright © 2019, 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -95,19 +95,14 @@ Import and convert the crate.io package for PACKAGE-NAME.\n")) (package-name->name+version spec)) (if (assoc-ref opts 'recursive) - (map (match-lambda - ((and ('package ('name name) . rest) pkg) - `(define-public ,(string->symbol name) - ,pkg)) - (_ #f)) - (crate-recursive-import name #:version version)) + (crate-recursive-import name #:version version) (let ((sexp (crate->guix-package name #:version version))) (unless sexp (leave (G_ "failed to download meta-data for package '~a'~%") (if version (string-append name "@" version) name))) - sexp))) + (list sexp)))) (() (leave (G_ "too few arguments~%"))) ((many ...) diff --git a/tests/crate.scm b/tests/crate.scm index 61a04f986b..4465e12767 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2014 David Thompson ;;; Copyright © 2016 David Craven ;;; Copyright © 2019, 2020 Ludovic Courtès +;;; Copyright © 2020 Martin Becze ;;; ;;; This file is part of GNU Guix. ;;; @@ -28,23 +29,67 @@ #:use-module (ice-9 match) #:use-module (srfi srfi-64)) + +;; crate versions and dependencies used here +;; foo-0.8.1 +;; foo-1.0.0 +;; foo-1.0.3 +;; leaf-alice 0.7.5 +;; +;; root-1.0.0 +;; root-1.0.4 +;; intermediate-a 1.0.42 +;; intermeidate-b ^1.0.0 +;; leaf-alice ^0.7 +;; leaf-bob ^3 +;; +;; intermediate-a-1.0.40 +;; intermediate-a-1.0.42 +;; intermediate-a-1.1.0-alpha.1 +;; intermediate-a 1.2.3 +;; leaf-alice 0.7.5 +;; leaf-bob ^3 +;; +;; intermediate-b-1.2.3 +;; leaf-bob 3.0.1 +;; +;; leaf-alice-0.7.3 +;; leaf-alice-0.7.5 +;; +;; leaf-bob-3.0.1 + + (define test-foo-crate "{ \"crate\": { - \"max_version\": \"1.0.0\", + \"max_version\": \"1.0.3\", \"name\": \"foo\", \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ - { \"id\": \"foo\", + { \"id\": 234210, + \"num\": \"0.8.1\", + \"license\": \"MIT OR Apache-2.0\", + \"links\": { + \"dependencies\": \"/api/v1/crates/foo/0.8.1/dependencies\" + } + }, + { \"id\": 234212, \"num\": \"1.0.0\", \"license\": \"MIT OR Apache-2.0\", \"links\": { \"dependencies\": \"/api/v1/crates/foo/1.0.0/dependencies\" } + }, + { \"id\": 234214, + \"num\": \"1.0.3\", + \"license\": \"MIT OR Apache-2.0\", + \"links\": { + \"dependencies\": \"/api/v1/crates/foo/1.0.3/dependencies\" + } } ] } @@ -54,8 +99,9 @@ "{ \"dependencies\": [ { - \"crate_id\": \"bar\", - \"kind\": \"normal\" + \"crate_id\": \"leaf-alice\", + \"kind\": \"normal\", + \"req\": \"0.7.5\" } ] }") @@ -63,20 +109,27 @@ (define test-root-crate "{ \"crate\": { - \"max_version\": \"1.0.0\", + \"max_version\": \"1.0.4\", \"name\": \"root\", \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ - { \"id\": \"foo\", + { \"id\": 234240, \"num\": \"1.0.0\", \"license\": \"MIT OR Apache-2.0\", \"links\": { \"dependencies\": \"/api/v1/crates/root/1.0.0/dependencies\" } + }, + { \"id\": 234242, + \"num\": \"1.0.4\", + \"license\": \"MIT OR Apache-2.0\", + \"links\": { + \"dependencies\": \"/api/v1/crates/root/1.0.4/dependencies\" + } } ] } @@ -86,92 +139,114 @@ "{ \"dependencies\": [ { - \"crate_id\": \"intermediate-1\", - \"kind\": \"normal\" + \"crate_id\": \"intermediate-a\", + \"kind\": \"normal\", + \"req\": \"1.0.42\" }, { - \"crate_id\": \"intermediate-2\", - \"kind\": \"normal\" + \"crate_id\": \"intermediate-b\", + \"kind\": \"normal\", + \"req\": \"^1.0.0\" } { \"crate_id\": \"leaf-alice\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"^0.7\" }, { \"crate_id\": \"leaf-bob\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"^3\" } ] }") -(define test-intermediate-1-crate +(define test-intermediate-a-crate "{ \"crate\": { - \"max_version\": \"1.0.0\", - \"name\": \"intermediate-1\", + \"max_version\": \"1.1.0-alpha.1\", + \"name\": \"intermediate-a\", \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ - { \"id\": \"intermediate-1\", - \"num\": \"1.0.0\", + { \"id\": 234251, + \"num\": \"1.0.40\", + \"license\": \"MIT OR Apache-2.0\", + \"links\": { + \"dependencies\": \"/api/v1/crates/intermediate-a/1.0.40/dependencies\" + } + }, + { \"id\": 234250, + \"num\": \"1.0.42\", + \"license\": \"MIT OR Apache-2.0\", + \"links\": { + \"dependencies\": \"/api/v1/crates/intermediate-a/1.0.42/dependencies\" + } + }, + { \"id\": 234252, + \"num\": \"1.1.0-alpha.1\", \"license\": \"MIT OR Apache-2.0\", \"links\": { - \"dependencies\": \"/api/v1/crates/intermediate-1/1.0.0/dependencies\" + \"dependencies\": \"/api/v1/crates/intermediate-a/1.1.0-alpha.1/dependencies\" } } ] } }") -(define test-intermediate-1-dependencies +(define test-intermediate-a-dependencies "{ \"dependencies\": [ { - \"crate_id\": \"intermediate-2\", - \"kind\": \"normal\" + \"crate_id\": \"intermediate-b\", + \"kind\": \"normal\", + \"req\": \"1.2.3\" }, { \"crate_id\": \"leaf-alice\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"0.7.5\" }, { \"crate_id\": \"leaf-bob\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"^3\" } ] }") -(define test-intermediate-2-crate +(define test-intermediate-b-crate "{ \"crate\": { - \"max_version\": \"1.0.0\", - \"name\": \"intermediate-2\", + \"max_version\": \"1.2.3\", + \"name\": \"intermediate-b\", \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ - { \"id\": \"intermediate-2\", - \"num\": \"1.0.0\", + { \"id\": 234260, + \"num\": \"1.2.3\", \"license\": \"MIT OR Apache-2.0\", \"links\": { - \"dependencies\": \"/api/v1/crates/intermediate-2/1.0.0/dependencies\" + \"dependencies\": \"/api/v1/crates/intermediate-b/1.2.3/dependencies\" } } ] } }") -(define test-intermediate-2-dependencies +(define test-intermediate-b-dependencies "{ \"dependencies\": [ { \"crate_id\": \"leaf-bob\", - \"kind\": \"normal\" + \"kind\": \"normal\", + \"req\": \"3.0.1\" } ] }") @@ -179,19 +254,26 @@ (define test-leaf-alice-crate "{ \"crate\": { - \"max_version\": \"1.0.0\", + \"max_version\": \"0.7.5\", \"name\": \"leaf-alice\", \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], - \"categories\": [\"test\"] + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"], \"actual_versions\": [ - { \"id\": \"leaf-alice\", - \"num\": \"1.0.0\", + { \"id\": 234270, + \"num\": \"0.7.3\", \"license\": \"MIT OR Apache-2.0\", \"links\": { - \"dependencies\": \"/api/v1/crates/leaf-alice/1.0.0/dependencies\" + \"dependencies\": \"/api/v1/crates/leaf-alice/0.7.3/dependencies\" + } + }, + { \"id\": 234272, + \"num\": \"0.7.5\", + \"license\": \"MIT OR Apache-2.0\", + \"links\": { + \"dependencies\": \"/api/v1/crates/leaf-alice/0.7.5/dependencies\" } } ] @@ -206,19 +288,19 @@ (define test-leaf-bob-crate "{ \"crate\": { - \"max_version\": \"1.0.0\", + \"max_version\": \"3.0.1\", \"name\": \"leaf-bob\", \"description\": \"summary\", \"homepage\": \"http://example.com\", \"repository\": \"http://example.com\", - \"keywords\": [\"dummy\" \"test\"], + \"keywords\": [\"dummy\", \"test\"], \"categories\": [\"test\"] \"actual_versions\": [ - { \"id\": \"leaf-bob\", - \"num\": \"1.0.0\", + { \"id\": 234280, + \"num\": \"3.0.1\", \"license\": \"MIT OR Apache-2.0\", \"links\": { - \"dependencies\": \"/api/v1/crates/leaf-bob/1.0.0/dependencies\" + \"dependencies\": \"/api/v1/crates/leaf-bob/3.0.1/dependencies\" } } ] @@ -251,36 +333,51 @@ (match url ("https://crates.io/api/v1/crates/foo" (open-input-string test-foo-crate)) - ("https://crates.io/api/v1/crates/foo/1.0.0/download" + ("https://crates.io/api/v1/crates/foo/1.0.3/download" (set! test-source-hash - (bytevector->nix-base32-string - (sha256 (string->bytevector "empty file\n" "utf-8")))) + (bytevector->nix-base32-string + (sha256 (string->bytevector "empty file\n" "utf-8")))) (open-input-string "empty file\n")) - ("https://crates.io/api/v1/crates/foo/1.0.0/dependencies" + ("https://crates.io/api/v1/crates/foo/1.0.3/dependencies" (open-input-string test-foo-dependencies)) + ("https://crates.io/api/v1/crates/leaf-alice" + (open-input-string test-leaf-alice-crate)) + ("https://crates.io/api/v1/crates/leaf-alice/0.7.5/download" + (set! test-source-hash + (bytevector->nix-base32-string + (sha256 (string->bytevector "empty file\n" "utf-8")))) + (open-input-string "empty file\n")) + ("https://crates.io/api/v1/crates/leaf-alice/0.7.5/dependencies" + (open-input-string test-leaf-alice-dependencies)) (_ (error "Unexpected URL: " url))))) - (match (crate->guix-package "foo") - (('package - ('name "rust-foo") - ('version "1.0.0") - ('source ('origin - ('method 'url-fetch) - ('uri ('crate-uri "foo" 'version)) - ('file-name ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-bar" ('unquote rust-bar)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - (string=? test-source-hash hash)) - (x - (pk 'fail x #f))))) + + (match (crate->guix-package "foo") + ((define-public 'rust-foo-1.0.3 + (package (name "rust-foo") + (version "1.0.3") + (source + (origin + (method url-fetch) + (uri (crate-uri "foo" 'version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system 'cargo-build-system) + (arguments + ('quasiquote + (#:skip-build? #t + #:cargo-inputs + (("rust-leaf-alice" + ('unquote 'rust-leaf-alice-0.7.5)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + + (string=? test-source-hash hash)) + (x + (pk 'fail x #f))))) (test-assert "cargo-recursive-import" ;; Replace network resources with sample data. @@ -289,151 +386,170 @@ (match url ("https://crates.io/api/v1/crates/root" (open-input-string test-root-crate)) - ("https://crates.io/api/v1/crates/root/1.0.0/download" + ("https://crates.io/api/v1/crates/root/1.0.4/download" (set! test-source-hash (bytevector->nix-base32-string (sha256 (string->bytevector "empty file\n" "utf-8")))) (open-input-string "empty file\n")) - ("https://crates.io/api/v1/crates/root/1.0.0/dependencies" + ("https://crates.io/api/v1/crates/root/1.0.4/dependencies" (open-input-string test-root-dependencies)) - ("https://crates.io/api/v1/crates/intermediate-1" - (open-input-string test-intermediate-1-crate)) - ("https://crates.io/api/v1/crates/intermediate-1/1.0.0/download" + ("https://crates.io/api/v1/crates/intermediate-a" + (open-input-string test-intermediate-a-crate)) + ("https://crates.io/api/v1/crates/intermediate-a/1.0.42/download" (set! test-source-hash (bytevector->nix-base32-string (sha256 (string->bytevector "empty file\n" "utf-8")))) (open-input-string "empty file\n")) - ("https://crates.io/api/v1/crates/intermediate-1/1.0.0/dependencies" - (open-input-string test-intermediate-1-dependencies)) - ("https://crates.io/api/v1/crates/intermediate-2" - (open-input-string test-intermediate-2-crate)) - ("https://crates.io/api/v1/crates/intermediate-2/1.0.0/download" + ("https://crates.io/api/v1/crates/intermediate-a/1.0.42/dependencies" + (open-input-string test-intermediate-a-dependencies)) + ("https://crates.io/api/v1/crates/intermediate-b" + (open-input-string test-intermediate-b-crate)) + ("https://crates.io/api/v1/crates/intermediate-b/1.2.3/download" (set! test-source-hash (bytevector->nix-base32-string (sha256 (string->bytevector "empty file\n" "utf-8")))) (open-input-string "empty file\n")) - ("https://crates.io/api/v1/crates/intermediate-2/1.0.0/dependencies" - (open-input-string test-intermediate-2-dependencies)) + ("https://crates.io/api/v1/crates/intermediate-b/1.2.3/dependencies" + (open-input-string test-intermediate-b-dependencies)) ("https://crates.io/api/v1/crates/leaf-alice" (open-input-string test-leaf-alice-crate)) - ("https://crates.io/api/v1/crates/leaf-alice/1.0.0/download" + ("https://crates.io/api/v1/crates/leaf-alice/0.7.5/download" (set! test-source-hash (bytevector->nix-base32-string (sha256 (string->bytevector "empty file\n" "utf-8")))) (open-input-string "empty file\n")) - ("https://crates.io/api/v1/crates/leaf-alice/1.0.0/dependencies" + ("https://crates.io/api/v1/crates/leaf-alice/0.7.5/dependencies" (open-input-string test-leaf-alice-dependencies)) ("https://crates.io/api/v1/crates/leaf-bob" (open-input-string test-leaf-bob-crate)) - ("https://crates.io/api/v1/crates/leaf-bob/1.0.0/download" + ("https://crates.io/api/v1/crates/leaf-bob/3.0.1/download" (set! test-source-hash (bytevector->nix-base32-string (sha256 (string->bytevector "empty file\n" "utf-8")))) (open-input-string "empty file\n")) - ("https://crates.io/api/v1/crates/leaf-bob/1.0.0/dependencies" + ("https://crates.io/api/v1/crates/leaf-bob/3.0.1/dependencies" (open-input-string test-leaf-bob-dependencies)) (_ (error "Unexpected URL: " url))))) (match (crate-recursive-import "root") - ;; rust-intermediate-2 has no dependency on the rust-leaf-alice package, so this is a valid ordering - ((('package - ('name "rust-leaf-alice") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "leaf-alice" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-leaf-bob") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "leaf-bob" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-intermediate-2") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "intermediate-2" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-leaf-bob" ('unquote rust-leaf-bob)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-intermediate-1") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "intermediate-1" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-intermediate-2" ('unquote rust-intermediate-2)) - ("rust-leaf-alice" ('unquote rust-leaf-alice)) - ("rust-leaf-bob" ('unquote rust-leaf-bob)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0))) - ('package - ('name "rust-root") - ('version (? string? ver)) - ('source - ('origin - ('method 'url-fetch) - ('uri ('crate-uri "root" 'version)) - ('file-name - ('string-append 'name "-" 'version ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'cargo-build-system) - ('arguments - ('quasiquote - ('#:cargo-inputs (("rust-intermediate-1" ('unquote rust-intermediate-1)) - ("rust-intermediate-2" ('unquote rust-intermediate-2)) - ("rust-leaf-alice" ('unquote rust-leaf-alice)) - ("rust-leaf-bob" ('unquote rust-leaf-bob)))))) - ('home-page "http://example.com") - ('synopsis "summary") - ('description "summary") - ('license ('list 'license:expat 'license:asl2.0)))) + ;; rust-intermediate-b has no dependency on the rust-leaf-alice + ;; package, so this is a valid ordering + (((define-public 'rust-leaf-alice-0.7.5 + (package + (name "rust-leaf-alice") + (version "0.7.5") + (source + (origin + (method url-fetch) + (uri (crate-uri "leaf-alice" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments ('quasiquote (#:skip-build? #t))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public 'rust-leaf-bob-3.0.1 + (package + (name "rust-leaf-bob") + (version "3.0.1") + (source + (origin + (method url-fetch) + (uri (crate-uri "leaf-bob" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments ('quasiquote (#:skip-build? #t))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public 'rust-intermediate-b-1.2.3 + (package + (name "rust-intermediate-b") + (version "1.2.3") + (source + (origin + (method url-fetch) + (uri (crate-uri "intermediate-b" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments + ('quasiquote (#:skip-build? #t + #:cargo-inputs + (("rust-leaf-bob" + ('unquote 'rust-leaf-bob-3.0.1)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public 'rust-intermediate-a-1.0.42 + (package + (name "rust-intermediate-a") + (version "1.0.42") + (source + (origin + (method url-fetch) + (uri (crate-uri "intermediate-a" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments + ('quasiquote (#:skip-build? #t + #:cargo-inputs + (("rust-intermediate-b" + ('unquote 'rust-intermediate-b-1.2.3)) + ("rust-leaf-alice" + ('unquote 'rust-leaf-alice-0.7.5)) + ("rust-leaf-bob" + ('unquote 'rust-leaf-bob-3.0.1)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0)))) + (define-public 'rust-root-1.0.4 + (package + (name "rust-root") + (version "1.0.4") + (source + (origin + (method url-fetch) + (uri (crate-uri "root" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments + ('quasiquote (#:skip-build? #t + #:cargo-inputs + (("rust-intermediate-a" + ('unquote 'rust-intermediate-a-1.0.42)) + ("rust-intermediate-b" + ('unquote 'rust-intermediate-b-1.2.3)) + ("rust-leaf-alice" + ('unquote 'rust-leaf-alice-0.7.5)) + ("rust-leaf-bob" + ('unquote 'rust-leaf-bob-3.0.1)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0))))) #t) (x (pk 'fail x #f))))) -- cgit v1.2.3 From 50fbb3f032b46a565ca225daddf4eeb7c8edfab4 Mon Sep 17 00:00:00 2001 From: Martin Becze Date: Mon, 3 Feb 2020 16:19:49 -0500 Subject: import: crate: Parameterized importing of dev dependencies. The recursive crate importer will now include development dependencies only for the top level package, but not for any of the recursively imported packages. Also #:skip-build will be false for the top-most package. * guix/import/crate.scm (make-crate-sexp): Add the key BUILD?. (crate->guix-package): Add the key INCLUDE-DEV-DEPS?. (crate-recursive-import): Likewise. * guix/scripts/import/crate.scm (guix-import-crate): Likewise. * tests/crate.scm (cargo-recursive-import): Likewise. --- guix/import/crate.scm | 25 ++++++++++++++++++------- guix/scripts/import/crate.scm | 2 +- tests/crate.scm | 3 +-- 3 files changed, 20 insertions(+), 10 deletions(-) (limited to 'tests') diff --git a/guix/import/crate.scm b/guix/import/crate.scm index c830449555..9704b3087b 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -151,7 +151,7 @@ record or #f if it was not found." `((arguments (,'quasiquote ,args)))))) (define* (make-crate-sexp #:key name version cargo-inputs cargo-development-inputs - home-page synopsis description license) + home-page synopsis description license build?) "Return the `package' s-expression for a rust package with the given NAME, VERSION, CARGO-INPUTS, CARGO-DEVELOPMENT-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." @@ -177,7 +177,9 @@ and LICENSE." (base32 ,(bytevector->nix-base32-string (port-sha256 port)))))) (build-system cargo-build-system) - ,@(maybe-arguments (append '(#:skip-build? #t) + ,@(maybe-arguments (append (if build? + '() + '(#:skip-build? #t)) (maybe-cargo-inputs cargo-inputs) (maybe-cargo-development-inputs cargo-development-inputs))) @@ -202,12 +204,13 @@ and LICENSE." 'unknown-license!))) (string-split string (string->char-set " /")))) -(define* (crate->guix-package crate-name #:key version repo) +(define* (crate->guix-package crate-name #:key version include-dev-deps? repo) "Fetch the metadata for CRATE-NAME from crates.io, and return the `package' s-expression corresponding to that package, or #f on failure. When VERSION is specified, convert it into a semver range and attempt to fetch the latest version matching this semver range; otherwise fetch the latest -version of CRATE-NAME." +version of CRATE-NAME. If INCLUDE-DEV-DEPS is true then this will also +look up the development dependencs for the given crate." (define (semver-range-contains-string? range version) (semver-range-contains? (string->semver-range range) @@ -263,9 +266,12 @@ version of CRATE-NAME." (let* ((dependencies (crate-version-dependencies version*)) (dep-crates dev-dep-crates (partition normal-dependency? dependencies)) (cargo-inputs (sort-map-dependencies dep-crates)) - (cargo-development-inputs '())) + (cargo-development-inputs (if include-dev-deps? + (sort-map-dependencies dev-dep-crates) + '()))) (values - (make-crate-sexp #:name crate-name + (make-crate-sexp #:build? include-dev-deps? + #:name crate-name #:version (crate-version-number version*) #:cargo-inputs cargo-inputs #:cargo-development-inputs cargo-development-inputs @@ -279,7 +285,12 @@ version of CRATE-NAME." (define* (crate-recursive-import crate-name #:key version) (recursive-import crate-name - #:repo->guix-package (memoize crate->guix-package) + #:repo->guix-package (lambda* params + ;; download development dependencies only for the top level package + (let ((include-dev-deps? (equal? (car params) crate-name)) + (crate->guix-package* (memoize crate->guix-package))) + (apply crate->guix-package* + (append params `(#:include-dev-deps? ,include-dev-deps?))))) #:version version #:guix-name crate-name->package-name)) diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm index 33dae56561..9252c52dfa 100644 --- a/guix/scripts/import/crate.scm +++ b/guix/scripts/import/crate.scm @@ -96,7 +96,7 @@ Import and convert the crate.io package for PACKAGE-NAME.\n")) (if (assoc-ref opts 'recursive) (crate-recursive-import name #:version version) - (let ((sexp (crate->guix-package name #:version version))) + (let ((sexp (crate->guix-package name #:version version #:include-dev-deps? #t))) (unless sexp (leave (G_ "failed to download meta-data for package '~a'~%") (if version diff --git a/tests/crate.scm b/tests/crate.scm index 4465e12767..b6cd577552 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -536,8 +536,7 @@ (? string? hash))))) (build-system cargo-build-system) (arguments - ('quasiquote (#:skip-build? #t - #:cargo-inputs + ('quasiquote (#:cargo-inputs (("rust-intermediate-a" ('unquote 'rust-intermediate-a-1.0.42)) ("rust-intermediate-b" -- cgit v1.2.3 From 9a48e35be853e10ba9b21bb91ef52a66b4264c84 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Sat, 14 Nov 2020 19:47:32 +0100 Subject: import: utils: Trim patch version from names. This remove the patch version from generated package names. For example 'rust-my-crate-1.1.2' now becomes 'rust-my-crate-1.1'. * guix/import/utils.scm (package->definition): Trim patch version from generated package names. * tests/crate.scm: (cargo>guix-package, cargo-recursive-import): Likewise. --- guix/import/crate.scm | 3 ++- guix/import/utils.scm | 7 ++++--- tests/crate.scm | 30 +++++++++++++++--------------- 3 files changed, 21 insertions(+), 19 deletions(-) (limited to 'tests') diff --git a/guix/import/crate.scm b/guix/import/crate.scm index 9704b3087b..20efa131d5 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -159,7 +159,8 @@ and LICENSE." (map (match-lambda ((name version) - (list (crate-name->package-name name) version))) + (list (crate-name->package-name name) + (version-major+minor version)))) inputs)) (let* ((port (http-fetch (crate-uri name version))) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 10eb030188..b74393e617 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -269,9 +269,10 @@ package definition." ('package ('name name) ('version version) . rest) ('let _ ('package ('name name) ('version version) . rest))) - `(define-public ,(string->symbol (if append-version? - (string-append name "-" version) - version)) + `(define-public ,(string->symbol + (if append-version? + (string-append name "-" (version-major+minor version)) + version)) ,guix-package)))) (define (build-system-modules) diff --git a/tests/crate.scm b/tests/crate.scm index b6cd577552..3fbf6762c5 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -352,7 +352,7 @@ (_ (error "Unexpected URL: " url))))) (match (crate->guix-package "foo") - ((define-public 'rust-foo-1.0.3 + ((define-public 'rust-foo-1.0 (package (name "rust-foo") (version "1.0.3") (source @@ -369,7 +369,7 @@ (#:skip-build? #t #:cargo-inputs (("rust-leaf-alice" - ('unquote 'rust-leaf-alice-0.7.5)))))) + ('unquote 'rust-leaf-alice-0.7)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") @@ -433,7 +433,7 @@ (match (crate-recursive-import "root") ;; rust-intermediate-b has no dependency on the rust-leaf-alice ;; package, so this is a valid ordering - (((define-public 'rust-leaf-alice-0.7.5 + (((define-public 'rust-leaf-alice-0.7 (package (name "rust-leaf-alice") (version "0.7.5") @@ -452,7 +452,7 @@ (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-leaf-bob-3.0.1 + (define-public 'rust-leaf-bob-3.0 (package (name "rust-leaf-bob") (version "3.0.1") @@ -471,7 +471,7 @@ (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-intermediate-b-1.2.3 + (define-public 'rust-intermediate-b-1.2 (package (name "rust-intermediate-b") (version "1.2.3") @@ -489,12 +489,12 @@ ('quasiquote (#:skip-build? #t #:cargo-inputs (("rust-leaf-bob" - ('unquote 'rust-leaf-bob-3.0.1)))))) + ('unquote 'rust-leaf-bob-3.0)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-intermediate-a-1.0.42 + (define-public 'rust-intermediate-a-1.0 (package (name "rust-intermediate-a") (version "1.0.42") @@ -512,16 +512,16 @@ ('quasiquote (#:skip-build? #t #:cargo-inputs (("rust-intermediate-b" - ('unquote 'rust-intermediate-b-1.2.3)) + ('unquote 'rust-intermediate-b-1.2)) ("rust-leaf-alice" - ('unquote 'rust-leaf-alice-0.7.5)) + ('unquote 'rust-leaf-alice-0.7)) ("rust-leaf-bob" - ('unquote 'rust-leaf-bob-3.0.1)))))) + ('unquote 'rust-leaf-bob-3.0)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-root-1.0.4 + (define-public 'rust-root-1.0 (package (name "rust-root") (version "1.0.4") @@ -538,13 +538,13 @@ (arguments ('quasiquote (#:cargo-inputs (("rust-intermediate-a" - ('unquote 'rust-intermediate-a-1.0.42)) + ('unquote 'rust-intermediate-a-1.0)) ("rust-intermediate-b" - ('unquote 'rust-intermediate-b-1.2.3)) + ('unquote 'rust-intermediate-b-1.2)) ("rust-leaf-alice" - ('unquote 'rust-leaf-alice-0.7.5)) + ('unquote 'rust-leaf-alice-0.7)) ("rust-leaf-bob" - ('unquote 'rust-leaf-bob-3.0.1)))))) + ('unquote 'rust-leaf-bob-3.0)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") -- cgit v1.2.3 From 45584061a9ebe961e2be08ee94c3fe8ad74e2713 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Sat, 14 Nov 2020 21:31:33 +0100 Subject: import: crate: Trim version for names after left-most non-zero part. This complies to how versions are matched for caret requirements in crates: An update is allowed if the new version number does not modify the left-most non-zero digit in the major, minor, patch grouping. * guix/import/crate.scm (version->semver-prefix): New function. (make-crate-sexp)[format-inputs]: Use it. (make-crate-sexp): Use it to pass shorter version to package->definition. * guix/import/utils.scm (package->definition): Change optional parameter APPEND-VERSION? into APPEND-VERSION?/STRING. If it is a string, append its value to name. * tests/crate.scm: Adjust tests accordingly. --- guix/import/crate.scm | 10 ++++++++-- guix/import/utils.scm | 13 +++++++++---- tests/crate.scm | 25 ++++++++++++------------- 3 files changed, 29 insertions(+), 19 deletions(-) (limited to 'tests') diff --git a/guix/import/crate.scm b/guix/import/crate.scm index 20efa131d5..b133529ba7 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -150,6 +150,12 @@ record or #f if it was not found." ((args ...) `((arguments (,'quasiquote ,args)))))) +(define (version->semver-prefix version) + "Return the version up to and including the first non-zero part" + (first + (map match:substring + (list-matches "^(0+\\.){,2}[0-9]+" version)))) + (define* (make-crate-sexp #:key name version cargo-inputs cargo-development-inputs home-page synopsis description license build?) "Return the `package' s-expression for a rust package with the given NAME, @@ -160,7 +166,7 @@ and LICENSE." (match-lambda ((name version) (list (crate-name->package-name name) - (version-major+minor version)))) + (version->semver-prefix version)))) inputs)) (let* ((port (http-fetch (crate-uri name version))) @@ -194,7 +200,7 @@ and LICENSE." ((license) license) (_ `(list ,@license))))))) (close-port port) - (package->definition pkg #t))) + (package->definition pkg (version->semver-prefix version)))) (define (string->license string) (filter-map (lambda (license) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index b74393e617..7de95349cd 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -263,16 +263,21 @@ package definition." ((package-inputs ...) `((native-inputs (,'quasiquote ,package-inputs)))))) -(define* (package->definition guix-package #:optional append-version?) +(define* (package->definition guix-package #:optional append-version?/string) + "If APPEND-VERSION?/STRING is #t, append the package's major+minor +version. If APPEND-VERSION?/string is a string, append this string." (match guix-package ((or ('package ('name name) ('version version) . rest) ('let _ ('package ('name name) ('version version) . rest))) `(define-public ,(string->symbol - (if append-version? - (string-append name "-" (version-major+minor version)) - version)) + (cond + ((string? append-version?/string) + (string-append name "-" append-version?/string)) + ((= append-version?/string #t) + (string-append name "-" (version-major+minor version))) + ((#t) version))) ,guix-package)))) (define (build-system-modules) diff --git a/tests/crate.scm b/tests/crate.scm index 3fbf6762c5..1506daeadd 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -352,7 +352,7 @@ (_ (error "Unexpected URL: " url))))) (match (crate->guix-package "foo") - ((define-public 'rust-foo-1.0 + ((define-public 'rust-foo-1 (package (name "rust-foo") (version "1.0.3") (source @@ -368,8 +368,7 @@ ('quasiquote (#:skip-build? #t #:cargo-inputs - (("rust-leaf-alice" - ('unquote 'rust-leaf-alice-0.7)))))) + (("rust-leaf-alice" ('unquote 'rust-leaf-alice-0.7)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") @@ -452,7 +451,7 @@ (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-leaf-bob-3.0 + (define-public 'rust-leaf-bob-3 (package (name "rust-leaf-bob") (version "3.0.1") @@ -471,7 +470,7 @@ (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-intermediate-b-1.2 + (define-public 'rust-intermediate-b-1 (package (name "rust-intermediate-b") (version "1.2.3") @@ -489,12 +488,12 @@ ('quasiquote (#:skip-build? #t #:cargo-inputs (("rust-leaf-bob" - ('unquote 'rust-leaf-bob-3.0)))))) + ('unquote rust-leaf-bob-3)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-intermediate-a-1.0 + (define-public 'rust-intermediate-a-1 (package (name "rust-intermediate-a") (version "1.0.42") @@ -512,16 +511,16 @@ ('quasiquote (#:skip-build? #t #:cargo-inputs (("rust-intermediate-b" - ('unquote 'rust-intermediate-b-1.2)) + ('unquote rust-intermediate-b-1)) ("rust-leaf-alice" ('unquote 'rust-leaf-alice-0.7)) ("rust-leaf-bob" - ('unquote 'rust-leaf-bob-3.0)))))) + ('unquote rust-leaf-bob-3)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") (license (list license:expat license:asl2.0)))) - (define-public 'rust-root-1.0 + (define-public 'rust-root-1 (package (name "rust-root") (version "1.0.4") @@ -538,13 +537,13 @@ (arguments ('quasiquote (#:cargo-inputs (("rust-intermediate-a" - ('unquote 'rust-intermediate-a-1.0)) + ('unquote rust-intermediate-a-1)) ("rust-intermediate-b" - ('unquote 'rust-intermediate-b-1.2)) + ('unquote rust-intermediate-b-1)) ("rust-leaf-alice" ('unquote 'rust-leaf-alice-0.7)) ("rust-leaf-bob" - ('unquote 'rust-leaf-bob-3.0)))))) + ('unquote rust-leaf-bob-3)))))) (home-page "http://example.com") (synopsis "summary") (description "summary") -- cgit v1.2.3 From 054e308f5d85ca96327861a577d69c6e18fdc9dc Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Sun, 15 Nov 2020 21:25:16 +0100 Subject: import: crate: Use existing package satisfying semver requirement. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a package satisfying the dependency's semver requirement already exists, use it. Prior to this change the highest version matching the semver requirement was used (and imported in case it was not defined as package already). When resolving a dependency (now done in `sort-map-dependencies`), first search for a package matching the semver requirement and only if this fails reach out for a crate. * guix/import/crate.scm (crate->guix-package)[find-package-version]: New function. [dependency-name+version]: New function. [sort-map-dependencies]: Use it instead of lambda function. * tests/crate.scm (test-doctool-crate, test-doctool-dependencies): New variables. ("self-test …", "cargo-recursive-import-hoors-existing-packages"): New tests. --- guix/import/crate.scm | 37 ++++++++++++++++++----- tests/crate.scm | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 8 deletions(-) (limited to 'tests') diff --git a/guix/import/crate.scm b/guix/import/crate.scm index b133529ba7..3bc261b04e 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -32,6 +32,7 @@ #:use-module (guix packages) #:use-module (guix upstream) #:use-module (guix utils) + #:use-module (gnu packages) #:use-module (ice-9 match) #:use-module (ice-9 regex) #:use-module (json) @@ -92,7 +93,7 @@ (requirement crate-dependency-requirement "req")) ;string (module-autoload! (current-module) - '(semver) '(string->semver semversemver semver->string semversemver-range semver-range-contains?)) @@ -235,6 +236,21 @@ look up the development dependencs for the given crate." (or version (crate-latest-version crate)))) + ;; find the highest existing package that fulfills the semver + (define (find-package-version name range) + (let* ((semver-range (string->semver-range range)) + (versions + (sort + (filter (lambda (version) + (semver-range-contains? semver-range version)) + (map (lambda (pkg) + (string->semver (package-version pkg))) + (find-packages-by-name + (crate-name->package-name name)))) + semverstring (last versions))))) + ;; find the highest version of a crate that fulfills the semver (define (find-crate-version crate range) (let* ((semver-range (string->semver-range range)) @@ -251,6 +267,17 @@ look up the development dependencs for the given crate." (and (not (null-list? versions)) (second (last versions))))) + (define (dependency-name+version dep) + (let* ((name (crate-dependency-id dep)) + (req (crate-dependency-requirement dep)) + (existing-version (find-package-version name req))) + (if existing-version + (list name existing-version) + (let* ((crate (lookup-crate* name)) + (ver (find-crate-version crate req))) + (list name + (crate-version-number ver)))))) + (define version* (and crate (find-crate-version crate version-number))) @@ -258,13 +285,7 @@ look up the development dependencs for the given crate." ;; sort and map the dependencies to a list containing ;; pairs of (name version) (define (sort-map-dependencies deps) - (sort (map (lambda (dep) - (let* ((name (crate-dependency-id dep)) - (crate (lookup-crate* name)) - (req (crate-dependency-requirement dep)) - (ver (find-crate-version crate req))) - (list name - (crate-version-number ver)))) + (sort (map dependency-name+version deps) (match-lambda* (((name _) ...) (apply string-cilicense "MIT/Apache-2.0")) + + +(define test-doctool-crate + "{ + \"crate\": { + \"max_version\": \"2.2.2\", + \"name\": \"leaf-bob\", + \"description\": \"summary\", + \"homepage\": \"http://example.com\", + \"repository\": \"http://example.com\", + \"keywords\": [\"dummy\", \"test\"], + \"categories\": [\"test\"] + \"actual_versions\": [ + { \"id\": 234280, + \"num\": \"2.2.2\", + \"license\": \"MIT OR Apache-2.0\", + \"links\": { + \"dependencies\": \"/api/v1/crates/doctool/2.2.2/dependencies\" + } + } + ] + } +}") + +;; FIXME: This test depends on some existing packages +(define test-doctool-dependencies + "{ + \"dependencies\": [ + { + \"crate_id\": \"docopt\", + \"kind\": \"normal\", + \"req\": \"^0.8.1\" + } + ] +}") + + +(test-assert "self-test: rust-docopt 0.8.x is gone, please adjust the test case" + (not (null? (find-packages-by-name "rust-docopt" "0.8")))) + +(test-assert "cargo-recursive-import-hoors-existing-packages" + (mock ((guix http-client) http-fetch + (lambda (url . rest) + (match url + ("https://crates.io/api/v1/crates/doctool" + (open-input-string test-doctool-crate)) + ("https://crates.io/api/v1/crates/doctool/2.2.2/download" + (set! test-source-hash + (bytevector->nix-base32-string + (sha256 (string->bytevector "empty file\n" "utf-8")))) + (open-input-string "empty file\n")) + ("https://crates.io/api/v1/crates/doctool/2.2.2/dependencies" + (open-input-string test-doctool-dependencies)) + (_ (error "Unexpected URL: " url))))) + (match (crate-recursive-import "doctool") + (((define-public 'rust-doctool-2 + (package + (name "rust-doctool") + (version "2.2.2") + (source + (origin + (method url-fetch) + (uri (crate-uri "doctool" version)) + (file-name + (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + (? string? hash))))) + (build-system cargo-build-system) + (arguments + ('quasiquote (#:cargo-inputs + (("rust-docopt" + ('unquote 'rust-docopt-0.8)))))) + (home-page "http://example.com") + (synopsis "summary") + (description "summary") + (license (list license:expat license:asl2.0))))) + #t) + (x + (pk 'fail x #f))))) + (test-end "crate") -- cgit v1.2.3 From a67b82475dc14d0be7e14a5438ce70fa94de1907 Mon Sep 17 00:00:00 2001 From: Paul Garlick Date: Fri, 4 Dec 2020 12:20:58 +0000 Subject: tests: pack-relocatable: Ensure commands can run in the current namespace. * tests/guix-pack-relocatable (run_without_store): Use subshell to run commands in the current namespace. This avoids shell syntax and parsing errors. --- tests/guix-pack-relocatable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/guix-pack-relocatable.sh b/tests/guix-pack-relocatable.sh index 2beb1b1eb6..b90bc7f891 100644 --- a/tests/guix-pack-relocatable.sh +++ b/tests/guix-pack-relocatable.sh @@ -55,7 +55,7 @@ run_without_store () # Run the relocatable program in the current namespaces. This is a # weak test because we're going to access store items from the host # store. - $* + sh -c "$*" fi } -- cgit v1.2.3 From e3065ec11742eef58ca688db240d86031941a39a Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 4 Dec 2020 16:53:14 +0100 Subject: import: crate: Skip tests when Guile-Semver is missing. * guix/import/crate.scm: Add comment for the 'module-autoload!' calls. * tests/crate.scm (have-guile-semver?): New variable. ("crate->guix-package", "cargo-recursive-import") ("cargo-recursive-import-hoors-existing-packages"): Skip when HAVE-GUILE-SEMVER? is false. --- guix/import/crate.scm | 1 + tests/crate.scm | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/guix/import/crate.scm b/guix/import/crate.scm index 3bc261b04e..c10c0d55ea 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -92,6 +92,7 @@ string->symbol) (requirement crate-dependency-requirement "req")) ;string +;; Autoload Guile-Semver so we only have a soft dependency. (module-autoload! (current-module) '(semver) '(string->semver semver->string semverguix-package" ;; Replace network resources with sample data. (mock ((guix http-client) http-fetch @@ -380,6 +384,7 @@ (x (pk 'fail x #f))))) +(unless have-guile-semver? (test-skip 1)) (test-assert "cargo-recursive-import" ;; Replace network resources with sample data. (mock ((guix http-client) http-fetch @@ -614,6 +619,7 @@ (test-assert "self-test: rust-docopt 0.8.x is gone, please adjust the test case" (not (null? (find-packages-by-name "rust-docopt" "0.8")))) +(unless have-guile-semver? (test-skip 1)) (test-assert "cargo-recursive-import-hoors-existing-packages" (mock ((guix http-client) http-fetch (lambda (url . rest) -- cgit v1.2.3 From f5d952c5f50cd9c6005cdf47dda5bccb6e428119 Mon Sep 17 00:00:00 2001 From: Leo Prikler Date: Sat, 5 Dec 2020 17:20:09 +0100 Subject: profiles: Remove duplicates in manifest transactions. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes . Reported by Ricardo Wurmus . * guix/profiles.scm (manifest-transaction-effects): Delete duplicates in install and remove. Let multiple upgrades and downgrades shadow previous transactions of the same kind. * tests/profiles.scm ("manifest-transaction-effects no double install or upgrades") ("manifest-transaction-effects no double downgrade") ("manifest-transaction-effects no double removal"): New tests. Signed-off-by: Ludovic Courtès --- guix/profiles.scm | 18 ++++++++++++++++-- tests/profiles.scm | 28 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/guix/profiles.scm b/guix/profiles.scm index 1b15257210..034591eb79 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -716,6 +716,12 @@ replace it." (manifest-pattern (name (manifest-entry-name entry)) (output (manifest-entry-output entry)))) + (define manifest-entry-pair=? + (match-lambda* + (((m1a . m2a) (m1b . m2b)) + (and (manifest-entry=? m1a m1b) + (manifest-entry=? m2a m2b))) + (_ #f))) (let loop ((input (manifest-transaction-install transaction)) (install '()) @@ -724,8 +730,16 @@ replace it." (match input (() (let ((remove (manifest-transaction-remove transaction))) - (values (manifest-matching-entries manifest remove) - (reverse install) (reverse upgrade) (reverse downgrade)))) + (values (delete-duplicates + (manifest-matching-entries manifest remove) + manifest-entry=?) + (delete-duplicates (reverse install) manifest-entry=?) + (delete-duplicates + (reverse upgrade) + manifest-entry-pair=?) + (delete-duplicates + (reverse downgrade) + manifest-entry-pair=?)))) ((entry rest ...) ;; Check whether installing ENTRY corresponds to the installation of a ;; new package or to an upgrade. diff --git a/tests/profiles.scm b/tests/profiles.scm index 055924ba3e..f0a1a1d11c 100644 --- a/tests/profiles.scm +++ b/tests/profiles.scm @@ -183,6 +183,16 @@ (equal? (list glibc) install) (equal? (list (cons guile-1.8.8 guile-2.0.9)) upgrade))))) +(test-assert "manifest-transaction-effects no double install or upgrades" + (let* ((m0 (manifest (list guile-1.8.8))) + (t (manifest-transaction + (install (list guile-2.0.9 glibc glibc))))) + (let-values (((remove install upgrade downgrade) + (manifest-transaction-effects m0 t))) + (and (null? remove) (null? downgrade) + (equal? (list glibc) install) + (equal? (list (cons guile-1.8.8 guile-2.0.9)) upgrade))))) + (test-assert "manifest-transaction-effects and downgrades" (let* ((m0 (manifest (list guile-2.0.9))) (t (manifest-transaction (install (list guile-1.8.8))))) @@ -191,6 +201,14 @@ (and (null? remove) (null? install) (null? upgrade) (equal? (list (cons guile-2.0.9 guile-1.8.8)) downgrade))))) +(test-assert "manifest-transaction-effects no double downgrade" + (let* ((m0 (manifest (list guile-2.0.9))) + (t (manifest-transaction (install (list guile-1.8.8 guile-1.8.8))))) + (let-values (((remove install upgrade downgrade) + (manifest-transaction-effects m0 t))) + (and (null? remove) (null? install) (null? upgrade) + (equal? (list (cons guile-2.0.9 guile-1.8.8)) downgrade))))) + (test-assert "manifest-transaction-effects and pseudo-upgrades" (let* ((m0 (manifest (list guile-2.0.9))) (t (manifest-transaction (install (list guile-2.0.9))))) @@ -209,6 +227,16 @@ (and (manifest-transaction-removal-candidate? guile-2.0.9 t) (not (manifest-transaction-removal-candidate? glibc t))))) +(test-assert "manifest-transaction-effects no double removal" + (let* ((m0 (manifest (list guile-2.0.9))) + (t (manifest-transaction + (remove (list (manifest-pattern (name "guile"))))))) + (let-values (((remove install upgrade downgrade) + (manifest-transaction-effects m0 t))) + (and (= 1 (length remove)) + (manifest-transaction-removal-candidate? guile-2.0.9 t) + (null? install) (null? downgrade) (null? upgrade))))) + (test-assertm "profile-derivation" (mlet* %store-monad ((entry -> (package->manifest-entry %bootstrap-guile)) -- cgit v1.2.3 From 07340cbebc1ef97c0196c631144775f5ce61d4f6 Mon Sep 17 00:00:00 2001 From: Leo Prikler Date: Sat, 5 Dec 2020 17:20:10 +0100 Subject: profiles: Delete duplicate manifest entries in packages->manifest. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gnu/profiles.scm (packages->manifest): Delete duplicate entries. * tests/profiles.scm ("packages->manifest, no duplicates"): New test. Signed-off-by: Ludovic Courtès --- guix/profiles.scm | 34 ++++++++++++++++++---------------- tests/profiles.scm | 10 ++++++++++ 2 files changed, 28 insertions(+), 16 deletions(-) (limited to 'tests') diff --git a/guix/profiles.scm b/guix/profiles.scm index 034591eb79..59a313ea08 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -399,22 +399,24 @@ denoting a specific output of a package." 'inferior-package->manifest-entry)) (manifest - (map (match-lambda - (((? package? package) output) - (package->manifest-entry package output)) - ((? package? package) - (package->manifest-entry package)) - ((thing output) - (if inferiors-loaded? - ((inferior->entry) thing output) - (throw 'wrong-type-arg 'packages->manifest - "Wrong package object: ~S" (list thing) (list thing)))) - (thing - (if inferiors-loaded? - ((inferior->entry) thing) - (throw 'wrong-type-arg 'packages->manifest - "Wrong package object: ~S" (list thing) (list thing))))) - packages))) + (delete-duplicates + (map (match-lambda + (((? package? package) output) + (package->manifest-entry package output)) + ((? package? package) + (package->manifest-entry package)) + ((thing output) + (if inferiors-loaded? + ((inferior->entry) thing output) + (throw 'wrong-type-arg 'packages->manifest + "Wrong package object: ~S" (list thing) (list thing)))) + (thing + (if inferiors-loaded? + ((inferior->entry) thing) + (throw 'wrong-type-arg 'packages->manifest + "Wrong package object: ~S" (list thing) (list thing))))) + packages) + manifest-entry=?))) (define (manifest->gexp manifest) "Return a representation of MANIFEST as a gexp." diff --git a/tests/profiles.scm b/tests/profiles.scm index f0a1a1d11c..2dec42bec1 100644 --- a/tests/profiles.scm +++ b/tests/profiles.scm @@ -384,6 +384,16 @@ (manifest-entry-search-paths (package->manifest-entry mpl))))) +(test-assert "packages->manifest, no duplicates" + (let ((expected + (manifest + (list + (package->manifest-entry packages:guile-2.2)))) + (manifest (packages->manifest + (list packages:guile-2.2 packages:guile-2.2)))) + (every manifest-entry=? (manifest-entries expected) + (manifest-entries manifest)))) + (test-equal "packages->manifest, propagated inputs" (map (match-lambda ((label package) -- cgit v1.2.3 From d8ae7852057d5c1818c9c8bb77e8c41407a0d985 Mon Sep 17 00:00:00 2001 From: Chris Marusich Date: Sat, 13 Jun 2020 22:09:46 -0700 Subject: tests: lint: Add origin patch file name test cases. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In particular, " patches: same file name -> no warnings" would have caught the issue which was fixed in commit 21887021b9acf60157b1b0a39c16f2ec6498021b. * tests/lint.scm (patches: file names): Rename this test case... ("file patches: different file name -> warning"): ... to this. ("file patches: same file name -> no warnings") (" patches: different file name -> warning") (" patches: same file name -> no warnings"): New test cases. Signed-off-by: Ludovic Courtès --- tests/lint.scm | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/lint.scm b/tests/lint.scm index 9b230814a5..7c24611934 100644 --- a/tests/lint.scm +++ b/tests/lint.scm @@ -315,7 +315,7 @@ `(("python-setuptools" ,python-setuptools)))))) (check-inputs-should-not-be-an-input-at-all pkg)))) -(test-equal "patches: file names" +(test-equal "file patches: different file name -> warning" "file names of patches should start with the package name" (single-lint-warning-message (let ((pkg (dummy-package "x" @@ -324,6 +324,37 @@ (patches (list "/path/to/y.patch"))))))) (check-patch-file-names pkg)))) +(test-equal "file patches: same file name -> no warnings" + '() + (let ((pkg (dummy-package "x" + (source + (dummy-origin + (patches (list "/path/to/x.patch"))))))) + (check-patch-file-names pkg))) + +(test-equal " patches: different file name -> warning" + "file names of patches should start with the package name" + (single-lint-warning-message + (let ((pkg (dummy-package "x" + (source + (dummy-origin + (patches + (list + (dummy-origin + (file-name "y.patch"))))))))) + (check-patch-file-names pkg)))) + +(test-equal " patches: same file name -> no warnings" + '() + (let ((pkg (dummy-package "x" + (source + (dummy-origin + (patches + (list + (dummy-origin + (file-name "x.patch"))))))))) + (check-patch-file-names pkg))) + (test-equal "patches: file name too long" (string-append "x-" (make-string 100 #\a) -- cgit v1.2.3 From 15ee1b8317adeb74938f3bb7f0c7cb635a818292 Mon Sep 17 00:00:00 2001 From: Julien Lepiller Date: Tue, 8 Dec 2020 13:58:49 +0100 Subject: guix: opam: Pass default repository to recursive importer. * guix/import/opam.scm (opam->guix-package): Rename #:repository key to #:repo. (opam-recursive-import): Pass #:repo keyword. * tests/opam.scm (opam->guix-package): Rename #:repository to #:repo. --- guix/import/opam.scm | 9 +++++---- tests/opam.scm | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/guix/import/opam.scm b/guix/import/opam.scm index 4f366a9384..6e98822104 100644 --- a/guix/import/opam.scm +++ b/guix/import/opam.scm @@ -264,11 +264,11 @@ path to the repository." (substring version 1) version))))) -(define* (opam->guix-package name #:key (repository (get-opam-repository)) version) +(define* (opam->guix-package name #:key (repo (get-opam-repository)) version) "Import OPAM package NAME from REPOSITORY (a directory name) or, if REPOSITORY is #f, from the official OPAM repository. Return a 'package' sexp or #f on failure." - (and-let* ((opam-file (opam-fetch name repository)) + (and-let* ((opam-file (opam-fetch name repo)) (version (assoc-ref opam-file "version")) (opam-content (assoc-ref opam-file "metadata")) (url-dict (metadata-ref opam-content "url")) @@ -323,10 +323,11 @@ or #f on failure." (not (member name '("dune" "jbuilder")))) dependencies)))))))) -(define (opam-recursive-import package-name) +(define* (opam-recursive-import package-name #:repo (get-opam-repository)) (recursive-import package-name #:repo->guix-package opam->guix-package - #:guix-name ocaml-name->guix-name)) + #:guix-name ocaml-name->guix-name + #:repo repo)) (define (guix-name->opam-name name) (if (string-prefix? "ocaml-" name) diff --git a/tests/opam.scm b/tests/opam.scm index ec2a668307..8d43e2ce70 100644 --- a/tests/opam.scm +++ b/tests/opam.scm @@ -85,7 +85,7 @@ url { (with-output-to-file (string-append my-package "/opam") (lambda _ (format #t "~a" test-opam-file)))) - (match (opam->guix-package "foo" #:repository test-repo) + (match (opam->guix-package "foo" #:repo test-repo) (('package ('name "ocaml-foo") ('version "1.0.0") -- cgit v1.2.3 From 79c6614f58a57b985daf8940766319e440311db0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 1 Dec 2020 15:01:40 +0100 Subject: daemon: Use 'Agent' to spawn 'guix substitute --query'. * nix/libstore/local-store.hh (RunningSubstituter): Remove. (LocalStore)[runningSubstituter]: Change to unique_ptr. [setSubstituterEnv, didSetSubstituterEnv]: Remove. [getLineFromSubstituter, getIntLineFromSubstituter]: Take an 'Agent'. * nix/libstore/local-store.cc (LocalStore::~LocalStore): Remove reference to 'runningSubstituter'. (LocalStore::setSubstituterEnv, LocalStore::startSubstituter): Remove. (LocalStore::getLineFromSubstituter): Adjust to 'run' being an 'Agent'. (LocalStore::querySubstitutablePaths): Spawn substituter agent if needed. Adjust to 'Agent' interface. (LocalStore::querySubstitutablePathInfos): Likewise. * nix/libstore/build.cc (SubstitutionGoal::tryToRun): Remove call to 'setSubstituterEnv' and add 'setenv' call for "_NIX_OPTIONS" instead. (SubstitutionGoal::finished): Remove 'readLine' call for 'dummy'. * guix/scripts/substitute.scm (%allow-unauthenticated-substitutes?): Remove second argument to 'make-parameter'. (process-query): Call 'warn-about-missing-authentication' when (%allow-unauthenticated-substitutes?) is #t. (guix-substitute): Wrap body in 'parameterize'. Set 'guix-warning-port' too. No longer exit when 'substitute-urls' returns the empty list. No longer print newline initially. * tests/substitute.scm (test-quit): Parameterize 'current-error-port' to account for the port changes in 'guix-substitute'. --- guix/scripts/substitute.scm | 132 ++++++++++++++++++------------------ nix/libstore/build.cc | 6 +- nix/libstore/local-store.cc | 159 +++++++++++--------------------------------- nix/libstore/local-store.hh | 22 +----- tests/substitute.scm | 3 +- 5 files changed, 110 insertions(+), 212 deletions(-) (limited to 'tests') diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index feae2df9cb..5e392eaa8b 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -124,11 +124,7 @@ disabled!~%")) ;; purposes, and should be avoided otherwise. (make-parameter (and=> (getenv "GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES") - (cut string-ci=? <> "yes")) - (lambda (value) - (when value - (warn-about-missing-authentication)) - value))) + (cut string-ci=? <> "yes")))) (define %narinfo-ttl ;; Number of seconds during which cached narinfo lookups are considered @@ -893,6 +889,9 @@ authorized substitutes." (define (valid? obj) (valid-narinfo? obj acl)) + (when (%allow-unauthenticated-substitutes?) + (warn-about-missing-authentication)) + (match (string-tokenize command) (("have" paths ..1) ;; Return the subset of PATHS available in CACHE-URLS. @@ -1139,68 +1138,67 @@ default value." ((= string->number number) (> number 0)) (_ #f))) - (mkdir-p %narinfo-cache-directory) - (maybe-remove-expired-cache-entries %narinfo-cache-directory - cached-narinfo-files - #:entry-expiration - cached-narinfo-expiration-time - #:cleanup-period - %narinfo-expired-cache-entry-removal-delay) - (check-acl-initialized) - - ;; Starting from commit 22144afa in Nix, we are allowed to bail out directly - ;; when we know we cannot substitute, but we must emit a newline on stdout - ;; when everything is alright. - (when (null? (substitute-urls)) - (exit 0)) - - ;; Say hello (see above.) - (newline) - (force-output (current-output-port)) - - ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error message. - (for-each validate-uri (substitute-urls)) - - ;; Attempt to install the client's locale so that messages are suitably - ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default so - ;; don't change it. - (match (or (find-daemon-option "untrusted-locale") - (find-daemon-option "locale")) - (#f #f) - (locale (false-if-exception (setlocale LC_MESSAGES locale)))) - - (catch 'system-error - (lambda () - (set-thread-name "guix substitute")) - (const #t)) ;GNU/Hurd lacks 'prctl' - - (with-networking - (with-error-handling ; for signature errors - (match args - (("--query") - (let ((acl (current-acl))) - (let loop ((command (read-line))) - (or (eof-object? command) - (begin - (process-query command - #:cache-urls (substitute-urls) - #:acl acl) - (loop (read-line))))))) - (("--substitute" store-path destination) - ;; Download STORE-PATH and add store it as a Nar in file DESTINATION. - ;; Specify the number of columns of the terminal so the progress - ;; report displays nicely. - (parameterize ((current-terminal-columns (client-terminal-columns))) - (process-substitution store-path destination - #:cache-urls (substitute-urls) - #:acl (current-acl) - #:print-build-trace? print-build-trace?))) - ((or ("-V") ("--version")) - (show-version-and-exit "guix substitute")) - (("--help") - (show-help)) - (opts - (leave (G_ "~a: unrecognized options~%") opts)))))) + ;; The daemon's agent code opens file descriptor 4 for us and this is where + ;; stderr should go. + (parameterize ((current-error-port (match args + (("--query") (fdopen 4 "wl")) + (_ (current-error-port))))) + ;; Redirect diagnostics to file descriptor 4 as well. + (guix-warning-port (current-error-port)) + + (mkdir-p %narinfo-cache-directory) + (maybe-remove-expired-cache-entries %narinfo-cache-directory + cached-narinfo-files + #:entry-expiration + cached-narinfo-expiration-time + #:cleanup-period + %narinfo-expired-cache-entry-removal-delay) + (check-acl-initialized) + + ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error + ;; message. + (for-each validate-uri (substitute-urls)) + + ;; Attempt to install the client's locale so that messages are suitably + ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default + ;; so don't change it. + (match (or (find-daemon-option "untrusted-locale") + (find-daemon-option "locale")) + (#f #f) + (locale (false-if-exception (setlocale LC_MESSAGES locale)))) + + (catch 'system-error + (lambda () + (set-thread-name "guix substitute")) + (const #t)) ;GNU/Hurd lacks 'prctl' + + (with-networking + (with-error-handling ; for signature errors + (match args + (("--query") + (let ((acl (current-acl))) + (let loop ((command (read-line))) + (or (eof-object? command) + (begin + (process-query command + #:cache-urls (substitute-urls) + #:acl acl) + (loop (read-line))))))) + (("--substitute" store-path destination) + ;; Download STORE-PATH and store it as a Nar in file DESTINATION. + ;; Specify the number of columns of the terminal so the progress + ;; report displays nicely. + (parameterize ((current-terminal-columns (client-terminal-columns))) + (process-substitution store-path destination + #:cache-urls (substitute-urls) + #:acl (current-acl) + #:print-build-trace? print-build-trace?))) + ((or ("-V") ("--version")) + (show-version-and-exit "guix substitute")) + (("--help") + (show-help)) + (opts + (leave (G_ "~a: unrecognized options~%") opts))))))) ;;; Local Variables: ;;; eval: (put 'with-timeout 'scheme-indent-function 1) diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index 8413819114..7e9ab3f39c 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -2986,8 +2986,6 @@ void SubstitutionGoal::tryToRun() if (pathExists(destPath)) deletePath(destPath); - worker.store.setSubstituterEnv(); - /* Fill in the arguments. */ Strings args; args.push_back("guix"); @@ -2999,6 +2997,9 @@ void SubstitutionGoal::tryToRun() /* Fork the substitute program. */ pid = startProcess([&]() { + /* Communicate substitute-urls & co. to 'guix substitute'. */ + setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); + commonChildInit(logPipe); if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) @@ -3041,7 +3042,6 @@ void SubstitutionGoal::finished() logPipe.readSide.close(); /* Get the hash info from stdout. */ - string dummy = readLine(outPipe.readSide); string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; outPipe.readSide.close(); diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 8c479002ec..4219573a56 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -57,7 +57,6 @@ void checkStoreNotSymlink() LocalStore::LocalStore(bool reserveSpace) - : didSetSubstituterEnv(false) { schemaPath = settings.nixDBPath + "/schema"; @@ -182,21 +181,6 @@ LocalStore::LocalStore(bool reserveSpace) LocalStore::~LocalStore() { - try { - if (runningSubstituter) { - RunningSubstituter &i = *runningSubstituter; - if (!i.disabled) { - i.to.close(); - i.from.close(); - i.error.close(); - if (i.pid != -1) - i.pid.wait(true); - } - } - } catch (...) { - ignoreException(); - } - try { if (fdTempRoots != -1) { fdTempRoots.close(); @@ -796,96 +780,31 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) }); } - -void LocalStore::setSubstituterEnv() -{ - if (didSetSubstituterEnv) return; - - /* Pass configuration options (including those overridden with - --option) to substituters. */ - setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; -} - - -void LocalStore::startSubstituter(RunningSubstituter & run) -{ - if (run.disabled || run.pid != -1) return; - - debug(format("starting substituter program `%1% substitute'") - % settings.guixProgram); - - Pipe toPipe, fromPipe, errorPipe; - - toPipe.create(); - fromPipe.create(); - errorPipe.create(); - - setSubstituterEnv(); - - run.pid = startProcess([&]() { - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(settings.guixProgram.c_str(), "guix", "substitute", "--query", NULL); - throw SysError(format("executing `%1%'") % settings.guixProgram); - }); - - run.to = toPipe.writeSide.borrow(); - run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); - run.error = errorPipe.readSide.borrow(); - - toPipe.readSide.close(); - fromPipe.writeSide.close(); - errorPipe.writeSide.close(); - - /* The substituter may exit right away if it's disabled in any way - (e.g. copy-from-other-stores.pl will exit if no other stores - are configured). */ - try { - getLineFromSubstituter(run); - } catch (EndOfFile & e) { - run.to.close(); - run.from.close(); - run.error.close(); - run.disabled = true; - if (run.pid.wait(true) != 0) throw; - } -} - - /* Read a line from the substituter's stdout, while also processing its stderr. */ -string LocalStore::getLineFromSubstituter(RunningSubstituter & run) +string LocalStore::getLineFromSubstituter(Agent & run) { string res, err; - /* We might have stdout data left over from the last time. */ - if (run.fromBuf.hasData()) goto haveData; - while (1) { checkInterrupt(); fd_set fds; FD_ZERO(&fds); - FD_SET(run.from, &fds); - FD_SET(run.error, &fds); + FD_SET(run.fromAgent.readSide, &fds); + FD_SET(run.builderOut.readSide, &fds); /* Wait for data to appear on the substituter's stdout or stderr. */ - if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { + if (select(std::max(run.fromAgent.readSide, run.builderOut.readSide) + 1, &fds, 0, 0, 0) == -1) { if (errno == EINTR) continue; throw SysError("waiting for input from the substituter"); } /* Completely drain stderr before dealing with stdout. */ - if (FD_ISSET(run.error, &fds)) { + if (FD_ISSET(run.builderOut.readSide, &fds)) { char buf[4096]; - ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); + ssize_t n = read(run.builderOut.readSide, (unsigned char *) buf, sizeof(buf)); if (n == -1) { if (errno == EINTR) continue; throw SysError("reading from substituter's stderr"); @@ -903,23 +822,20 @@ string LocalStore::getLineFromSubstituter(RunningSubstituter & run) } /* Read from stdout until we get a newline or the buffer is empty. */ - else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { - haveData: - do { - unsigned char c; - run.fromBuf(&c, 1); - if (c == '\n') { - if (!err.empty()) printMsg(lvlError, "substitute: " + err); - return res; - } - res += c; - } while (run.fromBuf.hasData()); + else if (FD_ISSET(run.fromAgent.readSide, &fds)) { + unsigned char c; + readFull(run.fromAgent.readSide, (unsigned char *) &c, 1); + if (c == '\n') { + if (!err.empty()) printMsg(lvlError, "substitute: " + err); + return res; + } + res += c; } } } -template T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) +template T LocalStore::getIntLineFromSubstituter(Agent & run) { string s = getLineFromSubstituter(run); T res; @@ -935,27 +851,26 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) if (!settings.useSubstitutes || paths.empty()) return res; if (!runningSubstituter) { - std::unique_ptrfresh(new RunningSubstituter); + const Strings args = { "substitute", "--query" }; + const std::map env = { { "_NIX_OPTIONS", settings.pack() } }; + std::unique_ptr fresh(new Agent(settings.guixProgram, args, env)); runningSubstituter.swap(fresh); } - RunningSubstituter & run = *runningSubstituter; - startSubstituter(run); - - if (!run.disabled) { - string s = "have "; - foreach (PathSet::const_iterator, j, paths) - if (res.find(*j) == res.end()) { s += *j; s += " "; } - writeLine(run.to, s); - while (true) { - /* FIXME: we only read stderr when an error occurs, so - substituters should only write (short) messages to - stderr when they fail. I.e. they shouldn't write debug - output. */ - Path path = getLineFromSubstituter(run); - if (path == "") break; - res.insert(path); - } + Agent & run = *runningSubstituter; + + string s = "have "; + foreach (PathSet::const_iterator, j, paths) + if (res.find(*j) == res.end()) { s += *j; s += " "; } + writeLine(run.toAgent.writeSide, s); + while (true) { + /* FIXME: we only read stderr when an error occurs, so + substituters should only write (short) messages to + stderr when they fail. I.e. they shouldn't write debug + output. */ + Path path = getLineFromSubstituter(run); + if (path == "") break; + res.insert(path); } return res; @@ -967,18 +882,18 @@ void LocalStore::querySubstitutablePathInfos(PathSet & paths, SubstitutablePathI if (!settings.useSubstitutes) return; if (!runningSubstituter) { - std::unique_ptrfresh(new RunningSubstituter); + const Strings args = { "substitute", "--query" }; + const std::map env = { { "_NIX_OPTIONS", settings.pack() } }; + std::unique_ptr fresh(new Agent(settings.guixProgram, args, env)); runningSubstituter.swap(fresh); } - RunningSubstituter & run = *runningSubstituter; - startSubstituter(run); - if (run.disabled) return; + Agent & run = *runningSubstituter; string s = "info "; foreach (PathSet::const_iterator, i, paths) if (infos.find(*i) == infos.end()) { s += *i; s += " "; } - writeLine(run.to, s); + writeLine(run.toAgent.writeSide, s); while (true) { Path path = getLineFromSubstituter(run); diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index 2e48cf03e6..57d15bac7e 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -38,21 +38,11 @@ struct OptimiseStats }; -struct RunningSubstituter -{ - Pid pid; - AutoCloseFD to, from, error; - FdSource fromBuf; - bool disabled; - RunningSubstituter() : disabled(false) { }; -}; - - class LocalStore : public StoreAPI { private: /* The currently running substituter or empty. */ - std::unique_ptr runningSubstituter; + std::unique_ptr runningSubstituter; Path linksDir; @@ -178,8 +168,6 @@ public: void markContentsGood(const Path & path); - void setSubstituterEnv(); - void createUser(const std::string & userName, uid_t userId); private: @@ -213,8 +201,6 @@ private: /* Cache for pathContentsGood(). */ std::map pathContentsGoodCache; - bool didSetSubstituterEnv; - /* The file to which we write our temporary roots. */ Path fnTempRoots; AutoCloseFD fdTempRoots; @@ -262,11 +248,9 @@ private: void removeUnusedLinks(const GCState & state); - void startSubstituter(RunningSubstituter & runningSubstituter); - - string getLineFromSubstituter(RunningSubstituter & run); + string getLineFromSubstituter(Agent & run); - template T getIntLineFromSubstituter(RunningSubstituter & run); + template T getIntLineFromSubstituter(Agent & run); Path createTempDirInStore(); diff --git a/tests/substitute.scm b/tests/substitute.scm index 6560612c40..bd5b6305b0 100644 --- a/tests/substitute.scm +++ b/tests/substitute.scm @@ -47,7 +47,8 @@ it writes to GUIX-WARNING-PORT a messages that matches ERROR-RX." (test-equal name '(1 #t) (let ((error-output (open-output-string))) - (parameterize ((guix-warning-port error-output)) + (parameterize ((current-error-port error-output) + (guix-warning-port error-output)) (catch 'quit (lambda () exp -- cgit v1.2.3 From 711df9ef3c04a0e0d7e844bed4c6b260ea1f65c1 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 2 Dec 2020 16:27:34 +0100 Subject: daemon: Run 'guix substitute --substitute' as an agent. This avoids spawning one substitute process per substitution. * nix/libstore/build.cc (class Worker)[substituter]: New field. [outPipe, logPipe, pid]: Remove. (class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields. (SubstitutionGoal::timedOut): Adjust to check 'substituter'. (SubstitutionGoal::tryToRun): Remove references to 'outPipe' and 'logPipe'. Run "guix substitute --substitute" as an 'Agent'. Send the request with 'writeLine'. (SubstitutionGoal::finished): Likewise. (SubstitutionGoal::handleChildOutput): Change to fill in 'expectedHashStr' and 'status'. (SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally. (SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'. * guix/scripts/substitute.scm (process-substitution): Write "success\n" to stdout upon success. (%error-to-file-descriptor-4?): New variable. (guix-substitute): Set 'current-error-port' to file descriptor 4 unless (%error-to-file-descriptor-4?) is false. Remove "--substitute" arguments. Loop reading line from stdin. * tests/substitute.scm : Call '%error-to-file-descriptor-4?'. (request-substitution): New procedure. ("substitute, no signature") ("substitute, invalid hash") ("substitute, unauthorized key") ("substitute, authorized key") ("substitute, unauthorized narinfo comes first") ("substitute, unsigned narinfo comes first") ("substitute, first narinfo is unsigned and has wrong hash") ("substitute, first narinfo is unsigned and has wrong refs") ("substitute, two invalid narinfos") ("substitute, narinfo with several URLs"): Adjust to new "guix substitute --substitute" calling convention. --- guix/scripts/substitute.scm | 34 ++++++++---- nix/libstore/build.cc | 129 ++++++++++++++++++++++---------------------- tests/substitute.scm | 95 ++++++++++++++++++-------------- 3 files changed, 145 insertions(+), 113 deletions(-) (limited to 'tests') diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 5e392eaa8b..73abd3f029 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -88,6 +88,7 @@ write-narinfo %allow-unauthenticated-substitutes? + %error-to-file-descriptor-4? substitute-urls guix-substitute)) @@ -1016,7 +1017,10 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; Skip a line after what 'progress-reporter/file' printed, and another ;; one to visually separate substitutions. - (display "\n\n" (current-error-port))))) + (display "\n\n" (current-error-port)) + + ;; Tell the daemon that we're done. + (display "success\n" (current-output-port))))) ;;; @@ -1127,6 +1131,11 @@ default value." (unless (string->uri uri) (leave (G_ "~a: invalid URI~%") uri))) +(define %error-to-file-descriptor-4? + ;; Whether to direct 'current-error-port' to file descriptor 4 like + ;; 'guix-daemon' expects. + (make-parameter #t)) + (define-command (guix-substitute . args) (category internal) (synopsis "implement the build daemon's substituter protocol") @@ -1140,9 +1149,9 @@ default value." ;; The daemon's agent code opens file descriptor 4 for us and this is where ;; stderr should go. - (parameterize ((current-error-port (match args - (("--query") (fdopen 4 "wl")) - (_ (current-error-port))))) + (parameterize ((current-error-port (if (%error-to-file-descriptor-4?) + (fdopen 4 "wl") + (current-error-port)))) ;; Redirect diagnostics to file descriptor 4 as well. (guix-warning-port (current-error-port)) @@ -1184,15 +1193,22 @@ default value." #:cache-urls (substitute-urls) #:acl acl) (loop (read-line))))))) - (("--substitute" store-path destination) + (("--substitute") ;; Download STORE-PATH and store it as a Nar in file DESTINATION. ;; Specify the number of columns of the terminal so the progress ;; report displays nicely. (parameterize ((current-terminal-columns (client-terminal-columns))) - (process-substitution store-path destination - #:cache-urls (substitute-urls) - #:acl (current-acl) - #:print-build-trace? print-build-trace?))) + (let loop () + (match (read-line) + ((? eof-object?) + #t) + ((= string-tokenize ("substitute" store-path destination)) + (process-substitution store-path destination + #:cache-urls (substitute-urls) + #:acl (current-acl) + #:print-build-trace? + print-build-trace?) + (loop)))))) ((or ("-V") ("--version")) (show-version-and-exit "guix substitute")) (("--help") diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index 7e9ab3f39c..50d300253d 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -262,6 +262,7 @@ public: LocalStore & store; std::shared_ptr hook; + std::shared_ptr substituter; Worker(LocalStore & store); ~Worker(); @@ -2773,15 +2774,6 @@ private: /* Path info returned by the substituter's query info operation. */ SubstitutablePathInfo info; - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* Pipe for the substituter's standard error. */ - Pipe logPipe; - - /* The process ID of the builder. */ - Pid pid; - /* Lock on the store path. */ std::shared_ptr outputLock; @@ -2795,6 +2787,17 @@ private: typedef void (SubstitutionGoal::*GoalState)(); GoalState state; + /* The substituter. */ + std::shared_ptr substituter; + + /* Either the empty string, or the expected hash as returned by the + substituter. */ + string expectedHashStr; + + /* Either the empty string, or the status phrase returned by the + substituter. */ + string status; + void tryNext(); public: @@ -2840,7 +2843,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool SubstitutionGoal::~SubstitutionGoal() { - if (pid != -1) worker.childTerminated(pid); + if (substituter) worker.childTerminated(substituter->pid); } @@ -2848,9 +2851,9 @@ void SubstitutionGoal::timedOut() { if (settings.printBuildTrace) printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); - if (pid != -1) { - pid_t savedPid = pid; - pid.kill(); + if (substituter) { + pid_t savedPid = substituter->pid; + substituter.reset(); worker.childTerminated(savedPid); } amDone(ecFailed); @@ -2977,45 +2980,29 @@ void SubstitutionGoal::tryToRun() printMsg(lvlInfo, format("fetching path `%1%'...") % storePath); - outPipe.create(); - logPipe.create(); - destPath = repair ? storePath + ".tmp" : storePath; /* Remove the (stale) output path if it exists. */ if (pathExists(destPath)) deletePath(destPath); - /* Fill in the arguments. */ - Strings args; - args.push_back("guix"); - args.push_back("substitute"); - args.push_back("--substitute"); - args.push_back(storePath); - args.push_back(destPath); - - /* Fork the substitute program. */ - pid = startProcess([&]() { - - /* Communicate substitute-urls & co. to 'guix substitute'. */ - setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - commonChildInit(logPipe); - - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); + if (!worker.substituter) { + const Strings args = { "substitute", "--substitute" }; + const std::map env = { { "_NIX_OPTIONS", settings.pack() } }; + worker.substituter = std::make_shared(settings.guixProgram, args, env); + } - execv(settings.guixProgram.c_str(), stringsToCharPtrs(args).data()); + /* Borrow the worker's substituter. */ + if (!substituter) substituter.swap(worker.substituter); - throw SysError(format("executing `%1% substitute'") % settings.guixProgram); - }); + /* Send the request to the substituter. */ + writeLine(substituter->toAgent.writeSide, + (format("substitute %1% %2%") % storePath % destPath).str()); - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - outPipe.writeSide.close(); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton >(logPipe.readSide), true, true); + set fds; + fds.insert(substituter->fromAgent.readSide); + fds.insert(substituter->builderOut.readSide); + worker.childStarted(shared_from_this(), substituter->pid, fds, true, true); state = &SubstitutionGoal::finished; @@ -3030,28 +3017,25 @@ void SubstitutionGoal::finished() { trace("substitute finished"); - /* Since we got an EOF on the logger pipe, the substitute is - presumed to have terminated. */ - pid_t savedPid = pid; - int status = pid.wait(true); - - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); + /* Remove the 'guix substitute' process from the list of children. */ + worker.childTerminated(substituter->pid); - /* Get the hash info from stdout. */ - string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; - outPipe.readSide.close(); + /* If max-jobs > 1, the worker might have created a new 'substitute' + process in the meantime. If that is the case, terminate ours; + otherwise, give it back to the worker. */ + if (worker.substituter) { + substituter.reset (); + } else { + worker.substituter.swap(substituter); + } /* Check the exit status and the build result. */ HashResult hash; try { - if (!statusOk(status)) - throw SubstError(format("fetching path `%1%' %2%") - % storePath % statusToString(status)); + if (status != "success") + throw SubstError(format("fetching path `%1%' (status: '%2%')") + % storePath % status); if (!pathExists(destPath)) throw SubstError(format("substitute did not produce path `%1%'") % destPath); @@ -3122,16 +3106,33 @@ void SubstitutionGoal::finished() void SubstitutionGoal::handleChildOutput(int fd, const string & data) { - assert(fd == logPipe.readSide); - if (verbosity >= settings.buildVerbosity) writeToStderr(data); - /* Don't write substitution output to a log file for now. We - probably should, though. */ + if (verbosity >= settings.buildVerbosity + && fd == substituter->builderOut.readSide) { + writeToStderr(data); + /* Don't write substitution output to a log file for now. We + probably should, though. */ + } + + if (fd == substituter->fromAgent.readSide) { + /* Trim whitespace to the right. */ + size_t end = data.find_last_not_of(" \t\n"); + string trimmed = (end != string::npos) ? data.substr(0, end + 1) : data; + + if (expectedHashStr == "") { + expectedHashStr = trimmed; + } else if (status == "") { + status = trimmed; + worker.wakeUp(shared_from_this()); + } else { + printMsg(lvlError, format("unexpected substituter message '%1%'") % data); + } + } } void SubstitutionGoal::handleEOF(int fd) { - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/tests/substitute.scm b/tests/substitute.scm index bd5b6305b0..b86ce09425 100644 --- a/tests/substitute.scm +++ b/tests/substitute.scm @@ -58,6 +58,14 @@ it writes to GUIX-WARNING-PORT a messages that matches ERROR-RX." (let ((message (get-output-string error-output))) (->bool (string-match error-rx message)))))))))) +(define (request-substitution item destination) + "Run 'guix substitute --substitute' to fetch ITEM to DESTINATION." + (parameterize ((guix-warning-port (current-error-port))) + (with-input-from-string (string-append "substitute " item " " + destination "\n") + (lambda () + (guix-substitute "--substitute"))))) + (define %public-key ;; This key is known to be in the ACL by default. (call-with-input-file (string-append %config-directory "/signing-key.pub") @@ -184,6 +192,11 @@ a file for NARINFO." ;; Transmit these options to 'guix substitute'. (substitute-urls (list (getenv "GUIX_BINARY_SUBSTITUTE_URL"))) +;; Never use file descriptor 4, unlike what happens when invoked by the +;; daemon. +(%error-to-file-descriptor-4? #f) + + (test-equal "query narinfo without signature" "" ; not substitutable @@ -284,10 +297,12 @@ System: mips64el-linux\n") (test-quit "substitute, no signature" "no valid substitute" (with-narinfo %narinfo - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "foo"))) + (with-input-from-string (string-append "substitute " + (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo" + " foo\n") + (lambda () + (guix-substitute "--substitute"))))) (test-quit "substitute, invalid hash" "no valid substitute" @@ -295,10 +310,12 @@ System: mips64el-linux\n") (with-narinfo (string-append %narinfo "Signature: " (signature-field "different body") "\n") - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "foo"))) + (with-input-from-string (string-append "substitute " + (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo" + " foo\n") + (lambda () + (guix-substitute "--substitute"))))) (test-quit "substitute, unauthorized key" "no valid substitute" @@ -307,10 +324,12 @@ System: mips64el-linux\n") %narinfo #:public-key %wrong-public-key) "\n") - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "foo"))) + (with-input-from-string (string-append "substitute " + (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo" + " foo\n") + (lambda () + (guix-substitute "--substitute"))))) (test-equal "substitute, authorized key" "Substitutable data." @@ -319,10 +338,9 @@ System: mips64el-linux\n") (dynamic-wind (const #t) (lambda () - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "substitute-retrieved") + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved") (call-with-input-file "substitute-retrieved" get-string-all)) (lambda () (false-if-exception (delete-file "substitute-retrieved")))))) @@ -352,10 +370,9 @@ System: mips64el-linux\n") (map (cut string-append "file://" <>) (list %alternate-substitute-directory %main-substitute-directory)))) - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "substitute-retrieved")) + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) (call-with-input-file "substitute-retrieved" get-string-all)) (lambda () (false-if-exception (delete-file "substitute-retrieved"))))))) @@ -381,10 +398,9 @@ System: mips64el-linux\n") (map (cut string-append "file://" <>) (list %alternate-substitute-directory %main-substitute-directory)))) - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "substitute-retrieved")) + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) (call-with-input-file "substitute-retrieved" get-string-all)) (lambda () (false-if-exception (delete-file "substitute-retrieved"))))))) @@ -417,10 +433,9 @@ System: mips64el-linux\n") (map (cut string-append "file://" <>) (list %alternate-substitute-directory %main-substitute-directory)))) - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "substitute-retrieved")) + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) (call-with-input-file "substitute-retrieved" get-string-all)) (lambda () (false-if-exception (delete-file "substitute-retrieved"))))))) @@ -451,10 +466,9 @@ System: mips64el-linux\n") (map (cut string-append "file://" <>) (list %alternate-substitute-directory %main-substitute-directory)))) - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "substitute-retrieved")) + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) (call-with-input-file "substitute-retrieved" get-string-all)) (lambda () (false-if-exception (delete-file "substitute-retrieved"))))))) @@ -470,10 +484,12 @@ System: mips64el-linux\n") #:public-key %wrong-public-key)) %main-substitute-directory - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "substitute-retrieved")))) + (with-input-from-string (string-append "substitute " + (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo" + " substitute-retrieved\n") + (lambda () + (guix-substitute "--substitute")))))) (test-equal "substitute, narinfo with several URLs" "Substitutable data." @@ -513,10 +529,9 @@ System: mips64el-linux\n"))) (parameterize ((substitute-urls (list (string-append "file://" %main-substitute-directory)))) - (guix-substitute "--substitute" - (string-append (%store-prefix) - "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") - "substitute-retrieved")) + (request-substitution (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) (call-with-input-file "substitute-retrieved" get-string-all)) (lambda () (false-if-exception (delete-file "substitute-retrieved"))))))) -- cgit v1.2.3 From 799f066768bacb321ebad84c75b2bbfd269e7cd8 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 8 Dec 2020 22:58:37 +0100 Subject: import: opam: Adjust test to latest 'opam->guix-package' changes. This is a followup to a8dccd4bdc1e58219d4ba08fe1649bf0b8325f44, which broke the test. * guix/import/opam.scm (get-opam-repository): Prevent inlining. * tests/opam.scm ("opam->guix-package"): Mock 'get-opam-repository'. --- guix/import/opam.scm | 3 +++ tests/opam.scm | 67 +++++++++++++++++++++++++++------------------------- 2 files changed, 38 insertions(+), 32 deletions(-) (limited to 'tests') diff --git a/guix/import/opam.scm b/guix/import/opam.scm index 54143f83ca..670973b193 100644 --- a/guix/import/opam.scm +++ b/guix/import/opam.scm @@ -141,6 +141,9 @@ path to the repository." (string-append location "/" (substring (symbol->string repo) 4))) (else location))))) +;; Prevent Guile 3 from inlining this procedure so we can mock it in tests. +(set! get-opam-repository get-opam-repository) + (define (latest-version versions) "Find the most recent version from a list of versions." (fold (lambda (a b) (if (version>? a b) a b)) (car versions) versions)) diff --git a/tests/opam.scm b/tests/opam.scm index 8d43e2ce70..11984b56a6 100644 --- a/tests/opam.scm +++ b/tests/opam.scm @@ -80,38 +80,41 @@ url { (set! test-source-hash (call-with-input-file file-name port-sha256)))) (_ (error "Unexpected URL: " url))))) - (let ((my-package (string-append test-repo "/packages/foo/foo.1.0.0"))) - (mkdir-p my-package) - (with-output-to-file (string-append my-package "/opam") - (lambda _ - (format #t "~a" test-opam-file)))) - (match (opam->guix-package "foo" #:repo test-repo) - (('package - ('name "ocaml-foo") - ('version "1.0.0") - ('source ('origin - ('method 'url-fetch) - ('uri "https://example.org/foo-1.0.0.tar.gz") - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'ocaml-build-system) - ('propagated-inputs - ('quasiquote - (("ocaml-zarith" ('unquote 'ocaml-zarith))))) - ('native-inputs - ('quasiquote - (("ocaml-alcotest" ('unquote 'ocaml-alcotest)) - ("ocamlbuild" ('unquote 'ocamlbuild))))) - ('home-page "https://example.org/") - ('synopsis "Some example package") - ('description "This package is just an example.") - ('license #f)) - (string=? (bytevector->nix-base32-string - test-source-hash) - hash)) - (x - (pk 'fail x #f))))) + (mock ((guix import opam) get-opam-repository + (const test-repo)) + (let ((my-package (string-append test-repo + "/packages/foo/foo.1.0.0"))) + (mkdir-p my-package) + (with-output-to-file (string-append my-package "/opam") + (lambda _ + (format #t "~a" test-opam-file)))) + (match (opam->guix-package "foo" #:repo test-repo) + (('package + ('name "ocaml-foo") + ('version "1.0.0") + ('source ('origin + ('method 'url-fetch) + ('uri "https://example.org/foo-1.0.0.tar.gz") + ('sha256 + ('base32 + (? string? hash))))) + ('build-system 'ocaml-build-system) + ('propagated-inputs + ('quasiquote + (("ocaml-zarith" ('unquote 'ocaml-zarith))))) + ('native-inputs + ('quasiquote + (("ocaml-alcotest" ('unquote 'ocaml-alcotest)) + ("ocamlbuild" ('unquote 'ocamlbuild))))) + ('home-page "https://example.org/") + ('synopsis "Some example package") + ('description "This package is just an example.") + ('license #f)) + (string=? (bytevector->nix-base32-string + test-source-hash) + hash)) + (x + (pk 'fail x #f)))))) ;; Test the opam file parser ;; We fold over some test cases. Each case is a pair of the string to parse and the -- cgit v1.2.3 From 371ba7b4be81895d202519824bf19893bfdb2526 Mon Sep 17 00:00:00 2001 From: Giacomo Leidi Date: Tue, 12 May 2020 23:31:30 +0200 Subject: guix: Add globstar support. * guix/glob.scm (string->sglob) (glob-match?): Add globstar support. * tests/glob.scm: Update accordingly. Signed-off-by: Jelle Licht --- guix/glob.scm | 15 +++++++++++++++ tests/glob.scm | 8 ++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/guix/glob.scm b/guix/glob.scm index a9fc744802..d73783cd30 100644 --- a/guix/glob.scm +++ b/guix/glob.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018 Ludovic Courtès +;;; Copyright © 2020 Giacomo Leidi ;;; ;;; This file is part of GNU Guix. ;;; @@ -61,6 +62,11 @@ STR, a glob pattern such as \"foo*\" or \"foo??bar\"." (flatten (reverse (if (null? pending) result (cons-string pending result))))) + ((#\* #\* #\/ . rest) + (if (zero? brackets) + (loop rest '() 0 + (cons* '**/ (cons-string pending result))) + (loop rest (cons '**/ pending) brackets result))) (((and chr (or #\? #\*)) . rest) (let ((wildcard (match chr (#\? '?) @@ -121,6 +127,15 @@ STR, a glob pattern such as \"foo*\" or \"foo??bar\"." (string-null? str)) (('*) #t) + (('**/) + #t) + (('**/ suffix . rest) + (let ((rest (if (eq? '* suffix) (cdr rest) rest)) + (suffix (if (eq? '* suffix) (car rest) suffix))) + (match (string-contains str suffix) + (#f #f) + (index (loop rest (string-drop str + (+ index (string-length suffix)))))))) (('* suffix . rest) (match (string-contains str suffix) (#f #f) diff --git a/tests/glob.scm b/tests/glob.scm index 3134069789..2a5a40c3c6 100644 --- a/tests/glob.scm +++ b/tests/glob.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018 Ludovic Courtès +;;; Copyright © 2020 Giacomo Leidi ;;; ;;; This file is part of GNU Guix. ;;; @@ -53,7 +54,8 @@ "foo[abc]bar" => '("foo" (set #\a #\b #\c) "bar") "foo[a[b]c]bar" => '("foo" (set #\a #\[ #\b #\] #\c) "bar") "[123]x" => '((set #\1 #\2 #\3) "x") - "[a-z]" => '((range #\a #\z))) + "[a-z]" => '((range #\a #\z)) + "**/*.scm" => '(**/ * ".scm")) (test-glob-match ("foo" matches "foo" (and not "foobar" "barfoo")) @@ -64,6 +66,8 @@ ("ab[0-9]c" matches "ab0c" "ab7c" "ab9c" (and not "ab-c" "ab00c" "ab3")) ("ab[cdefg]" matches "abc" "abd" "abg" - (and not "abh" "abcd" "ab["))) + (and not "abh" "abcd" "ab[")) + ("foo/**/*.scm" matches "foo/bar/baz.scm" "foo/bar.scm" "foo/bar/baz/zab.scm" + (and not "foo/bar/baz.java" "foo/bar.smc"))) (test-end "glob") -- cgit v1.2.3 From 465d2cb286170933577de045e6e6dad7205bfe10 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 9 Dec 2020 21:50:21 +0100 Subject: serialization: 'fold-archive' notifies about directory processing completion. * guix/serialization.scm (fold-archive): Call PROC with a 'directory-complete tag when done with a directory. (restore-file): Handle it. * guix/scripts/archive.scm (list-contents): Likewise. * guix/scripts/challenge.scm (archive-contents): Likewise. * tests/nar.scm ("write-file-tree + fold-archive"): Adjust accordingly. --- guix/scripts/archive.scm | 2 ++ guix/scripts/challenge.scm | 1 + guix/serialization.scm | 5 ++++- tests/nar.scm | 6 ++++-- 4 files changed, 11 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/guix/scripts/archive.scm b/guix/scripts/archive.scm index c04baf9784..1f73fff711 100644 --- a/guix/scripts/archive.scm +++ b/guix/scripts/archive.scm @@ -347,6 +347,8 @@ output port." (match type ('directory (format #t "D ~a~%" file)) + ('directory-complete + #t) ('symlink (format #t "S ~a -> ~a~%" file content)) ((or 'regular 'executable) diff --git a/guix/scripts/challenge.scm b/guix/scripts/challenge.scm index 39bd2c1c0f..d0a456ac1d 100644 --- a/guix/scripts/challenge.scm +++ b/guix/scripts/challenge.scm @@ -210,6 +210,7 @@ taken since we do not import the archives." (cons `(,file ,type ,(port-sha256* port size)) result)))) ('directory result) + ('directory-complete result) ('symlink (cons `(,file ,type ,contents) result)))) '() diff --git a/guix/serialization.scm b/guix/serialization.scm index 836ad06caf..cc56134ef4 100644 --- a/guix/serialization.scm +++ b/guix/serialization.scm @@ -444,7 +444,8 @@ depends on TYPE." (file file) (token x)))))) (loop (read-string port) result))))) - (")" result) ;done with DIR + (")" ;done with DIR + (proc file 'directory-complete #f result)) (x (raise (condition @@ -463,6 +464,8 @@ Restore it as FILE." (match type ('directory (mkdir file)) + ('directory-complete + #t) ('symlink (symlink content file)) ((or 'regular 'executable) diff --git a/tests/nar.scm b/tests/nar.scm index aeff3d3330..b542ebd47c 100644 --- a/tests/nar.scm +++ b/tests/nar.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -218,8 +218,10 @@ '(("R" directory #f) ("R/dir" directory #f) ("R/dir/exe" executable "1234") + ("R/dir" directory-complete #f) ("R/foo" regular "abcdefg") - ("R/lnk" symlink "foo")) + ("R/lnk" symlink "foo") + ("R" directory-complete #f)) (let () (define-values (port get-bytevector) -- cgit v1.2.3 From ed7d02f7c198970ce3fe94bcee47592963326446 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 9 Dec 2020 22:16:35 +0100 Subject: serialization: 'restore-file' sets canonical timestamp and permissions. * guix/serialization.scm (restore-file): Set the permissions and mtime of FILE. * guix/nar.scm (finalize-store-file): Pass #:reset-timestamps? #f to 'register-items'. * tests/nar.scm (rm-rf): Add 'chmod' calls to ensure files are writable. ("write-file + restore-file with symlinks"): Ensure every file in OUTPUT passes 'canonical-file?'. * tests/guix-archive.sh: Run "chmod -R +w" before "rm -rf". --- guix/nar.scm | 8 +++++--- guix/serialization.scm | 14 +++++++++----- tests/guix-archive.sh | 4 ++-- tests/nar.scm | 12 ++++++++++-- 4 files changed, 26 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/guix/nar.scm b/guix/nar.scm index a23af2e5de..edfcc9aab5 100644 --- a/guix/nar.scm +++ b/guix/nar.scm @@ -114,10 +114,12 @@ held." ;; Install the new TARGET. (rename-file source target) - ;; Register TARGET. As a side effect, it resets the timestamps of all - ;; its files, recursively, and runs a deduplication pass. + ;; Register TARGET. As a side effect, run a deduplication pass. + ;; Timestamps and permissions are already correct thanks to + ;; 'restore-file'. (register-items db - (list (store-info target deriver references)))) + (list (store-info target deriver references)) + #:reset-timestamps? #f)) (when lock? (delete-file (string-append target ".lock")) diff --git a/guix/serialization.scm b/guix/serialization.scm index cc56134ef4..677ca60b66 100644 --- a/guix/serialization.scm +++ b/guix/serialization.scm @@ -459,23 +459,27 @@ depends on TYPE." (define (restore-file port file) "Read a file (possibly a directory structure) in Nar format from PORT. -Restore it as FILE." +Restore it as FILE with canonical permissions and timestamps." (fold-archive (lambda (file type content result) (match type ('directory (mkdir file)) ('directory-complete - #t) + (chmod file #o555) + (utime file 1 1 0 0)) ('symlink - (symlink content file)) + (symlink content file) + (utime file 1 1 0 0 AT_SYMLINK_NOFOLLOW)) ((or 'regular 'executable) (match content ((input . size) (call-with-output-file file (lambda (output) (dump input output size) - (when (eq? type 'executable) - (chmod output #o755))))))))) + (chmod output (if (eq? type 'executable) + #o555 + #o444)))) + (utime file 1 1 0 0)))))) #t port file)) diff --git a/tests/guix-archive.sh b/tests/guix-archive.sh index e796c62f9a..00b87ff0ac 100644 --- a/tests/guix-archive.sh +++ b/tests/guix-archive.sh @@ -1,5 +1,5 @@ # GNU Guix --- Functional package management for GNU -# Copyright © 2013, 2014, 2015, 2019 Ludovic Courtès +# Copyright © 2013, 2014, 2015, 2019, 2020 Ludovic Courtès # # This file is part of GNU Guix. # @@ -28,7 +28,7 @@ tmpdir="t-archive-dir-$$" rm -f "$archive" "$archive_alt" rm -rf "$tmpdir" -trap 'rm -f "$archive" "$archive_alt"; rm -rf "$tmpdir"' EXIT +trap 'rm -f "$archive" "$archive_alt"; chmod -R +w "$tmpdir"; rm -rf "$tmpdir"' EXIT guix archive --export guile-bootstrap > "$archive" guix archive --export guile-bootstrap:out > "$archive_alt" diff --git a/tests/nar.scm b/tests/nar.scm index b542ebd47c..59616659c8 100644 --- a/tests/nar.scm +++ b/tests/nar.scm @@ -136,8 +136,11 @@ (define (rm-rf dir) (file-system-fold (const #t) ; enter? (lambda (file stat result) ; leaf + (unless (eq? 'symlink (stat:type stat)) + (chmod file #o644)) (delete-file file)) - (const #t) ; down + (lambda (dir stat result) ; down + (chmod dir #o755)) (lambda (dir stat result) ; up (rmdir dir)) (const #t) ; skip @@ -363,7 +366,12 @@ (cut write-file input <>)) (call-with-input-file nar (cut restore-file <> output)) - (file-tree-equal? input output)) + + (and (file-tree-equal? input output) + (every (lambda (file) + (canonical-file? + (string-append output "/" file))) + '("root" "root/reg" "root/exe")))) (lambda () (false-if-exception (delete-file nar)) (false-if-exception (rm-rf output))))))) -- cgit v1.2.3 From 2718c29c3fb9f9de2ec897248ad49ae11ca39b7a Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 10 Dec 2020 11:21:14 +0100 Subject: nar: Deduplicate files right as they are restored. This avoids having to traverse and re-read the files that we have just restored, thereby reducing I/O. * guix/serialization.scm (dump-file): New procedure. (restore-file): Add #:dump-file parameter and honor it. * guix/store/deduplication.scm (tee, dump-file/deduplicate): New procedures. * guix/nar.scm (restore-one-item): Pass #:dump-file to 'restore-file'. (finalize-store-file): Pass #:deduplicate? #f to 'register-items'. * tests/nar.scm : Call 'setenv' to set "NIX_STORE". --- guix/nar.scm | 12 ++++++---- guix/serialization.scm | 27 ++++++++++++++------- guix/store/deduplication.scm | 57 +++++++++++++++++++++++++++++++++++++++++++- tests/nar.scm | 3 +++ 4 files changed, 85 insertions(+), 14 deletions(-) (limited to 'tests') diff --git a/guix/nar.scm b/guix/nar.scm index edfcc9aab5..ba035ca6dc 100644 --- a/guix/nar.scm +++ b/guix/nar.scm @@ -27,6 +27,7 @@ ;; (guix store) since this is "daemon-side" code. #:use-module (guix store) #:use-module (guix store database) + #:use-module ((guix store deduplication) #:select (dump-file/deduplicate)) #:use-module ((guix build store-copy) #:select (store-info)) #:use-module (guix i18n) @@ -114,12 +115,12 @@ held." ;; Install the new TARGET. (rename-file source target) - ;; Register TARGET. As a side effect, run a deduplication pass. - ;; Timestamps and permissions are already correct thanks to - ;; 'restore-file'. + ;; Register TARGET. The 'restore-file' call took care of + ;; deduplication, timestamps, and permissions. (register-items db (list (store-info target deriver references)) - #:reset-timestamps? #f)) + #:reset-timestamps? #f + #:deduplicate? #f)) (when lock? (delete-file (string-append target ".lock")) @@ -212,7 +213,8 @@ s-expression")) (let-values (((port get-hash) (open-sha256-input-port port))) (with-temporary-store-file temp - (restore-file port temp) + (restore-file port temp + #:dump-file dump-file/deduplicate) (let ((magic (read-int port))) (unless (= magic %export-magic) diff --git a/guix/serialization.scm b/guix/serialization.scm index 677ca60b66..9e2dce8bb0 100644 --- a/guix/serialization.scm +++ b/guix/serialization.scm @@ -457,9 +457,22 @@ depends on TYPE." (&message (message "unsupported nar entry type")) (&nar-read-error (port port) (file file) (token x))))))))) -(define (restore-file port file) +(define (dump-file file input size type) + "Dump SIZE bytes from INPUT to FILE." + (call-with-output-file file + (lambda (output) + (dump input output size)))) + +(define* (restore-file port file + #:key (dump-file dump-file)) "Read a file (possibly a directory structure) in Nar format from PORT. -Restore it as FILE with canonical permissions and timestamps." +Restore it as FILE with canonical permissions and timestamps. To write a +regular or executable file, call: + + (DUMP-FILE FILE INPUT SIZE TYPE) + +The default is to dump SIZE bytes from INPUT to FILE, but callers can provide +a custom procedure, for instance to deduplicate FILE on the fly." (fold-archive (lambda (file type content result) (match type ('directory @@ -473,12 +486,10 @@ Restore it as FILE with canonical permissions and timestamps." ((or 'regular 'executable) (match content ((input . size) - (call-with-output-file file - (lambda (output) - (dump input output size) - (chmod output (if (eq? type 'executable) - #o555 - #o444)))) + (dump-file file input size type) + (chmod file (if (eq? type 'executable) + #o555 + #o444)) (utime file 1 1 0 0)))))) #t port diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm index 0655ceb890..b4d37d4525 100644 --- a/guix/store/deduplication.scm +++ b/guix/store/deduplication.scm @@ -26,12 +26,15 @@ #:use-module (guix build syscalls) #:use-module (guix base32) #:use-module (srfi srfi-11) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) #:use-module (rnrs io ports) #:use-module (ice-9 ftw) #:use-module (ice-9 match) #:use-module (guix serialization) #:export (nar-sha256 - deduplicate)) + deduplicate + dump-file/deduplicate)) ;; XXX: This port is used as a workaround on Guile <= 2.2.4 where ;; 'port-position' throws to 'out-of-range' when the offset is great than or @@ -201,3 +204,55 @@ under STORE." ;; that's OK: we just can't deduplicate it more. #f) (else (apply throw args))))))))))) + +(define (tee input len output) + "Return a port that reads up to LEN bytes from INPUT and writes them to +OUTPUT as it goes." + (define bytes-read 0) + + (define (fail) + ;; Reached EOF before we had read LEN bytes from INPUT. + (raise (condition + (&nar-error (port input) + (file (port-filename output)))))) + + (define (read! bv start count) + ;; Read at most LEN bytes in total. + (let ((count (min count (- len bytes-read)))) + (let loop ((ret (get-bytevector-n! input bv start count))) + (cond ((eof-object? ret) + (if (= bytes-read len) + 0 ; EOF + (fail))) + ((and (zero? ret) (> count 0)) + ;; Do not return zero since zero means EOF, so try again. + (loop (get-bytevector-n! input bv start count))) + (else + (put-bytevector output bv start ret) + (set! bytes-read (+ bytes-read ret)) + ret))))) + + (make-custom-binary-input-port "tee input port" read! #f #f #f)) + +(define* (dump-file/deduplicate file input size type + #:key (store (%store-directory))) + "Write SIZE bytes read from INPUT to FILE. TYPE is a symbol, either +'regular or 'executable. + +This procedure is suitable as a #:dump-file argument to 'restore-file'. When +used that way, it deduplicates files on the fly as they are restored, thereby +removing the need to a deduplication pass that would re-read all the files +down the road." + (define hash + (call-with-output-file file + (lambda (output) + (let-values (((hash-port get-hash) + (open-hash-port (hash-algorithm sha256)))) + (write-file-tree file hash-port + #:file-type+size (lambda (_) (values type size)) + #:file-port + (const (tee input size output))) + (close-port hash-port) + (get-hash))))) + + (deduplicate file hash #:store store)) diff --git a/tests/nar.scm b/tests/nar.scm index 59616659c8..ba4881caaa 100644 --- a/tests/nar.scm +++ b/tests/nar.scm @@ -452,6 +452,9 @@ (false-if-exception (rm-rf %test-dir)) (setlocale LC_ALL locale))))) +;; XXX: Tell the 'deduplicate' procedure what store we're actually using. +(setenv "NIX_STORE" (%store-prefix)) + (test-assert "restore-file-set (signed, valid)" (with-store store (let* ((texts (unfold (cut >= <> 10) -- cgit v1.2.3 From 7b8d239ec241b9663820fed3bfde4344366f9d19 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 10 Dec 2020 13:37:59 +0100 Subject: store-copy: 'populate-store' resets timestamps. Until now, 'populate-store' would reset permissions but not timestamps, so callers would resort to going through an extra directory traversal to reset timestamps. * guix/build/store-copy.scm (reset-permissions): Remove. (copy-recursively): New procedure. (populate-store): Pass #:keep-permissions? to 'copy-recursively'. Remove call to 'reset-permissions'. * tests/gexp.scm ("gexp->derivation, store copy"): In BUILD-DRV, check whether 'populate-store' canonicalizes permissions and timestamps. * gnu/build/image.scm (initialize-root-partition): Pass #:reset-timestamps? #f to 'register-closure'. * gnu/build/vm.scm (root-partition-initializer): Likewise. --- gnu/build/image.scm | 5 +-- gnu/build/vm.scm | 2 +- guix/build/store-copy.scm | 103 +++++++++++++++++++++++++++++++++------------- tests/gexp.scm | 19 ++++++++- 4 files changed, 95 insertions(+), 34 deletions(-) (limited to 'tests') diff --git a/gnu/build/image.scm b/gnu/build/image.scm index 640a784204..2857362914 100644 --- a/gnu/build/image.scm +++ b/gnu/build/image.scm @@ -196,9 +196,8 @@ register-closure." (when register-closures? (for-each (lambda (closure) - (register-closure root - closure - #:reset-timestamps? #t + (register-closure root closure + #:reset-timestamps? #f #:deduplicate? deduplicate? #:wal-mode? wal-mode?)) references-graphs)) diff --git a/gnu/build/vm.scm b/gnu/build/vm.scm index 287d099f79..30feaf800f 100644 --- a/gnu/build/vm.scm +++ b/gnu/build/vm.scm @@ -414,7 +414,7 @@ system that is passed to 'populate-root-file-system'." (for-each (lambda (closure) (register-closure target (string-append "/xchg/" closure) - #:reset-timestamps? copy-closures? + #:reset-timestamps? #f #:deduplicate? deduplicate?)) closures) (unless copy-closures? diff --git a/guix/build/store-copy.scm b/guix/build/store-copy.scm index ad551bca98..95dcb8e114 100644 --- a/guix/build/store-copy.scm +++ b/guix/build/store-copy.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2017, 2018 Ludovic Courtès +;;; Copyright © 2013, 2014, 2017, 2018, 2020 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -17,7 +17,7 @@ ;;; along with GNU Guix. If not, see . (define-module (guix build store-copy) - #:use-module (guix build utils) + #:use-module ((guix build utils) #:hide (copy-recursively)) #:use-module (guix sets) #:use-module (guix progress) #:use-module (srfi srfi-1) @@ -169,32 +169,83 @@ REFERENCE-GRAPHS, a list of reference-graph files." (reduce + 0 (map file-size items))) -(define (reset-permissions file) - "Reset the permissions on FILE and its sub-directories so that they are all -read-only." - ;; XXX: This procedure exists just to work around the inability of - ;; 'copy-recursively' to preserve permissions. - (file-system-fold (const #t) ;enter? - (lambda (file stat _) ;leaf - (unless (eq? 'symlink (stat:type stat)) - (chmod file - (if (zero? (logand (stat:mode stat) - #o100)) - #o444 - #o555)))) - (const #t) ;down - (lambda (directory stat _) ;up - (chmod directory #o555)) - (const #f) ;skip - (const #f) ;error +;; TODO: Remove when the one in (guix build utils) has #:keep-permissions?, +;; the fix for , and when #:keep-mtime? works for +;; symlinks. +(define* (copy-recursively source destination + #:key + (log (current-output-port)) + (follow-symlinks? #f) + (copy-file copy-file) + keep-mtime? keep-permissions?) + "Copy SOURCE directory to DESTINATION. Follow symlinks if FOLLOW-SYMLINKS? +is true; otherwise, just preserve them. Call COPY-FILE to copy regular files. +When KEEP-MTIME? is true, keep the modification time of the files in SOURCE on +those of DESTINATION. When KEEP-PERMISSIONS? is true, preserve file +permissions. Write verbose output to the LOG port." + (define AT_SYMLINK_NOFOLLOW + ;; Guile 2.0 did not define this constant, hence this hack. + (let ((variable (module-variable the-root-module 'AT_SYMLINK_NOFOLLOW))) + (if variable + (variable-ref variable) + 256))) ;for GNU/Linux + + (define (set-file-time file stat) + (utime file + (stat:atime stat) + (stat:mtime stat) + (stat:atimensec stat) + (stat:mtimensec stat) + AT_SYMLINK_NOFOLLOW)) + + (define strip-source + (let ((len (string-length source))) + (lambda (file) + (substring file len)))) + + (file-system-fold (const #t) ; enter? + (lambda (file stat result) ; leaf + (let ((dest (string-append destination + (strip-source file)))) + (format log "`~a' -> `~a'~%" file dest) + (case (stat:type stat) + ((symlink) + (let ((target (readlink file))) + (symlink target dest))) + (else + (copy-file file dest) + (when keep-permissions? + (chmod dest (stat:perms stat))))) + (when keep-mtime? + (set-file-time dest stat)))) + (lambda (dir stat result) ; down + (let ((target (string-append destination + (strip-source dir)))) + (mkdir-p target))) + (lambda (dir stat result) ; up + (let ((target (string-append destination + (strip-source dir)))) + (when keep-mtime? + (set-file-time target stat)) + (when keep-permissions? + (chmod target (stat:perms stat))))) + (const #t) ; skip + (lambda (file stat errno result) + (format (current-error-port) "i/o error: ~a: ~a~%" + file (strerror errno)) + #f) #t - file - lstat)) + source + + (if follow-symlinks? + stat + lstat))) (define* (populate-store reference-graphs target #:key (log-port (current-error-port))) "Populate the store under directory TARGET with the items specified in -REFERENCE-GRAPHS, a list of reference-graph files." +REFERENCE-GRAPHS, a list of reference-graph files. Items copied to TARGET +maintain timestamps and permissions." (define store (string-append target (%store-directory))) @@ -221,12 +272,8 @@ REFERENCE-GRAPHS, a list of reference-graph files." (copy-recursively thing (string-append target thing) #:keep-mtime? #t + #:keep-permissions? #t #:log (%make-void-port "w")) - - ;; XXX: Since 'copy-recursively' doesn't allow us to - ;; preserve permissions, we have to traverse TARGET to - ;; make sure everything is read-only. - (reset-permissions (string-append target thing)) (report)) things))))) diff --git a/tests/gexp.scm b/tests/gexp.scm index 686334af61..a0e55178fa 100644 --- a/tests/gexp.scm +++ b/tests/gexp.scm @@ -723,10 +723,25 @@ (lambda (port) (display "This is the second one." port)))))) (build-drv #~(begin - (use-modules (guix build store-copy)) + (use-modules (guix build store-copy) + (guix build utils) + (srfi srfi-1)) + + (define (canonical-file? file) + ;; Copied from (guix tests). + (let ((st (lstat file))) + (or (not (string-prefix? (%store-directory) file)) + (eq? 'symlink (stat:type st)) + (and (= 1 (stat:mtime st)) + (zero? (logand #o222 (stat:mode st))))))) (mkdir #$output) - (populate-store '("graph") #$output)))) + (populate-store '("graph") #$output) + + ;; Check whether 'populate-store' canonicalizes + ;; permissions and timestamps. + (unless (every canonical-file? (find-files #$output)) + (error "not canonical!" #$output))))) (mlet* %store-monad ((one (gexp->derivation "one" build-one)) (two (gexp->derivation "two" (build-two one))) (drv (gexp->derivation "store-copy" build-drv -- cgit v1.2.3 From 6a060ff27ff68384d7c90076baa36c349fff689d Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 10 Dec 2020 15:12:34 +0100 Subject: store-copy: 'populate-store' can optionally deduplicate files. Until now deduplication was performed as an additional pass after copying files, which involve re-traversing all the files that had just been copied. * guix/store/deduplication.scm (copy-file/deduplicate): New procedure. * tests/store-deduplication.scm ("copy-file/deduplicate"): New test. * guix/build/store-copy.scm (populate-store): Add #:deduplicate? parameter and honor it. * tests/gexp.scm ("gexp->derivation, store copy"): Pass #:deduplicate? #f to 'populate-store'. * gnu/build/image.scm (initialize-root-partition): Pass #:deduplicate? to 'populate-store'. Pass #:deduplicate? #f to 'register-closure'. * gnu/build/vm.scm (root-partition-initializer): Likewise. * gnu/build/install.scm (populate-single-profile-directory): Pass #:deduplicate? #f to 'populate-store'. * gnu/build/linux-initrd.scm (build-initrd): Likewise. * guix/scripts/pack.scm (self-contained-tarball)[import-module?]: New procedure. [build]: Pass it as an argument to 'source-module-closure'. * guix/scripts/pack.scm (squashfs-image)[build]: Wrap in 'with-extensions'. * gnu/system/linux-initrd.scm (expression->initrd)[import-module?]: New procedure. [builder]: Pass it to 'source-module-closure'. * gnu/system/install.scm (cow-store-service-type)[import-module?]: New procedure. Pass it to 'source-module-closure'. --- gnu/build/image.scm | 5 +- gnu/build/install.scm | 3 +- gnu/build/linux-initrd.scm | 3 +- gnu/build/vm.scm | 5 +- gnu/system/install.scm | 12 +- gnu/system/linux-initrd.scm | 10 +- guix/build/store-copy.scm | 13 ++- guix/scripts/pack.scm | 258 ++++++++++++++++++++++-------------------- guix/store/deduplication.scm | 16 ++- tests/gexp.scm | 3 +- tests/store-deduplication.scm | 18 ++- 11 files changed, 207 insertions(+), 139 deletions(-) (limited to 'tests') diff --git a/gnu/build/image.scm b/gnu/build/image.scm index 0deea10a9d..8f50f27f78 100644 --- a/gnu/build/image.scm +++ b/gnu/build/image.scm @@ -186,7 +186,8 @@ rest of the store when registering the closures. SYSTEM-DIRECTORY is the name of the directory of the 'system' derivation. Pass WAL-MODE? to register-closure." (populate-root-file-system system-directory root) - (populate-store references-graphs root) + (populate-store references-graphs root + #:deduplicate? deduplicate?) ;; Populate /dev. (when make-device-nodes @@ -195,7 +196,7 @@ register-closure." (when register-closures? (for-each (lambda (closure) (register-closure root closure - #:deduplicate? deduplicate? + #:deduplicate? #f #:wal-mode? wal-mode?)) references-graphs)) diff --git a/gnu/build/install.scm b/gnu/build/install.scm index 63995e1d09..f5c8407b89 100644 --- a/gnu/build/install.scm +++ b/gnu/build/install.scm @@ -214,7 +214,8 @@ This is used to create the self-contained tarballs with 'guix pack'." (symlink old (scope new))) ;; Populate the store. - (populate-store (list closure) directory) + (populate-store (list closure) directory + #:deduplicate? #f) (when database (install-database-and-gc-roots directory database profile diff --git a/gnu/build/linux-initrd.scm b/gnu/build/linux-initrd.scm index 99796adba6..bb2ed0db0c 100644 --- a/gnu/build/linux-initrd.scm +++ b/gnu/build/linux-initrd.scm @@ -127,7 +127,8 @@ REFERENCES-GRAPHS." (mkdir "contents") ;; Copy the closures of all the items referenced in REFERENCES-GRAPHS. - (populate-store references-graphs "contents") + (populate-store references-graphs "contents" + #:deduplicate? #f) (with-directory-excursion "contents" ;; Make '/init'. diff --git a/gnu/build/vm.scm b/gnu/build/vm.scm index abb0317faf..03be5697b7 100644 --- a/gnu/build/vm.scm +++ b/gnu/build/vm.scm @@ -395,7 +395,8 @@ system that is passed to 'populate-root-file-system'." (when copy-closures? ;; Populate the store. (populate-store (map (cut string-append "/xchg/" <>) closures) - target)) + target + #:deduplicate? deduplicate?)) ;; Populate /dev. (make-device-nodes target) @@ -412,7 +413,7 @@ system that is passed to 'populate-root-file-system'." (for-each (lambda (closure) (register-closure target (string-append "/xchg/" closure) - #:deduplicate? deduplicate?)) + #:deduplicate? #f)) closures) (unless copy-closures? (umount target-store))) diff --git a/gnu/system/install.scm b/gnu/system/install.scm index a6b9e3d952..e753463473 100644 --- a/gnu/system/install.scm +++ b/gnu/system/install.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès +;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès ;;; Copyright © 2015 Mark H Weaver ;;; Copyright © 2016 Andreas Enge ;;; Copyright © 2017 Marius Bakke @@ -176,6 +176,13 @@ manual." (shepherd-service-type 'cow-store (lambda _ + (define (import-module? module) + ;; Since we don't use deduplication support in 'populate-store', don't + ;; import (guix store deduplication) and its dependencies, which + ;; includes Guile-Gcrypt. + (and (guix-module-name? module) + (not (equal? module '(guix store deduplication))))) + (shepherd-service (requirement '(root-file-system user-processes)) (provision '(cow-store)) @@ -190,7 +197,8 @@ the given target.") ,@%default-modules)) (start (with-imported-modules (source-module-closure - '((gnu build install))) + '((gnu build install)) + #:select? import-module?) #~(case-lambda ((target) (mount-cow-store target #$%backing-directory) diff --git a/gnu/system/linux-initrd.scm b/gnu/system/linux-initrd.scm index 4fb1d863c9..c6ba9bb560 100644 --- a/gnu/system/linux-initrd.scm +++ b/gnu/system/linux-initrd.scm @@ -76,12 +76,20 @@ the derivations referenced by EXP are automatically copied to the initrd." (define init (program-file "init" exp #:guile guile)) + (define (import-module? module) + ;; Since we don't use deduplication support in 'populate-store', don't + ;; import (guix store deduplication) and its dependencies, which includes + ;; Guile-Gcrypt. That way we can run tests with '--bootstrap'. + (and (guix-module-name? module) + (not (equal? module '(guix store deduplication))))) + (define builder ;; Do not use "guile-zlib" extension here, otherwise it would drag the ;; non-static "zlib" package to the initrd closure. It is not needed ;; anyway because the modules are stored uncompressed within the initrd. (with-imported-modules (source-module-closure - '((gnu build linux-initrd))) + '((gnu build linux-initrd)) + #:select? import-module?) #~(begin (use-modules (gnu build linux-initrd)) diff --git a/guix/build/store-copy.scm b/guix/build/store-copy.scm index 95dcb8e114..7f0672cd9d 100644 --- a/guix/build/store-copy.scm +++ b/guix/build/store-copy.scm @@ -20,6 +20,7 @@ #:use-module ((guix build utils) #:hide (copy-recursively)) #:use-module (guix sets) #:use-module (guix progress) + #:autoload (guix store deduplication) (copy-file/deduplicate) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) #:use-module (srfi srfi-26) @@ -242,10 +243,13 @@ permissions. Write verbose output to the LOG port." lstat))) (define* (populate-store reference-graphs target - #:key (log-port (current-error-port))) + #:key + (deduplicate? #t) + (log-port (current-error-port))) "Populate the store under directory TARGET with the items specified in REFERENCE-GRAPHS, a list of reference-graph files. Items copied to TARGET -maintain timestamps and permissions." +maintain timestamps and permissions. When DEDUPLICATE? is true, deduplicate +regular files as they are copied to TARGET." (define store (string-append target (%store-directory))) @@ -273,6 +277,11 @@ maintain timestamps and permissions." (string-append target thing) #:keep-mtime? #t #:keep-permissions? #t + #:copy-file + (if deduplicate? + (cut copy-file/deduplicate <> <> + #:store store) + copy-file) #:log (%make-void-port "w")) (report)) things))))) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 1612ec8f04..440c4b0903 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -203,12 +203,19 @@ added to the pack." #+(file-append glibc-utf8-locales "/lib/locale")) (setlocale LC_ALL "en_US.utf8")))) + (define (import-module? module) + ;; Since we don't use deduplication support in 'populate-store', don't + ;; import (guix store deduplication) and its dependencies, which includes + ;; Guile-Gcrypt. That way we can run tests with '--bootstrap'. + (and (not-config? module) + (not (equal? '(guix store deduplication) module)))) + (define build (with-imported-modules (source-module-closure `((guix build utils) (guix build union) (gnu build install)) - #:select? not-config?) + #:select? import-module?) #~(begin (use-modules (guix build utils) ((guix build union) #:select (relative-file-name)) @@ -382,138 +389,139 @@ added to the pack." `(("/bin" -> "bin") ,@symlinks))) (define build - (with-imported-modules (source-module-closure - '((guix build utils) - (guix build store-copy) - (guix build union) - (gnu build install)) - #:select? not-config?) - #~(begin - (use-modules (guix build utils) - (guix build store-copy) - ((guix build union) #:select (relative-file-name)) - (gnu build install) - (srfi srfi-1) - (srfi srfi-26) - (ice-9 match)) + (with-extensions (list guile-gcrypt) + (with-imported-modules (source-module-closure + '((guix build utils) + (guix build store-copy) + (guix build union) + (gnu build install)) + #:select? not-config?) + #~(begin + (use-modules (guix build utils) + (guix build store-copy) + ((guix build union) #:select (relative-file-name)) + (gnu build install) + (srfi srfi-1) + (srfi srfi-26) + (ice-9 match)) - (define database #+database) - (define entry-point #$entry-point) + (define database #+database) + (define entry-point #$entry-point) - (define (mksquashfs args) - (apply invoke "mksquashfs" - `(,@args + (define (mksquashfs args) + (apply invoke "mksquashfs" + `(,@args - ;; Do not create a "recovery file" when appending to the - ;; file system since it's useless in this case. - "-no-recovery" + ;; Do not create a "recovery file" when appending to the + ;; file system since it's useless in this case. + "-no-recovery" - ;; Do not attempt to store extended attributes. - ;; See . - "-no-xattrs" + ;; Do not attempt to store extended attributes. + ;; See . + "-no-xattrs" - ;; Set file times and the file system creation time to - ;; one second after the Epoch. - "-all-time" "1" "-mkfs-time" "1" + ;; Set file times and the file system creation time to + ;; one second after the Epoch. + "-all-time" "1" "-mkfs-time" "1" - ;; Reset all UIDs and GIDs. - "-force-uid" "0" "-force-gid" "0"))) + ;; Reset all UIDs and GIDs. + "-force-uid" "0" "-force-gid" "0"))) - (setenv "PATH" #+(file-append archiver "/bin")) + (setenv "PATH" #+(file-append archiver "/bin")) - ;; We need an empty file in order to have a valid file argument when - ;; we reparent the root file system. Read on for why that's - ;; necessary. - (with-output-to-file ".empty" (lambda () (display ""))) - - ;; Create the squashfs image in several steps. - ;; Add all store items. Unfortunately mksquashfs throws away all - ;; ancestor directories and only keeps the basename. We fix this - ;; in the following invocations of mksquashfs. - (mksquashfs `(,@(map store-info-item - (call-with-input-file "profile" - read-reference-graph)) - #$environment - ,#$output - - ;; Do not perform duplicate checking because we - ;; don't have any dupes. - "-no-duplicates" - "-comp" - ,#+(compressor-name compressor))) - - ;; Here we reparent the store items. For each sub-directory of - ;; the store prefix we need one invocation of "mksquashfs". - (for-each (lambda (dir) - (mksquashfs `(".empty" - ,#$output - "-root-becomes" ,dir))) - (reverse (string-tokenize (%store-directory) - (char-set-complement (char-set #\/))))) - - ;; Add symlinks and mount points. - (mksquashfs - `(".empty" - ,#$output - ;; Create SYMLINKS via pseudo file definitions. - ,@(append-map - (match-lambda - ((source '-> target) - ;; Create relative symlinks to work around a bug in - ;; Singularity 2.x: - ;; https://bugs.gnu.org/34913 - ;; https://github.com/sylabs/singularity/issues/1487 - (let ((target (string-append #$profile "/" target))) - (list "-p" - (string-join - ;; name s mode uid gid symlink - (list source - "s" "777" "0" "0" - (relative-file-name (dirname source) - target))))))) - '#$symlinks*) - - "-p" "/.singularity.d d 555 0 0" - - ;; Create the environment file. - "-p" "/.singularity.d/env d 555 0 0" - "-p" ,(string-append - "/.singularity.d/env/90-environment.sh s 777 0 0 " - (relative-file-name "/.singularity.d/env" - #$environment)) - - ;; Create /.singularity.d/actions, and optionally the 'run' - ;; script, used by 'singularity run'. - "-p" "/.singularity.d/actions d 555 0 0" - - ,@(if entry-point - `(;; This one if for Singularity 2.x. - "-p" - ,(string-append - "/.singularity.d/actions/run s 777 0 0 " - (relative-file-name "/.singularity.d/actions" - (string-append #$profile "/" - entry-point))) - - ;; This one is for Singularity 3.x. - "-p" - ,(string-append - "/.singularity.d/runscript s 777 0 0 " - (relative-file-name "/.singularity.d" - (string-append #$profile "/" - entry-point)))) - '()) - - ;; Create empty mount points. - "-p" "/proc d 555 0 0" - "-p" "/sys d 555 0 0" - "-p" "/dev d 555 0 0" - "-p" "/home d 555 0 0")) - - (when database - ;; Initialize /var/guix. - (install-database-and-gc-roots "var-etc" database #$profile) - (mksquashfs `("var-etc" ,#$output)))))) + ;; We need an empty file in order to have a valid file argument when + ;; we reparent the root file system. Read on for why that's + ;; necessary. + (with-output-to-file ".empty" (lambda () (display ""))) + + ;; Create the squashfs image in several steps. + ;; Add all store items. Unfortunately mksquashfs throws away all + ;; ancestor directories and only keeps the basename. We fix this + ;; in the following invocations of mksquashfs. + (mksquashfs `(,@(map store-info-item + (call-with-input-file "profile" + read-reference-graph)) + #$environment + ,#$output + + ;; Do not perform duplicate checking because we + ;; don't have any dupes. + "-no-duplicates" + "-comp" + ,#+(compressor-name compressor))) + + ;; Here we reparent the store items. For each sub-directory of + ;; the store prefix we need one invocation of "mksquashfs". + (for-each (lambda (dir) + (mksquashfs `(".empty" + ,#$output + "-root-becomes" ,dir))) + (reverse (string-tokenize (%store-directory) + (char-set-complement (char-set #\/))))) + + ;; Add symlinks and mount points. + (mksquashfs + `(".empty" + ,#$output + ;; Create SYMLINKS via pseudo file definitions. + ,@(append-map + (match-lambda + ((source '-> target) + ;; Create relative symlinks to work around a bug in + ;; Singularity 2.x: + ;; https://bugs.gnu.org/34913 + ;; https://github.com/sylabs/singularity/issues/1487 + (let ((target (string-append #$profile "/" target))) + (list "-p" + (string-join + ;; name s mode uid gid symlink + (list source + "s" "777" "0" "0" + (relative-file-name (dirname source) + target))))))) + '#$symlinks*) + + "-p" "/.singularity.d d 555 0 0" + + ;; Create the environment file. + "-p" "/.singularity.d/env d 555 0 0" + "-p" ,(string-append + "/.singularity.d/env/90-environment.sh s 777 0 0 " + (relative-file-name "/.singularity.d/env" + #$environment)) + + ;; Create /.singularity.d/actions, and optionally the 'run' + ;; script, used by 'singularity run'. + "-p" "/.singularity.d/actions d 555 0 0" + + ,@(if entry-point + `( ;; This one if for Singularity 2.x. + "-p" + ,(string-append + "/.singularity.d/actions/run s 777 0 0 " + (relative-file-name "/.singularity.d/actions" + (string-append #$profile "/" + entry-point))) + + ;; This one is for Singularity 3.x. + "-p" + ,(string-append + "/.singularity.d/runscript s 777 0 0 " + (relative-file-name "/.singularity.d" + (string-append #$profile "/" + entry-point)))) + '()) + + ;; Create empty mount points. + "-p" "/proc d 555 0 0" + "-p" "/sys d 555 0 0" + "-p" "/dev d 555 0 0" + "-p" "/home d 555 0 0")) + + (when database + ;; Initialize /var/guix. + (install-database-and-gc-roots "var-etc" database #$profile) + (mksquashfs `("var-etc" ,#$output))))))) (gexp->derivation (string-append name (compressor-extension compressor) diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm index b4d37d4525..8564f12107 100644 --- a/guix/store/deduplication.scm +++ b/guix/store/deduplication.scm @@ -34,7 +34,8 @@ #:use-module (guix serialization) #:export (nar-sha256 deduplicate - dump-file/deduplicate)) + dump-file/deduplicate + copy-file/deduplicate)) ;; XXX: This port is used as a workaround on Guile <= 2.2.4 where ;; 'port-position' throws to 'out-of-range' when the offset is great than or @@ -256,3 +257,16 @@ down the road." (get-hash))))) (deduplicate file hash #:store store)) + +(define* (copy-file/deduplicate source target + #:key (store (%store-directory))) + "Like 'copy-file', but additionally deduplicate TARGET in STORE." + (call-with-input-file source + (lambda (input) + (let ((stat (stat input))) + (dump-file/deduplicate target input (stat:size stat) + (if (zero? (logand (stat:mode stat) + #o100)) + 'regular + 'executable) + #:store store))))) diff --git a/tests/gexp.scm b/tests/gexp.scm index a0e55178fa..6e92f0e4b3 100644 --- a/tests/gexp.scm +++ b/tests/gexp.scm @@ -736,7 +736,8 @@ (zero? (logand #o222 (stat:mode st))))))) (mkdir #$output) - (populate-store '("graph") #$output) + (populate-store '("graph") #$output + #:deduplicate? #f) ;; Check whether 'populate-store' canonicalizes ;; permissions and timestamps. diff --git a/tests/store-deduplication.scm b/tests/store-deduplication.scm index e2870a363d..7b01acae24 100644 --- a/tests/store-deduplication.scm +++ b/tests/store-deduplication.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2018 Ludovic Courtès +;;; Copyright © 2018, 2020 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,6 +25,7 @@ #:use-module (rnrs bytevectors) #:use-module (ice-9 binary-ports) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) #:use-module (srfi srfi-64)) (test-begin "store-deduplication") @@ -106,4 +107,19 @@ (cons (apply = (map (compose stat:ino stat) identical)) (map (compose stat:nlink stat) identical)))))) +(test-assert "copy-file/deduplicate" + (call-with-temporary-directory + (lambda (store) + (let ((source (search-path %load-path "gnu/packages/emacs-xyz.scm"))) + (for-each (lambda (target) + (copy-file/deduplicate source + (string-append store target) + #:store store)) + '("/a" "/b" "/c")) + (and (directory-exists? (string-append store "/.links")) + (file=? source (string-append store "/a")) + (apply = (map (compose stat:ino stat + (cut string-append store <>)) + '("/a" "/b" "/c")))))))) + (test-end "store-deduplication") -- cgit v1.2.3 From 0682cc593688e7d9a435ca69f05320aa87df06d0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 11 Dec 2020 12:03:25 +0100 Subject: database: Remove #:deduplicate? and #:reset-timestamps? from 'register-path'. * guix/store/database.scm (register-path): Remove #:deduplicate? and #:reset-timestamps?. * guix/scripts/system.scm (copy-item): Adjust accordingly. * tests/store-database.scm ("register-path") ("register-path, directory"): Call 'reset-timestamps'. --- guix/scripts/system.scm | 6 +----- guix/store/database.scm | 17 ++--------------- tests/store-database.scm | 5 +++-- 3 files changed, 6 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index c08929066b..0e543d9460 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -158,11 +158,7 @@ REFERENCES as its set of references." (unless (register-path item #:prefix target #:state-directory state - #:references references - - ;; Those are taken care of by 'copy-store-item'. - #:reset-timestamps? #f - #:deduplicate? #f) + #:references references) (leave (G_ "failed to register '~a' under '~a'~%") item target)))) diff --git a/guix/store/database.scm b/guix/store/database.scm index 31ea9add78..c0010b72b9 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -384,16 +384,14 @@ is true." (define* (register-path path #:key (references '()) deriver prefix - state-directory (deduplicate? #t) - (reset-timestamps? #t) + state-directory (schema (sql-schema))) "Register PATH as a valid store file, with REFERENCES as its list of references, and DERIVER as its deriver (.drv that led to it.) If PREFIX is given, it must be the name of the directory containing the new store to initialize; if STATE-DIRECTORY is given, it must be a string containing the absolute file name to the state directory of the store being initialized. -Return #t on success. As a side effect, reset timestamps on PATH, unless -RESET-TIMESTAMPS? is false. +Return #t on success. Use with care as it directly modifies the store! This is primarily meant to be used internally by the daemon's build hook. @@ -404,17 +402,6 @@ by adding it as a temp-root." (store-database-file #:prefix prefix #:state-directory state-directory)) - (define real-file-name - (string-append (or prefix "") path)) - - (when deduplicate? - (deduplicate real-file-name (nar-sha256 real-file-name) - #:store (string-append (or prefix "") - %store-directory))) - - (when reset-timestamps? - (reset-timestamps real-file-name)) - (parameterize ((sql-schema schema)) (with-database db-file db (register-items db (list (store-info path deriver references)) diff --git a/tests/store-database.scm b/tests/store-database.scm index 3b4ef43f6d..33fd6cfbad 100644 --- a/tests/store-database.scm +++ b/tests/store-database.scm @@ -34,8 +34,7 @@ (test-begin "store-database") -(test-equal "register-path" - '(1 1) +(test-assert "register-path" (let ((file (string-append (%store-prefix) "/" (make-string 32 #\f) "-fake"))) (when (valid-path? %store file) @@ -46,6 +45,7 @@ (drv (string-append file ".drv"))) (call-with-output-file file (cut display "This is a fake store item.\n" <>)) + (reset-timestamps file) (register-path file #:references (list ref) #:deriver drv) @@ -69,6 +69,7 @@ (mkdir-p (string-append file "/a")) (call-with-output-file (string-append file "/a/b") (const #t)) + (reset-timestamps file) (register-path file #:deriver drv) (and (valid-path? %store file) -- cgit v1.2.3 From 3169c93903c20cea000335d59560eac7f28e8f92 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 11 Dec 2020 14:46:20 +0100 Subject: database: Remove 'register-path'. * guix/store/database.scm (register-path): Remove. * tests/store-database.scm ("register-path"): Rename to... ("register-items"): ... this, and use 'register-items' instead of 'register-path'. ("register-path, directory"): Rename to... ("register-items, directory"): ... this, and use 'register-items' instead of 'register-path'. ("register-path with unregistered references"): Rename to... ("sqlite-register with unregistered references"): ... this. --- guix/store/database.scm | 27 --------------------------- tests/store-database.scm | 15 ++++++++------- 2 files changed, 8 insertions(+), 34 deletions(-) (limited to 'tests') diff --git a/guix/store/database.scm b/guix/store/database.scm index 9d5bc531bb..4579b05261 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -43,7 +43,6 @@ with-database path-id sqlite-register - register-path register-items %epoch reset-timestamps)) @@ -383,32 +382,6 @@ is true." (chmod file (if (executable-file? file) #o555 #o444))) (utime file 1 1 0 0))))) -(define* (register-path path - #:key (references '()) deriver prefix - state-directory - (schema (sql-schema))) - "Register PATH as a valid store file, with REFERENCES as its list of -references, and DERIVER as its deriver (.drv that led to it.) If PREFIX is -given, it must be the name of the directory containing the new store to -initialize; if STATE-DIRECTORY is given, it must be a string containing the -absolute file name to the state directory of the store being initialized. -Return #t on success. - -Use with care as it directly modifies the store! This is primarily meant to -be used internally by the daemon's build hook. - -PATH must be protected from GC and locked during execution of this, typically -by adding it as a temp-root." - (define db-file - (store-database-file #:prefix prefix - #:state-directory state-directory)) - - (parameterize ((sql-schema schema)) - (with-database db-file db - (register-items db (list (store-info path deriver references)) - #:prefix prefix - #:log-port (%make-void-port "w"))))) - (define %epoch ;; When it all began. (make-time time-utc 0 1)) diff --git a/tests/store-database.scm b/tests/store-database.scm index 33fd6cfbad..17eea38c63 100644 --- a/tests/store-database.scm +++ b/tests/store-database.scm @@ -20,6 +20,7 @@ #:use-module (guix tests) #:use-module (guix store) #:use-module (guix store database) + #:use-module (guix build store-copy) #:use-module ((guix utils) #:select (call-with-temporary-output-file)) #:use-module ((guix build utils) #:select (mkdir-p delete-file-recursively)) @@ -34,7 +35,7 @@ (test-begin "store-database") -(test-assert "register-path" +(test-assert "register-items" (let ((file (string-append (%store-prefix) "/" (make-string 32 #\f) "-fake"))) (when (valid-path? %store file) @@ -46,9 +47,8 @@ (call-with-output-file file (cut display "This is a fake store item.\n" <>)) (reset-timestamps file) - (register-path file - #:references (list ref) - #:deriver drv) + (with-database (store-database-file) db + (register-items db (list (store-info file drv (list ref))))) (and (valid-path? %store file) (equal? (references %store file) (list ref)) @@ -57,7 +57,7 @@ (list (stat:mtime (lstat file)) (stat:mtime (lstat ref))))))) -(test-equal "register-path, directory" +(test-equal "register-items, directory" '(1 1 1) (let ((file (string-append (%store-prefix) "/" (make-string 32 #\f) "-fake-directory"))) @@ -70,7 +70,8 @@ (call-with-output-file (string-append file "/a/b") (const #t)) (reset-timestamps file) - (register-path file #:deriver drv) + (with-database (store-database-file) db + (register-items db (list (store-info file drv '())))) (and (valid-path? %store file) (null? (references %store file)) @@ -102,7 +103,7 @@ (list (path-id db "/gnu/foo") (path-id db "/gnu/bar"))))))) -(test-assert "register-path with unregistered references" +(test-assert "sqlite-register with unregistered references" ;; Make sure we get a "NOT NULL constraint failed: Refs.reference" error ;; when we try to add references that are not registered yet. Better safe ;; than sorry. -- cgit v1.2.3 From 7530e491b517497b7b8166b5ccecdc3d4cdb468d Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 11 Dec 2020 15:48:02 +0100 Subject: deduplicate: Create the '.links' directory lazily. This avoids repeated (mkdir-p "/gnu/store/.links") calls when deduplicating lots of files. * guix/store/deduplication.scm (deduplicate): Remove initial call to 'mkdir-p'. Add ENOENT case in 'link' exception handler. Reindent. * tests/store-deduplication.scm ("deduplicate, ENOSPC"): Check for (<= links 4) to account for the initial 'link' call. --- guix/store/deduplication.scm | 96 ++++++++++++++++++++++--------------------- tests/store-deduplication.scm | 2 +- 2 files changed, 51 insertions(+), 47 deletions(-) (limited to 'tests') diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm index 8564f12107..a72a43bf79 100644 --- a/guix/store/deduplication.scm +++ b/guix/store/deduplication.scm @@ -159,52 +159,56 @@ under STORE." (define links-directory (string-append store "/.links")) - (mkdir-p links-directory) - (let loop ((path path) - (type (stat:type (lstat path))) - (hash hash)) - (if (eq? 'directory type) - ;; Can't hardlink directories, so hardlink their atoms. - (for-each (match-lambda - ((file . properties) - (unless (member file '("." "..")) - (let* ((file (string-append path "/" file)) - (type (match (assoc-ref properties 'type) - ((or 'unknown #f) - (stat:type (lstat file))) - (type type)))) - (loop file type - (and (not (eq? 'directory type)) - (nar-sha256 file))))))) - (scandir* path)) - (let ((link-file (string-append links-directory "/" - (bytevector->nix-base32-string hash)))) - (if (file-exists? link-file) - (replace-with-link link-file path - #:swap-directory links-directory - #:store store) - (catch 'system-error - (lambda () - (link path link-file)) - (lambda args - (let ((errno (system-error-errno args))) - (cond ((= errno EEXIST) - ;; Someone else put an entry for PATH in - ;; LINKS-DIRECTORY before we could. Let's use it. - (replace-with-link path link-file - #:swap-directory - links-directory - #:store store)) - ((= errno ENOSPC) - ;; There's not enough room in the directory index for - ;; more entries in .links, but that's fine: we can - ;; just stop. - #f) - ((= errno EMLINK) - ;; PATH has reached the maximum number of links, but - ;; that's OK: we just can't deduplicate it more. - #f) - (else (apply throw args))))))))))) + (let loop ((path path) + (type (stat:type (lstat path))) + (hash hash)) + (if (eq? 'directory type) + ;; Can't hardlink directories, so hardlink their atoms. + (for-each (match-lambda + ((file . properties) + (unless (member file '("." "..")) + (let* ((file (string-append path "/" file)) + (type (match (assoc-ref properties 'type) + ((or 'unknown #f) + (stat:type (lstat file))) + (type type)))) + (loop file type + (and (not (eq? 'directory type)) + (nar-sha256 file))))))) + (scandir* path)) + (let ((link-file (string-append links-directory "/" + (bytevector->nix-base32-string hash)))) + (if (file-exists? link-file) + (replace-with-link link-file path + #:swap-directory links-directory + #:store store) + (catch 'system-error + (lambda () + (link path link-file)) + (lambda args + (let ((errno (system-error-errno args))) + (cond ((= errno EEXIST) + ;; Someone else put an entry for PATH in + ;; LINKS-DIRECTORY before we could. Let's use it. + (replace-with-link path link-file + #:swap-directory + links-directory + #:store store)) + ((= errno ENOENT) + ;; This most likely means that LINKS-DIRECTORY does + ;; not exist. Attempt to create it and try again. + (mkdir-p links-directory) + (loop path type hash)) + ((= errno ENOSPC) + ;; There's not enough room in the directory index for + ;; more entries in .links, but that's fine: we can + ;; just stop. + #f) + ((= errno EMLINK) + ;; PATH has reached the maximum number of links, but + ;; that's OK: we just can't deduplicate it more. + #f) + (else (apply throw args))))))))))) (define (tee input len output) "Return a port that reads up to LEN bytes from INPUT and writes them to diff --git a/tests/store-deduplication.scm b/tests/store-deduplication.scm index 7b01acae24..b1c2d93bbd 100644 --- a/tests/store-deduplication.scm +++ b/tests/store-deduplication.scm @@ -95,7 +95,7 @@ (lambda () (set! link (lambda (old new) (set! links (+ links 1)) - (if (<= links 3) + (if (<= links 4) (true-link old new) (throw 'system-error "link" "~A" '("Whaaat?!") (list ENOSPC)))))) -- cgit v1.2.3 From 6d955f1731dc593a51625b455882102a67d95e1a Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 13 Dec 2020 22:20:08 +0100 Subject: tests: Check the build trace for hash mismatches on substitutes. * tests/store.scm ("substitute, corrupt output hash, build trace"): New test. --- tests/store.scm | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'tests') diff --git a/tests/store.scm b/tests/store.scm index 38051bf5e5..7f1ec51875 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -787,6 +787,61 @@ (build-derivations s (list d)) #f)))))) +(test-assert "substitute, corrupt output hash, build trace" + ;; Likewise, and check the build trace. + (with-store s + (let* ((c "hello, world") ; contents of the output + (d (build-expression->derivation + s "corrupt-substitute" + `(mkdir %output) + #:guile-for-build + (package-derivation s %bootstrap-guile (%current-system)))) + (o (derivation->output-path d))) + ;; Make sure we use 'guix substitute'. + (set-build-options s + #:print-build-trace #t + #:use-substitutes? #t + #:fallback? #f + #:substitute-urls (%test-substitute-urls)) + + (with-derivation-substitute d c + (sha256 => (make-bytevector 32 0)) ;select a hash that doesn't match C + + (define output + (call-with-output-string + (lambda (port) + (parameterize ((current-build-output-port port)) + (guard (c ((store-protocol-error? c) #t)) + (build-derivations s (list d)) + #f))))) + + (define actual-hash + (let-values (((port get-hash) + (gcrypt:open-hash-port + (gcrypt:hash-algorithm gcrypt:sha256)))) + (write-file-tree "foo" port + #:file-type+size + (lambda _ + (values 'regular (string-length c))) + #:file-port + (lambda _ + (open-input-string c))) + (close-port port) + (bytevector->nix-base32-string (get-hash)))) + + (define expected-hash + (bytevector->nix-base32-string (make-bytevector 32 0))) + + (define mismatch + (string-append "@ hash-mismatch " o " sha256 " + expected-hash " " actual-hash "\n")) + + (define failure + (string-append "@ substituter-failed " o)) + + (and (string-contains output mismatch) + (string-contains output failure)))))) + (test-assert "substitute --fallback" (with-store s (let* ((t (random-text)) ; contents of the output -- cgit v1.2.3 From 9dfa20a22ae0be3d3b01a7b3d422af97428c627e Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 13 Dec 2020 22:46:03 +0100 Subject: daemon: Let 'guix substitute' perform hash checks. This way, the hash of the store item can be computed as it is restored, thereby avoiding an additional file tree traversal ('hashPath' call) later on in the daemon. Consequently, it should reduce latency between subsequent substitute downloads. This is a followup to 5ff521452b9ec2aae9ed8e4bb7bdc250a581f203. * guix/scripts/substitute.scm (narinfo-hash-algorithm+value): New procedure. (process-substitution): Wrap INPUT into a hash input port, 'hashed', and read from it. Compare the actual and expected hashes, and print a "hash-mismatch" status line when they differ. When they match, print not just "success" but also the nar hash and size. * nix/libstore/build.cc (class SubstitutionGoal)[expectedHashStr]: Remove. (SubstitutionGoal::finished): Tokenize 'status'. Parse it and handle "success" and "hash-mismatch" accordingly. Call 'hashPath' only when the returned hash is not SHA256. (SubstitutionGoal::handleChildOutput): Remove 'expectedHashStr' handling. * tests/substitute.scm ("substitute, invalid hash"): Rename to... ("substitute, invalid narinfo hash"): ... this. ("substitute, invalid hash"): New test. --- guix/scripts/substitute.scm | 45 +++++++++++++++++++++++----- nix/libstore/build.cc | 73 ++++++++++++++++++++++++--------------------- tests/substitute.scm | 50 +++++++++++++++++++++++++++++-- 3 files changed, 124 insertions(+), 44 deletions(-) (limited to 'tests') diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 25075eedff..17d0002b9f 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -26,6 +26,8 @@ #:use-module (guix combinators) #:use-module (guix config) #:use-module (guix records) + #:use-module (guix diagnostics) + #:use-module (guix i18n) #:use-module ((guix serialization) #:select (restore-file)) #:autoload (guix scripts discover) (read-substitute-urls) #:use-module (gcrypt hash) @@ -256,6 +258,18 @@ connection (typically PORT) is kept open once data has been fetched from URI." ;; for more information. (contents narinfo-contents)) +(define (narinfo-hash-algorithm+value narinfo) + "Return two values: the hash algorithm used by NARINFO and its value as a +bytevector." + (match (string-tokenize (narinfo-hash narinfo) + (char-set-complement (char-set #\:))) + ((algorithm base32) + (values (lookup-hash-algorithm (string->symbol algorithm)) + (nix-base32-string->bytevector base32))) + (_ + (raise (formatted-message + (G_ "invalid narinfo hash: ~s") (narinfo-hash narinfo)))))) + (define (narinfo-hash->sha256 hash) "If the string HASH denotes a sha256 hash, return it as a bytevector. Otherwise return #f." @@ -1033,7 +1047,9 @@ one. Return #f if URI's scheme is 'file' or #f." (define* (process-substitution store-item destination #:key cache-urls acl print-build-trace?) "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to -DESTINATION as a nar file. Verify the substitute against ACL." +DESTINATION as a nar file. Verify the substitute against ACL, and verify its +hash against what appears in the narinfo. Print a status line on the current +output port." (define narinfo (lookup-narinfo cache-urls store-item (cut valid-narinfo? <> acl))) @@ -1044,9 +1060,6 @@ DESTINATION as a nar file. Verify the substitute against ACL." (let-values (((uri compression file-size) (narinfo-best-uri narinfo))) - ;; Tell the daemon what the expected hash of the Nar itself is. - (format #t "~a~%" (narinfo-hash narinfo)) - (unless print-build-trace? (format (current-error-port) (G_ "Downloading ~a...~%") (uri->string uri))) @@ -1079,9 +1092,16 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; closed here, while the child process doing the ;; reporting will close it upon exit. (decompressed-port (string->symbol compression) - progress))) + progress)) + + ;; Compute the actual nar hash as we read it. + ((algorithm expected) + (narinfo-hash-algorithm+value narinfo)) + ((hashed get-hash) + (open-hash-input-port algorithm input))) ;; Unpack the Nar at INPUT into DESTINATION. - (restore-file input destination) + (restore-file hashed destination) + (close-port hashed) (close-port input) ;; Wait for the reporter to finish. @@ -1091,8 +1111,17 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; one to visually separate substitutions. (display "\n\n" (current-error-port)) - ;; Tell the daemon that we're done. - (display "success\n" (current-output-port))))) + ;; Check whether we got the data announced in NARINFO. + (let ((actual (get-hash))) + (if (bytevector=? actual expected) + ;; Tell the daemon that we're done. + (format (current-output-port) "success ~a ~a~%" + (narinfo-hash narinfo) (narinfo-size narinfo)) + ;; The actual data has a different hash than that in NARINFO. + (format (current-output-port) "hash-mismatch ~a ~a ~a~%" + (hash-algorithm-name algorithm) + (bytevector->nix-base32-string expected) + (bytevector->nix-base32-string actual))))))) ;;; diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index b5551b87ae..b19471a68f 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -2790,10 +2790,6 @@ private: /* The substituter. */ std::shared_ptr substituter; - /* Either the empty string, or the expected hash as returned by the - substituter. */ - string expectedHashStr; - /* Either the empty string, or the status phrase returned by the substituter. */ string status; @@ -3032,36 +3028,47 @@ void SubstitutionGoal::finished() /* Check the exit status and the build result. */ HashResult hash; try { - - if (status != "success") - throw SubstError(format("fetching path `%1%' (status: '%2%')") - % storePath % status); - - if (!pathExists(destPath)) - throw SubstError(format("substitute did not produce path `%1%'") % destPath); - - if (expectedHashStr == "") - throw SubstError(format("substituter did not communicate hash for `%1'") % storePath); - - hash = hashPath(htSHA256, destPath); - - /* Verify the expected hash we got from the substituer. */ - size_t n = expectedHashStr.find(':'); - if (n == string::npos) - throw Error(format("bad hash from substituter: %1%") % expectedHashStr); - HashType hashType = parseHashType(string(expectedHashStr, 0, n)); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr); - Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); - Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; - if (expectedHash != actualHash) { - if (settings.printBuildTrace) + auto statusList = tokenizeString >(status); + + if (statusList.empty()) { + throw SubstError(format("fetching path `%1%' (empty status: '%2%')") + % storePath % status); + } else if (statusList[0] == "hash-mismatch") { + if (settings.printBuildTrace) { + auto hashType = statusList[1]; + auto expectedHash = statusList[2]; + auto actualHash = statusList[3]; printMsg(lvlError, format("@ hash-mismatch %1% %2% %3% %4%") - % storePath % "sha256" - % printHash16or32(expectedHash) - % printHash16or32(actualHash)); + % storePath + % hashType % expectedHash % actualHash); + } throw SubstError(format("hash mismatch for substituted item `%1%'") % storePath); + } else if (statusList[0] == "success") { + if (!pathExists(destPath)) + throw SubstError(format("substitute did not produce path `%1%'") % destPath); + + std::string hashStr = statusList[1]; + size_t n = hashStr.find(':'); + if (n == string::npos) + throw Error(format("bad hash from substituter: %1%") % hashStr); + + HashType hashType = parseHashType(string(hashStr, 0, n)); + switch (hashType) { + case htUnknown: + throw Error(format("unknown hash algorithm in `%1%'") % hashStr); + case htSHA256: + hash.first = parseHash16or32(hashType, string(hashStr, n + 1)); + hash.second = std::atoi(statusList[2].c_str()); + break; + default: + /* The database only stores SHA256 hashes, so compute it. */ + hash = hashPath(htSHA256, destPath); + break; + } } + else + throw SubstError(format("fetching path `%1%' (status: '%2%')") + % storePath % status); } catch (SubstError & e) { @@ -3123,9 +3130,7 @@ void SubstitutionGoal::handleChildOutput(int fd, const string & data) string trimmed = (end != string::npos) ? input.substr(0, end) : input; /* Update the goal's state accordingly. */ - if (expectedHashStr == "") { - expectedHashStr = trimmed; - } else if (status == "") { + if (status == "") { status = trimmed; worker.wakeUp(shared_from_this()); } else { diff --git a/tests/substitute.scm b/tests/substitute.scm index b86ce09425..5b42632552 100644 --- a/tests/substitute.scm +++ b/tests/substitute.scm @@ -28,7 +28,9 @@ #:use-module (guix base32) #:use-module ((guix store) #:select (%store-prefix)) #:use-module ((guix ui) #:select (guix-warning-port)) - #:use-module ((guix utils) #:select (call-with-compressed-output-port)) + #:use-module ((guix utils) + #:select (call-with-temporary-directory + call-with-compressed-output-port)) #:use-module ((guix build utils) #:select (mkdir-p delete-file-recursively dump-port)) #:use-module (guix tests http) @@ -36,6 +38,7 @@ #:use-module (rnrs io ports) #:use-module (web uri) #:use-module (ice-9 regex) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) @@ -304,7 +307,7 @@ System: mips64el-linux\n") (lambda () (guix-substitute "--substitute"))))) -(test-quit "substitute, invalid hash" +(test-quit "substitute, invalid narinfo hash" "no valid substitute" ;; The hash in the signature differs from the hash of %NARINFO. (with-narinfo (string-append %narinfo "Signature: " @@ -317,6 +320,49 @@ System: mips64el-linux\n") (lambda () (guix-substitute "--substitute"))))) +(test-equal "substitute, invalid hash" + (string-append "hash-mismatch sha256 " + (bytevector->nix-base32-string (sha256 #vu8())) " " + (let-values (((port get-hash) + (open-hash-port (hash-algorithm sha256))) + ((content) + "Substitutable data.")) + (write-file-tree "foo" port + #:file-type+size + (lambda _ + (values 'regular + (string-length content))) + #:file-port + (lambda _ + (open-input-string content))) + (close-port port) + (bytevector->nix-base32-string (get-hash))) + "\n") + + ;; Arrange so the actual data hash does not match the 'NarHash' field in the + ;; narinfo. + (with-output-to-string + (lambda () + (let ((narinfo (string-append "StorePath: " (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-wrong-hash +URL: example.nar +Compression: none +NarHash: sha256:" (bytevector->nix-base32-string (sha256 #vu8())) " +NarSize: 42 +References: +Deriver: " (%store-prefix) "/foo.drv +System: mips64el-linux\n"))) + (with-narinfo (string-append narinfo "Signature: " + (signature-field narinfo) "\n") + (call-with-temporary-directory + (lambda (directory) + (with-input-from-string (string-append + "substitute " (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-wrong-hash " + directory "/wrong-hash\n") + (lambda () + (guix-substitute "--substitute")))))))))) + (test-quit "substitute, unauthorized key" "no valid substitute" (with-narinfo (string-append %narinfo "Signature: " -- cgit v1.2.3 From 77a1efed9e12ce0e2c470d7b0601ae70c72b010b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 14 Dec 2020 14:33:06 +0100 Subject: tests: Check the mtime and permissions of substituted items. * tests/store.scm ("substitute") ("substitute + build-things with output path") ("substitute + build-things with specific output"): Call 'canonical-file?'. * tests/substitute.scm ("substitute, authorized key"): Check the mtime and permissions of "substitute-retrieved". --- tests/store.scm | 3 +++ tests/substitute.scm | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/store.scm b/tests/store.scm index 7f1ec51875..4dc125bcb9 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -715,6 +715,7 @@ #:substitute-urls (%test-substitute-urls)) (and (has-substitutes? s o) (build-derivations s (list d)) + (canonical-file? o) (equal? c (call-with-input-file o get-string-all))))))) (test-assert "substitute + build-things with output path" @@ -735,6 +736,7 @@ (and (has-substitutes? s o) (build-things s (list o)) ;give the output path (valid-path? s o) + (canonical-file? o) (equal? c (call-with-input-file o get-string-all))))))) (test-assert "substitute + build-things with specific output" @@ -755,6 +757,7 @@ (build-things s `((,(derivation-file-name d) . "out"))) (valid-path? s o) + (canonical-file? o) (equal? c (call-with-input-file o get-string-all))))))) (test-assert "substitute, corrupt output hash" diff --git a/tests/substitute.scm b/tests/substitute.scm index 5b42632552..542aaf603f 100644 --- a/tests/substitute.scm +++ b/tests/substitute.scm @@ -378,7 +378,7 @@ System: mips64el-linux\n"))) (guix-substitute "--substitute"))))) (test-equal "substitute, authorized key" - "Substitutable data." + '("Substitutable data." 1 #o444) (with-narinfo (string-append %narinfo "Signature: " (signature-field %narinfo)) (dynamic-wind @@ -387,7 +387,9 @@ System: mips64el-linux\n"))) (request-substitution (string-append (%store-prefix) "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") "substitute-retrieved") - (call-with-input-file "substitute-retrieved" get-string-all)) + (list (call-with-input-file "substitute-retrieved" get-string-all) + (stat:mtime (lstat "substitute-retrieved")) + (stat:perms (lstat "substitute-retrieved")))) (lambda () (false-if-exception (delete-file "substitute-retrieved")))))) -- cgit v1.2.3 From 3c799ccb98ba2ea4c19747306289586e42ae493b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 14 Dec 2020 15:33:00 +0100 Subject: tests: Make sure substituted items are deduplicated. * tests/store.scm ("substitute, deduplication"): New test. --- tests/store.scm | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'tests') diff --git a/tests/store.scm b/tests/store.scm index 4dc125bcb9..c9a08ac690 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -718,6 +718,30 @@ (canonical-file? o) (equal? c (call-with-input-file o get-string-all))))))) +(test-assert "substitute, deduplication" + (with-store s + (let* ((c (random-text)) ; contents of the output + (g (package-derivation s %bootstrap-guile)) + (d1 (build-expression->derivation s "substitute-me" + `(begin ,c (exit 1)) + #:guile-for-build g)) + (d2 (build-expression->derivation s "build-me" + `(call-with-output-file %output + (lambda (p) + (display ,c p))) + #:guile-for-build g)) + (o1 (derivation->output-path d1)) + (o2 (derivation->output-path d2))) + (with-derivation-substitute d1 c + (set-build-options s #:use-substitutes? #t + #:substitute-urls (%test-substitute-urls)) + (and (has-substitutes? s o1) + (build-derivations s (list d2)) ;build + (build-derivations s (list d1)) ;substitute + (canonical-file? o1) + (equal? c (call-with-input-file o1 get-string-all)) + (= (stat:ino (stat o1)) (stat:ino (stat o2)))))))) + (test-assert "substitute + build-things with output path" (with-store s (let* ((c (random-text)) ;contents of the output -- cgit v1.2.3 From 9608f4003dedd8dfe99327c15668ca1a43ebd93b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 21 Dec 2020 11:44:19 +0100 Subject: tests: Fix malformed JSON. Guile-JSON 4.3.2 would parse in spite of these typos, but 4.4.1 is stricter. * tests/swh.scm (%directory-entries): Add missing comma. * tests/cve-sample.json: Likewise. --- tests/cve-sample.json | 2 +- tests/swh.scm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/cve-sample.json b/tests/cve-sample.json index 39816f9dd4..11b71817bb 100644 --- a/tests/cve-sample.json +++ b/tests/cve-sample.json @@ -49,7 +49,7 @@ "vulnerable" : true, "cpe23Uri" : "cpe:2.3:o:juniper:junos:16.1:*:*:*:*:*:*:*" } ] - } { + }, { "operator" : "OR", "cpe_match" : [ { "vulnerable" : true, diff --git a/tests/swh.scm b/tests/swh.scm index aef68acbe7..06984b2a80 100644 --- a/tests/swh.scm +++ b/tests/swh.scm @@ -33,7 +33,7 @@ "[ { \"name\": \"one\", \"type\": \"regular\", \"length\": 123, - \"dir_id\": 1 } + \"dir_id\": 1 }, { \"name\": \"two\", \"type\": \"regular\", \"length\": 456, -- cgit v1.2.3 From 527551287011888c2ba13fe92e636300ce625430 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 21 Dec 2020 12:06:11 +0100 Subject: tests: Check the effect of '--without-tests' on implicit inputs. * tests/transformations.scm ("options->transformation, without-tests"): Ensure TAR has #:tests? #f. --- tests/transformations.scm | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/transformations.scm b/tests/transformations.scm index 07ed8b1234..2d33bed7ae 100644 --- a/tests/transformations.scm +++ b/tests/transformations.scm @@ -368,10 +368,9 @@ (let ((new (t p))) (match (bag-direct-inputs (package->bag new)) ((("dep" dep) ("tar" tar) _ ...) - ;; TODO: Check whether TAR has #:tests? #f when transformations - ;; apply to implicit inputs. - (equal? (package-arguments dep) - '(#:tests? #f))))))) + (and (equal? (package-arguments dep) '(#:tests? #f)) + (match (memq #:tests? (package-arguments tar)) + ((#:tests? #f _ ...) #t)))))))) (test-end) -- cgit v1.2.3 From f00e68ace070fd5240a4b5874e61c26f6e909b6c Mon Sep 17 00:00:00 2001 From: Miguel Ángel Arruga Vivas Date: Mon, 21 Dec 2020 13:02:01 +0100 Subject: system: Allow separated /boot and encrypted root. * gnu/bootloader/grub.scm (grub-configuration-file): New parameter store-crypto-devices. [crypto-devices]: New helper function. [builder]: Use crypto-devices. * gnu/machine/ssh.scm (roll-back-managed-host): Use boot-parameters-store-crypto-devices to provide its contents to the bootloader configuration generation process. * gnu/tests/install.scm (%encrypted-root-not-boot-os, %encrypted-root-not-boot-os): New os declaration. (%encrypted-root-not-boot-installation-script): New script, whose contents were initially taken from %encrypted-root-installation-script. (%test-encrypted-root-not-boot-os): New test. * gnu/system.scm (define-module): Export operating-system-bootoader-crypto-devices and boot-parameters-store-crypto-devices. (): Add field store-crypto-devices. (read-boot-parameters): Parse store-crypto-devices field. [uuid-sexp->uuid]: New helper function extracted from device-sexp->device. (operating-system-bootloader-crypto-devices): New function. (operating-system-bootcfg): Use operating-system-bootloader-crypto-devices to provide its contents to the bootloader configuration generation process. (operating-system-boot-parameters): Add store-crypto-devices to the generated boot-parameters. (operating-system-boot-parameters-file): Likewise to the file with the serialized structure. * guix/scripts/system.scm (reinstall-bootloader): Use boot-parameters-store-crypto-devices to provide its contents to the bootloader configuration generation process. * tests/boot-parameters.scm (%default-store-crypto-devices): New variable. (%grub-boot-parameters, test-read-boot-parameters): Use %default-store-crypto-devices. (tests store-crypto-devices): New tests. --- gnu/bootloader/grub.scm | 21 +++++++++- gnu/machine/ssh.scm | 3 ++ gnu/system.scm | 59 ++++++++++++++++++++++++++- gnu/tests/install.scm | 102 ++++++++++++++++++++++++++++++++++++++++++++++ guix/scripts/system.scm | 2 + tests/boot-parameters.scm | 30 +++++++++++++- 6 files changed, 212 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/gnu/bootloader/grub.scm b/gnu/bootloader/grub.scm index af7b7561ff..29c81ae641 100644 --- a/gnu/bootloader/grub.scm +++ b/gnu/bootloader/grub.scm @@ -4,7 +4,7 @@ ;;; Copyright © 2017 Leo Famulari ;;; Copyright © 2017, 2020 Mathieu Othacehe ;;; Copyright © 2019, 2020 Jan (janneke) Nieuwenhuizen -;;; Copyright © 2019 Miguel Ángel Arruga Vivas +;;; Copyright © 2019, 2020 Miguel Ángel Arruga Vivas ;;; Copyright © 2020 Maxim Cournoyer ;;; Copyright © 2020 Stefan ;;; @@ -359,11 +359,14 @@ code." (locale #f) (system (%current-system)) (old-entries '()) + (store-crypto-devices '()) store-directory-prefix) "Return the GRUB configuration file corresponding to CONFIG, a object, and where the store is available at STORE-FS, a object. OLD-ENTRIES is taken to be a list of menu entries corresponding to old generations of the system. +STORE-CRYPTO-DEVICES contain the UUIDs of the encrypted units that must +be unlocked to access the store contents. STORE-DIRECTORY-PREFIX may be used to specify a store prefix, as is required when booting a root file system on a Btrfs subvolume." (define all-entries @@ -411,6 +414,21 @@ menuentry ~s { (string-join (map string-join '#$modules) "\n module " 'prefix)))))) + (define (crypto-devices) + (define (crypto-device->cryptomount dev) + (if (uuid? dev) + #~(format port "cryptomount -u ~a~%" + ;; cryptomount only accepts UUID without the hypen. + #$(string-delete #\- (uuid->string dev))) + ;; Other type of devices aren't implemented. + #~())) + (let ((devices (map crypto-device->cryptomount store-crypto-devices)) + ;; XXX: Add luks2 when grub 2.06 is packaged. + (modules #~(format port "insmod luks~%"))) + (if (null? devices) + devices + (cons modules devices)))) + (define (sugar) (let* ((entry (first all-entries)) (device (menu-entry-device entry)) @@ -474,6 +492,7 @@ keymap ~a~%" #$keymap)))) "# This file was generated from your Guix configuration. Any changes # will be lost upon reconfiguration. ") + #$@(crypto-devices) #$(sugar) #$locale-config #$keyboard-layout-config diff --git a/gnu/machine/ssh.scm b/gnu/machine/ssh.scm index 1b748c8da7..08c653ba17 100644 --- a/gnu/machine/ssh.scm +++ b/gnu/machine/ssh.scm @@ -484,6 +484,8 @@ an environment type of 'managed-host." (list (second boot-parameters)))) (locale -> (boot-parameters-locale (second boot-parameters))) + (crypto-dev -> (boot-parameters-store-crypto-devices + (second boot-parameters))) (store-dir -> (boot-parameters-store-directory-prefix (second boot-parameters))) (old-entries -> (map boot-parameters->menu-entry @@ -496,6 +498,7 @@ an environment type of 'managed-host." bootloader)) bootloader entries #:locale locale + #:store-crypto-devices crypto-dev #:store-directory-prefix store-dir #:old-entries old-entries))) (remote-result (machine-remote-eval machine remote-exp))) diff --git a/gnu/system.scm b/gnu/system.scm index fcf3310fa3..c284a18379 100644 --- a/gnu/system.scm +++ b/gnu/system.scm @@ -5,7 +5,7 @@ ;;; Copyright © 2016 Chris Marusich ;;; Copyright © 2017 Mathieu Othacehe ;;; Copyright © 2019 Meiyo Peng -;;; Copyright © 2019 Miguel Ángel Arruga Vivas +;;; Copyright © 2019, 2020 Miguel Ángel Arruga Vivas ;;; Copyright © 2020 Danny Milosavljevic ;;; Copyright © 2020 Brice Waegeneire ;;; Copyright © 2020 Florian Pelz @@ -112,6 +112,7 @@ operating-system-store-file-system operating-system-user-mapped-devices operating-system-boot-mapped-devices + operating-system-bootloader-crypto-devices operating-system-activation-script operating-system-user-accounts operating-system-shepherd-service-names @@ -147,6 +148,7 @@ boot-parameters-root-device boot-parameters-bootloader-name boot-parameters-bootloader-menu-entries + boot-parameters-store-crypto-devices boot-parameters-store-device boot-parameters-store-directory-prefix boot-parameters-store-mount-point @@ -305,6 +307,8 @@ directly by the user." (store-device boot-parameters-store-device) (store-mount-point boot-parameters-store-mount-point) (store-directory-prefix boot-parameters-store-directory-prefix) + (store-crypto-devices boot-parameters-store-crypto-devices + (default '())) (locale boot-parameters-locale) (kernel boot-parameters-kernel) (kernel-arguments boot-parameters-kernel-arguments) @@ -338,6 +342,13 @@ file system labels." (if (string-prefix? "/" device) device (file-system-label device)))))) + (define uuid-sexp->uuid + (match-lambda + (('uuid (? symbol? type) (? bytevector? bv)) + (bytevector->uuid bv type)) + (x + (warning (G_ "unrecognized uuid ~a at '~a'~%") x (port-filename port)) + #f))) (match (read port) (('boot-parameters ('version 0) @@ -411,6 +422,23 @@ file system labels." ;; No store found, old format. #f))) + (store-crypto-devices + (match (assq 'store rest) + (('store . store-data) + (match (assq 'crypto-devices store-data) + (('crypto-devices (devices ...)) + (map uuid-sexp->uuid devices)) + (('crypto-devices dev) + (warning (G_ "unrecognized crypto-devices ~S at '~a'~%") + dev (port-filename port)) + '()) + (_ + ;; No crypto-devices found. + '()))) + (_ + ;; No store found, old format. + '()))) + (store-mount-point (match (assq 'store rest) (('store ('device _) ('mount-point mount-point) _ ...) @@ -525,6 +553,26 @@ from the initrd." (any file-system-needed-for-boot? users))) devices))) +(define (operating-system-bootloader-crypto-devices os) + "Return the subset of mapped devices that the bootloader must open. +Only devices specified by uuid are supported." + (define (valid-crypto-device? dev) + (or (uuid? dev) + (begin + (warning (G_ "\ +mapped-device '~a' may not be mounted by the bootloader.~%") + dev) + #f))) + (filter-map (match-lambda + ((and (= mapped-device-type type) + (= mapped-device-source source)) + (and (eq? luks-device-mapping type) + (valid-crypto-device? source) + source)) + (_ #f)) + ;; XXX: Ordering is important, we trust the returned one. + (operating-system-boot-mapped-devices os))) + (define (device-mapping-services os) "Return the list of device-mapping services for OS as a list." (map device-mapping-service @@ -1261,6 +1309,7 @@ a list of , to populate the \"old entries\" menu." (root-fs (operating-system-root-file-system os)) (root-device (file-system-device root-fs)) (locale (operating-system-locale os)) + (crypto-devices (operating-system-bootloader-crypto-devices os)) (params (operating-system-boot-parameters os root-device #:system-kernel-arguments? #t)) @@ -1274,6 +1323,7 @@ a list of , to populate the \"old entries\" menu." (generate-config-file bootloader-conf (list entry) #:old-entries old-entries #:locale locale + #:store-crypto-devices crypto-devices #:store-directory-prefix (btrfs-store-subvolume-file-name file-systems)))) @@ -1313,6 +1363,7 @@ such as '--root' and '--load' to ." (operating-system-initrd-file os))) (store (operating-system-store-file-system os)) (file-systems (operating-system-file-systems os)) + (crypto-devices (operating-system-bootloader-crypto-devices os)) (locale (operating-system-locale os)) (bootloader (bootloader-configuration-bootloader (operating-system-bootloader os))) @@ -1335,6 +1386,7 @@ such as '--root' and '--load' to ." (locale locale) (store-device (ensure-not-/dev (file-system-device store))) (store-directory-prefix (btrfs-store-subvolume-file-name file-systems)) + (store-crypto-devices crypto-devices) (store-mount-point (file-system-mount-point store))))) (define (device->sexp device) @@ -1393,7 +1445,10 @@ being stored into the \"parameters\" file)." (mount-point #$(boot-parameters-store-mount-point params)) (directory-prefix - #$(boot-parameters-store-directory-prefix params)))) + #$(boot-parameters-store-directory-prefix params)) + (crypto-devices + #$(map device->sexp + (boot-parameters-store-crypto-devices params))))) #:set-load-path? #f))) (define-gexp-compiler (operating-system-compiler (os ) diff --git a/gnu/tests/install.scm b/gnu/tests/install.scm index 71caa3a493..bf94e97c2a 100644 --- a/gnu/tests/install.scm +++ b/gnu/tests/install.scm @@ -63,6 +63,7 @@ %test-separate-home-os %test-raid-root-os %test-encrypted-root-os + %test-encrypted-root-not-boot-os %test-btrfs-root-os %test-btrfs-root-on-subvolume-os %test-jfs-root-os @@ -883,6 +884,107 @@ reboot\n") (run-basic-test %lvm-separate-home-os `(,@command) "lvm-separate-home-os"))))) + +;;; +;;; LUKS-encrypted root file system and /boot in a non-encrypted partition. +;;; + +(define-os-with-source (%encrypted-root-not-boot-os + %encrypted-root-not-boot-os-source) + ;; The OS we want to install. + (use-modules (gnu) (gnu tests) (srfi srfi-1)) + + (operating-system + (host-name "bootroot") + (timezone "Europe/Madrid") + (locale "en_US.UTF-8") + + (bootloader (bootloader-configuration + (bootloader grub-bootloader) + (target "/dev/vdb"))) + + (mapped-devices (list (mapped-device + (source + (uuid "12345678-1234-1234-1234-123456789abc")) + (target "root") + (type luks-device-mapping)))) + (file-systems (cons* (file-system + (device (file-system-label "my-boot")) + (mount-point "/boot") + (type "ext4")) + (file-system + (device "/dev/mapper/root") + (mount-point "/") + (type "ext4")) + %base-file-systems)) + (users (cons (user-account + (name "alice") + (group "users") + (supplementary-groups '("wheel" "audio" "video"))) + %base-user-accounts)) + (services (cons (service marionette-service-type + (marionette-configuration + (imported-modules '((gnu services herd) + (guix combinators))))) + %base-services)))) + +(define %encrypted-root-not-boot-installation-script + ;; Shell script for an installation with boot not encrypted but root + ;; encrypted. + (format #f "\ +. /etc/profile +set -e -x +guix --version + +export GUIX_BUILD_OPTIONS=--no-grafts +ls -l /run/current-system/gc-roots +parted --script /dev/vdb mklabel gpt \\ + mkpart primary ext2 1M 3M \\ + mkpart primary ext2 3M 50M \\ + mkpart primary ext2 50M 1.6G \\ + set 1 boot on \\ + set 1 bios_grub on +echo -n \"~a\" | cryptsetup luksFormat --uuid=\"~a\" -q /dev/vdb3 - +echo -n \"~a\" | cryptsetup open --type luks --key-file - /dev/vdb3 root +mkfs.ext4 -L my-root /dev/mapper/root +mkfs.ext4 -L my-boot /dev/vdb2 +mount LABEL=my-root /mnt +mkdir /mnt/boot +mount LABEL=my-boot /mnt/boot +echo \"Checking mounts\" +mount +herd start cow-store /mnt +mkdir /mnt/etc +cp /etc/target-config.scm /mnt/etc/config.scm +guix system build /mnt/etc/config.scm +guix system init /mnt/etc/config.scm /mnt --no-substitutes +sync +echo \"Debugging info\" +blkid +cat /mnt/boot/grub/grub.cfg +reboot\n" + %luks-passphrase "12345678-1234-1234-1234-123456789abc" + %luks-passphrase)) + +(define %test-encrypted-root-not-boot-os + (system-test + (name "encrypted-root-not-boot-os") + (description + "Test the manual installation on an OS with / in an encrypted partition +but /boot on a different, non-encrypted partition. This test is expensive in +terms of CPU and storage usage since we need to build (current-guix) and then +store a couple of full system images.") + (value + (mlet* %store-monad + ((image (run-install %encrypted-root-not-boot-os + %encrypted-root-not-boot-os-source + #:script + %encrypted-root-not-boot-installation-script)) + (command (qemu-command/writable-image image))) + (run-basic-test %encrypted-root-not-boot-os command + "encrypted-root-not-boot-os" + #:initialization enter-luks-passphrase))))) + ;;; ;;; Btrfs root file system. diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 5427f875ec..0dcf2b3afe 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -391,6 +391,7 @@ STORE is an open connection to the store." (params (first (profile-boot-parameters %system-profile (list number)))) (locale (boot-parameters-locale params)) + (store-crypto-devices (boot-parameters-store-crypto-devices params)) (store-directory-prefix (boot-parameters-store-directory-prefix params)) (old-generations @@ -406,6 +407,7 @@ STORE is an open connection to the store." ((bootloader-configuration-file-generator bootloader) bootloader-config entries #:locale locale + #:store-crypto-devices store-crypto-devices #:store-directory-prefix store-directory-prefix #:old-entries old-entries))) (drvs -> (list bootcfg))) diff --git a/tests/boot-parameters.scm b/tests/boot-parameters.scm index a00b227551..3deae564c4 100644 --- a/tests/boot-parameters.scm +++ b/tests/boot-parameters.scm @@ -50,6 +50,9 @@ (define %default-store-directory-prefix (string-append "/" %default-btrfs-subvolume)) (define %default-store-mount-point (%store-prefix)) +(define %default-store-crypto-devices + (list (uuid "00000000-1111-2222-3333-444444444444") + (uuid "55555555-6666-7777-8888-999999999999"))) (define %default-multiboot-modules '()) (define %default-locale "es_ES.utf8") (define %root-path "/") @@ -67,6 +70,7 @@ (locale %default-locale) (store-device %default-store-device) (store-directory-prefix %default-store-directory-prefix) + (store-crypto-devices %default-store-crypto-devices) (store-mount-point %default-store-mount-point))) (define %default-operating-system @@ -110,6 +114,8 @@ (with-store #t) (store-device (quote-uuid %default-store-device)) + (store-crypto-devices + (map quote-uuid %default-store-crypto-devices)) (store-directory-prefix %default-store-directory-prefix) (store-mount-point %default-store-mount-point)) (define (generate-boot-parameters) @@ -125,12 +131,14 @@ (sexp-or-nothing " (kernel-arguments ~S)" kernel-arguments) (sexp-or-nothing " (initrd ~S)" initrd) (if with-store - (format #false " (store~a~a~a)" + (format #false " (store~a~a~a~a)" (sexp-or-nothing " (device ~S)" store-device) (sexp-or-nothing " (mount-point ~S)" store-mount-point) (sexp-or-nothing " (directory-prefix ~S)" - store-directory-prefix)) + store-directory-prefix) + (sexp-or-nothing " (crypto-devices ~S)" + store-crypto-devices)) "") (sexp-or-nothing " (locale ~S)" locale) (sexp-or-nothing " (bootloader-name ~a)" bootloader-name) @@ -158,6 +166,7 @@ (test-read-boot-parameters #:with-store #false) (test-read-boot-parameters #:store-device #false) (test-read-boot-parameters #:store-device 'false) + (test-read-boot-parameters #:store-crypto-devices #false) (test-read-boot-parameters #:store-mount-point #false) (test-read-boot-parameters #:store-directory-prefix #false) (test-read-boot-parameters #:multiboot-modules #false) @@ -254,6 +263,23 @@ (boot-parameters-store-mount-point (test-read-boot-parameters #:with-store #false))) +(test-equal "read, store-crypto-devices, default" + '() + (boot-parameters-store-crypto-devices + (test-read-boot-parameters #:store-crypto-devices #false))) + +;; XXX: +(test-equal "read, store-crypto-devices, false" + '() + (boot-parameters-store-crypto-devices + (test-read-boot-parameters #:store-crypto-devices 'false))) + +;; XXX: +(test-equal "read, store-crypto-devices, string" + '() + (boot-parameters-store-crypto-devices + (test-read-boot-parameters #:store-crypto-devices "bad"))) + ;; For whitebox testing (define operating-system-boot-parameters (@@ (gnu system) operating-system-boot-parameters)) -- cgit v1.2.3 From 5aae614868b8a7e3097ae70f6024352aa5c8de21 Mon Sep 17 00:00:00 2001 From: Ricardo Wurmus Date: Sun, 27 Dec 2020 12:05:24 +0100 Subject: import/utils: alist->package: Handle SPDX license names only as fallback. Fixes . * guix/import/utils.scm (alist->package): Find plain license names in (guix licenses) first, and only fall back to SPDX names on error. * tests/import-utils.scm ("alist->package with SPDX license name 1/2", "alist->package with SPDX license name 2/2"): New tests. --- guix/import/utils.scm | 8 ++++++-- tests/import-utils.scm | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/guix/import/utils.scm b/guix/import/utils.scm index e227c2e42d..cdbcf6bfa5 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -372,8 +372,12 @@ specifications to look up and replace them with plain symbols instead." (match (assoc-ref meta "license") (#f #f) (l - (or (module-ref (resolve-interface '(guix licenses) #:prefix 'license:) - (spdx-string->license l)) + (or (false-if-exception + (module-ref (resolve-interface '(guix licenses)) + (string->symbol l))) + (false-if-exception + (module-ref (resolve-interface '(guix licenses) #:prefix 'license:) + (spdx-string->license l))) (license:fsdg-compatible l))))))) (define* (read-lines #:optional (port (current-input-port))) diff --git a/tests/import-utils.scm b/tests/import-utils.scm index 2357ea5c40..874816442e 100644 --- a/tests/import-utils.scm +++ b/tests/import-utils.scm @@ -122,6 +122,38 @@ (or (package-license (alist->package meta)) 'license-is-false))) +(test-equal "alist->package with SPDX license name 1/2" ; + license:expat + (let* ((meta '(("name" . "hello") + ("version" . "2.10") + ("source" . (("method" . "url-fetch") + ("uri" . "mirror://gnu/hello/hello-2.10.tar.gz") + ("sha256" . + (("base32" . + "0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i"))))) + ("build-system" . "gnu") + ("home-page" . "https://gnu.org") + ("synopsis" . "Say hi") + ("description" . "This package says hi.") + ("license" . "expat")))) + (package-license (alist->package meta)))) + +(test-equal "alist->package with SPDX license name 2/2" ; + license:expat + (let* ((meta '(("name" . "hello") + ("version" . "2.10") + ("source" . (("method" . "url-fetch") + ("uri" . "mirror://gnu/hello/hello-2.10.tar.gz") + ("sha256" . + (("base32" . + "0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i"))))) + ("build-system" . "gnu") + ("home-page" . "https://gnu.org") + ("synopsis" . "Say hi") + ("description" . "This package says hi.") + ("license" . "MIT")))) + (package-license (alist->package meta)))) + (test-equal "alist->package with dependencies" `(("gettext" ,(specification->package "gettext"))) (let* ((meta '(("name" . "hello") -- cgit v1.2.3 From e38d90d497e19e00263fa28961c688a433154386 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 21 Dec 2020 14:52:38 +0100 Subject: transformations: Add '--with-patch'. Suggested by Philippe Swartvagher . * guix/transformations.scm (transform-package-patches): New procedure. (%transformations): Add it as 'with-patch'. (%transformation-options, show-transformation-options-help/detailed): Add '--with-patch'. * tests/transformations.scm ("options->transformation, with-patch"): New test. * doc/guix.texi (Package Transformation Options): Document it. --- doc/guix.texi | 18 ++++++++++++++ guix/transformations.scm | 63 ++++++++++++++++++++++++++++++++++++++++++++++- tests/transformations.scm | 24 ++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/doc/guix.texi b/doc/guix.texi index b12cb11bdf..6c681494a2 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -10357,6 +10357,24 @@ This is similar to @option{--with-branch}, except that it builds from @var{commit} rather than the tip of a branch. @var{commit} must be a valid Git commit SHA1 identifier or a tag. +@item --with-patch=@var{package}=@var{file} +Add @var{file} to the list of patches applied to @var{package}, where +@var{package} is a spec such as @code{python@@3.8} or @code{glibc}. +@var{file} must contain a patch; it is applied with the flags specified +in the @code{origin} of @var{package} (@pxref{origin Reference}), which +by default includes @code{-p1} (@pxref{patch Directories,,, diffutils, +Comparing and Merging Files}). + +As an example, the command below rebuilds Coreutils with the GNU C +Library (glibc) patched with the given patch: + +@example +guix build coreutils --with-patch=glibc=./glibc-frob.patch +@end example + +In this example, glibc itself as well as everything that leads to +Coreutils in the dependency graph is rebuilt. + @cindex test suite, skipping @item --without-tests=@var{package} Build @var{package} without running its tests. This can be useful in diff --git a/guix/transformations.scm b/guix/transformations.scm index d49041cf59..2385d3231e 100644 --- a/guix/transformations.scm +++ b/guix/transformations.scm @@ -41,6 +41,7 @@ #:use-module (srfi srfi-34) #:use-module (srfi srfi-37) #:use-module (ice-9 match) + #:use-module (ice-9 vlist) #:export (options->transformation manifest-entry-with-transformations @@ -456,6 +457,60 @@ to the same package but with #:strip-binaries? #f in its 'arguments' field." (rewrite obj) obj))) +(define (transform-package-patches specs) + "Return a procedure that, when passed a package, returns a package with +additional patches." + (define (package-with-extra-patches p patches) + (if (origin? (package-source p)) + (package/inherit p + (source (origin + (inherit (package-source p)) + (patches (append (map (lambda (file) + (local-file file)) + patches) + (origin-patches (package-source p))))))) + p)) + + (define (coalesce-alist alist) + ;; Coalesce multiple occurrences of the same key in ALIST. + (let loop ((alist alist) + (keys '()) + (mapping vlist-null)) + (match alist + (() + (map (lambda (key) + (cons key (vhash-fold* cons '() key mapping))) + (delete-duplicates (reverse keys)))) + (((key . value) . rest) + (loop rest + (cons key keys) + (vhash-cons key value mapping)))))) + + (define patches + ;; Spec/patch alist. + (coalesce-alist + (map (lambda (spec) + (match (string-tokenize spec %not-equal) + ((spec patch) + (cons spec (canonicalize-path patch))) + (_ + (raise (formatted-message + (G_ "~a: invalid package patch specification") + spec))))) + specs))) + + (define rewrite + (package-input-rewriting/spec + (map (match-lambda + ((spec . patches) + (cons spec (cut package-with-extra-patches <> patches)))) + patches))) + + (lambda (obj) + (if (package? obj) + (rewrite obj) + obj))) + (define %transformations ;; Transformations that can be applied to things to build. The car is the ;; key used in the option alist, and the cdr is the transformation @@ -469,7 +524,8 @@ to the same package but with #:strip-binaries? #f in its 'arguments' field." (with-git-url . ,transform-package-source-git-url) (with-c-toolchain . ,transform-package-toolchain) (with-debug-info . ,transform-package-with-debug-info) - (without-tests . ,transform-package-tests))) + (without-tests . ,transform-package-tests) + (with-patch . ,transform-package-patches))) (define (transformation-procedure key) "Return the transformation procedure associated with KEY, a symbol such as @@ -509,6 +565,8 @@ to the same package but with #:strip-binaries? #f in its 'arguments' field." (parser 'with-debug-info)) (option '("without-tests") #t #f (parser 'without-tests)) + (option '("with-patch") #t #f + (parser 'with-patch)) (option '("help-transform") #f #f (lambda _ @@ -537,6 +595,9 @@ to the same package but with #:strip-binaries? #f in its 'arguments' field." (display (G_ " --with-git-url=PACKAGE=URL build PACKAGE from the repository at URL")) + (display (G_ " + --with-patch=PACKAGE=FILE + add FILE to the list of patches of PACKAGE")) (display (G_ " --with-c-toolchain=PACKAGE=TOOLCHAIN build PACKAGE and its dependents with TOOLCHAIN")) diff --git a/tests/transformations.scm b/tests/transformations.scm index 2d33bed7ae..9053deba41 100644 --- a/tests/transformations.scm +++ b/tests/transformations.scm @@ -26,6 +26,7 @@ #:use-module (guix build-system) #:use-module (guix build-system gnu) #:use-module (guix transformations) + #:use-module ((guix gexp) #:select (local-file? local-file-file)) #:use-module (guix ui) #:use-module (guix utils) #:use-module (guix git) @@ -372,6 +373,29 @@ (match (memq #:tests? (package-arguments tar)) ((#:tests? #f _ ...) #t)))))))) +(test-equal "options->transformation, with-patch" + (search-patches "glibc-locales.patch" "guile-relocatable.patch") + (let* ((dep (dummy-package "dep" + (source (dummy-origin)))) + (p (dummy-package "foo" + (inputs `(("dep" ,dep))))) + (patch1 (search-patch "glibc-locales.patch")) + (patch2 (search-patch "guile-relocatable.patch")) + (t (options->transformation + `((with-patch . ,(string-append "dep=" patch1)) + (with-patch . ,(string-append "dep=" patch2)) + (with-patch . ,(string-append "tar=" patch1)))))) + (let ((new (t p))) + (match (bag-direct-inputs (package->bag new)) + ((("dep" dep) ("tar" tar) _ ...) + (and (member patch1 + (filter-map (lambda (patch) + (and (local-file? patch) + (local-file-file patch))) + (origin-patches (package-source tar)))) + (map local-file-file + (origin-patches (package-source dep))))))))) + (test-end) ;;; Local Variables: -- cgit v1.2.3 From ed63b7f87e88b0ce9c4a78bf0529599de9172b1f Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 3 Jan 2021 21:59:49 +0100 Subject: guix hash: Honor '-H' when used alongside '-r'. * guix/scripts/hash.scm (guix-hash): When 'recursive? is true, use 'open-hash-port' instead of 'open-sha256-port'. * tests/guix-hash.sh: Add test for 'guix hash -r -H sha512'. --- guix/scripts/hash.scm | 5 +++-- tests/guix-hash.sh | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/guix/scripts/hash.scm b/guix/scripts/hash.scm index 797b99f053..b8622373cc 100644 --- a/guix/scripts/hash.scm +++ b/guix/scripts/hash.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2016, 2017, 2020 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2016, 2017, 2020, 2021 Ludovic Courtès ;;; Copyright © 2013 Nikita Karetnikov ;;; Copyright © 2016 Jan Nieuwenhuizen ;;; Copyright © 2018 Tim Gesthuizen @@ -151,7 +151,8 @@ and 'base16' ('hex' and 'hexadecimal' can be used as well).\n")) ;; Catch and gracefully report possible '&nar-error' conditions. (with-error-handling (if (assoc-ref opts 'recursive?) - (let-values (((port get-hash) (open-sha256-port))) + (let-values (((port get-hash) + (open-hash-port (assoc-ref opts 'hash-algorithm)))) (write-file file port #:select? select?) (force-output port) (get-hash)) diff --git a/tests/guix-hash.sh b/tests/guix-hash.sh index 346355539f..c4461fa955 100644 --- a/tests/guix-hash.sh +++ b/tests/guix-hash.sh @@ -1,5 +1,5 @@ # GNU Guix --- Functional package management for GNU -# Copyright © 2013, 2014, 2016, 2020 Ludovic Courtès +# Copyright © 2013, 2014, 2016, 2020, 2021 Ludovic Courtès # Copyright © 2016 Jan Nieuwenhuizen # # This file is part of GNU Guix. @@ -43,6 +43,7 @@ chmod +x "$tmpdir/exe" mkdir "$tmpdir/subdir" test `guix hash -r "$tmpdir"` = 10k1lw41wyrjf9mxydi0is5nkpynlsvgslinics4ppir13g7d74p +test `guix hash -r "$tmpdir" -H sha512` = 301ra58c2vahczzxiyfin41mpyb0ljh4dh9zn3ijvwviaw1j40sfzw5skh9x945da88n3785ggifzig7acd6k72h0mpsc20m1f66m9n # Without '-r', this should fail. ! guix hash "$tmpdir" -- cgit v1.2.3