From 2be5c2652a5fd79089048905ff6be60d74244d7b Mon Sep 17 00:00:00 2001 From: Danny Milosavljevic Date: Mon, 14 Sep 2020 00:24:13 +0200 Subject: build-system: linux-module: Delete some huge items that we probably don't need. * guix/build-system/linux-module.scm (make-linux-module-builder): Delete some huge items that we probably don't need. --- guix/build-system/linux-module.scm | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/build-system/linux-module.scm b/guix/build-system/linux-module.scm index 1077215671..fc3d959ce7 100644 --- a/guix/build-system/linux-module.scm +++ b/guix/build-system/linux-module.scm @@ -68,14 +68,41 @@ (lambda* (#:key inputs outputs #:allow-other-keys) (let* ((out (assoc-ref outputs "out")) (out-lib-build (string-append out "/lib/modules/build"))) + ;; Delete some huge items that we probably don't need. ;; TODO: Only preserve the minimum, i.e. [Kbuild], Kconfig, ;; scripts, include, ".config". (copy-recursively "." out-lib-build) + (for-each (lambda (name) + (when (file-exists? name) + (delete-file-recursively name))) + (map (lambda (name) + (string-append out-lib-build "/" name)) + '("arch" ; 137 MB + ;"tools" ; 44 MB ; Note: is built by our 'build phase. + "tools/testing" ; 14 MB + "tools/perf" ; 17 MB + "drivers" ; 600 MB + "Documentation" ; 52 MB + "fs" ; 43 MB + "net" ; 33 MB + "samples" ; 2 MB + "sound"))) ; 40 MB + ;; Reinstate arch/**/dts since "scripts/dtc" depends on it. + ;; Reinstate arch/**/include directories. + ;; Reinstate arch/**/Makefile. + ;; Reinstate arch/**/module.lds. + (for-each + (lambda (name) + (mkdir-p (dirname (string-append out-lib-build "/" name))) + (copy-recursively name + (string-append out-lib-build "/" name))) + (append (find-files "arch" "^(dts|include)$" #:directories? #t) + (find-files "arch" "^(Makefile|module.lds)$"))) (let* ((linux (assoc-ref inputs "linux"))) (install-file (string-append linux "/System.map") out-lib-build) (let ((source (string-append linux "/Module.symvers"))) - (if (file-exists? source) + (when (file-exists? source) (install-file source out-lib-build)))) #t))))))))) -- cgit v1.2.3 From 14c422c12c86126cfb5ca7e1641bbcd78d02f711 Mon Sep 17 00:00:00 2001 From: Caleb Ristvedt Date: Sat, 8 Aug 2020 10:05:22 -0500 Subject: deduplication: pass store directory to replace-with-link. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This causes with-writable-file to take into consideration the actual store being used, as passed to 'deduplicate', rather than whatever (%store-directory) may return. * guix/store/deduplication.scm (replace-with-link): new keyword argument 'store'. Pass to with-writable-file. (with-writable-file, call-with-writable-file): new store argument. (deduplicate): pass store to replace-with-link. Signed-off-by: Ludovic Courtès --- .dir-locals.el | 2 +- guix/store/deduplication.scm | 102 ++++++++++++++++++++++--------------------- 2 files changed, 54 insertions(+), 50 deletions(-) (limited to 'guix') diff --git a/.dir-locals.el b/.dir-locals.el index 5954e313f2..7f310d2612 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -37,7 +37,7 @@ (eval . (put 'with-file-lock 'scheme-indent-function 1)) (eval . (put 'with-file-lock/no-wait 'scheme-indent-function 1)) (eval . (put 'with-profile-lock 'scheme-indent-function 1)) - (eval . (put 'with-writable-file 'scheme-indent-function 1)) + (eval . (put 'with-writable-file 'scheme-indent-function 2)) (eval . (put 'package 'scheme-indent-function 0)) (eval . (put 'package/inherit 'scheme-indent-function 1)) diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm index df959bdd06..0655ceb890 100644 --- a/guix/store/deduplication.scm +++ b/guix/store/deduplication.scm @@ -94,8 +94,8 @@ LINK-PREFIX." (try (tempname-in link-prefix)) (apply throw args)))))) -(define (call-with-writable-file file thunk) - (if (string=? file (%store-directory)) +(define (call-with-writable-file file store thunk) + (if (string=? file store) (thunk) ;don't meddle with the store's permissions (let ((stat (lstat file))) (dynamic-wind @@ -106,17 +106,18 @@ LINK-PREFIX." (set-file-time file stat) (chmod file (stat:mode stat))))))) -(define-syntax-rule (with-writable-file file exp ...) +(define-syntax-rule (with-writable-file file store exp ...) "Make FILE writable for the dynamic extent of EXP..., except if FILE is the store." - (call-with-writable-file file (lambda () exp ...))) + (call-with-writable-file file store (lambda () exp ...))) ;; There are 3 main kinds of errors we can get from hardlinking: "Too many ;; things link to this" (EMLINK), "this link already exists" (EEXIST), and ;; "can't fit more stuff in this directory" (ENOSPC). (define* (replace-with-link target to-replace - #:key (swap-directory (dirname target))) + #:key (swap-directory (dirname target)) + (store (%store-directory))) "Atomically replace the file TO-REPLACE with a link to TARGET. Use SWAP-DIRECTORY as the directory to store temporary hard links. Upon ENOSPC and EMLINK, TO-REPLACE is left unchanged. @@ -137,7 +138,7 @@ Note: TARGET, TO-REPLACE, and SWAP-DIRECTORY must be on the same file system." ;; If we couldn't create TEMP-LINK, that's OK: just don't do the ;; replacement, which means TO-REPLACE won't be deduplicated. (when temp-link - (with-writable-file (dirname to-replace) + (with-writable-file (dirname to-replace) store (catch 'system-error (lambda () (rename-file temp-link to-replace)) @@ -154,46 +155,49 @@ under STORE." (define links-directory (string-append store "/.links")) - (mkdir-p links-directory) - (let loop ((path path) - (type (stat:type (lstat path))) - (hash hash)) - (if (eq? 'directory type) - ;; Can't hardlink directories, so hardlink their atoms. - (for-each (match-lambda - ((file . properties) - (unless (member file '("." "..")) - (let* ((file (string-append path "/" file)) - (type (match (assoc-ref properties 'type) - ((or 'unknown #f) - (stat:type (lstat file))) - (type type)))) - (loop file type - (and (not (eq? 'directory type)) - (nar-sha256 file))))))) - (scandir* path)) - (let ((link-file (string-append links-directory "/" - (bytevector->nix-base32-string hash)))) - (if (file-exists? link-file) - (replace-with-link link-file path - #:swap-directory links-directory) - (catch 'system-error - (lambda () - (link path link-file)) - (lambda args - (let ((errno (system-error-errno args))) - (cond ((= errno EEXIST) - ;; Someone else put an entry for PATH in - ;; LINKS-DIRECTORY before we could. Let's use it. - (replace-with-link path link-file - #:swap-directory links-directory)) - ((= errno ENOSPC) - ;; There's not enough room in the directory index for - ;; more entries in .links, but that's fine: we can - ;; just stop. - #f) - ((= errno EMLINK) - ;; PATH has reached the maximum number of links, but - ;; that's OK: we just can't deduplicate it more. - #f) - (else (apply throw args))))))))))) + (mkdir-p links-directory) + (let loop ((path path) + (type (stat:type (lstat path))) + (hash hash)) + (if (eq? 'directory type) + ;; Can't hardlink directories, so hardlink their atoms. + (for-each (match-lambda + ((file . properties) + (unless (member file '("." "..")) + (let* ((file (string-append path "/" file)) + (type (match (assoc-ref properties 'type) + ((or 'unknown #f) + (stat:type (lstat file))) + (type type)))) + (loop file type + (and (not (eq? 'directory type)) + (nar-sha256 file))))))) + (scandir* path)) + (let ((link-file (string-append links-directory "/" + (bytevector->nix-base32-string hash)))) + (if (file-exists? link-file) + (replace-with-link link-file path + #:swap-directory links-directory + #:store store) + (catch 'system-error + (lambda () + (link path link-file)) + (lambda args + (let ((errno (system-error-errno args))) + (cond ((= errno EEXIST) + ;; Someone else put an entry for PATH in + ;; LINKS-DIRECTORY before we could. Let's use it. + (replace-with-link path link-file + #:swap-directory + links-directory + #:store store)) + ((= errno ENOSPC) + ;; There's not enough room in the directory index for + ;; more entries in .links, but that's fine: we can + ;; just stop. + #f) + ((= errno EMLINK) + ;; PATH has reached the maximum number of links, but + ;; that's OK: we just can't deduplicate it more. + #f) + (else (apply throw args))))))))))) -- cgit v1.2.3 From a05c31ab3007a7a60f735ed7405fb9c54be1728d Mon Sep 17 00:00:00 2001 From: Caleb Ristvedt Date: Wed, 8 Jul 2020 11:33:23 -0500 Subject: database: document extra registration requirements. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's necessary that store items be locked and protected from garbage collection while they are being registered. This documents that. * guix/store/database.scm (register-path, register-items): document GC protection and locking requirements. Signed-off-by: Ludovic Courtès --- guix/store/database.scm | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'guix') diff --git a/guix/store/database.scm b/guix/store/database.scm index 50b66ce282..e39a1603af 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -397,7 +397,10 @@ absolute file name to the state directory of the store being initialized. Return #t on success. Use with care as it directly modifies the store! This is primarily meant to -be used internally by the daemon's build hook." +be used internally by the daemon's build hook. + +PATH must be protected from GC and locked during execution of this, typically +by adding it as a temp-root." (define db-file (store-database-file #:prefix prefix #:state-directory state-directory)) @@ -423,7 +426,9 @@ be used internally by the daemon's build hook." "Register all of ITEMS, a list of records as returned by 'read-reference-graph', in DB. ITEMS must be in topological order (with leaves first.) REGISTRATION-TIME must be the registration time to be recorded -in the database; #f means \"now\". Write a progress report to LOG-PORT." +in the database; #f means \"now\". Write a progress report to LOG-PORT. All +of ITEMS must be protected from GC and locked during execution of this, +typically by adding them as temp-roots." (define store-dir (if prefix (string-append prefix %storedir) -- cgit v1.2.3 From 2932591b8aeec89732c8f8faa0d3f8ef900e68d2 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Tue, 23 Jun 2020 17:36:49 +0100 Subject: database: register-items: reduce transaction scope. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was made transactional in a4678c6ba18d8dbd79d931f80426eebf61be7ebe, with the reasoning to prevent broken intermediate states from being visible. I think this means something like an entry being in ValidPaths, but the Refs not being inserted. Using a transaction for this makes sense, but I think using one single transaction for the whole register-items call is unnecessary to avoid broken states from being visible, and could block other writes to the store database while register-items is running. Because the deduplication and resetting timestamps happens within the transaction as well, even though these things don't involve the database, writes to the database will still be blocked while this is happening. To reduce the potential for register-items to block other writers to the database for extended periods, this commit moves the transaction to just wrap the call to sqlite-register. This is the one place where writes occur, so that should prevent the broken intermediate states issue above. The one difference this will make is some of the registered items will be visible to other connections while others may be still being added. I think this is OK, as it's equivalent to just registering different items. * guix/store/database.scm (register-items): Reduce transaction scope. Signed-off-by: Ludovic Courtès --- guix/store/database.scm | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) (limited to 'guix') diff --git a/guix/store/database.scm b/guix/store/database.scm index e39a1603af..2ea63b17aa 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -457,24 +457,25 @@ typically by adding them as temp-roots." (when reset-timestamps? (reset-timestamps real-file-name)) (let-values (((hash nar-size) (nar-sha256 real-file-name))) - (sqlite-register db #:path to-register - #:references (store-info-references item) - #:deriver (store-info-deriver item) - #:hash (string-append "sha256:" - (bytevector->base16-string hash)) - #:nar-size nar-size - #:time registration-time) + (call-with-retrying-transaction db + (lambda () + (sqlite-register db #:path to-register + #:references (store-info-references item) + #:deriver (store-info-deriver item) + #:hash (string-append + "sha256:" + (bytevector->base16-string hash)) + #:nar-size nar-size + #:time registration-time))) (when deduplicate? (deduplicate real-file-name hash #:store store-dir))))) - (call-with-retrying-transaction db - (lambda () - (let* ((prefix (format #f "registering ~a items" (length items))) - (progress (progress-reporter/bar (length items) - prefix log-port))) - (call-with-progress-reporter progress - (lambda (report) - (for-each (lambda (item) - (register db item) - (report)) - items))))))) + (let* ((prefix (format #f "registering ~a items" (length items))) + (progress (progress-reporter/bar (length items) + prefix log-port))) + (call-with-progress-reporter progress + (lambda (report) + (for-each (lambda (item) + (register db item) + (report)) + items))))) -- cgit v1.2.3 From 735808b12cc23909b421e10e212a07e7aa69a5eb Mon Sep 17 00:00:00 2001 From: Paul Garlick Date: Mon, 14 Sep 2020 14:01:15 +0100 Subject: guix: Fix download-svn-to-store. * guix/svn-download.scm (download-svn-to-store): Add a subdirectory to the download path. The subdirectory is used as the target for the 'svn export' command, avoiding a 'directory exists' error when attempting to use the parent directory directly. --- guix/svn-download.scm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'guix') diff --git a/guix/svn-download.scm b/guix/svn-download.scm index 59e2eb8d07..b96151234c 100644 --- a/guix/svn-download.scm +++ b/guix/svn-download.scm @@ -159,10 +159,11 @@ reports to LOG." (parameterize ((current-output-port log)) (build:svn-fetch (svn-reference-url ref) (svn-reference-revision ref) - temp + (string-append temp "/svn") #:user-name (svn-reference-user-name ref) #:password (svn-reference-password ref))))) (and result - (add-to-store store name #t "sha256" temp)))))) + (add-to-store store name #t "sha256" + (string-append temp "/svn"))))))) ;;; svn-download.scm ends here -- cgit v1.2.3 From 64cf660f872fb7aaf0d2b463e45b4c756297f743 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 11 Sep 2020 12:51:36 +0200 Subject: daemon: Spawn 'guix authenticate' once for all. Previously, we'd spawn 'guix authenticate' once for each item that has to be signed (when exporting) or authenticated (when importing). Now, we spawn it once for all and then follow a request/reply protocol. This reduces the wall-clock time of: guix archive --export -r $(guix build coreutils -d) from 30s to 2s. * guix/scripts/authenticate.scm (sign-with-key): Return the signature instead of displaying it. Raise a &formatted-message instead of calling 'leave'. (validate-signature): Likewise. (read-command): New procedure. (define-enumerate-type, reply-code): New macros. (guix-authenticate)[send-reply]: New procedure. Change to read commands from current-input-port. * nix/libstore/local-store.cc (runAuthenticationProgram): Remove. (authenticationAgent, readInteger, readAuthenticateReply): New functions. (signHash, verifySignature): Rewrite in terms of the agent. * tests/store.scm ("import not signed"): Remove 'pk' call. ("import signed by unauthorized key"): Check the error message of C. * tests/guix-authenticate.sh: Rewrite using the new protocol. fixlet --- guix/scripts/authenticate.scm | 138 +++++++++++++++++++++++++++++++++--------- nix/libstore/local-store.cc | 87 ++++++++++++++++++++------ tests/guix-authenticate.sh | 45 ++++++++------ tests/store.scm | 8 +-- 4 files changed, 208 insertions(+), 70 deletions(-) (limited to 'guix') diff --git a/guix/scripts/authenticate.scm b/guix/scripts/authenticate.scm index 37e6cef53c..dc73981092 100644 --- a/guix/scripts/authenticate.scm +++ b/guix/scripts/authenticate.scm @@ -22,6 +22,10 @@ #:use-module (gcrypt pk-crypto) #:use-module (guix pki) #:use-module (guix ui) + #:use-module (guix diagnostics) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) + #:use-module (rnrs bytevectors) #:use-module (ice-9 binary-ports) #:use-module (ice-9 rdelim) #:use-module (ice-9 match) @@ -40,41 +44,90 @@ (compose string->canonical-sexp read-string)) (define (sign-with-key key-file sha256) - "Sign the hash SHA256 (a bytevector) with KEY-FILE, and write an sexp that -includes both the hash and the actual signature." + "Sign the hash SHA256 (a bytevector) with KEY-FILE, and return the signature +as a canonical sexp that includes both the hash and the actual signature." (let* ((secret-key (call-with-input-file key-file read-canonical-sexp)) (public-key (if (string-suffix? ".sec" key-file) (call-with-input-file (string-append (string-drop-right key-file 4) ".pub") read-canonical-sexp) - (leave - (G_ "cannot find public key for secret key '~a'~%") - key-file))) + (raise + (formatted-message + (G_ "cannot find public key for secret key '~a'~%") + key-file)))) (data (bytevector->hash-data sha256 #:key-type (key-type public-key))) (signature (signature-sexp data secret-key public-key))) - (display (canonical-sexp->string signature)) - #t)) + signature)) (define (validate-signature signature) "Validate SIGNATURE, a canonical sexp. Check whether its public key is -authorized, verify the signature, and print the signed data to stdout upon -success." +authorized, verify the signature, and return the signed data (a bytevector) +upon success." (let* ((subject (signature-subject signature)) (data (signature-signed-data signature))) (if (and data subject) (if (authorized-key? subject) (if (valid-signature? signature) - (let ((hash (hash-data->bytevector data))) - (display (bytevector->base16-string hash)) - #t) ; success - (leave (G_ "error: invalid signature: ~a~%") - (canonical-sexp->string signature))) - (leave (G_ "error: unauthorized public key: ~a~%") - (canonical-sexp->string subject))) - (leave (G_ "error: corrupt signature data: ~a~%") - (canonical-sexp->string signature))))) + (hash-data->bytevector data) ; success + (raise + (formatted-message (G_ "invalid signature: ~a") + (canonical-sexp->string signature)))) + (raise + (formatted-message (G_ "unauthorized public key: ~a") + (canonical-sexp->string subject)))) + (raise + (formatted-message (G_ "corrupt signature data: ~a") + (canonical-sexp->string signature)))))) + +(define (read-command port) + "Read a command from PORT and return the command and arguments as a list of +strings. Return the empty list when the end-of-file is reached. + +Commands are newline-terminated and must look something like this: + + COMMAND 3:abc 5:abcde 1:x + +where COMMAND is an alphanumeric sequence and the remainder is the command +arguments. Each argument is written as its length (in characters), followed +by colon, followed by the given number of characters." + (define (consume-whitespace port) + (let ((chr (lookahead-u8 port))) + (when (eqv? chr (char->integer #\space)) + (get-u8 port) + (consume-whitespace port)))) + + (match (read-delimited " \t\n\r" port) + ((? eof-object?) + '()) + (command + (let loop ((result (list command))) + (consume-whitespace port) + (let ((next (lookahead-u8 port))) + (cond ((eqv? next (char->integer #\newline)) + (get-u8 port) + (reverse result)) + ((eof-object? next) + (reverse result)) + (else + (let* ((len (string->number (read-delimited ":" port))) + (str (utf8->string + (get-bytevector-n port len)))) + (loop (cons str result)))))))))) + +(define-syntax define-enumerate-type ;TODO: factorize + (syntax-rules () + ((_ name->int (name id) ...) + (define-syntax name->int + (syntax-rules (name ...) + ((_ name) id) ...))))) + +;; Codes used when reply to requests. +(define-enumerate-type reply-code + (success 0) + (command-not-found 404) + (command-failed 500)) ;;; @@ -85,6 +138,13 @@ success." (category internal) (synopsis "sign or verify signatures on normalized archives (nars)") + (define (send-reply code str) + ;; Send CODE and STR as a reply to our client. + (let ((bv (string->utf8 str))) + (format #t "~a ~a:" code (bytevector-length bv)) + (put-bytevector (current-output-port) bv) + (force-output (current-output-port)))) + ;; Signature sexps written to stdout may contain binary data, so force ;; ISO-8859-1 encoding so that things are not mangled. See ;; for details. @@ -95,21 +155,39 @@ success." (with-fluids ((%default-port-encoding "ISO-8859-1") (%default-port-conversion-strategy 'error)) (match args - (("sign" key-file hash) - (sign-with-key key-file (base16-string->bytevector hash))) - (("verify" signature-file) - (call-with-input-file signature-file - (lambda (port) - (validate-signature (string->canonical-sexp - (read-string port)))))) - (("--help") (display (G_ "Usage: guix authenticate OPTION... -Sign or verify the signature on the given file. This tool is meant to -be used internally by 'guix-daemon'.\n"))) +Sign data or verify signatures. This tool is meant to be used internally by +'guix-daemon'.\n"))) (("--version") (show-version-and-exit "guix authenticate")) - (else - (leave (G_ "wrong arguments")))))) + (() + (let loop () + (guard (c ((formatted-message? c) + (send-reply (reply-code command-failed) + (apply format #f + (G_ (formatted-message-string c)) + (formatted-message-arguments c))))) + ;; Read a request on standard input and reply. + (match (read-command (current-input-port)) + (("sign" signing-key (= base16-string->bytevector hash)) + (let ((signature (sign-with-key signing-key hash))) + (send-reply (reply-code success) + (canonical-sexp->string signature)))) + (("verify" signature) + (send-reply (reply-code success) + (bytevector->base16-string + (validate-signature + (string->canonical-sexp signature))))) + (() + (exit 0)) + (commands + (warning (G_ "~s: invalid command; ignoring~%") commands) + (send-reply (reply-code command-not-found) + "invalid command")))) + + (loop))) + (_ + (leave (G_ "wrong arguments~%")))))) ;;; authenticate.scm ends here diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index cbbd8e901d..8c479002ec 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -21,6 +21,7 @@ #include #include #include +#include #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H #include @@ -1231,39 +1232,91 @@ static void checkSecrecy(const Path & path) } -static std::string runAuthenticationProgram(const Strings & args) +/* Return the authentication agent, a "guix authenticate" process started + lazily. */ +static std::shared_ptr authenticationAgent() { - Strings fullArgs = { "authenticate" }; - fullArgs.insert(fullArgs.end(), args.begin(), args.end()); // append - return runProgram(settings.guixProgram, false, fullArgs); + static std::shared_ptr agent; + + if (!agent) { + Strings args = { "authenticate" }; + agent = std::make_shared(settings.guixProgram, args); + } + + return agent; +} + +/* Read an integer and the byte that immediately follows it from FD. Return + the integer. */ +static int readInteger(int fd) +{ + string str; + + while (1) { + char ch; + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading an integer"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading an integer"); + else { + if (isdigit(ch)) { + str += ch; + } else { + break; + } + } + } + + return stoi(str); +} + +/* Read from FD a reply coming from 'guix authenticate'. The reply has the + form "CODE LEN:STR". CODE is an integer, where zero indicates success. + LEN specifies the length in bytes of the string that immediately + follows. */ +static std::string readAuthenticateReply(int fd) +{ + int code = readInteger(fd); + int len = readInteger(fd); + + string str; + str.resize(len); + readFull(fd, (unsigned char *) &str[0], len); + + if (code == 0) + return str; + else + throw Error(str); } /* Sign HASH with the key stored in file SECRETKEY. Return the signature as a string, or raise an exception upon error. */ static std::string signHash(const string &secretKey, const Hash &hash) { - Strings args; - args.push_back("sign"); - args.push_back(secretKey); - args.push_back(printHash(hash)); + auto agent = authenticationAgent(); + auto hexHash = printHash(hash); - return runAuthenticationProgram(args); + writeLine(agent->toAgent.writeSide, + (format("sign %1%:%2% %3%:%4%") + % secretKey.size() % secretKey + % hexHash.size() % hexHash).str()); + + return readAuthenticateReply(agent->fromAgent.readSide); } /* Verify SIGNATURE and return the base16-encoded hash over which it was computed. */ static std::string verifySignature(const string &signature) { - Path tmpDir = createTempDir("", "guix", true, true, 0700); - AutoDelete delTmp(tmpDir); + auto agent = authenticationAgent(); - Path sigFile = tmpDir + "/sig"; - writeFile(sigFile, signature); + writeLine(agent->toAgent.writeSide, + (format("verify %1%:%2%") + % signature.size() % signature).str()); - Strings args; - args.push_back("verify"); - args.push_back(sigFile); - return runAuthenticationProgram(args); + return readAuthenticateReply(agent->fromAgent.readSide); } void LocalStore::exportPath(const Path & path, bool sign, diff --git a/tests/guix-authenticate.sh b/tests/guix-authenticate.sh index 773443453d..f3b36ee41d 100644 --- a/tests/guix-authenticate.sh +++ b/tests/guix-authenticate.sh @@ -28,33 +28,38 @@ rm -f "$sig" "$hash" trap 'rm -f "$sig" "$hash"' EXIT +key="$abs_top_srcdir/tests/signing-key.sec" +key_len="`echo -n $key | wc -c`" + # A hexadecimal string as long as a sha256 hash. hash="2749f0ea9f26c6c7be746a9cff8fa4c2f2a02b000070dba78429e9a11f87c6eb" +hash_len="`echo -n $hash | wc -c`" -guix authenticate sign \ - "$abs_top_srcdir/tests/signing-key.sec" \ - "$hash" > "$sig" +echo "sign $key_len:$key $hash_len:$hash" | guix authenticate > "$sig" test -f "$sig" +case "$(cat $sig)" in + "0 "*) ;; + *) echo "broken signature: $(cat $sig)" + exit 42;; +esac + +# Remove the leading "0". +sed -i "$sig" -e's/^0 //g' -hash2="`guix authenticate verify "$sig"`" -test "$hash2" = "$hash" +hash2="$(echo verify $(cat "$sig") | guix authenticate)" +test "$(echo $hash2 | cut -d : -f 2)" = "$hash" # Detect corrupt signatures. -if guix authenticate verify /dev/null -then false -else true -fi +code="$(echo "verify 5:wrong" | guix authenticate | cut -f1 -d ' ')" +test "$code" -ne 0 # Detect invalid signatures. # The signature has (payload (data ... (hash sha256 #...#))). We proceed by # modifying this hash. sed -i "$sig" \ -e's|#[A-Z0-9]\{64\}#|#0000000000000000000000000000000000000000000000000000000000000000#|g' -if guix authenticate verify "$sig" -then false -else true -fi - +code="$(echo "verify $(cat $sig)" | guix authenticate | cut -f1 -d ' ')" +test "$code" -ne 0 # Test for : make sure 'guix authenticate' produces # valid signatures when run in the C locale. @@ -63,9 +68,11 @@ hash="5eff0b55c9c5f5e87b4e34cd60a2d5654ca1eb78c7b3c67c3179fed1cff07b4c" LC_ALL=C export LC_ALL -guix authenticate sign "$abs_top_srcdir/tests/signing-key.sec" "$hash" \ - > "$sig" +echo "sign $key_len:$key $hash_len:$hash" | guix authenticate > "$sig" + +# Remove the leading "0". +sed -i "$sig" -e's/^0 //g' -guix authenticate verify "$sig" -hash2="`guix authenticate verify "$sig"`" -test "$hash2" = "$hash" +echo "verify $(cat $sig)" | guix authenticate +hash2="$(echo "verify $(cat $sig)" | guix authenticate | cut -f2 -d ' ')" +test "$(echo $hash2 | cut -d : -f 2)" = "$hash" diff --git a/tests/store.scm b/tests/store.scm index 8ff76e8f98..3a2a21a250 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -990,7 +990,7 @@ ;; Ensure 'import-paths' raises an exception. (guard (c ((store-protocol-error? c) - (and (not (zero? (store-protocol-error-status (pk 'C c)))) + (and (not (zero? (store-protocol-error-status c))) (string-contains (store-protocol-error-message c) "lacks a signature")))) (let* ((source (open-bytevector-input-port dump)) @@ -1030,9 +1030,9 @@ ;; Ensure 'import-paths' raises an exception. (guard (c ((store-protocol-error? c) - ;; XXX: The daemon-provided error message currently doesn't - ;; mention the reason of the failure. - (not (zero? (store-protocol-error-status c))))) + (and (not (zero? (store-protocol-error-status c))) + (string-contains (store-protocol-error-message c) + "unauthorized public key")))) (let* ((source (open-bytevector-input-port dump)) (imported (import-paths %store source))) (pk 'unauthorized-imported imported) -- cgit v1.2.3 From 7d516c17da50dfc8ce635a21c37533d1fe27b43b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 11 Sep 2020 14:35:07 +0200 Subject: authenticate: Cache the ACL and key pairs. In practice we're always using the same key pair, /etc/guix/signing-key.{pub,sec}. Keeping them in cache allows us to avoid redundant I/O and parsing when signing multiple store items in a row. * guix/scripts/authenticate.scm (load-key-pair): New procedure. (sign-with-key): Remove 'key-file' parameter and add 'public-key' and 'secret-key'. Adjust accordingly. (validate-signature): Add 'acl' parameter and pass it to 'authorized-key?'. (guix-authenticate)[call-with-reply]: New procedure. [with-reply]: New macro. Call 'current-acl' upfront and cache its result. Add 'key-pairs' as an argument to 'loop' and use it as a cache of key pairs. --- guix/scripts/authenticate.scm | 100 +++++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 35 deletions(-) (limited to 'guix') diff --git a/guix/scripts/authenticate.scm b/guix/scripts/authenticate.scm index dc73981092..0bac13edee 100644 --- a/guix/scripts/authenticate.scm +++ b/guix/scripts/authenticate.scm @@ -25,10 +25,12 @@ #:use-module (guix diagnostics) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) + #:use-module (srfi srfi-71) #:use-module (rnrs bytevectors) #:use-module (ice-9 binary-ports) #:use-module (ice-9 rdelim) #:use-module (ice-9 match) + #:use-module (ice-9 vlist) #:export (guix-authenticate)) ;;; Commentary: @@ -43,32 +45,40 @@ ;; Read a gcrypt sexp from a port and return it. (compose string->canonical-sexp read-string)) -(define (sign-with-key key-file sha256) - "Sign the hash SHA256 (a bytevector) with KEY-FILE, and return the signature -as a canonical sexp that includes both the hash and the actual signature." - (let* ((secret-key (call-with-input-file key-file read-canonical-sexp)) - (public-key (if (string-suffix? ".sec" key-file) - (call-with-input-file +(define (load-key-pair key-file) + "Load the key pair whose secret key lives at KEY-FILE. Return a pair of +canonical sexps representing those keys." + (catch 'system-error + (lambda () + (let* ((secret-key (call-with-input-file key-file read-canonical-sexp)) + (public-key (call-with-input-file (string-append (string-drop-right key-file 4) ".pub") - read-canonical-sexp) - (raise - (formatted-message - (G_ "cannot find public key for secret key '~a'~%") - key-file)))) - (data (bytevector->hash-data sha256 - #:key-type (key-type public-key))) - (signature (signature-sexp data secret-key public-key))) - signature)) - -(define (validate-signature signature) + read-canonical-sexp))) + (cons public-key secret-key))) + (lambda args + (let ((errno (system-error-errno args))) + (raise + (formatted-message + (G_ "failed to load key pair at '~a': ~a~%") + key-file (strerror errno))))))) + +(define (sign-with-key public-key secret-key sha256) + "Sign the hash SHA256 (a bytevector) with SECRET-KEY (a canonical sexp), and +return the signature as a canonical sexp that includes SHA256, PUBLIC-KEY, and +the actual signature." + (let ((data (bytevector->hash-data sha256 + #:key-type (key-type public-key)))) + (signature-sexp data secret-key public-key))) + +(define (validate-signature signature acl) "Validate SIGNATURE, a canonical sexp. Check whether its public key is -authorized, verify the signature, and return the signed data (a bytevector) -upon success." +authorized in ACL, verify the signature, and return the signed data (a +bytevector) upon success." (let* ((subject (signature-subject signature)) (data (signature-signed-data signature))) (if (and data subject) - (if (authorized-key? subject) + (if (authorized-key? subject acl) (if (valid-signature? signature) (hash-data->bytevector data) ; success (raise @@ -145,6 +155,19 @@ by colon, followed by the given number of characters." (put-bytevector (current-output-port) bv) (force-output (current-output-port)))) + (define (call-with-reply thunk) + ;; Send a reply for the result of THUNK or for any exception raised during + ;; its execution. + (guard (c ((formatted-message? c) + (send-reply (reply-code command-failed) + (apply format #f + (G_ (formatted-message-string c)) + (formatted-message-arguments c))))) + (send-reply (reply-code success) (thunk)))) + + (define-syntax-rule (with-reply exp ...) + (call-with-reply (lambda () exp ...))) + ;; Signature sexps written to stdout may contain binary data, so force ;; ISO-8859-1 encoding so that things are not mangled. See ;; for details. @@ -162,31 +185,38 @@ Sign data or verify signatures. This tool is meant to be used internally by (("--version") (show-version-and-exit "guix authenticate")) (() - (let loop () - (guard (c ((formatted-message? c) - (send-reply (reply-code command-failed) - (apply format #f - (G_ (formatted-message-string c)) - (formatted-message-arguments c))))) + (let ((acl (current-acl))) + (let loop ((key-pairs vlist-null)) ;; Read a request on standard input and reply. (match (read-command (current-input-port)) (("sign" signing-key (= base16-string->bytevector hash)) - (let ((signature (sign-with-key signing-key hash))) - (send-reply (reply-code success) - (canonical-sexp->string signature)))) + (let* ((key-pairs keys + (match (vhash-assoc signing-key key-pairs) + ((_ . keys) + (values key-pairs keys)) + (#f + (let ((keys (load-key-pair signing-key))) + (values (vhash-cons signing-key keys + key-pairs) + keys)))))) + (with-reply (canonical-sexp->string + (match keys + ((public . secret) + (sign-with-key public secret hash))))) + (loop key-pairs))) (("verify" signature) - (send-reply (reply-code success) - (bytevector->base16-string + (with-reply (bytevector->base16-string (validate-signature - (string->canonical-sexp signature))))) + (string->canonical-sexp signature) + acl))) + (loop key-pairs)) (() (exit 0)) (commands (warning (G_ "~s: invalid command; ignoring~%") commands) (send-reply (reply-code command-not-found) - "invalid command")))) - - (loop))) + "invalid command") + (loop key-pairs)))))) (_ (leave (G_ "wrong arguments~%")))))) -- cgit v1.2.3 From 846403ef62066b23c87b6f8a99fd8a551001fc52 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 14 Sep 2020 15:16:59 +0200 Subject: ui: 'show-what-to-build' displays download estimate more prominently. * guix/ui.scm (show-what-to-build): When VERBOSITY is 1, add a newline before the "would/will be downloaded" line, and wrap that message in 'highlight'. --- guix/ui.scm | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'guix') diff --git a/guix/ui.scm b/guix/ui.scm index 115d9801b2..ecaf975c1f 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -1075,16 +1075,19 @@ summary, and level 0 shows nothing." (null? hook) (map colorized-store-item hook))) ((= verbosity 1) ;; Display the bare minimum; don't mention grafts and hooks. + (unless (null? build) + (newline (current-error-port))) (if display-download-size? (format (current-error-port) ;; TRANSLATORS: "MB" is for "megabyte"; it should be ;; translated to the corresponding abbreviation. - (G_ "~:[~,1h MB would be downloaded~%~;~]") + (highlight (G_ "~:[~,1h MB would be downloaded~%~;~]")) (null? download) download-size) (format (current-error-port) - (N_ "~:[~h item would be downloaded~%~;~]" - "~:[~h items would be downloaded~%~;~]" - (length download)) + (highlight + (N_ "~:[~h item would be downloaded~%~;~]" + "~:[~h items would be downloaded~%~;~]" + (length download))) (null? download) (length download)))))) (begin @@ -1123,16 +1126,19 @@ summary, and level 0 shows nothing." (null? hook) (map colorized-store-item hook))) ((= verbosity 1) ;; Display the bare minimum; don't mention grafts and hooks. + (unless (null? build) + (newline (current-error-port))) (if display-download-size? (format (current-error-port) ;; TRANSLATORS: "MB" is for "megabyte"; it should be ;; translated to the corresponding abbreviation. - (G_ "~:[~,1h MB will be downloaded~%~;~]") + (highlight (G_ "~:[~,1h MB will be downloaded~%~;~]")) (null? download) download-size) (format (current-error-port) - (N_ "~:[~h item will be downloaded~%~;~]" - "~:[~h items will be downloaded~%~;~]" - (length download)) + (highlight + (N_ "~:[~h item will be downloaded~%~;~]" + "~:[~h items will be downloaded~%~;~]" + (length download))) (null? download) (length download))))))) (check-available-space installed-size) -- cgit v1.2.3 From b911d6547444b5f8d17b224bafa5ee1b5aafaff5 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 15 Sep 2020 14:24:05 +0200 Subject: authenticate: Encode strings as ISO-8859-1. Fixes . * guix/scripts/authenticate.scm (read-command): Decode strings as ISO-8859-1, not UTF-8. (guix-authenticate)[send-reply]: Encode strings as ISO-8859-1, not UTF-8. * tests/guix-authenticate.sh: Add test. --- guix/scripts/authenticate.scm | 8 +++++--- tests/guix-authenticate.sh | 9 +++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'guix') diff --git a/guix/scripts/authenticate.scm b/guix/scripts/authenticate.scm index 0bac13edee..45f62f6ebc 100644 --- a/guix/scripts/authenticate.scm +++ b/guix/scripts/authenticate.scm @@ -31,6 +31,7 @@ #:use-module (ice-9 rdelim) #:use-module (ice-9 match) #:use-module (ice-9 vlist) + #:use-module (ice-9 iconv) #:export (guix-authenticate)) ;;; Commentary: @@ -122,8 +123,9 @@ by colon, followed by the given number of characters." (reverse result)) (else (let* ((len (string->number (read-delimited ":" port))) - (str (utf8->string - (get-bytevector-n port len)))) + (str (bytevector->string + (get-bytevector-n port len) + "ISO-8859-1" 'error))) (loop (cons str result)))))))))) (define-syntax define-enumerate-type ;TODO: factorize @@ -150,7 +152,7 @@ by colon, followed by the given number of characters." (define (send-reply code str) ;; Send CODE and STR as a reply to our client. - (let ((bv (string->utf8 str))) + (let ((bv (string->bytevector str "ISO-8859-1" 'error))) (format #t "~a ~a:" code (bytevector-length bv)) (put-bytevector (current-output-port) bv) (force-output (current-output-port)))) diff --git a/tests/guix-authenticate.sh b/tests/guix-authenticate.sh index f3b36ee41d..3a05b232c1 100644 --- a/tests/guix-authenticate.sh +++ b/tests/guix-authenticate.sh @@ -61,6 +61,15 @@ sed -i "$sig" \ code="$(echo "verify $(cat $sig)" | guix authenticate | cut -f1 -d ' ')" test "$code" -ne 0 +# Make sure byte strings are correctly encoded. The hash string below is +# "café" repeated 8 times. Libgcrypt would normally choose to write it as a +# string rather than a hex sequence. We want that string to be Latin-1 +# encoded independently of the current locale: . +hash="636166e9636166e9636166e9636166e9636166e9636166e9636166e9636166e9" +latin1_cafe="caf$(printf '\351')" +echo "sign 21:tests/signing-key.sec 64:$hash" | guix authenticate \ + | LC_ALL=C grep "hash sha256 \"$latin1_cafe" + # Test for : make sure 'guix authenticate' produces # valid signatures when run in the C locale. hash="5eff0b55c9c5f5e87b4e34cd60a2d5654ca1eb78c7b3c67c3179fed1cff07b4c" -- cgit v1.2.3 From 9d1af83e0b87cde6dd914eba8e2eeb84ceda3bc0 Mon Sep 17 00:00:00 2001 From: Tobias Geerinckx-Rice Date: Mon, 14 Sep 2020 09:18:39 +0200 Subject: import: cpan: Export cpan-release-module. * guix/import/cpan.scm: Fix typo. --- guix/import/cpan.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/import/cpan.scm b/guix/import/cpan.scm index fd940415a2..514417f781 100644 --- a/guix/import/cpan.scm +++ b/guix/import/cpan.scm @@ -49,7 +49,7 @@ cpan-release-license cpan-release-author cpan-release-version - cpan-release-modle + cpan-release-module cpan-release-distribution cpan-release-download-url cpan-release-abstract -- cgit v1.2.3 From d7f7ed39be3be926b3c46c0ea15d416c593ef61f Mon Sep 17 00:00:00 2001 From: Konrad Hinsen Date: Fri, 11 Sep 2020 13:13:26 +0200 Subject: repl: Look for script files in (getcwd). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes . * guix/scripts/repl.scm (guix-repl): Replace "." by (getcwd) * tests/guix-repl.sh: Add test. Co-authored-by: Ludovic Courtès --- guix/scripts/repl.scm | 5 ++++- tests/guix-repl.sh | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/scripts/repl.scm b/guix/scripts/repl.scm index 3c79e89f8d..7d4e474e92 100644 --- a/guix/scripts/repl.scm +++ b/guix/scripts/repl.scm @@ -178,7 +178,10 @@ call THUNK." (lambda () (set-program-arguments script) (set-user-module) - (load-in-vicinity "." (car script))))) + + ;; When passed a relative file name, 'load-in-vicinity' searches the + ;; file in %LOAD-PATH. Thus, pass (getcwd) instead of ".". + (load-in-vicinity (getcwd) (car script))))) (when (null? script) ;; Start REPL diff --git a/tests/guix-repl.sh b/tests/guix-repl.sh index e1c2b8241f..d4ebb5f6c6 100644 --- a/tests/guix-repl.sh +++ b/tests/guix-repl.sh @@ -45,6 +45,10 @@ EOF test "`guix repl "$tmpfile"`" = "coreutils" +# Make sure that the file can also be loaded when passed as a relative file +# name. +(cd "$(dirname "$tmpfile")"; test "$(guix repl "$(basename "$tmpfile")")" = "coreutils") + cat > "$module_dir/foo.scm"< Date: Sat, 19 Sep 2020 16:26:44 +0200 Subject: describe: Save the original value of (program-arguments). Fixes . Reported by pkill9 . This ensures that 'guix repl -s SCRIPT' give SCRIPT the right value of (current-profile), which in turn ensures that (%package-module-path) is initialized with the right set of channels. * guix/describe.scm (initial-program-arguments): New variable. (current-profile): Use it. * guix/scripts/repl.scm (guix-repl): Call 'current-profile' before 'set-program-arguments'. --- guix/describe.scm | 10 ++++++++-- guix/scripts/repl.scm | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'guix') diff --git a/guix/describe.scm b/guix/describe.scm index 6b9b219113..05bf99eb58 100644 --- a/guix/describe.scm +++ b/guix/describe.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2018, 2019 Ludovic Courtès +;;; Copyright © 2018, 2019, 2020 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -43,11 +43,17 @@ ;;; ;;; Code: +(define initial-program-arguments + ;; Save the initial program arguments. This allows us to see the "real" + ;; 'guix' program, even if 'guix repl -s' calls 'set-program-arguments' + ;; later on. + (program-arguments)) + (define current-profile (mlambda () "Return the profile (created by 'guix pull') the calling process lives in, or #f if this is not applicable." - (match (command-line) + (match initial-program-arguments ((program . _) (and (string-suffix? "/bin/guix" program) ;; Note: We want to do _lexical dot-dot resolution_. Using ".." diff --git a/guix/scripts/repl.scm b/guix/scripts/repl.scm index 7d4e474e92..9f20803efc 100644 --- a/guix/scripts/repl.scm +++ b/guix/scripts/repl.scm @@ -27,6 +27,7 @@ #:use-module (srfi srfi-37) #:use-module (ice-9 match) #:use-module (rnrs bytevectors) + #:autoload (guix describe) (current-profile) #:autoload (system repl repl) (start-repl) #:autoload (system repl server) (make-tcp-server-socket make-unix-domain-server-socket) @@ -176,6 +177,13 @@ call THUNK." ;; Run script (save-module-excursion (lambda () + ;; Invoke 'current-profile' so that it memoizes the correct value + ;; based on (program-arguments), before we call + ;; 'set-program-arguments'. This in turn ensures that + ;; (%package-module-path) will contain entries for the channels + ;; available in the current profile. + (current-profile) + (set-program-arguments script) (set-user-module) -- cgit v1.2.3 From 9b65281de51bcb56714509524f5ae0731c9b96d0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 14 Sep 2020 22:49:06 +0200 Subject: environment: '--link-profile' uses ~/.guix-profile for environment variables. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this patch, we had: $ guix environment -CP --ad-hoc coreutils [env]$ echo $PATH /gnu/store/…-profile/bin [env]$ echo $GUIX_ENVIRONMENT /gnu/store/…-profile After this patch: $ guix environment -CP --ad-hoc coreutils [env]$ echo $PATH /home/ludo/.guix-profile/bin [env]$ echo $GUIX_ENVIRONMENT /home/ludo/.guix-profile * guix/scripts/environment.scm (launch-environment/container): When LINK-PROFILE? is true, pass ~/.guix-profile as the second argument to 'launch-environment'. * tests/guix-environment-container.sh: Adjust test accordingly. * doc/guix.texi (Invoking guix environment): Update accordingly. --- doc/guix.texi | 5 +++-- guix/scripts/environment.scm | 6 +++++- tests/guix-environment-container.sh | 10 +++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) (limited to 'guix') diff --git a/doc/guix.texi b/doc/guix.texi index f7e2204b53..949551a163 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -5420,8 +5420,9 @@ device. @item --link-profile @itemx -P For containers, link the environment profile to @file{~/.guix-profile} -within the container. This is equivalent to running the command -@samp{ln -s $GUIX_ENVIRONMENT ~/.guix-profile} within the container. +within the container and set @code{GUIX_ENVIRONMENT} to that. +This is equivalent to making @file{~/.guix-profile} a symlink to the +actual profile within the container. Linking will fail and abort the environment if the directory already exists, which will certainly be the case if @command{guix environment} was invoked in the user's home directory. diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index ad50281eb2..e2e481dd02 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -564,7 +564,11 @@ WHILE-LIST." (primitive-exit/status ;; A container's environment is already purified, so no need to ;; request it be purified again. - (launch-environment command profile manifest #:pure? #f))) + (launch-environment command + (if link-profile? + (string-append home-dir "/.guix-profile") + profile) + manifest #:pure? #f))) #:guest-uid uid #:guest-gid gid #:namespaces (if network? diff --git a/tests/guix-environment-container.sh b/tests/guix-environment-container.sh index 45264d4978..040f32cce9 100644 --- a/tests/guix-environment-container.sh +++ b/tests/guix-environment-container.sh @@ -127,11 +127,15 @@ grep -e "$NIX_STORE_DIR/.*-bash" $tmpdir/mounts # bootstrap bash rm $tmpdir/mounts -# Make sure 'GUIX_ENVIRONMENT' is linked to '~/.guix-profile' when requested +# Make sure 'GUIX_ENVIRONMENT' is set to '~/.guix-profile' when requested # within a container. ( - linktest='(exit (string=? (getenv "GUIX_ENVIRONMENT") -(readlink (string-append (getenv "HOME") "/.guix-profile"))))' + linktest=' +(exit (and (string=? (getenv "GUIX_ENVIRONMENT") + (string-append (getenv "HOME") "/.guix-profile")) + (string-prefix? "'"$NIX_STORE_DIR"'" + (readlink (string-append (getenv "HOME") + "/.guix-profile")))))' cd "$tmpdir" \ && guix environment --bootstrap --container --link-profile \ -- cgit v1.2.3 From 620681534a2a6f0505cb7a3e1b66e6c138b28769 Mon Sep 17 00:00:00 2001 From: zimoun Date: Wed, 16 Sep 2020 00:17:29 +0200 Subject: guix: scripts: build: Mention 'PACKAGE' in '--with-source' option. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * guix/scripts/build.scm (show-transformation-options-help): Mention 'PACKAGE' in '--with-source' option. Co-authored-by: Ludovic Courtès --- guix/scripts/build.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 25418661b9..38e0516c95 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -427,7 +427,7 @@ a checkout of the Git repository at the given URL." (define (show-transformation-options-help) (display (G_ " - --with-source=SOURCE + --with-source=[PACKAGE=]SOURCE use SOURCE when building the corresponding package")) (display (G_ " --with-input=PACKAGE=REPLACEMENT -- cgit v1.2.3 From bd16cc2902800932f58a34647e224734aa3647cd Mon Sep 17 00:00:00 2001 From: Tobias Geerinckx-Rice Date: Tue, 22 Sep 2020 15:24:10 +0200 Subject: import: Fix docstring typoes. * guix/import/cabal.scm (cabal-flags->alist): Fix typo in docstring. * guix/import/stackage.scm (lts-info-ghc-version): Likewise. * guix/scripts/import/hackage.scm (show-help): Likewise. --- guix/import/cabal.scm | 2 +- guix/import/stackage.scm | 4 ++-- guix/scripts/import/hackage.scm | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'guix') diff --git a/guix/import/cabal.scm b/guix/import/cabal.scm index 7dfe771e41..da00019297 100644 --- a/guix/import/cabal.scm +++ b/guix/import/cabal.scm @@ -718,7 +718,7 @@ If #f use the function 'port-filename' to obtain it." (dependencies cabal-custom-setup-dependencies)) ; list of (define (cabal-flags->alist flag-list) - "Retrun an alist associating the flag name to its default value from a + "Return an alist associating the flag name to its default value from a list of objects." (map (lambda (flag) (cons (cabal-flag-name flag) (cabal-flag-default flag))) flag-list)) diff --git a/guix/import/stackage.scm b/guix/import/stackage.scm index e04073d193..ee12108815 100644 --- a/guix/import/stackage.scm +++ b/guix/import/stackage.scm @@ -42,12 +42,12 @@ (define %stackage-url "http://www.stackage.org") (define (lts-info-ghc-version lts-info) - "Retruns the version of the GHC compiler contained in LTS-INFO." + "Returns the version of the GHC compiler contained in LTS-INFO." (and=> (assoc-ref lts-info "snapshot") (cut assoc-ref <> "ghc"))) (define (lts-info-packages lts-info) - "Retruns the alist of packages contained in LTS-INFO." + "Returns the alist of packages contained in LTS-INFO." (or (assoc-ref lts-info "packages") '())) (define (leave-with-message fmt . args) diff --git a/guix/scripts/import/hackage.scm b/guix/scripts/import/hackage.scm index 710e786a79..906dca24b1 100644 --- a/guix/scripts/import/hackage.scm +++ b/guix/scripts/import/hackage.scm @@ -49,7 +49,7 @@ Import and convert the Hackage package for PACKAGE-NAME. If PACKAGE-NAME includes a suffix constituted by a at-sign followed by a numerical version (as used with Guix packages), then a definition for the specified version of the -package will be generated. If no version suffix is pecified, then the +package will be generated. If no version suffix is specified, then the generated package definition will correspond to the latest available version.\n")) (display (G_ " -- cgit v1.2.3 From 9c4aaa630d97f9f29ca1b732fb265bd583e83e02 Mon Sep 17 00:00:00 2001 From: André Batista Date: Thu, 24 Sep 2020 21:29:49 -0300 Subject: licenses: Add Apple Public Source License 2.0. * guix/licenses.scm (apsl2): New variable. Signed-off-by: Mathieu Othacehe --- guix/licenses.scm | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'guix') diff --git a/guix/licenses.scm b/guix/licenses.scm index bf72a33c92..5038f75638 100644 --- a/guix/licenses.scm +++ b/guix/licenses.scm @@ -14,6 +14,7 @@ ;;; Copyright © 2017 Marius Bakke ;;; Copyright © 2017 Arun Isaac ;;; Copyright © 2017 Rutger Helling +;;; Copyright © 2020 André Batista ;;; ;;; This file is part of GNU Guix. ;;; @@ -34,6 +35,7 @@ #:use-module (srfi srfi-9) #:export (license? license-name license-uri license-comment agpl1 agpl3 agpl3+ + apsl2 asl1.1 asl2.0 boost1.0 bsd-2 bsd-3 bsd-4 @@ -132,6 +134,11 @@ "https://gnu.org/licenses/agpl.html" "https://gnu.org/licenses/why-affero-gpl.html")) +(define apsl2 + (license "APSL 2.0" + "https://directory.fsf.org/wiki/License:APSL-2.0" + "https://www.gnu.org/licenses/license-list.html#apsl2")) + (define asl1.1 (license "ASL 1.1" "http://directory.fsf.org/wiki/License:Apache1.1" -- cgit v1.2.3 From 4eeaae7994c6fb82e005acf290a3b81cba7bd871 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 24 Sep 2020 18:41:09 +0200 Subject: guix package: Simplify 'package->manifest-entry*'. * guix/scripts/package.scm (package->manifest-entry*): Rewrite in terms of 'manifest-entry-with-provenance'. --- guix/scripts/package.scm | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'guix') diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index 4eb968a49b..7e7c37eac4 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -585,14 +585,8 @@ upgrading, #f otherwise." (define (package->manifest-entry* package output) "Like 'package->manifest-entry', but attach PACKAGE provenance meta-data to the resulting manifest entry." - (define (provenance-properties package) - (match (package-provenance package) - (#f '()) - (sexp `((provenance ,@sexp))))) - - (package->manifest-entry package output - #:properties (provenance-properties package))) - + (manifest-entry-with-provenance + (package->manifest-entry package output))) (define (options->installable opts manifest transaction) "Given MANIFEST, the current manifest, and OPTS, the result of 'args-fold', -- cgit v1.2.3 From 795065533d3326e02326509d93d3bab7105d97a3 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 25 Sep 2020 19:02:13 +0200 Subject: gnu: Replace uses of 'guile3.0-gnutls' by 'gnutls'. * gnu/packages/package-management.scm (guix)[propagated-inputs]: Use GNUTLS instead of GUILE3.0-GNUTLS. (guix-daemon)[inputs]: Likewise. * guix/self.scm (specification->package): Likewise. --- gnu/packages/package-management.scm | 4 ++-- guix/self.scm | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'guix') diff --git a/gnu/packages/package-management.scm b/gnu/packages/package-management.scm index ec87226197..99f78f2ac8 100644 --- a/gnu/packages/package-management.scm +++ b/gnu/packages/package-management.scm @@ -384,7 +384,7 @@ $(prefix)/etc/init.d\n"))) ("glibc-utf8-locales" ,glibc-utf8-locales))) (propagated-inputs - `(("gnutls" ,(if (%current-target-system) gnutls-3.6.14 guile3.0-gnutls)) + `(("gnutls" ,(if (%current-target-system) gnutls-3.6.14 gnutls)) ("guile-gcrypt" ,guile-gcrypt) ("guile-json" ,guile-json-4) ("guile-sqlite3" ,guile-sqlite3) @@ -418,7 +418,7 @@ the Nix package manager.") (fold alist-delete (package-native-inputs guix) '("po4a" "graphviz" "help2man"))) (inputs - `(("gnutls" ,guile3.0-gnutls) + `(("gnutls" ,gnutls) ("guile-git" ,guile-git) ("guile-json" ,guile-json-3) ("guile-gcrypt" ,guile-gcrypt) diff --git a/guix/self.scm b/guix/self.scm index 02ef982c7c..5eb80f42fe 100644 --- a/guix/self.scm +++ b/guix/self.scm @@ -56,7 +56,7 @@ ("guile-zlib" (ref '(gnu packages guile) 'guile-zlib)) ("guile-lzlib" (ref '(gnu packages guile) 'guile-lzlib)) ("guile-gcrypt" (ref '(gnu packages gnupg) 'guile-gcrypt)) - ("gnutls" (ref '(gnu packages tls) 'guile3.0-gnutls)) + ("gnutls" (ref '(gnu packages tls) 'gnutls)) ("gzip" (ref '(gnu packages compression) 'gzip)) ("bzip2" (ref '(gnu packages compression) 'bzip2)) ("xz" (ref '(gnu packages compression) 'xz)) -- cgit v1.2.3 From f458cfbcc54ed87b1a87dd9e150ea276f17eab74 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 23 Sep 2020 22:29:17 +0200 Subject: guix build: Add '--without-tests'. * guix/scripts/build.scm (transform-package-tests): New procedure. (%transformations, %transformation-options) show-transformation-options-help): Add it. * tests/scripts-build.scm ("options->transformation, without-tests"): New test. * doc/guix.texi (Package Transformation Options): Document it. --- doc/guix.texi | 22 ++++++++++++++++++++++ guix/scripts/build.scm | 31 ++++++++++++++++++++++++++++--- tests/scripts-build.scm | 14 ++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) (limited to 'guix') diff --git a/doc/guix.texi b/doc/guix.texi index 538e7cceab..8384eee6c3 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -9271,6 +9271,28 @@ guix build --with-branch=guile-sqlite3=master cuirass This is similar to @option{--with-branch}, except that it builds from @var{commit} rather than the tip of a branch. @var{commit} must be a valid Git commit SHA1 identifier or a tag. + +@cindex test suite, skipping +@item --without-tests=@var{package} +Build @var{package} without running its tests. This can be useful in +situations where you want to skip the lengthy test suite of a +intermediate package, or if a package's test suite fails in a +non-deterministic fashion. It should be used with care because running +the test suite is a good way to ensure a package is working as intended. + +Turning off tests leads to a different store item. Consequently, when +using this option, anything that depends on @var{package} must be +rebuilt, as in this example: + +@example +guix install --without-tests=python python-notebook +@end example + +The command above installs @code{python-notebook} on top of +@code{python} built without running its test suite. To do so, it also +rebuilds everything that depends on @code{python}, including +@code{python-notebook} itself. + @end table @node Additional Build Options diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 38e0516c95..f238e9b876 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -393,6 +393,25 @@ a checkout of the Git repository at the given URL." (rewrite obj) obj))) +(define (transform-package-tests specs) + "Return a procedure that, when passed a package, sets #:tests? #f in its +'arguments' field." + (define (package-without-tests p) + (package/inherit p + (arguments + (substitute-keyword-arguments (package-arguments p) + ((#:tests? _ #f) #f))))) + + (define rewrite + (package-input-rewriting/spec (map (lambda (spec) + (cons spec package-without-tests)) + specs))) + + (lambda (store obj) + (if (package? obj) + (rewrite obj) + obj))) + (define %transformations ;; Transformations that can be applied to things to build. The car is the ;; key used in the option alist, and the cdr is the transformation @@ -403,7 +422,8 @@ a checkout of the Git repository at the given URL." (with-graft . ,transform-package-inputs/graft) (with-branch . ,transform-package-source-branch) (with-commit . ,transform-package-source-commit) - (with-git-url . ,transform-package-source-git-url))) + (with-git-url . ,transform-package-source-git-url) + (without-tests . ,transform-package-tests))) (define %transformation-options ;; The command-line interface to the above transformations. @@ -423,7 +443,9 @@ a checkout of the Git repository at the given URL." (option '("with-commit") #t #f (parser 'with-commit)) (option '("with-git-url") #t #f - (parser 'with-git-url))))) + (parser 'with-git-url)) + (option '("without-tests") #t #f + (parser 'without-tests))))) (define (show-transformation-options-help) (display (G_ " @@ -443,7 +465,10 @@ a checkout of the Git repository at the given URL." build PACKAGE from COMMIT")) (display (G_ " --with-git-url=PACKAGE=URL - build PACKAGE from the repository at URL"))) + build PACKAGE from the repository at URL")) + (display (G_ " + --without-tests=PACKAGE + build PACKAGE without running its tests"))) (define (options->transformation opts) diff --git a/tests/scripts-build.scm b/tests/scripts-build.scm index 32876e956a..12114fc8f5 100644 --- a/tests/scripts-build.scm +++ b/tests/scripts-build.scm @@ -264,5 +264,19 @@ ((("x" dep3)) (map package-source (list dep1 dep3)))))))))))) +(test-assert "options->transformation, without-tests" + (let* ((dep (dummy-package "dep")) + (p (dummy-package "foo" + (inputs `(("dep" ,dep))))) + (t (options->transformation '((without-tests . "dep") + (without-tests . "tar"))))) + (with-store store + (let ((new (t store p))) + (match (bag-direct-inputs (package->bag new)) + ((("dep" dep) ("tar" tar) _ ...) + ;; TODO: Check whether TAR has #:tests? #f when transformations + ;; apply to implicit inputs. + (equal? (package-arguments dep) + '(#:tests? #f)))))))) (test-end) -- cgit v1.2.3 From ff39361c80dfc67a9afe35f315a774140d8cf99b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 21 Sep 2020 17:44:29 +0200 Subject: packages: 'package-mapping' can recurse on implicit inputs. * guix/packages.scm (build-system-with-package-mapping): New procedure. (package-mapping): Add #:deep? and honor it. * tests/packages.scm ("package-mapping"): Compare the direct inputs of the bag of P0 and that of P1. ("package-mapping, deep"): New test. --- doc/guix.texi | 5 +++-- guix/packages.scm | 65 +++++++++++++++++++++++++++++++++++++++++------------- tests/packages.scm | 36 +++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 18 deletions(-) (limited to 'guix') diff --git a/doc/guix.texi b/doc/guix.texi index 8384eee6c3..054449d8d6 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -6296,10 +6296,11 @@ A more generic procedure to rewrite a package dependency graph is @code{package-mapping}: it supports arbitrary changes to nodes in the graph. -@deffn {Scheme Procedure} package-mapping @var{proc} [@var{cut?}] +@deffn {Scheme Procedure} package-mapping @var{proc} [@var{cut?}] [#:deep? #f] Return a procedure that, given a package, applies @var{proc} to all the packages depended on and returns the resulting package. The procedure stops recursion -when @var{cut?} returns true for a given package. +when @var{cut?} returns true for a given package. When @var{deep?} is true, @var{proc} is +applied to implicit inputs as well. @end deffn @menu diff --git a/guix/packages.scm b/guix/packages.scm index 6598bd3149..171fd048ef 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -968,10 +968,31 @@ packages they depend on, recursively." (vhash-consq package #t visited) (fold set-insert closure dependencies)))))))) -(define* (package-mapping proc #:optional (cut? (const #f))) +(define (build-system-with-package-mapping bs rewrite) + "Return a variant of BS, a build system, that rewrites a bag's inputs by +passing them through REWRITE, a procedure that takes an input tuplet and +returns a \"rewritten\" input tuplet." + (define lower + (build-system-lower bs)) + + (define (lower* . args) + (let ((lowered (apply lower args))) + (bag + (inherit lowered) + (build-inputs (map rewrite (bag-build-inputs lowered))) + (host-inputs (map rewrite (bag-host-inputs lowered))) + (target-inputs (map rewrite (bag-target-inputs lowered)))))) + + (build-system + (inherit bs) + (lower lower*))) + +(define* (package-mapping proc #:optional (cut? (const #f)) + #:key deep?) "Return a procedure that, given a package, applies PROC to all the packages depended on and returns the resulting package. The procedure stops recursion -when CUT? returns true for a given package." +when CUT? returns true for a given package. When DEEP? is true, PROC is +applied to implicit inputs as well." (define (rewrite input) (match input ((label (? package? package) outputs ...) @@ -980,21 +1001,35 @@ when CUT? returns true for a given package." (_ input))) + (define mapping-property + ;; Property indicating whether the package has already been processed. + (gensym " package-mapping-done")) + (define replace (mlambdaq (p) - ;; Return a variant of P with PROC applied to P and its explicit - ;; dependencies, recursively. Memoize the transformations. Failing to - ;; do that, we would build a huge object graph with lots of duplicates, - ;; which in turns prevents us from benefiting from memoization in - ;; 'package-derivation'. - (let ((p (proc p))) - (package - (inherit p) - (location (package-location p)) - (inputs (map rewrite (package-inputs p))) - (native-inputs (map rewrite (package-native-inputs p))) - (propagated-inputs (map rewrite (package-propagated-inputs p))) - (replacement (and=> (package-replacement p) proc)))))) + ;; If P is the result of a previous call, return it. + (if (assq-ref (package-properties p) mapping-property) + p + + ;; Return a variant of P with PROC applied to P and its explicit + ;; dependencies, recursively. Memoize the transformations. Failing + ;; to do that, we would build a huge object graph with lots of + ;; duplicates, which in turns prevents us from benefiting from + ;; memoization in 'package-derivation'. + (let ((p (proc p))) + (package + (inherit p) + (location (package-location p)) + (build-system (if deep? + (build-system-with-package-mapping + (package-build-system p) rewrite) + (package-build-system p))) + (inputs (map rewrite (package-inputs p))) + (native-inputs (map rewrite (package-native-inputs p))) + (propagated-inputs (map rewrite (package-propagated-inputs p))) + (replacement (and=> (package-replacement p) proc)) + (properties `((,mapping-property . #t) + ,@(package-properties p)))))))) replace) diff --git a/tests/packages.scm b/tests/packages.scm index cbd0503733..f33332a461 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -1172,15 +1172,24 @@ (let* ((dep (dummy-package "chbouib" (native-inputs `(("x" ,grep))))) (p0 (dummy-package "example" + (source 77) (inputs `(("foo" ,coreutils) ("bar" ,grep) ("baz" ,dep))))) (transform (lambda (p) (package (inherit p) (source 42)))) (rewrite (package-mapping transform)) - (p1 (rewrite p0))) + (p1 (rewrite p0)) + (bag0 (package->bag p0)) + (bag1 (package->bag p1))) (and (eq? p1 (rewrite p0)) (eqv? 42 (package-source p1)) + + ;; Implicit inputs should be left unchanged (skip "source", "foo", + ;; "bar", and "baz" in this comparison). + (equal? (drop (bag-direct-inputs bag0) 4) + (drop (bag-direct-inputs bag1) 4)) + (match (package-inputs p1) ((("foo" dep1) ("bar" dep2) ("baz" dep3)) (and (eq? dep1 (rewrite coreutils)) ;memoization @@ -1194,6 +1203,31 @@ (and (eq? dep (rewrite grep)) (package-source dep)))))))))) +(test-equal "package-mapping, deep" + '(42) + (let* ((p0 (dummy-package "example" + (inputs `(("foo" ,coreutils) + ("bar" ,grep))))) + (transform (lambda (p) + (package (inherit p) (source 42)))) + (rewrite (package-mapping transform #:deep? #t)) + (p1 (rewrite p0)) + (bag (package->bag p1))) + (and (eq? p1 (rewrite p0)) + (match (bag-direct-inputs bag) + ((("source" 42) ("foo" dep1) ("bar" dep2) rest ..1) + (and (eq? dep1 (rewrite coreutils)) ;memoization + (eq? dep2 (rewrite grep)) + (= 42 (package-source dep1)) + (= 42 (package-source dep2)) + + ;; Check that implicit inputs of P0 also got rewritten. + (delete-duplicates + (map (match-lambda + ((_ package . _) + (package-source package))) + rest)))))))) + (test-assert "package-input-rewriting" (let* ((dep (dummy-package "chbouib" (native-inputs `(("x" ,grep))))) -- cgit v1.2.3 From 2bf6f962b91123b0474c0f7123cd17efe7f09a66 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 23 Sep 2020 10:29:09 +0200 Subject: packages: 'package-input-rewriting/spec' can rewrite implicit dependencies. With this change, '--with-input', '--with-graft', etc. also apply to implicit dependencies. Thus, it's now possible to do: guix build python-itsdangerous --with-input=python-wrapper=python@2 or: guix build hello --with-graft=glibc=glibc@2.29 Additionally, before, implicit inputs were not rewritten, which could lead to duplicates in the output of 'bag-transitive-inputs' (packages that are not 'eq?' but lead to the same derivation). This in turn would lead to unnecessary rebuilds when using '--with-input' & co. This change fixes it by ensuring even implicit inputs are rewritten. Fixes . * guix/packages.scm (package-input-rewriting/spec): Add #:deep? defaulting to #true, and pass it to 'package-mapping'. [replacement-property]: New variable. [rewrite]: Check that property and set it on the result of PROC. [cut?]: New procedure. * tests/packages.scm ("package-input-rewriting/spec"): Ensure implicit inputs were unchanged. ("package-input-rewriting/spec, partial match"): Pass #:deep? #f. ("package-input-rewriting/spec, deep") ("package-input-rewriting/spec, no duplicates"): New tests. (package/inherit): Move before use. * tests/guix-build.sh: Add tests. * tests/scripts-build.scm ("options->transformation, with-graft"): Compare dependencies by package name or derivation file name. * doc/guix.texi (Defining Packages): Adjust accordingly. --- doc/guix.texi | 13 +++++----- guix/packages.scm | 55 ++++++++++++++++++++++++++--------------- tests/guix-build.sh | 11 +++++++++ tests/packages.scm | 66 ++++++++++++++++++++++++++++++++++++++++++++++--- tests/scripts-build.scm | 12 ++++++--- 5 files changed, 125 insertions(+), 32 deletions(-) (limited to 'guix') diff --git a/doc/guix.texi b/doc/guix.texi index 054449d8d6..e72e1ec130 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -6272,12 +6272,13 @@ This is exactly what the @option{--with-input} command-line option does The following variant of @code{package-input-rewriting} can match packages to be replaced by name rather than by identity. -@deffn {Scheme Procedure} package-input-rewriting/spec @var{replacements} -Return a procedure that, given a package, applies the given @var{replacements} to -all the package graph (excluding implicit inputs). @var{replacements} is a list of -spec/procedures pair; each spec is a package specification such as @code{"gcc"} or -@code{"guile@@2"}, and each procedure takes a matching package and returns a -replacement for that package. +@deffn {Scheme Procedure} package-input-rewriting/spec @var{replacements} [#:deep? #t] +Return a procedure that, given a package, applies the given +@var{replacements} to all the package graph, including implicit inputs +unless @var{deep?} is false. @var{replacements} is a list of +spec/procedures pair; each spec is a package specification such as +@code{"gcc"} or @code{"guile@@2"}, and each procedure takes a matching +package and returns a replacement for that package. @end deffn The example above could be rewritten this way: diff --git a/guix/packages.scm b/guix/packages.scm index 171fd048ef..f696945e30 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -422,6 +422,16 @@ name of its URI." package) 16))))) +(define-syntax-rule (package/inherit p overrides ...) + "Like (package (inherit P) OVERRIDES ...), except that the same +transformation is done to the package replacement, if any. P must be a bare +identifier, and will be bound to either P or its replacement when evaluating +OVERRIDES." + (let loop ((p p)) + (package (inherit p) + overrides ... + (replacement (and=> (package-replacement p) loop))))) + (define (package-upstream-name package) "Return the upstream name of PACKAGE, which could be different from the name it has in Guix." @@ -1051,12 +1061,12 @@ package and returns its new name after rewrite." (package-mapping rewrite (cut assq <> replacements))) -(define (package-input-rewriting/spec replacements) +(define* (package-input-rewriting/spec replacements #:key (deep? #t)) "Return a procedure that, given a package, applies the given REPLACEMENTS to -all the package graph (excluding implicit inputs). REPLACEMENTS is a list of -spec/procedures pair; each spec is a package specification such as \"gcc\" or -\"guile@2\", and each procedure takes a matching package and returns a -replacement for that package." +all the package graph, including implicit inputs unless DEEP? is false. +REPLACEMENTS is a list of spec/procedures pair; each spec is a package +specification such as \"gcc\" or \"guile@2\", and each procedure takes a +matching package and returns a replacement for that package." (define table (fold (lambda (replacement table) (match replacement @@ -1081,22 +1091,27 @@ replacement for that package." (package-name package) table)) - (define (rewrite package) - (match (find-replacement package) - (#f package) - (proc (proc package)))) - - (package-mapping rewrite find-replacement)) + (define replacement-property + (gensym " package-replacement")) -(define-syntax-rule (package/inherit p overrides ...) - "Like (package (inherit P) OVERRIDES ...), except that the same -transformation is done to the package replacement, if any. P must be a bare -identifier, and will be bound to either P or its replacement when evaluating -OVERRIDES." - (let loop ((p p)) - (package (inherit p) - overrides ... - (replacement (and=> (package-replacement p) loop))))) + (define (rewrite p) + (if (assq-ref (package-properties p) replacement-property) + p + (match (find-replacement p) + (#f p) + (proc + (let ((new (proc p))) + ;; Mark NEW as already processed. + (package/inherit new + (properties `((,replacement-property . #t) + ,@(package-properties new))))))))) + + (define (cut? p) + (or (assq-ref (package-properties p) replacement-property) + (find-replacement p))) + + (package-mapping rewrite cut? + #:deep? deep?)) ;;; diff --git a/tests/guix-build.sh b/tests/guix-build.sh index 6c08857358..ec2f736ccb 100644 --- a/tests/guix-build.sh +++ b/tests/guix-build.sh @@ -259,6 +259,17 @@ drv1=`guix build guile -d` drv2=`guix build guile --with-input=gimp=ruby -d` test "$drv1" = "$drv2" +# See . +drv1=`guix build glib -d` +drv2=`guix build glib -d --with-input=libreoffice=inkscape` +test "$drv1" = "$drv2" + +# Rewriting implicit inputs. +drv1=`guix build hello -d` +drv2=`guix build hello -d --with-input=gcc=gcc-toolchain` +test "$drv1" != "$drv2" +guix gc -R "$drv2" | grep `guix build -d gcc-toolchain` + if guix build guile --with-input=libunistring=something-really-silly then false; else true; fi diff --git a/tests/packages.scm b/tests/packages.scm index f33332a461..6fa4ad2f1b 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -38,6 +38,7 @@ #:use-module (guix build-system) #:use-module (guix build-system trivial) #:use-module (guix build-system gnu) + #:use-module (guix build-system python) #:use-module (guix memoization) #:use-module (guix profiles) #:use-module (guix scripts package) @@ -45,6 +46,7 @@ #:use-module (gnu packages base) #:use-module (gnu packages guile) #:use-module (gnu packages bootstrap) + #:use-module (gnu packages python) #:use-module (gnu packages version-control) #:use-module (gnu packages xml) #:use-module (srfi srfi-1) @@ -1262,7 +1264,8 @@ ("baz" ,dep))))) (rewrite (package-input-rewriting/spec `(("coreutils" . ,(const sed)) - ("grep" . ,(const findutils))))) + ("grep" . ,(const findutils))) + #:deep? #f)) (p1 (rewrite p0)) (p2 (rewrite p0))) (and (not (eq? p1 p0)) @@ -1279,7 +1282,11 @@ (match (package-native-inputs dep3) ((("x" dep)) (string=? (package-full-name dep) - (package-full-name findutils)))))))))) + (package-full-name findutils))))))) + + ;; Make sure implicit inputs were left unchanged. + (equal? (drop (bag-direct-inputs (package->bag p1)) 3) + (drop (bag-direct-inputs (package->bag p0)) 3))))) (test-assert "package-input-rewriting/spec, partial match" (let* ((dep (dummy-package "chbouib" @@ -1290,7 +1297,8 @@ ("bar" ,dep))))) (rewrite (package-input-rewriting/spec `(("chbouib@123" . ,(const sed)) ;not matched - ("grep" . ,(const findutils))))) + ("grep" . ,(const findutils))) + #:deep? #f)) (p1 (rewrite p0))) (and (not (eq? p1 p0)) (string=? "example" (package-name p1)) @@ -1304,6 +1312,58 @@ (string=? (package-full-name dep) (package-full-name findutils)))))))))) +(test-assert "package-input-rewriting/spec, deep" + (let* ((dep (dummy-package "chbouib")) + (p0 (dummy-package "example" + (build-system gnu-build-system) + (inputs `(("dep" ,dep))))) + (rewrite (package-input-rewriting/spec + `(("tar" . ,(const sed)) + ("gzip" . ,(const findutils))))) + (p1 (rewrite p0)) + (p2 (rewrite p0))) + (and (not (eq? p1 p0)) + (eq? p1 p2) ;memoization + (string=? "example" (package-name p1)) + (match (package-inputs p1) + ((("dep" dep1)) + (and (string=? (package-full-name dep1) + (package-full-name dep)) + (eq? dep1 (rewrite dep))))) ;memoization + + ;; Make sure implicit inputs were replaced. + (match (bag-direct-inputs (package->bag p1)) + ((("dep" dep1) ("tar" tar) ("gzip" gzip) _ ...) + (and (eq? dep1 (rewrite dep)) + (string=? (package-full-name tar) + (package-full-name sed)) + (string=? (package-full-name gzip) + (package-full-name findutils)))))))) + +(test-assert "package-input-rewriting/spec, no duplicates" + ;; Ensure that deep input rewriting does not forget implicit inputs. Doing + ;; so could lead to duplicates in a package's inputs: in the example below, + ;; P0's transitive inputs would contain one rewritten "python" and one + ;; original "python". These two "python" packages are thus not 'eq?' but + ;; they lower to the same derivation. See , + ;; which can be reproduced by passing #:deep? #f. + (let* ((dep0 (dummy-package "dep0" + (build-system trivial-build-system) + (propagated-inputs `(("python" ,python))))) + (p0 (dummy-package "chbouib" + (build-system python-build-system) + (arguments `(#:python ,python)) + (inputs `(("dep0" ,dep0))))) + (rewrite (package-input-rewriting/spec '() #:deep? #t)) + (p1 (rewrite p0)) + (bag1 (package->bag p1)) + (pythons (filter-map (match-lambda + (("python" python) python) + (_ #f)) + (bag-transitive-inputs bag1)))) + (match (delete-duplicates pythons eq?) + ((p) (eq? p (rewrite python)))))) + (test-equal "package-patched-vulnerabilities" '(("CVE-2015-1234") ("CVE-2016-1234" "CVE-2018-4567") diff --git a/tests/scripts-build.scm b/tests/scripts-build.scm index 12114fc8f5..5f91360953 100644 --- a/tests/scripts-build.scm +++ b/tests/scripts-build.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017, 2019 Ludovic Courtès +;;; Copyright © 2016, 2017, 2019, 2020 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -19,6 +19,7 @@ (define-module (test-scripts-build) #:use-module (guix tests) #:use-module (guix store) + #:use-module (guix derivations) #:use-module (guix packages) #:use-module (guix git-download) #:use-module (guix scripts build) @@ -163,11 +164,16 @@ ((("foo" dep1) ("bar" dep2)) (and (string=? (package-full-name dep1) (package-full-name grep)) - (eq? (package-replacement dep1) findutils) + (string=? (package-full-name (package-replacement dep1)) + (package-full-name findutils)) (string=? (package-name dep2) "chbouib") (match (package-native-inputs dep2) ((("x" dep)) - (eq? (package-replacement dep) findutils))))))))))) + (with-store store + (string=? (derivation-file-name + (package-derivation store findutils)) + (derivation-file-name + (package-derivation store dep)))))))))))))) (test-equal "options->transformation, with-branch" (git-checkout (url "https://example.org") -- cgit v1.2.3 From b3fc03ee266a5f6d810d780582d458e561efccf3 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 23 Sep 2020 14:40:15 +0200 Subject: packages: 'package-mapping' correctly recurses into 'replacement'. Previously, something like: guix build glib --with-graft=glibc=glibc@2.29 would produce a result showing that rewriting rules were not applied to libx11@1.6.A (a replacement). * guix/packages.scm (package-mapping): Call REPLACE instead of PROC to 'replacement'. * tests/packages.scm ("package-input-rewriting/spec, graft"): New test. --- guix/packages.scm | 2 +- tests/packages.scm | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/packages.scm b/guix/packages.scm index f696945e30..0d0d7492b6 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -1037,7 +1037,7 @@ applied to implicit inputs as well." (inputs (map rewrite (package-inputs p))) (native-inputs (map rewrite (package-native-inputs p))) (propagated-inputs (map rewrite (package-propagated-inputs p))) - (replacement (and=> (package-replacement p) proc)) + (replacement (and=> (package-replacement p) replace)) (properties `((,mapping-property . #t) ,@(package-properties p)))))))) diff --git a/tests/packages.scm b/tests/packages.scm index 6fa4ad2f1b..e31dea6f72 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -1364,6 +1364,33 @@ (match (delete-duplicates pythons eq?) ((p) (eq? p (rewrite python)))))) +(test-equal "package-input-rewriting/spec, graft" + (derivation-file-name (package-derivation %store sed)) + + ;; Make sure replacements are rewritten. + (let* ((dep0 (dummy-package "dep" + (version "1") + (build-system trivial-build-system) + (inputs `(("coreutils" ,coreutils))))) + (dep1 (dummy-package "dep" + (version "0") + (build-system trivial-build-system) + (replacement dep0))) + (p0 (dummy-package "p" + (build-system trivial-build-system) + (inputs `(("dep" ,dep1))))) + (rewrite (package-input-rewriting/spec + `(("coreutils" . ,(const sed))))) + (p1 (rewrite p0))) + (match (package-inputs p1) + ((("dep" dep)) + (match (package-inputs (package-replacement dep)) + ((("coreutils" coreutils)) + ;; COREUTILS is not 'eq?' to SED, so the most reliable way to check + ;; for equality is to lower to a derivation. + (derivation-file-name + (package-derivation %store coreutils)))))))) + (test-equal "package-patched-vulnerabilities" '(("CVE-2015-1234") ("CVE-2016-1234" "CVE-2018-4567") -- cgit v1.2.3 From 8819551c8d2a12cd4e84e09b51e434d05a012c9d Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 23 Sep 2020 14:56:38 +0200 Subject: packages: 'package-input-rewriting' has a #:deep? parameter. * guix/packages.scm (package-input-rewriting): Add #:deep? and pass it to 'package-mapping'. [replacement-property]: New variable. [rewrite]: Check it. [cut?]: New procedure. * tests/packages.scm ("package-input-rewriting"): Pass #:deep? #f and ensure implicit inputs were not rewritten. Avoid 'eq?' comparisons. ("package-input-rewriting, deep"): New test. * gnu/packages/guile.scm (package-for-guile-2.0, package-for-guile-3.0): Pass #:deep? #f. --- doc/guix.texi | 10 +++++----- gnu/packages/guile.scm | 6 ++++-- guix/packages.scm | 35 +++++++++++++++++++++++++---------- tests/packages.scm | 20 ++++++++++++++++++-- 4 files changed, 52 insertions(+), 19 deletions(-) (limited to 'guix') diff --git a/doc/guix.texi b/doc/guix.texi index e72e1ec130..0805e2d508 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -6238,12 +6238,12 @@ transformation is @dfn{input rewriting}, whereby the dependency tree of a package is rewritten by replacing specific inputs by others: @deffn {Scheme Procedure} package-input-rewriting @var{replacements} @ - [@var{rewrite-name}] + [@var{rewrite-name}] [#:deep? #t] Return a procedure that, when passed a package, replaces its direct and -indirect dependencies (but not its implicit inputs) according to -@var{replacements}. @var{replacements} is a list of package pairs; the -first element of each pair is the package to replace, and the second one -is the replacement. +indirect dependencies, including implicit inputs when @var{deep?} is +true, according to @var{replacements}. @var{replacements} is a list of +package pairs; the first element of each pair is the package to replace, +and the second one is the replacement. Optionally, @var{rewrite-name} is a one-argument procedure that takes the name of a package and returns its new name after rewrite. diff --git a/gnu/packages/guile.scm b/gnu/packages/guile.scm index c59daeebe2..280053bf06 100644 --- a/gnu/packages/guile.scm +++ b/gnu/packages/guile.scm @@ -420,11 +420,13 @@ GNU@tie{}Guile. Use the @code{(ice-9 readline)} module and call its ;; A procedure that rewrites the dependency tree of the given package to use ;; GUILE-2.0 instead of GUILE-3.0. (package-input-rewriting `((,guile-3.0 . ,guile-2.0)) - (guile-variant-package-name "guile2.0"))) + (guile-variant-package-name "guile2.0") + #:deep? #f)) (define package-for-guile-2.2 (package-input-rewriting `((,guile-3.0 . ,guile-2.2)) - (guile-variant-package-name "guile2.2"))) + (guile-variant-package-name "guile2.2") + #:deep? #f)) (define-syntax define-deprecated-guile3.0-package (lambda (s) diff --git a/guix/packages.scm b/guix/packages.scm index 0d0d7492b6..4f2bb432be 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -1044,22 +1044,37 @@ applied to implicit inputs as well." replace) (define* (package-input-rewriting replacements - #:optional (rewrite-name identity)) + #:optional (rewrite-name identity) + #:key (deep? #t)) "Return a procedure that, when passed a package, replaces its direct and -indirect dependencies (but not its implicit inputs) according to REPLACEMENTS. -REPLACEMENTS is a list of package pairs; the first element of each pair is the -package to replace, and the second one is the replacement. +indirect dependencies, including implicit inputs when DEEP? is true, according +to REPLACEMENTS. REPLACEMENTS is a list of package pairs; the first element +of each pair is the package to replace, and the second one is the replacement. Optionally, REWRITE-NAME is a one-argument procedure that takes the name of a package and returns its new name after rewrite." + (define replacement-property + ;; Property to tag right-hand sides in REPLACEMENTS. + (gensym " package-replacement")) + (define (rewrite p) - (match (assq-ref replacements p) - (#f (package - (inherit p) - (name (rewrite-name (package-name p))))) - (new new))) + (if (assq-ref (package-properties p) replacement-property) + p + (match (assq-ref replacements p) + (#f (package/inherit p + (name (rewrite-name (package-name p))))) + (new (if deep? + (package/inherit new + (properties `((,replacement-property . #t) + ,@(package-properties new)))) + new))))) - (package-mapping rewrite (cut assq <> replacements))) + (define (cut? p) + (or (assq-ref (package-properties p) replacement-property) + (assq-ref replacements p))) + + (package-mapping rewrite cut? + #:deep? deep?)) (define* (package-input-rewriting/spec replacements #:key (deep? #t)) "Return a procedure that, given a package, applies the given REPLACEMENTS to diff --git a/tests/packages.scm b/tests/packages.scm index e31dea6f72..af8941c2e2 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -1239,7 +1239,8 @@ ("baz" ,dep))))) (rewrite (package-input-rewriting `((,coreutils . ,sed) (,grep . ,findutils)) - (cut string-append "r-" <>))) + (cut string-append "r-" <>) + #:deep? #f)) (p1 (rewrite p0)) (p2 (rewrite p0))) (and (not (eq? p1 p0)) @@ -1253,7 +1254,22 @@ (eq? dep3 (rewrite dep)) ;memoization (match (package-native-inputs dep3) ((("x" dep)) - (eq? dep findutils))))))))) + (eq? dep findutils)))))) + + ;; Make sure implicit inputs were left unchanged. + (equal? (drop (bag-direct-inputs (package->bag p1)) 3) + (drop (bag-direct-inputs (package->bag p0)) 3))))) + +(test-eq "package-input-rewriting, deep" + (derivation-file-name (package-derivation %store sed)) + (let* ((p0 (dummy-package "chbouib" + (build-system python-build-system) + (arguments `(#:python ,python)))) + (rewrite (package-input-rewriting `((,python . ,sed)))) + (p1 (rewrite p0))) + (match (bag-direct-inputs (package->bag p1)) + ((("python" python) _ ...) + (derivation-file-name (package-derivation %store python)))))) (test-assert "package-input-rewriting/spec" (let* ((dep (dummy-package "chbouib" -- cgit v1.2.3 From e75443d4f28ff1aa97e545f2b47b311c3a5ac32a Mon Sep 17 00:00:00 2001 From: Julien Lepiller Date: Wed, 16 Sep 2020 23:32:00 +0200 Subject: guix build: Add a hint for unspecified value. * guix/scripts/build.scm (options->things-to-build): Add a hint when we cannot build something. --- guix/scripts/build.scm | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index f238e9b876..476e556618 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -38,6 +38,7 @@ #:use-module (guix monads) #:use-module (guix gexp) #:use-module (guix profiles) + #:use-module (guix diagnostics) #:autoload (guix http-client) (http-fetch http-get-error?) #:use-module (ice-9 format) #:use-module (ice-9 match) @@ -46,6 +47,7 @@ #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) #:use-module (srfi srfi-37) #:use-module (gnu packages) #:autoload (guix download) (download-to-store) @@ -830,7 +832,28 @@ must be one of 'package', 'all', or 'transitive'~%") build---packages, gexps, derivations, and so on." (define (validate-type x) (unless (or (derivation? x) (file-like? x) (gexp? x) (procedure? x)) - (leave (G_ "~s: not something we can build~%") x))) + (raise (make-compound-condition + (formatted-message (G_ "~s: not something we can build~%") x) + (condition + (&fix-hint + (hint + (if (unspecified? x) + (G_ "If you build from a file, make sure the last Scheme +expression returns a package value. @code{define-public} defines a variable, +but returns @code{#}. To fix this, add a Scheme expression at +the end of the file that consists only of the package's variable name you +defined, as in this example: + +@example +(define-public my-package + (package + ...)) + +my-package +@end example") + (G_ "If you build from a file, make sure the last +Scheme expression returns a package, gexp, derivation or a list of such +values."))))))))) (define (ensure-list x) (let ((lst (match x -- cgit v1.2.3 From 680b80e37453d4e23ad8188d60894916e1c07162 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Tue, 29 Sep 2020 10:29:23 +0200 Subject: openpgp: Fix argument order of 'fxbit-set?'. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * guix/openpgp.scm (fxbit-set?): Change to swap arguments compared to 'bit-set?'. * tests/openpgp.scm (%binary-sample): New test vector. ("port-ascii-armored?, #t"): Add test. ("port-ascii-armored?, #f"): Add another test. Co-authored-by: Ludovic Courtès --- guix/openpgp.scm | 2 +- tests/openpgp.scm | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/openpgp.scm b/guix/openpgp.scm index 33c851255b..153752ee73 100644 --- a/guix/openpgp.scm +++ b/guix/openpgp.scm @@ -110,7 +110,7 @@ (define-alias fx/ /) (define-alias fxdiv quotient) (define-alias fxand logand) -(define-alias fxbit-set? bit-set?) +(define-inlinable (fxbit-set? n index) (bit-set? index n)) (define-alias fxbit-field bit-field) (define-alias bitwise-bit-field bit-field) (define-alias fxarithmetic-shift-left ash) diff --git a/tests/openpgp.scm b/tests/openpgp.scm index 0beab6f88b..c2be26fa49 100644 --- a/tests/openpgp.scm +++ b/tests/openpgp.scm @@ -50,6 +50,12 @@ vBSFjNSiVHsuAA== =AAAA -----END PGP MESSAGE-----\n") +(define %binary-sample + ;; Same message as %radix-64-sample, decoded into bytevector. + (base16-string->bytevector + "c838013b6d96c411efecef17ecefe3ca0004ce8979ea250a897995f979a9\ +0ad9a9a9050a890ac5a9c945a940c1a2fcd2bc14858cd4a2547b2e00")) + (define %civodul-fingerprint "3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5") @@ -155,6 +161,12 @@ Pz7oopeN72xgggYUNT37ezqN3MeCqw0= read-radix-64)) list)) +(test-assert "port-ascii-armored?, #t" + (call-with-input-string %radix-64-sample port-ascii-armored?)) + +(test-assert "port-ascii-armored?, #f" + (not (port-ascii-armored? (open-bytevector-input-port %binary-sample)))) + (test-assert "get-openpgp-keyring" (let* ((key (search-path %load-path "tests/civodul.key")) (keyring (get-openpgp-keyring -- cgit v1.2.3 From 313f492657f1d0863c641fa5ee7f5b7028e27c94 Mon Sep 17 00:00:00 2001 From: Mathieu Othacehe Date: Fri, 31 Jul 2020 16:49:29 +0200 Subject: scripts: system: Add support for image-type. * guix/scripts/system.scm (list-image-types): New procedure, (%options): add "image-type" and "list-image-types" options, remove "file-system-type" option, (show-help): adapt accordingly, (%default-options): also adapt, and set the default "image-type" to "raw", (perform-action): add image-type argument and remove file-system-type argument, (process-action): adapt perform-action call, (system-derivation-for-action): remove base-image argument, add image-type argument, and use it to create the image passed to "system-image". * tests/guix-system.sh: Adapt accordingly and add a test for "--list-image-types" command. * doc/guix.texi (Building the Installation Image, Invoking guix system): Adapt accordingly. Signed-off-by: Mathieu Othacehe --- Makefile.am | 5 ++-- doc/guix.texi | 43 ++++++++++++++++-------------- guix/scripts/system.scm | 70 +++++++++++++++++++++++++++++++------------------ tests/guix-system.sh | 9 ++++--- 4 files changed, 75 insertions(+), 52 deletions(-) (limited to 'guix') diff --git a/Makefile.am b/Makefile.am index 8e91e1e558..9c3ff4420f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -833,9 +833,8 @@ release: dist-with-updated-version -v1 --no-grafts --fallback for system in $(GUIX_SYSTEM_SUPPORTED_SYSTEMS) ; do \ image=`$(top_builddir)/pre-inst-env \ - guix system disk-image \ - --file-system-type=iso9660 \ - --label="GUIX_$${system}_$(VERSION)" \ + guix system disk-image -t iso9660 \ + --label="GUIX_$${system}_$(VERSION)" \ --system=$$system --fallback \ gnu/system/install.scm` ; \ if [ ! -f "$$image" ] ; then \ diff --git a/doc/guix.texi b/doc/guix.texi index ff2e582347..e8458ad8d8 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -40,7 +40,7 @@ Copyright @copyright{} 2016, 2017, 2018, 2019, 2020 Julien Lepiller@* Copyright @copyright{} 2016 Alex ter Weele@* Copyright @copyright{} 2016, 2017, 2018, 2019 Christopher Baines@* Copyright @copyright{} 2017, 2018, 2019 Clément Lassieur@* -Copyright @copyright{} 2017, 2018 Mathieu Othacehe@* +Copyright @copyright{} 2017, 2018, 2020 Mathieu Othacehe@* Copyright @copyright{} 2017 Federico Beffa@* Copyright @copyright{} 2017, 2018 Carlo Zancanaro@* Copyright @copyright{} 2017 Thomas Danckaert@* @@ -2568,8 +2568,7 @@ The installation image described above was built using the @command{guix system} command, specifically: @example -guix system disk-image --file-system-type=iso9660 \ - gnu/system/install.scm +guix system disk-image -t iso9660 gnu/system/install.scm @end example Have a look at @file{gnu/system/install.scm} in the source tree, @@ -29375,24 +29374,28 @@ a value. Docker images are built to contain exactly what they need, so the @option{--image-size} option is ignored in the case of @code{docker-image}. -You can specify the root file system type by using the -@option{--file-system-type} option. It defaults to @code{ext4}. When its -value is @code{iso9660}, the @option{--label} option can be used to specify -a volume ID with @code{disk-image}. +The @code{disk-image} command can produce various image types. The +image type can be selected using the @command{--image-type} option. It +defaults to @code{raw}. When its value is @code{iso9660}, the +@option{--label} option can be used to specify a volume ID with +@code{disk-image}. -When using @code{vm-image}, the returned image is in qcow2 format, which -the QEMU emulator can efficiently use. @xref{Running Guix in a VM}, -for more information on how to run the image in a virtual machine. - -When using @code{disk-image}, a raw disk image is produced; it can be -copied as is to a USB stick, for instance. Assuming @code{/dev/sdc} is -the device corresponding to a USB stick, one can copy the image to it -using the following command: +When using the @code{raw} image type, a raw disk image is produced; it +can be copied as is to a USB stick, for instance. Assuming +@code{/dev/sdc} is the device corresponding to a USB stick, one can copy +the image to it using the following command: @example # dd if=$(guix system disk-image my-os.scm) of=/dev/sdc status=progress @end example +The @code{--list-image-types} command lists all the available image +types. + +When using @code{vm-image}, the returned image is in qcow2 format, which +the QEMU emulator can efficiently use. @xref{Running Guix in a VM}, +for more information on how to run the image in a virtual machine. + When using @code{docker-image}, a Docker image is produced. Guix builds the image from scratch, not from a pre-existing Docker base image. As a result, it contains @emph{exactly} what you define in the operating @@ -29494,17 +29497,17 @@ information, one can rebuild the image to make sure it really contains what it pretends to contain; or they could use that to derive a variant of the image. -@item --file-system-type=@var{type} +@item --image-type=@var{type} @itemx -t @var{type} -For the @code{disk-image} action, create a file system of the given -@var{type} on the image. +For the @code{disk-image} action, create an image with given @var{type}. -When this option is omitted, @command{guix system} uses @code{ext4}. +When this option is omitted, @command{guix system} uses the @code{raw} +image type. @cindex ISO-9660 format @cindex CD image format @cindex DVD image format -@option{--file-system-type=iso9660} produces an ISO-9660 image, suitable +@option{--image-type=iso9660} produces an ISO-9660 image, suitable for burning on CDs and DVDs. @item --image-size=@var{size} diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index bd5f84fc5b..7b3eacf2e1 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -666,8 +666,8 @@ checking this by themselves in their 'check' procedure." ;;; Action. ;;; -(define* (system-derivation-for-action os base-image action - #:key image-size file-system-type +(define* (system-derivation-for-action os action + #:key image-size image-type full-boot? container-shared-network? mappings label) "Return as a monadic value the derivation for OS according to ACTION." @@ -690,12 +690,15 @@ checking this by themselves in their 'check' procedure." (* 70 (expt 2 20))) #:mappings mappings)) ((disk-image) - (lower-object - (system-image - (image - (inherit (if label (image-with-label base-image label) base-image)) - (size image-size) - (operating-system os))))) + (let ((base-image (os->image os #:type image-type))) + (lower-object + (system-image + (image + (inherit (if label + (image-with-label base-image label) + base-image)) + (size image-size) + (operating-system os)))))) ((docker-image) (system-docker-image os #:shared-network? container-shared-network?)))) @@ -748,18 +751,19 @@ and TARGET arguments." install-bootloader? dry-run? derivations-only? use-substitutes? bootloader-target target - image-size file-system-type full-boot? label - container-shared-network? + image-size image-type + full-boot? label container-shared-network? (mappings '()) (gc-root #f)) "Perform ACTION for OS. INSTALL-BOOTLOADER? specifies whether to install bootloader; BOOTLOADER-TAGET is the target for the bootloader; TARGET is the target root directory; IMAGE-SIZE is the size of the image to be built, for -the 'vm-image' and 'disk-image' actions. The root file system is created as a -FILE-SYSTEM-TYPE file system. FULL-BOOT? is used for the 'vm' action; it -determines whether to boot directly to the kernel or to the bootloader. -CONTAINER-SHARED-NETWORK? determines if the container will use a separate -network namespace. +the 'vm-image' and 'disk-image' actions. IMAGE-TYPE is the type of image to +be built. + +FULL-BOOT? is used for the 'vm' action; it determines whether to +boot directly to the kernel or to the bootloader. CONTAINER-SHARED-NETWORK? +determines if the container will use a separate network namespace. When DERIVATIONS-ONLY? is true, print the derivation file name(s) without building anything. @@ -799,11 +803,9 @@ static checks." (check-initrd-modules os))) (mlet* %store-monad - ((target* (current-target-system)) - (image -> (find-image file-system-type target*)) - (sys (system-derivation-for-action os image action + ((sys (system-derivation-for-action os action #:label label - #:file-system-type file-system-type + #:image-type image-type #:image-size image-size #:full-boot? full-boot? #:container-shared-network? container-shared-network? @@ -886,6 +888,17 @@ Run 'herd status' to view the list of services on your system.\n")))))) #:node-type (shepherd-service-node-type shepherds) #:reverse-edges? #t))) + +;;; +;;; Images. +;;; + +(define (list-image-types) + "Print the available image types." + (display (G_ "The available image types are:\n")) + (newline) + (format #t "~{ - ~a ~%~}" (map image-type-name (force %image-types)))) + ;;; ;;; Options. @@ -945,9 +958,9 @@ Some ACTIONS support additional ARGS.\n")) apply STRATEGY (one of nothing-special, backtrace, or debug) when an error occurs while reading FILE")) (display (G_ " - --file-system-type=TYPE - for 'disk-image', produce a root file system of TYPE - (one of 'ext4', 'iso9660')")) + --list-image-types list available image types")) + (display (G_ " + -t, --image-type=TYPE for 'disk-image', produce an image of TYPE")) (display (G_ " --image-size=SIZE for 'vm-image', produce an image of SIZE")) (display (G_ " @@ -1008,10 +1021,14 @@ Some ACTIONS support additional ARGS.\n")) (lambda (opt name arg result) (alist-cons 'on-error (string->symbol arg) result))) - (option '(#\t "file-system-type") #t #f + (option '(#\t "image-type") #t #f (lambda (opt name arg result) - (alist-cons 'file-system-type arg + (alist-cons 'image-type (string->symbol arg) result))) + (option '("list-image-types") #f #f + (lambda (opt name arg result) + (list-image-types) + (exit 0))) (option '("image-size") #t #f (lambda (opt name arg result) (alist-cons 'image-size (size->number arg) @@ -1080,7 +1097,7 @@ Some ACTIONS support additional ARGS.\n")) (debug . 0) (verbosity . #f) ;default (validate-reconfigure . ,ensure-forward-reconfigure) - (file-system-type . "ext4") + (image-type . raw) (image-size . guess) (install-bootloader? . #t) (label . #f))) @@ -1177,7 +1194,8 @@ resulting from command-line parsing." (assoc-ref opts 'skip-safety-checks?) #:validate-reconfigure (assoc-ref opts 'validate-reconfigure) - #:file-system-type (assoc-ref opts 'file-system-type) + #:image-type (lookup-image-type-by-name + (assoc-ref opts 'image-type)) #:image-size (assoc-ref opts 'image-size) #:full-boot? (assoc-ref opts 'full-boot?) #:container-shared-network? diff --git a/tests/guix-system.sh b/tests/guix-system.sh index 0e22686a34..667e084fcf 100644 --- a/tests/guix-system.sh +++ b/tests/guix-system.sh @@ -261,8 +261,8 @@ guix system vm "$tmpfile" -d | grep '\.drv$' drv1="`guix system vm "$tmpfile" -d`" drv2="`guix system vm "$tmpfile" -d`" test "$drv1" = "$drv2" -drv1="`guix system disk-image --file-system-type=iso9660 "$tmpfile" -d`" -drv2="`guix system disk-image --file-system-type=iso9660 "$tmpfile" -d`" +drv1="`guix system disk-image -t iso9660 "$tmpfile" -d`" +drv2="`guix system disk-image -t iso9660 "$tmpfile" -d`" test "$drv1" = "$drv2" make_user_config "group-that-does-not-exist" "users" @@ -320,5 +320,8 @@ guix system -n vm gnu/system/examples/vm-image.tmpl guix system -n vm-image gnu/system/examples/vm-image.tmpl # This invocation was taken care of in the loop above: # guix system -n disk-image gnu/system/examples/bare-bones.tmpl -guix system -n disk-image --file-system-type=iso9660 gnu/system/examples/bare-bones.tmpl +guix system -n disk-image -t iso9660 gnu/system/examples/bare-bones.tmpl guix system -n docker-image gnu/system/examples/docker-image.tmpl + +# Verify that at least the raw image type is available. +guix system --list-image-types | grep "raw" -- cgit v1.2.3 From 58abd5873985e0cd9a2926867bf697c5e7bc01f9 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 1 Oct 2020 11:29:54 +0200 Subject: pack: Work around ld.so bug that affects the "fakechroot" engine. Fixes . * guix/scripts/pack.scm (wrapped-package): Use (runpath program) instead of (runpath #$(audit-module)). --- guix/scripts/pack.scm | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'guix') diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 379e6a3ac6..bab3a3e2e4 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -817,11 +817,17 @@ last resort for relocation." (string-append "-DLOADER_AUDIT_MODULE=\"" #$(audit-module) "\"") + + ;; XXX: Normally (runpath #$(audit-module)) is + ;; enough. However, to work around + ;; + ;; (glibc <= 2.32), pass the whole search path of + ;; PROGRAM, which presumably is a superset of that + ;; of the audit module. (string-append "-DLOADER_AUDIT_RUNPATH={ " (string-join (map object->string - (runpath - #$(audit-module))) + (runpath program)) ", " 'suffix) "NULL }") (if gconv -- cgit v1.2.3 From 7dc19c33fc71e17a1d7ddd4563aa6ffd73d1a2cf Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 1 Oct 2020 12:35:00 +0200 Subject: ui: "guix help" silently ignores EPIPE. This avoids a backtrace when running "guix help | head" or similar. * guix/ui.scm (run-guix): Wrap 'show-guix-help' calls in 'leave-on-EPIPE'. --- guix/ui.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'guix') diff --git a/guix/ui.scm b/guix/ui.scm index ecaf975c1f..e88b7b4015 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -2134,7 +2134,7 @@ and signal handling have already been set up." (G_ "guix: missing command name~%")) (show-guix-usage)) ((or ("-h") ("--help")) - (show-guix-help)) + (leave-on-EPIPE (show-guix-help))) ((or ("-V") ("--version")) (show-version-and-exit "guix")) (((? option? o) args ...) @@ -2145,7 +2145,7 @@ and signal handling have already been set up." (apply run-guix-command (string->symbol command) '("--help"))) (("help" args ...) - (show-guix-help)) + (leave-on-EPIPE (show-guix-help))) ((command args ...) (apply run-guix-command (string->symbol command) -- cgit v1.2.3 From f4390d391b5901735444cba21c94e1e23d3fc575 Mon Sep 17 00:00:00 2001 From: Julien Lepiller Date: Thu, 1 Oct 2020 15:05:05 +0200 Subject: guix: opam: Fix syntax. * guix/import/opam.scm (STRCHR, comment, choice): Fix syntax. (group-pat): Add syntax. (opam->guix-package): Suppport "archive" keyword. --- guix/import/opam.scm | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'guix') diff --git a/guix/import/opam.scm b/guix/import/opam.scm index 9cda3da006..7327ab6e29 100644 --- a/guix/import/opam.scm +++ b/guix/import/opam.scm @@ -49,16 +49,19 @@ condition)) ;; Define a PEG parser for the opam format -(define-peg-pattern comment none (and "#" (* STRCHR) "\n")) +(define-peg-pattern comment none (and "#" (* COMMCHR) "\n")) (define-peg-pattern SP none (or " " "\n" comment)) (define-peg-pattern SP2 body (or " " "\n")) (define-peg-pattern QUOTE none "\"") (define-peg-pattern QUOTE2 body "\"") (define-peg-pattern COLON none ":") ;; A string character is any character that is not a quote, or a quote preceded by a backslash. +(define-peg-pattern COMMCHR none + (or " " "!" "\\" "\"" (range #\# #\頋))) (define-peg-pattern STRCHR body (or " " "!" "\n" (and (ignore "\\") "\"") - (and (ignore "\\") "\\") (range #\# #\頋))) + (ignore "\\\n") (and (ignore "\\") "\\") + (range #\# #\頋))) (define-peg-pattern operator all (or "=" "!" "<" ">")) (define-peg-pattern records body (* (and (or record weird-record) (* SP)))) @@ -69,8 +72,12 @@ (define-peg-pattern choice-pat all (and (ignore "(") (* SP) choice (* SP) (ignore ")"))) (define-peg-pattern choice body (or (and (or conditional-value ground-value) (* SP) (ignore "|") (* SP) choice) + group-pat conditional-value ground-value)) +(define-peg-pattern group-pat body + (and ground-value (* SP) (ignore "&") (* SP) + (or group-pat conditional-value ground-value))) (define-peg-pattern ground-value body (and (or multiline-string string-pat choice-pat list-pat var) (* SP))) (define-peg-pattern conditional-value all (and ground-value (* SP) condition)) (define-peg-pattern string-pat all (and QUOTE (* STRCHR) QUOTE)) @@ -258,7 +265,8 @@ or #f on failure." (version (assoc-ref opam-file "version")) (opam-content (assoc-ref opam-file "metadata")) (url-dict (metadata-ref opam-content "url")) - (source-url (metadata-ref url-dict "src")) + (source-url (or (metadata-ref url-dict "src") + (metadata-ref url-dict "archive"))) (requirements (metadata-ref opam-content "depends")) (dependencies (dependency-list->names requirements)) (native-dependencies (depends->native-inputs requirements)) @@ -308,7 +316,7 @@ or #f on failure." (filter (lambda (name) (not (member name '("dune" "jbuilder")))) - dependencies)))))))) + dependencies)))))))) (define (opam-recursive-import package-name) (recursive-import package-name #f -- cgit v1.2.3 From a6816618fc1e48417a64c5f8ca67e3d64ebc5441 Mon Sep 17 00:00:00 2001 From: Julien Lepiller Date: Fri, 2 Oct 2020 00:27:24 +0200 Subject: import: opam: Report groups in syntax tree. * guix/import/opam.scm (group-pat): Report in syntax tree. (dependency->input, dependency->native-input, dependency->name): consider the case of a group. --- guix/import/opam.scm | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'guix') diff --git a/guix/import/opam.scm b/guix/import/opam.scm index 7327ab6e29..6d9eb0a092 100644 --- a/guix/import/opam.scm +++ b/guix/import/opam.scm @@ -75,8 +75,8 @@ group-pat conditional-value ground-value)) -(define-peg-pattern group-pat body - (and ground-value (* SP) (ignore "&") (* SP) +(define-peg-pattern group-pat all + (and (or conditional-value ground-value) (* SP) (ignore "&") (* SP) (or group-pat conditional-value ground-value))) (define-peg-pattern ground-value body (and (or multiline-string string-pat choice-pat list-pat var) (* SP))) (define-peg-pattern conditional-value all (and ground-value (* SP) condition)) @@ -196,6 +196,7 @@ path to the repository." (('string-pat str) str) ;; Arbitrary select the first dependency (('choice-pat choice ...) (dependency->input (car choice))) + (('group-pat val ...) (map dependency->input val)) (('conditional-value val condition) (if (native? condition) "" (dependency->input val))))) @@ -203,7 +204,8 @@ path to the repository." (match dependency (('string-pat str) "") ;; Arbitrary select the first dependency - (('choice-pat choice ...) (dependency->input (car choice))) + (('choice-pat choice ...) (dependency->native-input (car choice))) + (('group-pat val ...) (map dependency->native-input val)) (('conditional-value val condition) (if (native? condition) (dependency->input val) "")))) @@ -211,7 +213,8 @@ path to the repository." (match dependency (('string-pat str) str) ;; Arbitrary select the first dependency - (('choice-pat choice ...) (dependency->input (car choice))) + (('choice-pat choice ...) (dependency->name (car choice))) + (('group-pat val ...) (map dependency->name val)) (('conditional-value val condition) (dependency->name val)))) @@ -263,7 +266,7 @@ REPOSITORY is #f, from the official OPAM repository. Return a 'package' sexp or #f on failure." (and-let* ((opam-file (opam-fetch name repository)) (version (assoc-ref opam-file "version")) - (opam-content (assoc-ref opam-file "metadata")) + (opam-content (pk (assoc-ref opam-file "metadata"))) (url-dict (metadata-ref opam-content "url")) (source-url (or (metadata-ref url-dict "src") (metadata-ref url-dict "archive"))) -- cgit v1.2.3 From f43ffee90882c2d61b46d69728daa7432be297e4 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 1 Oct 2020 22:09:58 +0200 Subject: gexp: 'local-file' warns when passed a non-literal relative file name. Fixes . Reported by Vitaliy Shatrov . * guix/gexp.scm (%local-file): Add #:literal? and #:location. Emit a warning when LITERAL? is false and FILE is not absolute. (local-file): In the non-literal case, pass #:location and #:literal?. * po/guix/POTFILES.in: Add guix/gexp.scm. * tests/guix-system.sh: Add test for the warning. --- guix/gexp.scm | 19 +++++++++++++++---- po/guix/POTFILES.in | 1 + tests/guix-system.sh | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) (limited to 'guix') diff --git a/guix/gexp.scm b/guix/gexp.scm index 9d3c52e783..40346b61e1 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -26,6 +26,8 @@ #:use-module (guix derivations) #:use-module (guix grafts) #:use-module (guix utils) + #:use-module (guix diagnostics) + #:use-module (guix i18n) #:use-module (rnrs bytevectors) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) @@ -401,9 +403,15 @@ Here TARGET is bound to the cross-compilation triplet or #f." (define (true file stat) #t) (define* (%local-file file promise #:optional (name (basename file)) - #:key recursive? (select? true)) + #:key + (literal? #t) location + recursive? (select? true)) ;; This intermediate procedure is part of our ABI, but the underlying ;; %%LOCAL-FILE is not. + (when (and (not literal?) (not (string-prefix? "/" file))) + (warning (and=> location source-properties->location) + (G_ "resolving '~a' relative to current directory~%") + file)) (%%local-file file promise name recursive? select?)) (define (absolute-file-name file directory) @@ -443,9 +451,12 @@ appears." rest ...)) ((_ file rest ...) ;; Resolve FILE relative to the current directory. - #'(%local-file file - (delay (absolute-file-name file (getcwd))) - rest ...)) + (with-syntax ((location (datum->syntax s (syntax-source s)))) + #`(%local-file file + (delay (absolute-file-name file (getcwd))) + #:location 'location + #:literal? #f + rest ...))) ((_) #'(syntax-error "missing file name")) (id diff --git a/po/guix/POTFILES.in b/po/guix/POTFILES.in index f4d020782c..b877fac9df 100644 --- a/po/guix/POTFILES.in +++ b/po/guix/POTFILES.in @@ -76,6 +76,7 @@ guix/scripts/weather.scm guix/scripts/describe.scm guix/scripts/processes.scm guix/scripts/deploy.scm +guix/gexp.scm guix/gnu-maintenance.scm guix/scripts/container.scm guix/scripts/container/exec.scm diff --git a/tests/guix-system.sh b/tests/guix-system.sh index 667e084fcf..957479ede0 100644 --- a/tests/guix-system.sh +++ b/tests/guix-system.sh @@ -297,6 +297,20 @@ EOF guix system build "$tmpdir/config.scm" -n (cd "$tmpdir"; guix system build "config.scm" -n) +# Check that we get a warning when passing 'local-file' a non-literal relative +# file name. +cat > "$tmpdir/config.scm" <&1 | \ + grep "config\.scm:4:2: warning:.*whatever.*relative to current directory" + # Searching. guix system search tor | grep "^name: tor" guix system search tor | grep "^shepherdnames: tor" -- cgit v1.2.3 From dc749a0e91b15e7e65a95dac1a9341dc17e2ff12 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 1 Oct 2020 22:32:22 +0200 Subject: ui: Use "guix install" in locale hint. * guix/ui.scm (install-locale): Change "guix package -i" to "guix install". --- guix/ui.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/ui.scm b/guix/ui.scm index e88b7b4015..8213e8ebab 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -492,7 +492,7 @@ part." lines: @example -guix package -i glibc-utf8-locales +guix install glibc-utf8-locales export GUIX_LOCPATH=\"$HOME/.guix-profile/lib/locale\" @end example -- cgit v1.2.3 From 9471aea76ace5c0998d889fc5fbde7a6bcafc654 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 2 Oct 2020 09:29:26 +0200 Subject: gexp: Fix argument ordering in 'local-file' macro. Fixes a regression introduced in f43ffee90882c2d61b46d69728daa7432be297e4. Reported by jonsger on #guix. * guix/gexp.scm (local-file): In the non-literal case, add #:literal? and #:location after REST. --- guix/gexp.scm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'guix') diff --git a/guix/gexp.scm b/guix/gexp.scm index 40346b61e1..25e4881d21 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -454,9 +454,9 @@ appears." (with-syntax ((location (datum->syntax s (syntax-source s)))) #`(%local-file file (delay (absolute-file-name file (getcwd))) + rest ... #:location 'location - #:literal? #f - rest ...))) + #:literal? #f))) ((_) #'(syntax-error "missing file name")) (id -- cgit v1.2.3 From bdbd8bf9054c88aaf694a08e49270c95e6adad27 Mon Sep 17 00:00:00 2001 From: Mathieu Othacehe Date: Fri, 2 Oct 2020 09:53:45 +0200 Subject: scripts: system: Honor target argument. Since 313f492657f1d0863c641fa5ee7f5b7028e27c94 the target argument passed to "guix system" was not honored for 'disk-image' command. This forces the command line passed "target" to take precedence over the "target" field of the record returned by "os->image" procedure. * guix/scripts/system.scm (system-derivation-for-action): Override the "target" field of the "image" record using the "target" argument from the command line. --- guix/scripts/system.scm | 64 ++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 30 deletions(-) (limited to 'guix') diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 7b3eacf2e1..939559e719 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -671,36 +671,40 @@ checking this by themselves in their 'check' procedure." full-boot? container-shared-network? mappings label) "Return as a monadic value the derivation for OS according to ACTION." - (case action - ((build init reconfigure) - (operating-system-derivation os)) - ((container) - (container-script - os - #:mappings mappings - #:shared-network? container-shared-network?)) - ((vm-image) - (system-qemu-image os #:disk-image-size image-size)) - ((vm) - (system-qemu-image/shared-store-script os - #:full-boot? full-boot? - #:disk-image-size - (if full-boot? - image-size - (* 70 (expt 2 20))) - #:mappings mappings)) - ((disk-image) - (let ((base-image (os->image os #:type image-type))) - (lower-object - (system-image - (image - (inherit (if label - (image-with-label base-image label) - base-image)) - (size image-size) - (operating-system os)))))) - ((docker-image) - (system-docker-image os #:shared-network? container-shared-network?)))) + (mlet %store-monad ((target (current-target-system))) + (case action + ((build init reconfigure) + (operating-system-derivation os)) + ((container) + (container-script + os + #:mappings mappings + #:shared-network? container-shared-network?)) + ((vm-image) + (system-qemu-image os #:disk-image-size image-size)) + ((vm) + (system-qemu-image/shared-store-script os + #:full-boot? full-boot? + #:disk-image-size + (if full-boot? + image-size + (* 70 (expt 2 20))) + #:mappings mappings)) + ((disk-image) + (let* ((base-image (os->image os #:type image-type)) + (base-target (image-target base-image))) + (lower-object + (system-image + (image + (inherit (if label + (image-with-label base-image label) + base-image)) + (target (or base-target target)) + (size image-size) + (operating-system os)))))) + ((docker-image) + (system-docker-image os + #:shared-network? container-shared-network?))))) (define (maybe-suggest-running-guix-pull) "Suggest running 'guix pull' if this has never been done before." -- cgit v1.2.3 From ad54a73bb820a685f242976a86be63931789fa97 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 24 Sep 2020 22:13:06 +0200 Subject: guix build: Record package transformations in manifest entries. With this change, package transformation options used while building a manifest are saved in the metadata of the manifest entries. * guix/scripts/build.scm (transformation-procedure): New procedure. (options->transformation)[applicable]: Use it. Change to a list of key/value/proc tuples instead of key/proc pairs. [package-with-transformation-properties, tagged-object]: New procedures. Use them. (package-transformations, manifest-entry-with-transformations): New procedures. * guix/scripts/pack.scm (guix-pack)[with-transformations]: New procedure. Use it. * guix/scripts/package.scm (process-actions)[transform-entry]: Use it. * tests/guix-package-aliases.sh: Add test. --- guix/scripts/build.scm | 80 ++++++++++++++++++++++++++++++++++--------- guix/scripts/pack.scm | 29 +++++++++------- guix/scripts/package.scm | 13 +++---- tests/guix-package-aliases.sh | 6 ++++ 4 files changed, 93 insertions(+), 35 deletions(-) (limited to 'guix') diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 476e556618..72a5d46347 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -63,6 +63,7 @@ %transformation-options options->transformation + manifest-entry-with-transformations show-transformation-options-help guix-build @@ -427,6 +428,14 @@ a checkout of the Git repository at the given URL." (with-git-url . ,transform-package-source-git-url) (without-tests . ,transform-package-tests))) +(define (transformation-procedure key) + "Return the transformation procedure associated with KEY, a symbol such as +'with-source', or #f if there is none." + (any (match-lambda + ((k . proc) + (and (eq? k key) proc))) + %transformations)) + (define %transformation-options ;; The command-line interface to the above transformations. (let ((parser (lambda (symbol) @@ -481,32 +490,69 @@ derivation, etc.), applies the transformations specified by OPTS." ;; order in which they appear on the command line. (filter-map (match-lambda ((key . value) - (match (any (match-lambda - ((k . proc) - (and (eq? k key) proc))) - %transformations) + (match (transformation-procedure key) (#f #f) (transform ;; XXX: We used to pass TRANSFORM a list of several ;; arguments, but we now pass only one, assuming that ;; transform composes well. - (cons key (transform (list value))))))) + (list key value (transform (list value))))))) (reverse opts))) + (define (package-with-transformation-properties p) + (package/inherit p + (properties `((transformations + . ,(map (match-lambda + ((key value _) + (cons key value))) + applicable)) + ,@(package-properties p))))) + (lambda (store obj) - (fold (match-lambda* - (((name . transform) obj) - (let ((new (transform store obj))) - (when (eq? new obj) - (warning (G_ "transformation '~a' had no effect on ~a~%") - name - (if (package? obj) - (package-full-name obj) - obj))) - new))) - obj - applicable))) + (define (tagged-object new) + (if (and (not (eq? obj new)) + (package? new) (not (null? applicable))) + (package-with-transformation-properties new) + new)) + + (tagged-object + (fold (match-lambda* + (((name value transform) obj) + (let ((new (transform store obj))) + (when (eq? new obj) + (warning (G_ "transformation '~a' had no effect on ~a~%") + name + (if (package? obj) + (package-full-name obj) + obj))) + new))) + obj + applicable)))) + +(define (package-transformations package) + "Return the transformations applied to PACKAGE according to its properties." + (match (assq-ref (package-properties package) 'transformations) + (#f '()) + (transformations transformations))) + +(define (manifest-entry-with-transformations entry) + "Return ENTRY with an additional 'transformations' property if it's not +already there." + (let ((properties (manifest-entry-properties entry))) + (if (assq 'transformations properties) + entry + (let ((item (manifest-entry-item entry))) + (manifest-entry + (inherit entry) + (properties + (match (and (package? item) + (package-transformations item)) + ((or #f '()) + properties) + (transformations + `((transformations . ,transformations) + ,@properties))))))))) ;;; diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index bab3a3e2e4..0b66da01f9 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -1140,19 +1140,24 @@ Create a bundle of PACKAGE.\n")) manifest)) identity)) + (define (with-transformations manifest) + (map-manifest-entries manifest-entry-with-transformations + manifest)) + (with-provenance - (cond - ((and (not (null? manifests)) (not (null? packages))) - (leave (G_ "both a manifest and a package list were given~%"))) - ((not (null? manifests)) - (concatenate-manifests - (map (lambda (file) - (let ((user-module (make-user-module - '((guix profiles) (gnu))))) - (load* file user-module))) - manifests))) - (else - (packages->manifest packages)))))) + (with-transformations + (cond + ((and (not (null? manifests)) (not (null? packages))) + (leave (G_ "both a manifest and a package list were given~%"))) + ((not (null? manifests)) + (concatenate-manifests + (map (lambda (file) + (let ((user-module (make-user-module + '((guix profiles) (gnu))))) + (load* file user-module))) + manifests))) + (else + (packages->manifest packages))))))) (with-error-handling (with-store store diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index 7e7c37eac4..83f8c123d9 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -864,12 +864,13 @@ processed, #f otherwise." (define (transform-entry entry) (let ((item (transform store (manifest-entry-item entry)))) - (manifest-entry - (inherit entry) - (item item) - (version (if (package? item) - (package-version item) - (manifest-entry-version entry)))))) + (manifest-entry-with-transformations + (manifest-entry + (inherit entry) + (item item) + (version (if (package? item) + (package-version item) + (manifest-entry-version entry))))))) (when (equal? profile %current-profile) ;; Normally the daemon created %CURRENT-PROFILE when we connected, unless diff --git a/tests/guix-package-aliases.sh b/tests/guix-package-aliases.sh index e4ddace057..311838b768 100644 --- a/tests/guix-package-aliases.sh +++ b/tests/guix-package-aliases.sh @@ -39,6 +39,12 @@ test -x "$profile/bin/guile" ! guix install -r guile-bootstrap -p "$profile" --bootstrap test -x "$profile/bin/guile" +# Use a package transformation option and make sure it's recorded. +guix install --bootstrap guile-bootstrap -p "$profile" \ + --with-input=libreoffice=inkscape +test -x "$profile/bin/guile" +grep "libreoffice=inkscape" "$profile/manifest" + guix upgrade --version guix upgrade -n guix upgrade gui.e -n -- cgit v1.2.3 From 8e1907a72430aa989125b053573ef0897c480697 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 25 Sep 2020 17:16:34 +0200 Subject: guix package: Re-apply package transformation when upgrading. * guix/scripts/package.scm (transaction-upgrade-entry)[upgrade]: Add 'transform' parameter. Pass PKG through it. Use 'manifest-entry-with-transformations'. Call 'options->transformation' to get the transformation procedure. * tests/guix-package.sh: Add 'guix package -u' test. * tests/packages.scm ("transaction-upgrade-entry, transformation options preserved"): New test. * doc/guix.texi (Invoking guix package): Mention that transformations are preserved across upgrades. (Package Transformation Options): Likewise. --- doc/guix.texi | 27 +++++++++++++++++++++++++++ guix/scripts/package.scm | 20 +++++++++++++++----- tests/guix-package.sh | 15 +++++++++++++++ tests/packages.scm | 23 +++++++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) (limited to 'guix') diff --git a/doc/guix.texi b/doc/guix.texi index da48c8a72d..a6260a12aa 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -3101,6 +3101,29 @@ in the distribution currently installed. To update your distribution, you should regularly run @command{guix pull} (@pxref{Invoking guix pull}). +@cindex package transformations, upgrades +When upgrading, package transformations that were originally applied +when creating the profile are automatically re-applied (@pxref{Package +Transformation Options}). For example, assume you first installed Emacs +from the tip of its development branch with: + +@example +guix install emacs-next --with-branch=emacs-next=master +@end example + +Next time you run @command{guix upgrade}, Guix will again pull the tip +of the Emacs development branch and build @code{emacs-next} from that +checkout. + +Note that transformation options such as @option{--with-branch} and +@option{--with-source} depend on external state; it is up to you to +ensure that they work as expected. You can also discard a +transformations that apply to a package by running: + +@example +guix install @var{package} +@end example + @item --do-not-upgrade[=@var{regexp} @dots{}] When used together with the @option{--upgrade} option, do @emph{not} upgrade any packages whose name matches a @var{regexp}. For example, to @@ -9193,6 +9216,10 @@ This is a convenient way to create customized packages on the fly without having to type in the definitions of package variants (@pxref{Defining Packages}). +Package transformation options are preserved across upgrades: +@command{guix upgrade} attempts to apply transformation options +initially used when creating the profile to the upgraded packages. + @table @code @item --with-source=@var{source} diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index 83f8c123d9..2f04652634 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -218,12 +218,13 @@ non-zero relevance score." (output (manifest-entry-output old))) transaction))) - (define (upgrade entry) + (define (upgrade entry transform) (match entry (($ name version output (? string? path)) (match (find-best-packages-by-name name #f) ((pkg . rest) - (let ((candidate-version (package-version pkg))) + (let* ((pkg (transform store pkg)) + (candidate-version (package-version pkg))) (match (package-superseded pkg) ((? package? new) (supersede entry new)) @@ -231,12 +232,14 @@ non-zero relevance score." (case (version-compare candidate-version version) ((>) (manifest-transaction-install-entry - (package->manifest-entry* pkg output) + (manifest-entry-with-transformations + (package->manifest-entry* pkg output)) transaction)) ((<) transaction) ((=) - (let* ((new (package->manifest-entry* pkg output))) + (let* ((new (manifest-entry-with-transformations + (package->manifest-entry* pkg output)))) ;; Here we want to determine whether the NEW actually ;; differs from ENTRY, but we need to intercept ;; 'build-things' calls because they would prevent us from @@ -255,7 +258,14 @@ non-zero relevance score." (if (manifest-transaction-removal-candidate? entry transaction) transaction - (upgrade entry))) + + ;; Upgrade ENTRY, preserving transformation options listed in its + ;; properties. + (let ((transform (options->transformation + (or (assq-ref (manifest-entry-properties entry) + 'transformations) + '())))) + (upgrade entry transform)))) ;;; diff --git a/tests/guix-package.sh b/tests/guix-package.sh index a43496699b..3e5fa71d20 100644 --- a/tests/guix-package.sh +++ b/tests/guix-package.sh @@ -184,6 +184,21 @@ grep -E 'emacs[[:blank:]]+42\.5\.9rc7' "$tmpfile" rm "$emacs_tarball" "$tmpfile" rmdir "$module_dir" +# Install with package transformations. +guix install --bootstrap -p "$profile" sed --with-input=sed=guile-bootstrap +grep "sed=guile-bootstrap" "$profile/manifest" +test "$(readlink -f "$profile/bin/guile")" \ + = "$(guix build guile-bootstrap)/bin/guile" +test ! -f "$profile/bin/sed" + +# Make sure the package transformation is preserved. +guix package --bootstrap -p "$profile" -u +grep "sed=guile-bootstrap" "$profile/manifest" +test "$(readlink -f "$profile/bin/guile")" \ + = "$(guix build guile-bootstrap)/bin/guile" +test ! -f "$profile/bin/sed" +rm "$profile" "$profile"-[0-9]-link + # Profiles with a relative file name. Make sure we don't create dangling # symlinks--see bug report at # . diff --git a/tests/packages.scm b/tests/packages.scm index af8941c2e2..5d5abcbd76 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -187,6 +187,29 @@ (string=? (manifest-pattern-version pattern) "1") (string=? (manifest-pattern-output pattern) "out"))))))) +(test-equal "transaction-upgrade-entry, transformation options preserved" + (derivation-file-name (package-derivation %store grep)) + + (let* ((old (dummy-package "emacs" (version "1"))) + (props '((transformations . ((with-input . "emacs=grep"))))) + (tx (transaction-upgrade-entry + %store + (manifest-entry + (inherit (package->manifest-entry old)) + (properties props) + (item (string-append (%store-prefix) "/" + (make-string 32 #\e) "-foo-1"))) + (manifest-transaction)))) + (match (manifest-transaction-install tx) + (((? manifest-entry? entry)) + (and (string=? (manifest-entry-version entry) + (package-version grep)) + (string=? (manifest-entry-name entry) + (package-name grep)) + (equal? (manifest-entry-properties entry) props) + (derivation-file-name + (package-derivation %store (manifest-entry-item entry)))))))) + (test-assert "transaction-upgrade-entry, grafts" ;; Ensure that, when grafts are enabled, 'transaction-upgrade-entry' doesn't ;; try to build stuff. -- cgit v1.2.3 From 0f53c801b91919380a924b402d1ff822bb1dc6ea Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 2 Oct 2020 23:17:40 +0200 Subject: environment: Provide /etc/hosts in containers without '--network'. Fixes . * guix/scripts/environment.scm (launch-environment/container): Create /etc/hosts when NETWORK? is false. * tests/guix-environment-container.sh: Add "localhost" resolution test. --- guix/scripts/environment.scm | 7 +++++++ tests/guix-environment-container.sh | 4 ++++ 2 files changed, 11 insertions(+) (limited to 'guix') diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index e2e481dd02..9698111cd2 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -549,6 +549,13 @@ WHILE-LIST." (write-passwd (list passwd)) (write-group groups) + (unless network? + ;; When isolated from the network, provide a minimal /etc/hosts + ;; to resolve "localhost". + (call-with-output-file "/etc/hosts" + (lambda (port) + (display "127.0.0.1 localhost\n" port)))) + ;; For convenience, start in the user's current working ;; directory or, if unmapped, the home directory. (chdir (if map-cwd? diff --git a/tests/guix-environment-container.sh b/tests/guix-environment-container.sh index 040f32cce9..3674aa6026 100644 --- a/tests/guix-environment-container.sh +++ b/tests/guix-environment-container.sh @@ -44,6 +44,10 @@ else test $? = 42 fi +# Make sure "localhost" resolves. +guix environment --container --ad-hoc --bootstrap guile-bootstrap \ + -- guile -c '(exit (pair? (getaddrinfo "localhost" "80")))' + # Make sure '--preserve' is honored. result="`FOOBAR=42; export FOOBAR; guix environment -C --ad-hoc --bootstrap \ guile-bootstrap -E ^FOO -- guile -c '(display (getenv \"FOOBAR\"))'`" -- cgit v1.2.3 From b68d4106518abed20ba308831b65dcc69bf120a5 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 4 Oct 2020 22:40:26 +0200 Subject: environment: Turn "lo" up in network-less containers. This is a followup to 0f53c801b91919380a924b402d1ff822bb1dc6ea. * guix/scripts/environment.scm (launch-environment/container): Add call to 'set-network-interface-up'. * tests/guix-environment-container.sh: Add test. --- guix/scripts/environment.scm | 6 +++++- tests/guix-environment-container.sh | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'guix') diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index 9698111cd2..085f11a9d4 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -34,6 +34,7 @@ #:use-module (guix scripts build) #:use-module (gnu build linux-container) #:use-module (gnu build accounts) + #:use-module ((guix build syscalls) #:select (set-network-interface-up)) #:use-module (gnu system linux-container) #:use-module (gnu system file-systems) #:use-module (gnu packages) @@ -554,7 +555,10 @@ WHILE-LIST." ;; to resolve "localhost". (call-with-output-file "/etc/hosts" (lambda (port) - (display "127.0.0.1 localhost\n" port)))) + (display "127.0.0.1 localhost\n" port))) + + ;; Allow local AF_INET communications. + (set-network-interface-up "lo")) ;; For convenience, start in the user's current working ;; directory or, if unmapped, the home directory. diff --git a/tests/guix-environment-container.sh b/tests/guix-environment-container.sh index 3674aa6026..f2d15c8d0c 100644 --- a/tests/guix-environment-container.sh +++ b/tests/guix-environment-container.sh @@ -48,6 +48,17 @@ fi guix environment --container --ad-hoc --bootstrap guile-bootstrap \ -- guile -c '(exit (pair? (getaddrinfo "localhost" "80")))' +# We should get ECONNREFUSED, not ENETUNREACH, which would indicate that "lo" +# is down. +guix environment --container --ad-hoc --bootstrap guile-bootstrap \ + -- guile -c "(exit (= ECONNREFUSED + (catch 'system-error + (lambda () + (let ((sock (socket AF_INET SOCK_STREAM 0))) + (connect sock AF_INET INADDR_LOOPBACK 12345))) + (lambda args + (pk 'errno (system-error-errno args))))))" + # Make sure '--preserve' is honored. result="`FOOBAR=42; export FOOBAR; guix environment -C --ad-hoc --bootstrap \ guile-bootstrap -E ^FOO -- guile -c '(display (getenv \"FOOBAR\"))'`" -- cgit v1.2.3