From 493375cdb23fc1416348da584f17bec7171faadd Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 26 May 2019 01:18:53 +0200 Subject: publish: Maintain a hash-part-to-store-item mapping in cache. Fixes . * guix/scripts/publish.scm (hash-part-mapping-cache-file) (hash-part->path*): New procedures. * guix/scripts/publish.scm (render-narinfo/cached)[delete-entry]: Delete the 'hash-part-mapping-cache-file'. Use 'hash-part->path*' instead of 'hash-part->path'. * tests/publish.scm ("with cache, vanishing item"): New test. --- tests/publish.scm | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'tests') diff --git a/tests/publish.scm b/tests/publish.scm index 097ac036e0..7f44bc700f 100644 --- a/tests/publish.scm +++ b/tests/publish.scm @@ -469,6 +469,35 @@ FileSize: ~a~%" (assoc-ref narinfo "FileSize")) (response-code compressed)))))))))) +(test-equal "with cache, vanishing item" ; + 200 + (call-with-temporary-directory + (lambda (cache) + (let ((thread (with-separate-output-ports + (call-with-new-thread + (lambda () + (guix-publish "--port=6795" + (string-append "--cache=" cache))))))) + (wait-until-ready 6795) + + ;; Make sure that, even if ITEM disappears, we're still able to fetch + ;; it. + (let* ((base "http://localhost:6795/") + (item (add-text-to-store %store "random" (random-text))) + (part (store-path-hash-part item)) + (url (string-append base part ".narinfo")) + (cached (string-append cache + (if (zlib-available?) + "/gzip/" "/none/") + (basename item) + ".narinfo")) + (response (http-get url))) + (and (= 404 (response-code response)) + (wait-for-file cached) + (begin + (delete-paths %store (list item)) + (response-code (pk 'response (http-get url)))))))))) + (test-equal "/log/NAME" `(200 #t application/x-bzip2) (let ((drv (run-with-store %store -- cgit v1.2.3 From 002d17dcaacba0f86265b34f2509419d9e21224d Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Sat, 25 May 2019 08:40:38 +0200 Subject: discovery: 'all-modules' returns modules in path order. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A particular effect of this is that if there are ambiguous packages in a directory specified with `-L module_dir` and the distribution, the version from `module_dir` will be loaded, which is usually what would be expected. (E.g. for `guix build` or `guix package -i`.) * guix/discovery.scm (all-modules): Return modules in path order. * tests/guix-package.sh: Test local definitions take precedence. Signed-off-by: Ludovic Courtès --- guix/discovery.scm | 4 ++-- tests/guix-package.sh | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/guix/discovery.scm b/guix/discovery.scm index ef5ae73973..5bb494941b 100644 --- a/guix/discovery.scm +++ b/guix/discovery.scm @@ -145,8 +145,8 @@ Call (PROC MODULE RESULT) for each module that is found." "Return the list of package modules found in PATH, a list of directories to search. Entries in PATH can be directory names (strings) or (DIRECTORY . SUB-DIRECTORY) pairs, in which case modules are searched for beneath -SUB-DIRECTORY." - (fold-modules cons '() path #:warn warn)) +SUB-DIRECTORY. Modules are listed in the order they appear on the path." + (reverse (fold-modules cons '() path #:warn warn))) (define (fold-module-public-variables* proc init modules) "Call (PROC MODULE SYMBOL VARIABLE) for each variable exported by one of MODULES, diff --git a/tests/guix-package.sh b/tests/guix-package.sh index 767c3f8a66..79d6ec65e4 100644 --- a/tests/guix-package.sh +++ b/tests/guix-package.sh @@ -280,6 +280,20 @@ export GUIX_PACKAGE_PATH guix package -A emacs-foo-bar | grep 42 guix package -i emacs-foo-bar@42 -n +# Make sure GUIX_PACKAGE_PATH/'-L' takes precedence in case of duplicate packages. +cat > "$module_dir/bar.scm"<&1 | grep choosing.*bar.scm +( unset GUIX_PACKAGE_PATH; \ + guix package -i hello -n -L "$module_dir" 2>&1 | grep choosing.*bar.scm ) + # Make sure patches that live under $GUIX_PACKAGE_PATH are found. cat > "$module_dir/emacs.patch"< Date: Sun, 26 May 2019 23:18:21 +0200 Subject: import: hackage: Fix Cabal test. * guix/import/hackage.scm (hackage->guix-package): Remove call to 'memoize'. (hackage->guix-package/m): New procedure. (hackage-recursive-import): Use it. * tests/hackage.scm ("hackage->guix-package test 6"): Adjust. Co-authored-by: Robert Vollmert --- guix/import/hackage.scm | 32 +++++++++++++++++--------------- tests/hackage.scm | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 17 deletions(-) (limited to 'tests') diff --git a/guix/import/hackage.scm b/guix/import/hackage.scm index 2731b4cbee..bf7e99df18 100644 --- a/guix/import/hackage.scm +++ b/guix/import/hackage.scm @@ -279,13 +279,11 @@ representation of a Cabal file as produced by 'read-cabal'." (license ,(string->license (cabal-package-license cabal)))) (append hackage-dependencies hackage-native-dependencies)))) -(define hackage->guix-package - (memoize - (lambda* (package-name #:key - (include-test-dependencies? #t) - (port #f) - (cabal-environment '())) - "Fetch the Cabal file for PACKAGE-NAME from hackage.haskell.org, or, if the +(define* (hackage->guix-package package-name #:key + (include-test-dependencies? #t) + (port #f) + (cabal-environment '())) + "Fetch the Cabal file for PACKAGE-NAME from hackage.haskell.org, or, if the called with keyword parameter PORT, from PORT. Return the `package' S-expression corresponding to that package, or #f on failure. CABAL-ENVIRONMENT is an alist defining the environment in which the Cabal @@ -295,18 +293,22 @@ symbol 'true' or 'false'. The value associated with other keys has to conform to the Cabal file format definition. The default value associated with the keys \"os\", \"arch\" and \"impl\" is \"linux\", \"x86_64\" and \"ghc\" respectively." - (let ((cabal-meta (if port - (read-cabal (canonical-newline-port port)) - (hackage-fetch package-name)))) - (and=> cabal-meta (compose (cut hackage-module->sexp <> - #:include-test-dependencies? - include-test-dependencies?) - (cut eval-cabal <> cabal-environment))))))) + (let ((cabal-meta (if port + (read-cabal (canonical-newline-port port)) + (hackage-fetch package-name)))) + (and=> cabal-meta (compose (cut hackage-module->sexp <> + #:include-test-dependencies? + include-test-dependencies?) + (cut eval-cabal <> cabal-environment))))) + +(define hackage->guix-package/m ;memoized variant + (memoize hackage->guix-package)) (define* (hackage-recursive-import package-name . args) (recursive-import package-name #f #:repo->guix-package (lambda (name repo) - (apply hackage->guix-package (cons name args))) + (apply hackage->guix-package/m + (cons name args))) #:guix-name hackage-name->package-name)) (define (hackage-package? package) diff --git a/tests/hackage.scm b/tests/hackage.scm index e17851a213..0efad0638d 100644 --- a/tests/hackage.scm +++ b/tests/hackage.scm @@ -207,8 +207,41 @@ library #:cabal-environment '(("impl" . "ghc-7.8")))) (test-assert "hackage->guix-package test 6" - (eval-test-with-cabal test-cabal-6 - #:cabal-environment '(("impl" . "ghc-7.8")))) + (mock + ((guix import hackage) hackage-fetch + (lambda (name-version) + (call-with-input-string test-cabal-6 + read-cabal))) + (match (hackage->guix-package "foo") + (('package + ('name "ghc-foo") + ('version "1.0.0") + ('source + ('origin + ('method 'url-fetch) + ('uri ('string-append + "https://hackage.haskell.org/package/foo/foo-" + 'version + ".tar.gz")) + ('sha256 + ('base32 + (? string? hash))))) + ('build-system 'haskell-build-system) + ('inputs + ('quasiquote + (("ghc-b" ('unquote 'ghc-b)) + ("ghc-http" ('unquote 'ghc-http)) + ("ghc-mtl" ('unquote 'ghc-mtl))))) + ('native-inputs + ('quasiquote + (("ghc-haskell-gi" ('unquote 'ghc-haskell-gi))))) + ('home-page "http://test.org") + ('synopsis (? string?)) + ('description (? string?)) + ('license 'bsd-3)) + #t) + (x + (pk 'fail x #f))))) (test-assert "read-cabal test 1" (match (call-with-input-string test-read-cabal-1 read-cabal) -- cgit v1.2.3 From 2a991f3ae4823730395c7970d83159e93d5f5fe2 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 23 May 2019 21:35:47 +0200 Subject: lzlib: Add 'make-lzip-input-port/compressed'. * guix/lzlib.scm (lzwrite!, make-lzip-input-port/compressed): New procedures. * tests/lzlib.scm ("make-lzip-input-port/compressed"): New test. * guix/tests.scm (%seed): Export. --- guix/lzlib.scm | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ guix/tests.scm | 1 + tests/lzlib.scm | 10 ++++++++ 3 files changed, 82 insertions(+) (limited to 'tests') diff --git a/guix/lzlib.scm b/guix/lzlib.scm index deb900f352..a484315c0e 100644 --- a/guix/lzlib.scm +++ b/guix/lzlib.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019 Pierre Neidhardt +;;; Copyright © 2019 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -23,9 +24,11 @@ #:use-module (ice-9 match) #:use-module (system foreign) #:use-module (guix config) + #:use-module (srfi srfi-11) #:export (lzlib-available? make-lzip-input-port make-lzip-output-port + make-lzip-input-port/compressed call-with-lzip-input-port call-with-lzip-output-port %default-member-length-limit @@ -515,6 +518,24 @@ the end-of-stream has been reached." (loop rd))) read)) +(define (lzwrite! encoder source source-offset source-count + target target-offset target-count) + "Write up to SOURCE-COUNT bytes from SOURCE to ENCODER, and read up to +TARGET-COUNT bytes into TARGET at TARGET-OFFSET. Return two values: the +number of bytes read from SOURCE, and the number of bytes written to TARGET, +possibly zero." + (define read + (if (> (lz-compress-write-size encoder) 0) + (match (lz-compress-write encoder source source-offset source-count) + (0 (lz-compress-finish encoder) 0) + (n n)) + 0)) + + (define written + (lz-compress-read encoder target target-offset target-count)) + + (values read written)) + (define* (lzwrite encoder bv lz-port #:optional (start 0) (count (bytevector-length bv))) "Write up to COUNT bytes from BV at offset START into LZ-PORT. Return @@ -597,6 +618,56 @@ port is closed." (lz-compress-close encoder) (close-port port)))) +(define* (make-lzip-input-port/compressed port + #:key + (level %default-compression-level)) + "Return an input port that compresses data read from PORT, with the given LEVEL. +PORT is automatically closed when the resulting port is closed." + (define encoder (apply lz-compress-open + (car (assoc-ref %compression-levels level)))) + + (define input-buffer (make-bytevector 8192)) + (define input-len 0) + (define input-offset 0) + + (define input-eof? #f) + + (define (read! bv start count) + (cond + (input-eof? + (match (lz-compress-read encoder bv start count) + (0 (if (lz-compress-finished? encoder) + 0 + (read! bv start count))) + (n n))) + ((= input-offset input-len) + (match (get-bytevector-n! port input-buffer 0 + (bytevector-length input-buffer)) + ((? eof-object?) + (set! input-eof? #t) + (lz-compress-finish encoder)) + (count + (set! input-offset 0) + (set! input-len count))) + (read! bv start count)) + (else + (let-values (((read written) + (lzwrite! encoder + input-buffer input-offset + (- input-len input-offset) + bv start count))) + (set! input-offset (+ input-offset read)) + + ;; Make sure we don't return zero except on EOF. + (if (= 0 written) + (read! bv start count) + written))))) + + (make-custom-binary-input-port "lzip-input/compressed" + read! #f #f + (lambda () + (close-port port)))) + (define* (call-with-lzip-input-port port proc) "Call PROC with a port that wraps PORT and decompresses data read from it. PORT is closed upon completion." diff --git a/guix/tests.scm b/guix/tests.scm index 35ebf8464d..66d60e964e 100644 --- a/guix/tests.scm +++ b/guix/tests.scm @@ -33,6 +33,7 @@ #:use-module (web uri) #:export (open-connection-for-tests with-external-store + %seed random-text random-bytevector file=? diff --git a/tests/lzlib.scm b/tests/lzlib.scm index cf53a9417d..543622bb45 100644 --- a/tests/lzlib.scm +++ b/tests/lzlib.scm @@ -108,4 +108,14 @@ (test-assert* "Bytevector of size relative to Lzip internal buffers (1MiB+1)" (compress-and-decompress (random-bytevector (1+ (* 1024 1024))))) +(test-assert "make-lzip-input-port/compressed" + (let* ((len (pk 'len (+ 10 (random 4000 %seed)))) + (data (random-bytevector len)) + (compressed (make-lzip-input-port/compressed + (open-bytevector-input-port data))) + (result (call-with-lzip-input-port compressed + get-bytevector-all))) + (pk (bytevector-length result) (bytevector-length data)) + (bytevector=? result data))) + (test-end) -- cgit v1.2.3 From 4c7ebe318f6b3c4ddadcea8d08c9fb67ac46ec1e Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 23 May 2019 22:11:33 +0200 Subject: utils: Test 'compressed-port' and 'decompressed-port' for both gzip and xz. * tests/utils.scm (test-compression/decompression): New procedure. : Call it for both 'xz and 'gzip. --- tests/utils.scm | 67 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 25 deletions(-) (limited to 'tests') diff --git a/tests/utils.scm b/tests/utils.scm index 3015b21b23..a5141592a8 100644 --- a/tests/utils.scm +++ b/tests/utils.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès ;;; Copyright © 2014 Eric Bavier ;;; Copyright © 2016 Mathieu Lirzin ;;; @@ -174,30 +174,47 @@ (any (compose (negate zero?) cdr waitpid) pids)))) -(test-assert "compressed-port, decompressed-port, non-file" - (let ((data (call-with-input-file (search-path %load-path "guix.scm") - get-bytevector-all))) - (let*-values (((compressed pids1) - (compressed-port 'xz (open-bytevector-input-port data))) - ((decompressed pids2) - (decompressed-port 'xz compressed))) - (and (every (compose zero? cdr waitpid) - (append pids1 pids2)) - (equal? (get-bytevector-all decompressed) data))))) - -(false-if-exception (delete-file temp-file)) -(test-assert "compressed-output-port + decompressed-port" - (let* ((file (search-path %load-path "guix/derivations.scm")) - (data (call-with-input-file file get-bytevector-all)) - (port (open-file temp-file "w0b"))) - (call-with-compressed-output-port 'xz port - (lambda (compressed) - (put-bytevector compressed data))) - (close-port port) - - (bytevector=? data - (call-with-decompressed-port 'xz (open-file temp-file "r0b") - get-bytevector-all)))) +(define (test-compression/decompression method run?) + "Test METHOD, a symbol such as 'gzip. Call RUN? to determine whether to +skip these tests." + (unless (run?) (test-skip 1)) + (test-assert (format #f "compressed-port, decompressed-port, non-file [~a]" + method) + (let ((data (call-with-input-file (search-path %load-path "guix.scm") + get-bytevector-all))) + (let*-values (((compressed pids1) + (compressed-port method (open-bytevector-input-port data))) + ((decompressed pids2) + (decompressed-port method compressed))) + (and (every (compose zero? cdr waitpid) + (pk 'pids method (append pids1 pids2))) + (let ((result (get-bytevector-all decompressed))) + (pk 'len method + (if (bytevector? result) + (bytevector-length result) + result) + (bytevector-length data)) + (equal? result data)))))) + + (false-if-exception (delete-file temp-file)) + (unless (run?) (test-skip 1)) + (test-assert (format #f "compressed-output-port + decompressed-port [~a]" + method) + (let* ((file (search-path %load-path "guix/derivations.scm")) + (data (call-with-input-file file get-bytevector-all)) + (port (open-file temp-file "w0b"))) + (call-with-compressed-output-port method port + (lambda (compressed) + (put-bytevector compressed data))) + (close-port port) + + (bytevector=? data + (call-with-decompressed-port method (open-file temp-file "r0b") + get-bytevector-all))))) + +(for-each test-compression/decompression + '(gzip xz lzip) + (list (const #t) (const #t))) ;; This is actually in (guix store). (test-equal "store-path-package-name" -- cgit v1.2.3 From 4e48923e7523c863996bb616c6abb7e4cb78a3b5 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 23 May 2019 22:14:53 +0200 Subject: utils: Support compression and decompression with lzip. * guix/utils.scm (lzip-port): New procedure. (decompressed-port, compressed-port, compressed-output-port): Add 'lzip case. * tests/utils.scm : Call 'test-compression/decompression' for 'lzip as well. --- guix/utils.scm | 27 ++++++++++++++++++++++----- tests/utils.scm | 3 ++- 2 files changed, 24 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/guix/utils.scm b/guix/utils.scm index ed1a418cca..709cdf9353 100644 --- a/guix/utils.scm +++ b/guix/utils.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès ;;; Copyright © 2013, 2014, 2015 Mark H Weaver ;;; Copyright © 2014 Eric Bavier ;;; Copyright © 2014 Ian Denhardt @@ -169,6 +169,17 @@ buffered data is lost." (close-port out) (loop in (cons child pids))))))))) +(define (lzip-port proc port . args) + "Return the lzip port produced by calling PROC (a symbol) on PORT and ARGS. +Raise an error if lzlib support is missing." + (let* ((lzlib (false-if-exception (resolve-interface '(guix lzlib)))) + (supported? (and lzlib + ((module-ref lzlib 'lzlib-available?))))) + (if supported? + (let ((make-port (module-ref lzlib proc))) + (values (make-port port) '())) + (error "lzip compression not supported" lzlib)))) + (define (decompressed-port compression input) "Return an input port where INPUT is decompressed according to COMPRESSION, a symbol such as 'xz." @@ -177,17 +188,21 @@ a symbol such as 'xz." ('bzip2 (filtered-port `(,%bzip2 "-dc") input)) ('xz (filtered-port `(,%xz "-dc") input)) ('gzip (filtered-port `(,%gzip "-dc") input)) - (else (error "unsupported compression scheme" compression)))) + ('lzip (values (lzip-port 'make-lzip-input-port input) + '())) + (_ (error "unsupported compression scheme" compression)))) (define (compressed-port compression input) - "Return an input port where INPUT is decompressed according to COMPRESSION, + "Return an input port where INPUT is compressed according to COMPRESSION, a symbol such as 'xz." (match compression ((or #f 'none) (values input '())) ('bzip2 (filtered-port `(,%bzip2 "-c") input)) ('xz (filtered-port `(,%xz "-c") input)) ('gzip (filtered-port `(,%gzip "-c") input)) - (else (error "unsupported compression scheme" compression)))) + ('lzip (values (lzip-port 'make-lzip-input-port/compressed input) + '())) + (_ (error "unsupported compression scheme" compression)))) (define (call-with-decompressed-port compression port proc) "Call PROC with a wrapper around PORT, a file port, that decompresses data @@ -244,7 +259,9 @@ program--e.g., '(\"--fast\")." ('bzip2 (filtered-output-port `(,%bzip2 "-c" ,@options) output)) ('xz (filtered-output-port `(,%xz "-c" ,@options) output)) ('gzip (filtered-output-port `(,%gzip "-c" ,@options) output)) - (else (error "unsupported compression scheme" compression)))) + ('lzip (values (lzip-port 'make-lzip-output-port output) + '())) + (_ (error "unsupported compression scheme" compression)))) (define* (call-with-compressed-output-port compression port proc #:key (options '())) diff --git a/tests/utils.scm b/tests/utils.scm index a5141592a8..44861384ab 100644 --- a/tests/utils.scm +++ b/tests/utils.scm @@ -23,6 +23,7 @@ #:use-module (guix utils) #:use-module ((guix store) #:select (%store-prefix store-path-package-name)) #:use-module ((guix search-paths) #:select (string-tokenize*)) + #:use-module ((guix lzlib) #:select (lzlib-available?)) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-64) @@ -214,7 +215,7 @@ skip these tests." (for-each test-compression/decompression '(gzip xz lzip) - (list (const #t) (const #t))) + (list (const #t) (const #t) lzlib-available?)) ;; This is actually in (guix store). (test-equal "store-path-package-name" -- cgit v1.2.3 From 66229b04ae0ee05779b93d77900a062b8e0e8770 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 24 May 2019 08:26:38 +0200 Subject: publish: Add support for lzip. * guix/scripts/publish.scm (show-help, %options): Support '-C METHOD' and '-C METHOD:LEVEL'. (default-compression): New procedure. (bake-narinfo+nar): Add lzip. (nar-response-port): Likewise. (string->compression-type): New procedure. (make-request-handler): Generalize /nar/gzip handler to handle /nar/lzip as well. * tests/publish.scm ("/nar/lzip/*"): New test. ("/*.narinfo with lzip compression"): New test. * doc/guix.texi (Invoking guix publish): Document it. (Requirements): Mention lzlib. --- .dir-locals.el | 2 ++ doc/guix.texi | 25 ++++++++++---- guix/scripts/publish.scm | 84 +++++++++++++++++++++++++++++++++++------------- tests/publish.scm | 36 +++++++++++++++++++++ 4 files changed, 119 insertions(+), 28 deletions(-) (limited to 'tests') diff --git a/.dir-locals.el b/.dir-locals.el index 550e06ef09..f1196fd781 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -53,6 +53,8 @@ (eval . (put 'call-with-decompressed-port 'scheme-indent-function 2)) (eval . (put 'call-with-gzip-input-port 'scheme-indent-function 1)) (eval . (put 'call-with-gzip-output-port 'scheme-indent-function 1)) + (eval . (put 'call-with-lzip-input-port 'scheme-indent-function 1)) + (eval . (put 'call-with-lzip-output-port 'scheme-indent-function 1)) (eval . (put 'signature-case 'scheme-indent-function 1)) (eval . (put 'emacs-batch-eval 'scheme-indent-function 0)) (eval . (put 'emacs-batch-edit-file 'scheme-indent-function 1)) diff --git a/doc/guix.texi b/doc/guix.texi index 98c5d1e91d..340b806962 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -757,6 +757,11 @@ Support for build offloading (@pxref{Daemon Offload Setup}) and @uref{https://github.com/artyom-poptsov/guile-ssh, Guile-SSH}, version 0.10.2 or later. +@item +When @url{https://www.nongnu.org/lzip/lzlib.html, lzlib} is available, lzlib +substitutes can be used and @command{guix publish} can compress substitutes +with lzlib. + @item When @url{http://www.bzip.org, libbz2} is available, @command{guix-daemon} can use it to compress build logs. @@ -9656,12 +9661,20 @@ accept connections from any interface. Change privileges to @var{user} as soon as possible---i.e., once the server socket is open and the signing key has been read. -@item --compression[=@var{level}] -@itemx -C [@var{level}] -Compress data using the given @var{level}. When @var{level} is zero, -disable compression. The range 1 to 9 corresponds to different gzip -compression levels: 1 is the fastest, and 9 is the best (CPU-intensive). -The default is 3. +@item --compression[=@var{method}[:@var{level}]] +@itemx -C [@var{method}[:@var{level}]] +Compress data using the given @var{method} and @var{level}. @var{method} is +one of @code{lzip} and @code{gzip}; when @var{method} is omitted, @code{gzip} +is used. + +When @var{level} is zero, disable compression. The range 1 to 9 corresponds +to different compression levels: 1 is the fastest, and 9 is the best +(CPU-intensive). The default is 3. + +Usually, @code{lzip} compresses noticeably better than @code{gzip} for a small +increase in CPU usage; see +@uref{https://nongnu.org/lzip/lzip_benchmark.html,benchmarks on the lzip Web +page}. Unless @option{--cache} is used, compression occurs on the fly and the compressed streams are not diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index db64d6483e..11e7e985d1 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 David Thompson -;;; Copyright © 2015, 2016, 2017, 2018 Ludovic Courtès +;;; Copyright © 2015, 2016, 2017, 2018, 2019 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -51,6 +51,7 @@ #:use-module (guix store) #:use-module ((guix serialization) #:select (write-file)) #:use-module (guix zlib) + #:autoload (guix lzlib) (lzlib-available?) #:use-module (guix cache) #:use-module (guix ui) #:use-module (guix scripts) @@ -74,8 +75,8 @@ Publish ~a over HTTP.\n") %store-directory) (display (G_ " -u, --user=USER change privileges to USER as soon as possible")) (display (G_ " - -C, --compression[=LEVEL] - compress archives at LEVEL")) + -C, --compression[=METHOD:LEVEL] + compress archives with METHOD at LEVEL")) (display (G_ " -c, --cache=DIRECTORY cache published items to DIRECTORY")) (display (G_ " @@ -121,6 +122,9 @@ Publish ~a over HTTP.\n") %store-directory) ;; Since we compress on the fly, default to fast compression. (compression 'gzip 3)) +(define (default-compression type) + (compression type 3)) + (define (actual-compression item requested) "Return the actual compression used for ITEM, which may be %NO-COMPRESSION if ITEM is already compressed." @@ -153,18 +157,28 @@ if ITEM is already compressed." name))))) (option '(#\C "compression") #f #t (lambda (opt name arg result) - (match (if arg (string->number* arg) 3) - (0 - (alist-cons 'compression %no-compression result)) - (level - (if (zlib-available?) - (alist-cons 'compression - (compression 'gzip level) - result) - (begin - (warning (G_ "zlib support is missing; \ -compression disabled~%")) - result)))))) + (let* ((colon (string-index arg #\:)) + (type (cond + (colon (string-take arg colon)) + ((string->number arg) "gzip") + (else arg))) + (level (if colon + (string->number* + (string-drop arg (+ 1 colon))) + (or (string->number arg) 3)))) + (match level + (0 + (alist-cons 'compression %no-compression result)) + (level + (match (string->compression-type type) + ((? symbol? type) + (alist-cons 'compression + (compression type level) + result)) + (_ + (warning (G_ "~a: unsupported compression type~%") + type) + result))))))) (option '(#\c "cache") #t #f (lambda (opt name arg result) (alist-cons 'cache arg result))) @@ -511,6 +525,13 @@ requested using POOL." #:level (compression-level compression) #:buffer-size (* 128 1024)) (rename-file (string-append nar ".tmp") nar)) + ('lzip + ;; Note: the file port gets closed along with the lzip port. + (call-with-lzip-output-port (open-output-file (string-append nar ".tmp")) + (lambda (port) + (write-file item port)) + #:level (compression-level compression)) + (rename-file (string-append nar ".tmp") nar)) ('none ;; Cache nars even when compression is disabled so that we can ;; guarantee the TTL (see .) @@ -715,6 +736,9 @@ example: \"/foo/bar\" yields '(\"foo\" \"bar\")." (make-gzip-output-port (response-port response) #:level level #:buffer-size (* 64 1024))) + (($ 'lzip level) + (make-lzip-output-port (response-port response) + #:level level)) (($ 'none) (response-port response)) (#f @@ -789,12 +813,23 @@ blocking." http-write (@@ (web server http) http-close)) +(define (string->compression-type string) + "Return a symbol denoting the compression method expressed by STRING; return +#f if STRING doesn't match any supported method." + (match string + ("gzip" (and (zlib-available?) 'gzip)) + ("lzip" (and (lzlib-available?) 'lzip)) + (_ #f))) + (define* (make-request-handler store #:key cache pool narinfo-ttl (nar-path "nar") (compression %no-compression)) + (define compression-type? + string->compression-type) + (define nar-path? (let ((expected (split-and-decode-uri-path nar-path))) (cut equal? expected <>))) @@ -843,13 +878,18 @@ blocking." ;; is restarted with different compression parameters. ;; /nar/gzip/ - ((components ... "gzip" store-item) - (if (and (nar-path? components) (zlib-available?)) - (let ((compression (match compression - (($ 'gzip) - compression) - (_ - %default-gzip-compression)))) + ((components ... (? compression-type? type) store-item) + (if (nar-path? components) + (let* ((compression-type (string->compression-type type)) + (compression (match compression + (($ type) + (if (eq? type compression-type) + compression + (default-compression + compression-type))) + (_ + (default-compression + compression-type))))) (if cache (render-nar/cached store cache request store-item #:ttl narinfo-ttl diff --git a/tests/publish.scm b/tests/publish.scm index 7f44bc700f..80e0977cd5 100644 --- a/tests/publish.scm +++ b/tests/publish.scm @@ -36,6 +36,7 @@ #:use-module (gcrypt pk-crypto) #:use-module ((guix pki) #:select (%public-key-file %private-key-file)) #:use-module (guix zlib) + #:use-module (guix lzlib) #:use-module (web uri) #:use-module (web client) #:use-module (web response) @@ -229,6 +230,19 @@ FileSize: ~a~%" (string-append "/nar/gzip/" (basename %item)))))) (get-bytevector-n nar (bytevector-length %gzip-magic-bytes)))) +(unless (lzlib-available?) + (test-skip 1)) +(test-equal "/nar/lzip/*" + "bar" + (call-with-temporary-output-file + (lambda (temp port) + (let ((nar (http-get-port + (publish-uri + (string-append "/nar/lzip/" (basename %item)))))) + (call-with-lzip-input-port nar + (cut restore-file <> temp))) + (call-with-input-file temp read-string)))) + (unless (zlib-available?) (test-skip 1)) (test-equal "/*.narinfo with compression" @@ -251,6 +265,28 @@ FileSize: ~a~%" (_ #f))) (recutils->alist body))))) +(unless (lzlib-available?) + (test-skip 1)) +(test-equal "/*.narinfo with lzip compression" + `(("StorePath" . ,%item) + ("URL" . ,(string-append "nar/lzip/" (basename %item))) + ("Compression" . "lzip")) + (let ((thread (with-separate-output-ports + (call-with-new-thread + (lambda () + (guix-publish "--port=6790" "-Clzip")))))) + (wait-until-ready 6790) + (let* ((url (string-append "http://localhost:6790/" + (store-path-hash-part %item) ".narinfo")) + (body (http-get-port url))) + (filter (lambda (item) + (match item + (("Compression" . _) #t) + (("StorePath" . _) #t) + (("URL" . _) #t) + (_ #f))) + (recutils->alist body))))) + (unless (zlib-available?) (test-skip 1)) (test-equal "/*.narinfo for a compressed file" -- cgit v1.2.3 From 55c98f3261b6ced2c38e060566e1eb952bd3e42b Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Fri, 31 May 2019 23:22:41 +0200 Subject: tests: hackage: Factor out package pattern. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests/hackage.scm: Import result pattern matching via helper. Signed-off-by: Ludovic Courtès --- tests/hackage.scm | 133 +++++++++++++++++++++++++++--------------------------- 1 file changed, 66 insertions(+), 67 deletions(-) (limited to 'tests') diff --git a/tests/hackage.scm b/tests/hackage.scm index 0efad0638d..41e3b2dcd3 100644 --- a/tests/hackage.scm +++ b/tests/hackage.scm @@ -155,93 +155,92 @@ library (test-begin "hackage") -(define* (eval-test-with-cabal test-cabal #:key (cabal-environment '())) +(define-syntax-rule (define-package-matcher name pattern) + (define* (name obj) + (match obj + (pattern #t) + (x (pk 'fail x #f))))) + +(define-package-matcher match-ghc-foo + ('package + ('name "ghc-foo") + ('version "1.0.0") + ('source + ('origin + ('method 'url-fetch) + ('uri ('string-append + "https://hackage.haskell.org/package/foo/foo-" + 'version + ".tar.gz")) + ('sha256 + ('base32 + (? string? hash))))) + ('build-system 'haskell-build-system) + ('inputs + ('quasiquote + (("ghc-http" ('unquote 'ghc-http)) + ("ghc-mtl" ('unquote 'ghc-mtl))))) + ('home-page "http://test.org") + ('synopsis (? string?)) + ('description (? string?)) + ('license 'bsd-3))) + +(define* (eval-test-with-cabal test-cabal matcher #:key (cabal-environment '())) (mock ((guix import hackage) hackage-fetch (lambda (name-version) (call-with-input-string test-cabal read-cabal))) - (match (hackage->guix-package "foo" #:cabal-environment cabal-environment) - (('package - ('name "ghc-foo") - ('version "1.0.0") - ('source - ('origin - ('method 'url-fetch) - ('uri ('string-append - "https://hackage.haskell.org/package/foo/foo-" - 'version - ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'haskell-build-system) - ('inputs - ('quasiquote - (("ghc-http" ('unquote 'ghc-http)) - ("ghc-mtl" ('unquote 'ghc-mtl))))) - ('home-page "http://test.org") - ('synopsis (? string?)) - ('description (? string?)) - ('license 'bsd-3)) - #t) - (x - (pk 'fail x #f))))) + (matcher (hackage->guix-package "foo" #:cabal-environment cabal-environment)))) (test-assert "hackage->guix-package test 1" - (eval-test-with-cabal test-cabal-1)) + (eval-test-with-cabal test-cabal-1 match-ghc-foo)) (test-assert "hackage->guix-package test 2" - (eval-test-with-cabal test-cabal-2)) + (eval-test-with-cabal test-cabal-2 match-ghc-foo)) (test-assert "hackage->guix-package test 3" - (eval-test-with-cabal test-cabal-3 + (eval-test-with-cabal test-cabal-3 match-ghc-foo #:cabal-environment '(("impl" . "ghc-7.8")))) (test-assert "hackage->guix-package test 4" - (eval-test-with-cabal test-cabal-4 + (eval-test-with-cabal test-cabal-4 match-ghc-foo #:cabal-environment '(("impl" . "ghc-7.8")))) (test-assert "hackage->guix-package test 5" - (eval-test-with-cabal test-cabal-5 + (eval-test-with-cabal test-cabal-5 match-ghc-foo #:cabal-environment '(("impl" . "ghc-7.8")))) +(define-package-matcher match-ghc-foo-6 + ('package + ('name "ghc-foo") + ('version "1.0.0") + ('source + ('origin + ('method 'url-fetch) + ('uri ('string-append + "https://hackage.haskell.org/package/foo/foo-" + 'version + ".tar.gz")) + ('sha256 + ('base32 + (? string? hash))))) + ('build-system 'haskell-build-system) + ('inputs + ('quasiquote + (("ghc-b" ('unquote 'ghc-b)) + ("ghc-http" ('unquote 'ghc-http)) + ("ghc-mtl" ('unquote 'ghc-mtl))))) + ('native-inputs + ('quasiquote + (("ghc-haskell-gi" ('unquote 'ghc-haskell-gi))))) + ('home-page "http://test.org") + ('synopsis (? string?)) + ('description (? string?)) + ('license 'bsd-3))) + (test-assert "hackage->guix-package test 6" - (mock - ((guix import hackage) hackage-fetch - (lambda (name-version) - (call-with-input-string test-cabal-6 - read-cabal))) - (match (hackage->guix-package "foo") - (('package - ('name "ghc-foo") - ('version "1.0.0") - ('source - ('origin - ('method 'url-fetch) - ('uri ('string-append - "https://hackage.haskell.org/package/foo/foo-" - 'version - ".tar.gz")) - ('sha256 - ('base32 - (? string? hash))))) - ('build-system 'haskell-build-system) - ('inputs - ('quasiquote - (("ghc-b" ('unquote 'ghc-b)) - ("ghc-http" ('unquote 'ghc-http)) - ("ghc-mtl" ('unquote 'ghc-mtl))))) - ('native-inputs - ('quasiquote - (("ghc-haskell-gi" ('unquote 'ghc-haskell-gi))))) - ('home-page "http://test.org") - ('synopsis (? string?)) - ('description (? string?)) - ('license 'bsd-3)) - #t) - (x - (pk 'fail x #f))))) + (eval-test-with-cabal test-cabal-6 match-ghc-foo-6)) (test-assert "read-cabal test 1" (match (call-with-input-string test-read-cabal-1 read-cabal) -- cgit v1.2.3 From 4110cde00560bd97cc8d83c34b80c52f37c680a2 Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Fri, 31 May 2019 23:22:42 +0200 Subject: tests: hackage: Don't mock hackage-fetch. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests/hackage.scm: Pass a string input port to tests instead of mocking hackage download. Signed-off-by: Ludovic Courtès --- tests/hackage.scm | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/hackage.scm b/tests/hackage.scm index 41e3b2dcd3..1b4800164e 100644 --- a/tests/hackage.scm +++ b/tests/hackage.scm @@ -186,12 +186,8 @@ library ('license 'bsd-3))) (define* (eval-test-with-cabal test-cabal matcher #:key (cabal-environment '())) - (mock - ((guix import hackage) hackage-fetch - (lambda (name-version) - (call-with-input-string test-cabal - read-cabal))) - (matcher (hackage->guix-package "foo" #:cabal-environment cabal-environment)))) + (define port (open-input-string test-cabal)) + (matcher (hackage->guix-package "foo" #:port port #:cabal-environment cabal-environment))) (test-assert "hackage->guix-package test 1" (eval-test-with-cabal test-cabal-1 match-ghc-foo)) -- cgit v1.2.3 From 0be465924c6f745618a73eea816aa15aba7c8d30 Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Fri, 31 May 2019 23:22:43 +0200 Subject: tests: Indent hackage tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests/hackage.scm: Reindent using etc/indent-code.el. Signed-off-by: Ludovic Courtès --- tests/hackage.scm | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/tests/hackage.scm b/tests/hackage.scm index 1b4800164e..e5f3d6caed 100644 --- a/tests/hackage.scm +++ b/tests/hackage.scm @@ -242,19 +242,19 @@ library (match (call-with-input-string test-read-cabal-1 read-cabal) ((("name" ("test-me")) ('section 'library - (('if ('flag "base4point8") - (("build-depends" ("base >= 4.8 && < 5"))) - (('if ('flag "base4") - (("build-depends" ("base >= 4 && < 4.8"))) - (('if ('flag "base3") - (("build-depends" ("base >= 3 && < 4"))) - (("build-depends" ("base < 3")))))))) - ('if ('or ('flag "base4point8") - ('and ('flag "base4") ('flag "base3"))) - (("build-depends" ("random"))) - ()) - ("build-depends" ("containers")) - ("exposed-modules" ("Test.QuickCheck.Exception"))))) + (('if ('flag "base4point8") + (("build-depends" ("base >= 4.8 && < 5"))) + (('if ('flag "base4") + (("build-depends" ("base >= 4 && < 4.8"))) + (('if ('flag "base3") + (("build-depends" ("base >= 3 && < 4"))) + (("build-depends" ("base < 3")))))))) + ('if ('or ('flag "base4point8") + ('and ('flag "base4") ('flag "base3"))) + (("build-depends" ("random"))) + ()) + ("build-depends" ("containers")) + ("exposed-modules" ("Test.QuickCheck.Exception"))))) #t) (x (pk 'fail x #f)))) -- cgit v1.2.3 From b8fa86adfc01205f1d942af8cb57515eb3726c52 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 30 May 2019 18:36:37 +0200 Subject: publish: '--compression' can be repeated. This allows 'guix publish' to compress and advertise multiple compression methods from which users can choose. * guix/scripts/publish.scm (actual-compression): Rename to... (actual-compressions): ... this. Expect REQUESTED to be a list, and always return a list. (%default-options): Remove 'compression. (store-item->recutils): New procedure. (narinfo-string): Change #:compression to #:compressions (plural). Adjust accordingly. (render-narinfo, render-narinfo/cached): Likewise. (bake-narinfo+nar): Change #:compression to #:compressions. [compressed-nar-size]: New procedure. Call 'compress-nar' for each item returned by 'actual-compressions'. Create a narinfo for each compression. (effective-compression): New procedure. (make-request-handler): Change #:compression to #:compressions. Use 'effective-compression' to determine the applicable compression. (guix-publish): Adjust handling of '--compression'. Print a message for each compression that is enabled. * tests/publish.scm ("/*.narinfo"): Adjust to new narinfo field ordering. ("/*.narinfo with properly encoded '+' sign"): Likewise. ("/*.narinfo with lzip + gzip"): New test. ("with cache, lzip + gzip"): New test. * doc/guix.texi (Invoking guix publish): Document it. --- doc/guix.texi | 5 ++ guix/scripts/publish.scm | 206 ++++++++++++++++++++++++++++------------------- tests/publish.scm | 89 ++++++++++++++++++-- 3 files changed, 211 insertions(+), 89 deletions(-) (limited to 'tests') diff --git a/doc/guix.texi b/doc/guix.texi index 786788bad7..c01eb3a656 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -9685,6 +9685,11 @@ run @command{guix publish} behind a caching proxy, or to use allows @command{guix publish} to add @code{Content-Length} HTTP header to its responses. +This option can be repeated, in which case every substitute gets compressed +using all the selected methods, and all of them are advertised. This is +useful when users may not support all the compression methods: they can select +the one they support. + @item --cache=@var{directory} @itemx -c @var{directory} Cache archives and meta-data (@code{.narinfo} URLs) to @var{directory} diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index c55873db78..b4334b3f16 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -125,11 +125,11 @@ Publish ~a over HTTP.\n") %store-directory) (define (default-compression type) (compression type 3)) -(define (actual-compression item requested) - "Return the actual compression used for ITEM, which may be %NO-COMPRESSION +(define (actual-compressions item requested) + "Return the actual compressions used for ITEM, which may be %NO-COMPRESSION if ITEM is already compressed." (if (compressed-file? item) - %no-compression + (list %no-compression) requested)) (define %options @@ -217,11 +217,6 @@ if ITEM is already compressed." (public-key-file . ,%public-key-file) (private-key-file . ,%private-key-file) - ;; Default to fast & low compression. - (compression . ,(if (zlib-available?) - %default-gzip-compression - %no-compression)) - ;; Default number of workers when caching is enabled. (workers . ,(current-processor-count)) @@ -249,29 +244,40 @@ if ITEM is already compressed." (define base64-encode-string (compose base64-encode string->utf8)) +(define* (store-item->recutils store-item + #:key + (nar-path "nar") + (compression %no-compression) + file-size) + "Return the 'Compression' and 'URL' fields of the narinfo for STORE-ITEM, +with COMPRESSION, starting at NAR-PATH." + (let ((url (encode-and-join-uri-path + `(,@(split-and-decode-uri-path nar-path) + ,@(match compression + (($ 'none) + '()) + (($ type) + (list (symbol->string type)))) + ,(basename store-item))))) + (format #f "URL: ~a~%Compression: ~a~%~@[FileSize: ~a~%~]" + url (compression-type compression) file-size))) + (define* (narinfo-string store store-path key - #:key (compression %no-compression) - (nar-path "nar") file-size) + #:key (compressions (list %no-compression)) + (nar-path "nar") (file-sizes '())) "Generate a narinfo key/value string for STORE-PATH; an exception is raised if STORE-PATH is invalid. Produce a URL that corresponds to COMPRESSION. The narinfo is signed with KEY. NAR-PATH specifies the prefix for nar URLs. -Optionally, FILE-SIZE can specify the size in bytes of the compressed NAR; it -informs the client of how much needs to be downloaded." + +Optionally, FILE-SIZES is a list of compression/integer pairs, where the +integer is size in bytes of the compressed NAR; it informs the client of how +much needs to be downloaded." (let* ((path-info (query-path-info store store-path)) - (compression (actual-compression store-path compression)) - (url (encode-and-join-uri-path - `(,@(split-and-decode-uri-path nar-path) - ,@(match compression - (($ 'none) - '()) - (($ type) - (list (symbol->string type)))) - ,(basename store-path)))) + (compressions (actual-compressions store-path compressions)) (hash (bytevector->nix-base32-string (path-info-hash path-info))) (size (path-info-nar-size path-info)) - (file-size (or file-size - (and (eq? compression %no-compression) size))) + (file-sizes `((,%no-compression . ,size) ,@file-sizes)) (references (string-join (map basename (path-info-references path-info)) " ")) @@ -279,17 +285,21 @@ informs the client of how much needs to be downloaded." (base-info (format #f "\ StorePath: ~a -URL: ~a -Compression: ~a +~{~a~}\ NarHash: sha256:~a NarSize: ~d -References: ~a~%~a" - store-path url - (compression-type compression) - hash size references - (if file-size - (format #f "FileSize: ~a~%" file-size) - ""))) +References: ~a~%" + store-path + (map (lambda (compression) + (let ((size (assoc-ref file-sizes + compression))) + (store-item->recutils store-path + #:file-size size + #:nar-path nar-path + #:compression + compression))) + compressions) + hash size references)) ;; Do not render a "Deriver" or "System" line if we are rendering ;; info for a derivation. (info (if (not deriver) @@ -332,7 +342,7 @@ References: ~a~%~a" %nix-cache-info)))) (define* (render-narinfo store request hash - #:key ttl (compression %no-compression) + #:key ttl (compressions (list %no-compression)) (nar-path "nar")) "Render metadata for the store path corresponding to HASH. If TTL is true, advertise it as the maximum validity period (in seconds) via the @@ -348,7 +358,7 @@ appropriate duration. NAR-PATH specifies the prefix for nar URLs." (cut display (narinfo-string store store-path (%private-key) #:nar-path nar-path - #:compression compression) + #:compressions compressions) <>))))) (define* (nar-cache-file directory item @@ -442,7 +452,7 @@ vanished from the store in the meantime." (apply throw args)))))) (define* (render-narinfo/cached store request hash - #:key ttl (compression %no-compression) + #:key ttl (compressions (list %no-compression)) (nar-path "nar") cache pool) "Respond to the narinfo request for REQUEST. If the narinfo is available in @@ -460,11 +470,12 @@ requested using POOL." (delete-file* nar) (delete-file* mapping))) - (let* ((item (hash-part->path* store hash cache)) - (compression (actual-compression item compression)) - (cached (and (not (string-null? item)) - (narinfo-cache-file cache item - #:compression compression)))) + (let* ((item (hash-part->path* store hash cache)) + (compressions (actual-compressions item compressions)) + (cached (and (not (string-null? item)) + (narinfo-cache-file cache item + #:compression + (first compressions))))) (cond ((string-null? item) (not-found request)) ((file-exists? cached) @@ -488,7 +499,7 @@ requested using POOL." ;; (format #t "baking ~s~%" item) (bake-narinfo+nar cache item #:ttl ttl - #:compression compression + #:compressions compressions #:nar-path nar-path))) (when ttl @@ -535,30 +546,45 @@ requested using POOL." (write-file item port)))))) (define* (bake-narinfo+nar cache item - #:key ttl (compression %no-compression) + #:key ttl (compressions (list %no-compression)) (nar-path "/nar")) "Write the narinfo and nar for ITEM to CACHE." - (let* ((compression (actual-compression item compression)) - (nar (nar-cache-file cache item - #:compression compression)) - (narinfo (narinfo-cache-file cache item - #:compression compression))) - (compress-nar cache item compression) - - (mkdir-p (dirname narinfo)) - (with-atomic-file-output narinfo - (lambda (port) - ;; Open a new connection to the store. We cannot reuse the main - ;; thread's connection to the store since we would end up sending - ;; stuff concurrently on the same channel. - (with-store store - (display (narinfo-string store item - (%private-key) - #:nar-path nar-path - #:compression compression - #:file-size (and=> (stat nar #f) - stat:size)) - port)))))) + (define (compressed-nar-size compression) + (let* ((nar (nar-cache-file cache item #:compression compression)) + (stat (stat nar #f))) + (and stat + (cons compression (stat:size stat))))) + + (let ((compression (actual-compressions item compressions))) + + (for-each (cut compress-nar cache item <>) compressions) + + (match compressions + ((main others ...) + (let ((narinfo (narinfo-cache-file cache item + #:compression main))) + (with-atomic-file-output narinfo + (lambda (port) + ;; Open a new connection to the store. We cannot reuse the main + ;; thread's connection to the store since we would end up sending + ;; stuff concurrently on the same channel. + (with-store store + (let ((sizes (filter-map compressed-nar-size compression))) + (display (narinfo-string store item + (%private-key) + #:nar-path nar-path + #:compressions compressions + #:file-sizes sizes) + port))))) + + ;; Make narinfo files for OTHERS hard links to NARINFO such that the + ;; atime-based cache eviction considers either all the nars or none + ;; of them as candidates. + (for-each (lambda (other) + (let ((other (narinfo-cache-file cache item + #:compression other))) + (link narinfo other))) + others)))))) ;; XXX: Declare the 'X-Nar-Compression' HTTP header, which is in fact for ;; internal consumption: it allows us to pass the compression info to @@ -827,12 +853,22 @@ blocking." ("lzip" (and (lzlib-available?) 'lzip)) (_ #f))) +(define (effective-compression requested-type compressions) + "Given the REQUESTED-TYPE for compression and the set of chosen COMPRESSION +methods, return the applicable compression." + (or (find (match-lambda + (($ type) + (and (eq? type requested-type) + compression))) + compressions) + (default-compression requested-type))) + (define* (make-request-handler store #:key cache pool narinfo-ttl (nar-path "nar") - (compression %no-compression)) + (compressions (list %no-compression))) (define compression-type? string->compression-type) @@ -860,11 +896,11 @@ blocking." #:pool pool #:ttl narinfo-ttl #:nar-path nar-path - #:compression compression) + #:compressions compressions) (render-narinfo store request hash #:ttl narinfo-ttl #:nar-path nar-path - #:compression compression))) + #:compressions compressions))) ;; /nar/file/NAME/sha256/HASH (("file" name "sha256" hash) (guard (c ((invalid-base32-character? c) @@ -885,15 +921,8 @@ blocking." ((components ... (? compression-type? type) store-item) (if (nar-path? components) (let* ((compression-type (string->compression-type type)) - (compression (match compression - (($ type) - (if (eq? type compression-type) - compression - (default-compression - compression-type))) - (_ - (default-compression - compression-type))))) + (compression (effective-compression compression-type + compressions))) (if cache (render-nar/cached store cache request store-item #:ttl narinfo-ttl @@ -917,7 +946,8 @@ blocking." (not-found request)))) (define* (run-publish-server socket store - #:key (compression %no-compression) + #:key + (compressions (list %no-compression)) (nar-path "nar") narinfo-ttl cache pool) (run-server (make-request-handler store @@ -925,7 +955,7 @@ blocking." #:pool pool #:nar-path nar-path #:narinfo-ttl narinfo-ttl - #:compression compression) + #:compressions compressions) concurrent-http-server `(#:socket ,socket))) @@ -964,7 +994,17 @@ blocking." (user (assoc-ref opts 'user)) (port (assoc-ref opts 'port)) (ttl (assoc-ref opts 'narinfo-ttl)) - (compression (assoc-ref opts 'compression)) + (compressions (match (filter-map (match-lambda + (('compression . compression) + compression) + (_ #f)) + opts) + (() + ;; Default to fast & low compression. + (list (if (zlib-available?) + %default-gzip-compression + %no-compression))) + (lst (reverse lst)))) (address (let ((addr (assoc-ref opts 'address))) (make-socket-address (sockaddr:fam addr) (sockaddr:addr addr) @@ -996,9 +1036,11 @@ consider using the '--user' option!~%"))) (inet-ntop (sockaddr:fam address) (sockaddr:addr address)) (sockaddr:port address)) - (when compression - (info (G_ "using '~a' compression method, level ~a~%") - (compression-type compression) (compression-level compression))) + (for-each (lambda (compression) + (info (G_ "using '~a' compression method, level ~a~%") + (compression-type compression) + (compression-level compression))) + compressions) (when repl-port (repl:spawn-server (repl:make-tcp-server-socket #:port repl-port))) @@ -1013,7 +1055,7 @@ consider using the '--user' option!~%"))) #:thread-name "publish worker")) #:nar-path nar-path - #:compression compression + #:compressions compressions #:narinfo-ttl ttl)))))) ;;; Local Variables: diff --git a/tests/publish.scm b/tests/publish.scm index 80e0977cd5..64a8ff3cae 100644 --- a/tests/publish.scm +++ b/tests/publish.scm @@ -138,17 +138,17 @@ "StorePath: ~a URL: nar/~a Compression: none +FileSize: ~a NarHash: sha256:~a NarSize: ~d -References: ~a -FileSize: ~a~%" +References: ~a~%" %item (basename %item) + (path-info-nar-size info) (bytevector->nix-base32-string (path-info-hash info)) (path-info-nar-size info) - (basename (first (path-info-references info))) - (path-info-nar-size info))) + (basename (first (path-info-references info))))) (signature (base64-encode (string->utf8 (canonical-sexp->string @@ -170,15 +170,15 @@ FileSize: ~a~%" "StorePath: ~a URL: nar/~a Compression: none +FileSize: ~a NarHash: sha256:~a NarSize: ~d -References: ~%\ -FileSize: ~a~%" +References: ~%" item (uri-encode (basename item)) + (path-info-nar-size info) (bytevector->nix-base32-string (path-info-hash info)) - (path-info-nar-size info) (path-info-nar-size info))) (signature (base64-encode (string->utf8 @@ -301,6 +301,35 @@ FileSize: ~a~%" (list (assoc-ref info "Compression") (dirname (assoc-ref info "URL"))))) +(unless (and (zlib-available?) (lzlib-available?)) + (test-skip 1)) +(test-equal "/*.narinfo with lzip + gzip" + `((("StorePath" . ,%item) + ("URL" . ,(string-append "nar/gzip/" (basename %item))) + ("Compression" . "gzip") + ("URL" . ,(string-append "nar/lzip/" (basename %item))) + ("Compression" . "lzip")) + 200 + 200) + (call-with-temporary-directory + (lambda (cache) + (let ((thread (with-separate-output-ports + (call-with-new-thread + (lambda () + (guix-publish "--port=6793" "-Cgzip:2" "-Clzip:2")))))) + (wait-until-ready 6793) + (let* ((base "http://localhost:6793/") + (part (store-path-hash-part %item)) + (url (string-append base part ".narinfo")) + (body (http-get-port url))) + (list (take (recutils->alist body) 5) + (response-code + (http-get (string-append base "nar/gzip/" + (basename %item)))) + (response-code + (http-get (string-append base "nar/lzip/" + (basename %item)))))))))) + (test-equal "custom nar path" ;; Serve nars at /foo/bar/chbouib instead of /nar. (list `(("StorePath" . ,%item) @@ -441,6 +470,52 @@ FileSize: ~a~%" (stat:size (stat nar))) (response-code uncompressed))))))))) +(unless (and (zlib-available?) (lzlib-available?)) + (test-skip 1)) +(test-equal "with cache, lzip + gzip" + '(200 200 404) + (call-with-temporary-directory + (lambda (cache) + (let ((thread (with-separate-output-ports + (call-with-new-thread + (lambda () + (guix-publish "--port=6794" "-Cgzip:2" "-Clzip:2" + (string-append "--cache=" cache))))))) + (wait-until-ready 6794) + (let* ((base "http://localhost:6794/") + (part (store-path-hash-part %item)) + (url (string-append base part ".narinfo")) + (nar-url (cute string-append "nar/" <> "/" + (basename %item))) + (cached (cute string-append cache "/" <> "/" + (basename %item) ".narinfo")) + (nar (cute string-append cache "/" <> "/" + (basename %item) ".nar")) + (response (http-get url))) + (wait-for-file (cached "gzip")) + (let* ((body (http-get-port url)) + (narinfo (recutils->alist body)) + (uncompressed (string-append base "nar/" + (basename %item)))) + (and (file-exists? (nar "gzip")) + (file-exists? (nar "lzip")) + (equal? (take (pk 'narinfo/gzip+lzip narinfo) 7) + `(("StorePath" . ,%item) + ("URL" . ,(nar-url "gzip")) + ("Compression" . "gzip") + ("FileSize" . ,(number->string + (stat:size (stat (nar "gzip"))))) + ("URL" . ,(nar-url "lzip")) + ("Compression" . "lzip") + ("FileSize" . ,(number->string + (stat:size (stat (nar "lzip"))))))) + (list (response-code + (http-get (string-append base (nar-url "gzip")))) + (response-code + (http-get (string-append base (nar-url "lzip")))) + (response-code + (http-get uncompressed)))))))))) + (unless (zlib-available?) (test-skip 1)) (let ((item (add-text-to-store %store "fake-compressed-thing.tar.gz" -- cgit v1.2.3 From b90ae065b5a5fab4ed475bf2faa3a84476389a02 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 31 May 2019 16:26:08 +0200 Subject: substitute: Select the best compression methods. When a server publishes several URLs with different compression methods, 'guix substitute' can now choose the best one among the compression methods that it supports. * guix/scripts/substitute.scm ()[uri]: Replace with... [uris]: ... this. [compression]: Replace with... [compressions]: ... this. [file-size]: Replace with... [file-sizes]: ... this. [file-hash]: Replace with... [file-hashes]: ... this. (narinfo-maker): Adjust accordingly. Ensure 'file-sizes' and 'file-hashes' have the right length. (assert-valid-signature, valid-narinfo?): Use the first element of 'narinfo-uris' in error messages. (read-narinfo): Expect "URL", "Compression", "FileSize", and "FileHash" to occur multiple times. (display-narinfo-data): Call 'select-uri' to determine the file size. (%compression-methods): New variable. (supported-compression?, compresses-better?, select-uri): New procedures. (process-substitution): Call 'select-uri' to select the URI and compression. * guix/scripts/weather.scm (report-server-coverage): Account for all the values returned by 'narinfo-file-sizes'. * tests/substitute.scm ("substitute, narinfo with several URLs"): New test. --- guix/scripts/challenge.scm | 4 +- guix/scripts/substitute.scm | 141 ++++++++++++++++++++++++++++++++------------ guix/scripts/weather.scm | 5 +- tests/substitute.scm | 51 +++++++++++++++- 4 files changed, 160 insertions(+), 41 deletions(-) (limited to 'tests') diff --git a/guix/scripts/challenge.scm b/guix/scripts/challenge.scm index 65de42053d..17e87f0291 100644 --- a/guix/scripts/challenge.scm +++ b/guix/scripts/challenge.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2017 Ludovic Courtès +;;; Copyright © 2015, 2016, 2017, 2019 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -192,7 +192,7 @@ inconclusive reports." (report (G_ " no local build for '~a'~%") item)) (for-each (lambda (narinfo) (report (G_ " ~50a: ~a~%") - (uri->string (narinfo-uri narinfo)) + (uri->string (first (narinfo-uris narinfo))) (hash->string (narinfo-hash->sha256 (narinfo-hash narinfo))))) narinfos)) diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 135398ba48..dba08edf50 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -42,6 +42,7 @@ #:use-module (guix progress) #:use-module ((guix build syscalls) #:select (set-thread-name)) + #:autoload (guix lzlib) (lzlib-available?) #:use-module (ice-9 rdelim) #:use-module (ice-9 regex) #:use-module (ice-9 match) @@ -66,11 +67,11 @@ narinfo? narinfo-path - narinfo-uri + narinfo-uris narinfo-uri-base - narinfo-compression - narinfo-file-hash - narinfo-file-size + narinfo-compressions + narinfo-file-hashes + narinfo-file-sizes narinfo-hash narinfo-size narinfo-references @@ -280,15 +281,16 @@ failure, return #f and #f." (define-record-type - (%make-narinfo path uri uri-base compression file-hash file-size nar-hash nar-size - references deriver system signature contents) + (%make-narinfo path uri-base uris compressions file-sizes file-hashes + nar-hash nar-size references deriver system + signature contents) narinfo? (path narinfo-path) - (uri narinfo-uri) - (uri-base narinfo-uri-base) ; URI of the cache it originates from - (compression narinfo-compression) - (file-hash narinfo-file-hash) - (file-size narinfo-file-size) + (uri-base narinfo-uri-base) ;URI of the cache it originates from + (uris narinfo-uris) ;list of strings + (compressions narinfo-compressions) ;list of strings + (file-sizes narinfo-file-sizes) ;list of (integers | #f) + (file-hashes narinfo-file-hashes) (nar-hash narinfo-hash) (nar-size narinfo-size) (references narinfo-references) @@ -334,17 +336,25 @@ s-expression: ~s~%") (define (narinfo-maker str cache-url) "Return a narinfo constructor for narinfos originating from CACHE-URL. STR must contain the original contents of a narinfo file." - (lambda (path url compression file-hash file-size nar-hash nar-size - references deriver system signature) + (lambda (path urls compressions file-hashes file-sizes + nar-hash nar-size references deriver system + signature) "Return a new object." - (%make-narinfo path + (define len (length urls)) + (%make-narinfo path cache-url ;; Handle the case where URL is a relative URL. - (or (string->uri url) - (string->uri (string-append cache-url "/" url))) - cache-url - - compression file-hash - (and=> file-size string->number) + (map (lambda (url) + (or (string->uri url) + (string->uri + (string-append cache-url "/" url)))) + urls) + compressions + (match file-sizes + (() (make-list len #f)) + ((lst ...) (map string->number lst))) + (match file-hashes + (() (make-list len #f)) + ((lst ...) (map string->number lst))) nar-hash (and=> nar-size string->number) (string-tokenize references) @@ -360,7 +370,7 @@ must contain the original contents of a narinfo file." #:optional (acl (current-acl))) "Bail out if SIGNATURE, a canonical sexp representing the signature of NARINFO, doesn't match HASH, a bytevector containing the hash of NARINFO." - (let ((uri (uri->string (narinfo-uri narinfo)))) + (let ((uri (uri->string (first (narinfo-uris narinfo))))) (signature-case (signature hash acl) (valid-signature #t) (invalid-signature @@ -387,7 +397,8 @@ No authentication and authorization checks are performed here!" '("StorePath" "URL" "Compression" "FileHash" "FileSize" "NarHash" "NarSize" "References" "Deriver" "System" - "Signature")))) + "Signature") + '("URL" "Compression" "FileSize" "FileHash")))) (define (narinfo-sha256 narinfo) "Return the sha256 hash of NARINFO as a bytevector, or #f if NARINFO lacks a @@ -414,7 +425,7 @@ No authentication and authorization checks are performed here!" (or %allow-unauthenticated-substitutes? (let ((hash (narinfo-sha256 narinfo)) (signature (narinfo-signature narinfo)) - (uri (uri->string (narinfo-uri narinfo)))) + (uri (uri->string (first (narinfo-uris narinfo))))) (and hash signature (signature-case (signature hash acl) (valid-signature #t) @@ -919,9 +930,11 @@ expected by the daemon." (length (narinfo-references narinfo))) (for-each (cute format #t "~a/~a~%" (%store-prefix) <>) (narinfo-references narinfo)) - (format #t "~a\n~a\n" - (or (narinfo-file-size narinfo) 0) - (or (narinfo-size narinfo) 0))) + + (let-values (((uri compression file-size) (select-uri narinfo))) + (format #t "~a\n~a\n" + (or file-size 0) + (or (narinfo-size narinfo) 0)))) (define* (process-query command #:key cache-urls acl) @@ -947,17 +960,73 @@ authorized substitutes." (wtf (error "unknown `--query' command" wtf)))) +(define %compression-methods + ;; Known compression methods and a thunk to determine whether they're + ;; supported. See 'decompressed-port' in (guix utils). + `(("gzip" . ,(const #t)) + ("lzip" . ,lzlib-available?) + ("xz" . ,(const #t)) + ("bzip2" . ,(const #t)) + ("none" . ,(const #t)))) + +(define (supported-compression? compression) + "Return true if COMPRESSION, a string, denotes a supported compression +method." + (match (assoc-ref %compression-methods compression) + (#f #f) + (supported? (supported?)))) + +(define (compresses-better? compression1 compression2) + "Return true if COMPRESSION1 generally compresses better than COMPRESSION2; +this is a rough approximation." + (match compression1 + ("none" #f) + ("gzip" (string=? compression2 "none")) + (_ (or (string=? compression2 "none") + (string=? compression2 "gzip"))))) + +(define (select-uri narinfo) + "Select the \"best\" URI to download NARINFO's nar, and return three values: +the URI, its compression method (a string), and the compressed file size." + (define choices + (filter (match-lambda + ((uri compression file-size) + (supported-compression? compression))) + (zip (narinfo-uris narinfo) + (narinfo-compressions narinfo) + (narinfo-file-sizes narinfo)))) + + (define (file-size acl))) - (uri (and=> narinfo narinfo-uri))) - (unless uri - (leave (G_ "no valid substitute for '~a'~%") - store-item)) + (define narinfo + (lookup-narinfo cache-urls store-item + (cut valid-narinfo? <> acl))) + + (unless narinfo + (leave (G_ "no valid substitute for '~a'~%") + store-item)) + (let-values (((uri compression file-size) + (select-uri narinfo))) ;; Tell the daemon what the expected hash of the Nar itself is. (format #t "~a~%" (narinfo-hash narinfo)) @@ -971,9 +1040,8 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; DOWNLOAD-SIZE is #f in practice. (fetch uri #:buffered? #f #:timeout? #f)) ((progress) - (let* ((comp (narinfo-compression narinfo)) - (dl-size (or download-size - (and (equal? comp "none") + (let* ((dl-size (or download-size + (and (equal? compression "none") (narinfo-size narinfo)))) (reporter (if print-build-trace? (progress-reporter/trace @@ -989,8 +1057,7 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; NOTE: This 'progress' port of current process will be ;; closed here, while the child process doing the ;; reporting will close it upon exit. - (decompressed-port (and=> (narinfo-compression narinfo) - string->symbol) + (decompressed-port (string->symbol compression) progress))) ;; Unpack the Nar at INPUT into DESTINATION. (restore-file input destination) diff --git a/guix/scripts/weather.scm b/guix/scripts/weather.scm index 78b8674e0c..1701772bc1 100644 --- a/guix/scripts/weather.scm +++ b/guix/scripts/weather.scm @@ -175,7 +175,10 @@ about the derivations queued, as is the case with Hydra." (requested (length items)) (missing (lset-difference string=? items (map narinfo-path narinfos))) - (sizes (filter-map narinfo-file-size narinfos)) + (sizes (append-map (lambda (narinfo) + (filter integer? + (narinfo-file-sizes narinfo))) + narinfos)) (time (+ (time-second time) (/ (time-nanosecond time) 1e9)))) (format #t (G_ " ~2,1f% substitutes available (~h out of ~h)~%") diff --git a/tests/substitute.scm b/tests/substitute.scm index f4f2e9512d..ff2be662be 100644 --- a/tests/substitute.scm +++ b/tests/substitute.scm @@ -28,8 +28,10 @@ #:use-module (guix base32) #:use-module ((guix store) #:select (%store-prefix)) #:use-module ((guix ui) #:select (guix-warning-port)) + #:use-module ((guix utils) #:select (call-with-compressed-output-port)) + #:use-module ((guix lzlib) #:select (lzlib-available?)) #:use-module ((guix build utils) - #:select (mkdir-p delete-file-recursively)) + #:select (mkdir-p delete-file-recursively dump-port)) #:use-module (guix tests http) #:use-module (rnrs bytevectors) #:use-module (rnrs io ports) @@ -475,6 +477,53 @@ System: mips64el-linux\n") "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") "substitute-retrieved")))) +(test-equal "substitute, narinfo with several URLs" + "Substitutable data." + (let ((narinfo (string-append "StorePath: " (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo +URL: example.nar.gz +Compression: gzip +URL: example.nar.lz +Compression: lzip +URL: example.nar +Compression: none +NarHash: sha256:" (bytevector->nix-base32-string + (sha256 (string->utf8 "Substitutable data."))) " +NarSize: 42 +References: bar baz +Deriver: " (%store-prefix) "/foo.drv +System: mips64el-linux\n"))) + (with-narinfo (string-append narinfo "Signature: " + (signature-field narinfo)) + (dynamic-wind + (const #t) + (lambda () + (define (compress input output compression) + (call-with-output-file output + (lambda (port) + (call-with-compressed-output-port compression port + (lambda (port) + (call-with-input-file input + (lambda (input) + (dump-port input port)))))))) + + (let ((nar (string-append %main-substitute-directory + "/example.nar"))) + (compress nar (string-append nar ".gz") 'gzip) + (when (lzlib-available?) + (compress nar (string-append nar ".lz") 'lzip))) + + (parameterize ((substitute-urls + (list (string-append "file://" + %main-substitute-directory)))) + (guix-substitute "--substitute" + (string-append (%store-prefix) + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo") + "substitute-retrieved")) + (call-with-input-file "substitute-retrieved" get-string-all)) + (lambda () + (false-if-exception (delete-file "substitute-retrieved"))))))) + (test-end "substitute") ;;; Local Variables: -- cgit v1.2.3 From ea35f5c599a2fe4d6ab2925b1030f64e8b21e195 Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Sun, 2 Jun 2019 00:16:02 +0200 Subject: tests: Fix hackage tests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a followup to 1cc12357a65e4479c2f4735e915941382ef82d94. * tests/hackage.scm: ghc-mtl is no longer added as an input. Signed-off-by: Ludovic Courtès --- tests/hackage.scm | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/hackage.scm b/tests/hackage.scm index e5f3d6caed..269c1e1f9b 100644 --- a/tests/hackage.scm +++ b/tests/hackage.scm @@ -178,8 +178,7 @@ library ('build-system 'haskell-build-system) ('inputs ('quasiquote - (("ghc-http" ('unquote 'ghc-http)) - ("ghc-mtl" ('unquote 'ghc-mtl))))) + (("ghc-http" ('unquote 'ghc-http))))) ('home-page "http://test.org") ('synopsis (? string?)) ('description (? string?)) @@ -225,8 +224,7 @@ library ('inputs ('quasiquote (("ghc-b" ('unquote 'ghc-b)) - ("ghc-http" ('unquote 'ghc-http)) - ("ghc-mtl" ('unquote 'ghc-mtl))))) + ("ghc-http" ('unquote 'ghc-http))))) ('native-inputs ('quasiquote (("ghc-haskell-gi" ('unquote 'ghc-haskell-gi))))) -- cgit v1.2.3 From 64d31813577b7471f819652e3ec81abb285bb77c Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Sun, 2 Jun 2019 00:27:49 +0200 Subject: tests: hackage: Test multiline cabal description. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests/hackage.scm (test-cabal-multiline-desc): New variable. ("hackage->guix-package test multiline desc"): New test. Signed-off-by: Ludovic Courtès --- tests/hackage.scm | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'tests') diff --git a/tests/hackage.scm b/tests/hackage.scm index 269c1e1f9b..2f45194fab 100644 --- a/tests/hackage.scm +++ b/tests/hackage.scm @@ -236,6 +236,25 @@ library (test-assert "hackage->guix-package test 6" (eval-test-with-cabal test-cabal-6 match-ghc-foo-6)) +;; Check multi-line layouted description +(define test-cabal-multiline-desc + "name: foo +version: 1.0.0 +homepage: http://test.org +synopsis: synopsis +description: first line + second line +license: BSD3 +executable cabal + build-depends: + HTTP >= 4000.2.5 && < 4000.3, + mtl >= 2.0 && < 3 +") + +(test-assert "hackage->guix-package test multiline desc" + (eval-test-with-cabal test-cabal-multiline-desc match-ghc-foo)) + + (test-assert "read-cabal test 1" (match (call-with-input-string test-read-cabal-1 read-cabal) ((("name" ("test-me")) -- cgit v1.2.3 From 959c9d159da2c53b87ae0af1421aecac98b20f46 Mon Sep 17 00:00:00 2001 From: Robert Vollmert Date: Sun, 2 Jun 2019 00:27:50 +0200 Subject: import: hackage: Parse braced properties. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds partial support for Cabal properties that use curly braces instead of the layout rule. See for example https://hackage.haskell.org/package/cassava/ * guix/import/cabal.scm (read-braced-value): New procedure. (is-property): Remove. (is-layout-property, is-braced-property): New variables. (lex-property): Rename to... (lex-layout-property): ... this. (lex-braced-property, lex-property): New procedures. (lex-token): Add call to 'lex-property'. * guix/tests/hackage.scm: Test braced description import. * tests/hackage.scm (test-cabal-multiline-desc): Rename to... (test-cabal-multiline-layout): ... this. ("hackage->guix-package test multiline desc"): Rename to... ("hackage->guix-package test multiline desc (layout)"): ... this. (test-cabal-multiline-braced): New variable. ("hackage->guix-package test multiline desc (braced)"): New test. Signed-off-by: Ludovic Courtès --- guix/import/cabal.scm | 35 ++++++++++++++++++++++++++++------- tests/hackage.scm | 25 ++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 10 deletions(-) (limited to 'tests') diff --git a/guix/import/cabal.scm b/guix/import/cabal.scm index 13c2f3f48c..1a87be0b00 100644 --- a/guix/import/cabal.scm +++ b/guix/import/cabal.scm @@ -270,6 +270,10 @@ following lines with indentation larger than MIN-INDENT." (peek-next-line-indent port))) val))) +(define* (read-braced-value port) + "Read up to a closing brace." + (string-trim-both (read-delimited "}" port 'trim))) + (define (lex-white-space port bol) "Consume white spaces and comment lines on PORT. If a new line is started return #t, otherwise return BOL (beginning-of-line)." @@ -343,8 +347,11 @@ matching a string against the created regexp." (make-regexp pat)))) (cut regexp-exec rx <>))) -(define is-property (make-rx-matcher "([a-z0-9-]+)[ \t]*:[ \t]*(\\w?.*)$" - regexp/icase)) +(define is-layout-property (make-rx-matcher "([a-z0-9-]+)[ \t]*:[ \t]*(\\w?[^{}]*)$" + regexp/icase)) + +(define is-braced-property (make-rx-matcher "([a-z0-9-]+)[ \t]*:[ \t]*\\{[ \t]*$" + regexp/icase)) (define is-flag (make-rx-matcher "^flag +([a-z0-9_-]+)" regexp/icase)) @@ -435,13 +442,19 @@ string with the read characters." (begin (unread-char c) (list->string res))))) (else (list->string res))))) -(define (lex-property k-v-rx-res loc port) +(define (lex-layout-property k-v-rx-res loc port) (let ((key (string-downcase (match:substring k-v-rx-res 1))) (value (match:substring k-v-rx-res 2))) (make-lexical-token 'PROPERTY loc (list key `(,(read-value port value (current-indentation))))))) +(define (lex-braced-property k-rx-res loc port) + (let ((key (string-downcase (match:substring k-rx-res 1)))) + (make-lexical-token + 'PROPERTY loc + (list key `(,(read-braced-value port)))))) + (define (lex-rx-res rx-res token loc) (let ((name (string-downcase (match:substring rx-res 1)))) (make-lexical-token token loc name))) @@ -552,7 +565,6 @@ LOC is the current port location." the current port location." (let* ((s (read-delimited "\n{}" port 'peek))) (cond - ((is-property s) => (cut lex-property <> loc port)) ((is-flag s) => (cut lex-flag <> loc)) ((is-src-repo s) => (cut lex-src-repo <> loc)) ((is-exec s) => (cut lex-exec <> loc)) @@ -561,13 +573,22 @@ the current port location." ((is-benchmark s) => (cut lex-benchmark <> loc)) ((is-lib s) (lex-lib loc)) ((is-else s) (lex-else loc)) - (else - #f)))) + (else (unread-string s port) #f)))) + +(define (lex-property port loc) + (let* ((s (read-delimited "\n" port 'peek))) + (cond + ((is-braced-property s) => (cut lex-braced-property <> loc port)) + ((is-layout-property s) => (cut lex-layout-property <> loc port)) + (else #f)))) (define (lex-token port) (let* ((loc (make-source-location (cabal-file-name) (port-line port) (port-column port) -1 -1))) - (or (lex-single-char port loc) (lex-word port loc) (lex-line port loc)))) + (or (lex-single-char port loc) + (lex-word port loc) + (lex-line port loc) + (lex-property port loc)))) ;; Lexer- and error-function generators diff --git a/tests/hackage.scm b/tests/hackage.scm index 2f45194fab..38a5825af7 100644 --- a/tests/hackage.scm +++ b/tests/hackage.scm @@ -237,7 +237,7 @@ library (eval-test-with-cabal test-cabal-6 match-ghc-foo-6)) ;; Check multi-line layouted description -(define test-cabal-multiline-desc +(define test-cabal-multiline-layout "name: foo version: 1.0.0 homepage: http://test.org @@ -251,9 +251,28 @@ executable cabal mtl >= 2.0 && < 3 ") -(test-assert "hackage->guix-package test multiline desc" - (eval-test-with-cabal test-cabal-multiline-desc match-ghc-foo)) +(test-assert "hackage->guix-package test multiline desc (layout)" + (eval-test-with-cabal test-cabal-multiline-layout match-ghc-foo)) +;; Check multi-line braced description +(define test-cabal-multiline-braced + "name: foo +version: 1.0.0 +homepage: http://test.org +synopsis: synopsis +description: { +first line +second line +} +license: BSD3 +executable cabal + build-depends: + HTTP >= 4000.2.5 && < 4000.3, + mtl >= 2.0 && < 3 +") + +(test-assert "hackage->guix-package test multiline desc (braced)" + (eval-test-with-cabal test-cabal-multiline-braced match-ghc-foo)) (test-assert "read-cabal test 1" (match (call-with-input-string test-read-cabal-1 read-cabal) -- cgit v1.2.3 From c0a4db66976dc63decbd612aafb934f44629e321 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 7 Jun 2019 22:49:47 +0200 Subject: import: print: Honor the outputs of inputs (!). Fixes . Reported by Jesse Gibbons . * guix/import/print.scm (package->code)[package-lists->code]: Preserve OUT in the result. * tests/print.scm (define-with-source): New macro. (pkg): Use it. (pkg-source): New variable. (pkg-with-inputs, pkg-with-inputs-source): New variables. ("simple package"): Refer to 'pkg-source'. ("package with inputs"): New test. --- guix/import/print.scm | 13 +++++++------ tests/print.scm | 48 +++++++++++++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 23 deletions(-) (limited to 'tests') diff --git a/guix/import/print.scm b/guix/import/print.scm index 0bec32c8dc..4c2a91fa4f 100644 --- a/guix/import/print.scm +++ b/guix/import/print.scm @@ -94,12 +94,13 @@ when evaluated." (map (match-lambda ((label pkg . out) (let ((mod (package-module-name pkg))) - (list label - ;; FIXME: using '@ certainly isn't pretty, but it - ;; avoids having to import the individual package - ;; modules. - (list 'unquote - (list '@ mod (variable-name pkg mod))))))) + (cons* label + ;; FIXME: using '@ certainly isn't pretty, but it + ;; avoids having to import the individual package + ;; modules. + (list 'unquote + (list '@ mod (variable-name pkg mod))) + out)))) lsts))) (let ((name (package-name package)) diff --git a/tests/print.scm b/tests/print.scm index 305807c1d1..d4b2cca93f 100644 --- a/tests/print.scm +++ b/tests/print.scm @@ -24,9 +24,31 @@ #:use-module (guix licenses) #:use-module (srfi srfi-64)) +(define-syntax-rule (define-with-source object source expr) + (begin + (define object expr) + (define source 'expr))) + (test-begin "print") -(define pkg +(define-with-source pkg pkg-source + (package + (name "test") + (version "1.2.3") + (source (origin + (method url-fetch) + (uri (string-append "file:///tmp/test-" + version ".tar.gz")) + (sha256 + (base32 + "070pwb7brdcn1mfvplkd56vjc7lbz4iznzkqvfsakvgbv68k71ah")))) + (build-system gnu-build-system) + (home-page "http://gnu.org") + (synopsis "Dummy") + (description "This is a dummy package.") + (license gpl3+))) + +(define-with-source pkg-with-inputs pkg-with-inputs-source (package (name "test") (version "1.2.3") @@ -38,27 +60,19 @@ (base32 "070pwb7brdcn1mfvplkd56vjc7lbz4iznzkqvfsakvgbv68k71ah")))) (build-system gnu-build-system) + (inputs `(("coreutils" ,(@ (gnu packages base) coreutils)) + ("glibc" ,(@ (gnu packages base) glibc) "debug"))) (home-page "http://gnu.org") (synopsis "Dummy") (description "This is a dummy package.") (license gpl3+))) (test-equal "simple package" - (package->code pkg) - '(package - (name "test") - (version "1.2.3") - (source (origin - (method url-fetch) - (uri (string-append "file:///tmp/test-" - version ".tar.gz")) - (sha256 - (base32 - "070pwb7brdcn1mfvplkd56vjc7lbz4iznzkqvfsakvgbv68k71ah")))) - (build-system gnu-build-system) - (home-page "http://gnu.org") - (synopsis "Dummy") - (description "This is a dummy package.") - (license gpl3+))) + pkg-source + (package->code pkg)) + +(test-equal "package with inputs" + pkg-with-inputs-source + (package->code pkg-with-inputs)) (test-end "print") -- cgit v1.2.3 From f8a9f99cd602ce1dc5307cb0c21ae718ad8796bb Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 10 Jun 2019 22:10:21 +0200 Subject: store: 'build-things' accepts derivation/output pairs. This allows callers to request the substitution of a single derivation output. * guix/store.scm (build-things): Accept derivation/output pairs among THINGS. * guix/derivations.scm (build-derivations): Likewise. * tests/store.scm ("substitute + build-things with specific output"): New test. * tests/derivations.scm ("build-derivations with specific output"): New test. * doc/guix.texi (The Store): Adjust accordingly. --- doc/guix.texi | 9 +++++---- guix/derivations.scm | 13 +++++++++---- guix/store.scm | 26 ++++++++++++++++---------- tests/derivations.scm | 22 ++++++++++++++++++++++ tests/store.scm | 20 ++++++++++++++++++++ 5 files changed, 72 insertions(+), 18 deletions(-) (limited to 'tests') diff --git a/doc/guix.texi b/doc/guix.texi index 93bec28fc9..87dc6ea5c5 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -6466,10 +6466,11 @@ path. @var{references} is the list of store paths referred to by the resulting store path. @end deffn -@deffn {Scheme Procedure} build-derivations @var{server} @var{derivations} -Build @var{derivations} (a list of @code{} objects or -derivation paths), and return when the worker is done building them. -Return @code{#t} on success. +@deffn {Scheme Procedure} build-derivations @var{store} @var{derivations} @ + [@var{mode}] +Build @var{derivations}, a list of @code{} objects, @file{.drv} +file names, or derivation/output pairs, using the specified +@var{mode}---@code{(build-mode normal)} by default. @end deffn Note that the @code{(guix monads)} module provides a monad as well as diff --git a/guix/derivations.scm b/guix/derivations.scm index 7a5c3bca94..cad77bdb06 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -982,12 +982,17 @@ recursively." (define* (build-derivations store derivations #:optional (mode (build-mode normal))) - "Build DERIVATIONS, a list of objects or .drv file names, using -the specified MODE." + "Build DERIVATIONS, a list of objects, .drv file names, or +derivation/output pairs, using the specified MODE." (build-things store (map (match-lambda + ((? derivation? drv) + (derivation-file-name drv)) ((? string? file) file) - ((and drv ($ )) - (derivation-file-name drv))) + (((? derivation? drv) . output) + (cons (derivation-file-name drv) + output)) + (((? string? file) . output) + (cons file output))) derivations) mode)) diff --git a/guix/store.scm b/guix/store.scm index 738c0fb5f3..8fa16499f8 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -1211,16 +1211,22 @@ an arbitrary directory layout in the store without creating a derivation." "Build THINGS, a list of store items which may be either '.drv' files or outputs, and return when the worker is done building them. Elements of THINGS that are not derivations can only be substituted and not built locally. -Return #t on success." - (parameterize ((current-store-protocol-version - (store-connection-version store))) - (if (>= (store-connection-minor-version store) 15) - (build store things mode) - (if (= mode (build-mode normal)) - (build/old store things) - (raise (condition (&store-protocol-error - (message "unsupported build mode") - (status 1)))))))))) +Alternately, an element of THING can be a derivation/output name pair, in +which case the daemon will attempt to substitute just the requested output of +the derivation. Return #t on success." + (let ((things (map (match-lambda + ((drv . output) (string-append drv "!" output)) + (thing thing)) + things))) + (parameterize ((current-store-protocol-version + (store-connection-version store))) + (if (>= (store-connection-minor-version store) 15) + (build store things mode) + (if (= mode (build-mode normal)) + (build/old store things) + (raise (condition (&store-protocol-error + (message "unsupported build mode") + (status 1))))))))))) (define-operation (add-temp-root (store-path path)) "Make PATH a temporary root for the duration of the current session. diff --git a/tests/derivations.scm b/tests/derivations.scm index dbb5b584eb..c421d094a4 100644 --- a/tests/derivations.scm +++ b/tests/derivations.scm @@ -787,6 +787,28 @@ (build-derivations store (list drv)) #f))) +(test-assert "build-derivations with specific output" + (with-store store + (let* ((content (random-text)) ;contents of the output + (drv (build-expression->derivation + store "substitute-me" + `(begin ,content (exit 1)) ;would fail + #:outputs '("out" "one" "two") + #:guile-for-build + (package-derivation store %bootstrap-guile))) + (out (derivation->output-path drv))) + (with-derivation-substitute drv content + (set-build-options store #:use-substitutes? #t + #:substitute-urls (%test-substitute-urls)) + (and (has-substitutes? store out) + + ;; Ask for nothing but the "out" output of DRV. + (build-derivations store `((,drv . "out"))) + + (valid-path? store out) + (equal? (pk 'x content) (pk 'y (call-with-input-file out get-string-all))) + ))))) + (test-assert "build-expression->derivation and derivation-prerequisites-to-build" (let ((drv (build-expression->derivation %store "fail" #f))) ;; The only direct dependency is (%guile-for-build) and it's already diff --git a/tests/store.scm b/tests/store.scm index df66feaebb..518750d26a 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -599,6 +599,26 @@ (valid-path? s o) (equal? c (call-with-input-file o get-string-all))))))) +(test-assert "substitute + build-things with specific output" + (with-store s + (let* ((c (random-text)) ;contents of the output + (d (build-expression->derivation + s "substitute-me" `(begin ,c (exit 1)) ;would fail + #:outputs '("out" "one" "two") + #:guile-for-build + (package-derivation s %bootstrap-guile (%current-system)))) + (o (derivation->output-path d))) + (with-derivation-substitute d c + (set-build-options s #:use-substitutes? #t + #:substitute-urls (%test-substitute-urls)) + (and (has-substitutes? s o) + + ;; Ask for nothing but the "out" output of D. + (build-things s `((,(derivation-file-name d) . "out"))) + + (valid-path? s o) + (equal? c (call-with-input-file o get-string-all))))))) + (test-assert "substitute, corrupt output hash" ;; Tweak the substituter into installing a substitute whose hash doesn't ;; match the one announced in the narinfo. The daemon must notice this and -- cgit v1.2.3 From 5a9ef8a960706a55764f5bbc67ac83dd48516016 Mon Sep 17 00:00:00 2001 From: Ivan Petkov Date: Fri, 17 May 2019 00:26:07 -0700 Subject: import: crate: Define dependencies as arguments. * guix/import/crate.scm: (crate-fetch)[input-crates]: Rename to dev-crates. [native-input-crates]: Rename to dev-dep-crates. [inputs]: Rename to cargo-inputs. [native-inputs]: Rename to cargo-development-inputs. (maybe-cargo-inputs, maybe-cargo-development-inputs, maybe-arguments): Add them. (make-crate-sexp)[inputs]: Rename to cargo-inputs. [native-inputs]: Rename to cargo-development-inputs. [maybe-native-inputs, maybe-inputs]: Replace with maybe-arguments. * guix/import/utils.scm: (package-names->package-inputs): Make public. Add docstring. * tests/crate.scm (crate->guix-package): Update the match pattern. Signed-off-by: Chris Marusich --- guix/import/crate.scm | 47 ++++++++++++++++++++++++++++++++++++----------- guix/import/utils.scm | 4 ++++ tests/crate.scm | 4 ++-- 3 files changed, 42 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/guix/import/crate.scm b/guix/import/crate.scm index e0b400d054..9a73d9fe16 100644 --- a/guix/import/crate.scm +++ b/guix/import/crate.scm @@ -65,29 +65,53 @@ (path (string-append "/" version "/dependencies")) (deps-json (json-fetch-alist (string-append crate-url name path))) (deps (assoc-ref deps-json "dependencies")) - (input-crates (filter (crate-kind-predicate "normal") deps)) - (native-input-crates + (dep-crates (filter (crate-kind-predicate "normal") deps)) + (dev-dep-crates (filter (lambda (dep) (not ((crate-kind-predicate "normal") dep))) deps)) - (inputs (crates->inputs input-crates)) - (native-inputs (crates->inputs native-input-crates)) + (cargo-inputs (crates->inputs dep-crates)) + (cargo-development-inputs (crates->inputs dev-dep-crates)) (home-page (match homepage (() repository) (_ homepage)))) (callback #:name name #:version version - #:inputs inputs #:native-inputs native-inputs + #:cargo-inputs cargo-inputs + #:cargo-development-inputs cargo-development-inputs #:home-page home-page #:synopsis synopsis #:description description #:license license))) -(define* (make-crate-sexp #:key name version inputs native-inputs +(define (maybe-cargo-inputs package-names) + (match (package-names->package-inputs package-names) + (() + '()) + ((package-inputs ...) + `((#:cargo-inputs ,package-inputs))))) + +(define (maybe-cargo-development-inputs package-names) + (match (package-names->package-inputs package-names) + (() + '()) + ((package-inputs ...) + `((#:cargo-development-inputs ,package-inputs))))) + +(define (maybe-arguments arguments) + (match arguments + (() + '()) + ((args ...) + `((arguments (,'quasiquote ,args)))))) + +(define* (make-crate-sexp #:key name version cargo-inputs cargo-development-inputs home-page synopsis description license #:allow-other-keys) "Return the `package' s-expression for a rust package with the given NAME, -VERSION, INPUTS, NATIVE-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." +VERSION, CARGO-INPUTS, CARGO-DEVELOPMENT-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, +and LICENSE." (let* ((port (http-fetch (crate-uri name version))) (guix-name (crate-name->package-name name)) - (inputs (map crate-name->package-name inputs)) - (native-inputs (map crate-name->package-name native-inputs)) + (cargo-inputs (map crate-name->package-name cargo-inputs)) + (cargo-development-inputs (map crate-name->package-name + cargo-development-inputs)) (pkg `(package (name ,guix-name) (version ,version) @@ -99,8 +123,9 @@ VERSION, INPUTS, NATIVE-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." (base32 ,(bytevector->nix-base32-string (port-sha256 port)))))) (build-system cargo-build-system) - ,@(maybe-native-inputs native-inputs "src") - ,@(maybe-inputs inputs "src") + ,@(maybe-arguments (append (maybe-cargo-inputs cargo-inputs) + (maybe-cargo-development-inputs + cargo-development-inputs))) (home-page ,(match home-page (() "") (_ home-page))) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 63fc9bbb27..84503ab907 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -52,6 +52,7 @@ url-fetch guix-hash-url + package-names->package-inputs maybe-inputs maybe-native-inputs package->definition @@ -236,6 +237,9 @@ into a proper sentence and by using two spaces between sentences." cleaned 'pre ". " 'post))) (define* (package-names->package-inputs names #:optional (output #f)) + "Given a list of PACKAGE-NAMES, and an optional OUTPUT, tries to generate a +quoted list of inputs, as suitable to use in an 'inputs' field of a package +definition." (map (lambda (input) (cons* input (list 'unquote (string->symbol input)) (or (and output (list output)) diff --git a/tests/crate.scm b/tests/crate.scm index a1dcfd5e52..a4a328d507 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -89,9 +89,9 @@ ('base32 (? string? hash))))) ('build-system 'cargo-build-system) - ('inputs + ('arguments ('quasiquote - (("rust-bar" ('unquote 'rust-bar) "src")))) + (('#:cargo-inputs (("rust-bar" ('unquote rust-bar))))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") -- cgit v1.2.3