summaryrefslogtreecommitdiff
path: root/guix
diff options
context:
space:
mode:
Diffstat (limited to 'guix')
-rw-r--r--guix/build-system/cargo.scm2
-rw-r--r--guix/build-system/gnu.scm18
-rw-r--r--guix/build/copy-build-system.scm2
-rw-r--r--guix/build/emacs-build-system.scm16
-rw-r--r--guix/build/gnu-build-system.scm181
-rw-r--r--guix/build/gremlin.scm117
-rw-r--r--guix/build/lisp-utils.scm2
-rw-r--r--guix/build/maven/pom.scm2
-rw-r--r--guix/build/meson-build-system.scm2
-rw-r--r--guix/build/minify-build-system.scm11
-rw-r--r--guix/build/python-build-system.scm70
-rw-r--r--guix/build/rpath.scm59
-rw-r--r--guix/build/ruby-build-system.scm25
-rw-r--r--guix/build/texlive-build-system.scm12
-rw-r--r--guix/build/utils.scm169
-rw-r--r--guix/gexp.scm29
-rw-r--r--guix/packages.scm100
-rw-r--r--guix/scripts/pack.scm27
-rw-r--r--guix/store/roots.scm2
-rw-r--r--guix/utils.scm33
20 files changed, 559 insertions, 320 deletions
diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm
index 6c8edf6bac..ed69746a3b 100644
--- a/guix/build-system/cargo.scm
+++ b/guix/build-system/cargo.scm
@@ -217,7 +217,7 @@ any dependent crates. This can be a benefits:
- It avoids waiting for quadratic builds from source: cargo always builds
dependencies within the current workspace. This is largely due to Rust not
having a stable ABI and other resolutions that cargo applies. This means that
- if we have a depencency chain of X -> Y -> Z and we build each definition
+ if we have a dependency chain of X -> Y -> Z and we build each definition
independently the following will happen:
* Cargo will build and test crate Z
* Cargo will build crate Z in Y's workspace, then build and test Y
diff --git a/guix/build-system/gnu.scm b/guix/build-system/gnu.scm
index 6b481ad45c..abff7d6998 100644
--- a/guix/build-system/gnu.scm
+++ b/guix/build-system/gnu.scm
@@ -215,7 +215,7 @@ use `--strip-all' as the arguments to `strip'."
(arguments
(let ((a (default-keyword-arguments (package-arguments p)
'(#:configure-flags '()
- #:strip-flags '("--strip-debug")))))
+ #:strip-flags '("--strip-unneeded")))))
(substitute-keyword-arguments a
((#:configure-flags flags)
`(cons* "--disable-shared" "LDFLAGS=-static" ,flags))
@@ -324,10 +324,15 @@ standard packages used as implicit inputs of the GNU build system."
;; Regexp matching license files.
"^(COPYING.*|LICEN[CS]E.*|[Ll]icen[cs]e.*|Copy[Rr]ight(\\.(txt|md))?)$")
+(define %bootstrap-scripts
+ ;; Typical names of Autotools "bootstrap" scripts.
+ '("bootstrap" "bootstrap.sh" "autogen.sh"))
+
(define* (gnu-build store name input-drvs
#:key (guile #f)
(outputs '("out"))
(search-paths '())
+ (bootstrap-scripts (list 'quote %bootstrap-scripts))
(configure-flags ''())
(make-flags ''())
(out-of-source? #f)
@@ -337,11 +342,12 @@ standard packages used as implicit inputs of the GNU build system."
(parallel-tests? #t)
(patch-shebangs? #t)
(strip-binaries? #t)
- (strip-flags ''("--strip-debug"
+ (strip-flags ''("--strip-unneeded"
"--enable-deterministic-archives"))
(strip-directories ''("lib" "lib64" "libexec"
"bin" "sbin"))
(validate-runpath? #t)
+ (make-dynamic-linker-cache? #t)
(license-file-regexp %license-file-regexp)
(phases '%standard-phases)
(locale "en_US.utf8")
@@ -400,6 +406,7 @@ packages that must not be referenced."
search-paths)
#:phases ,phases
#:locale ,locale
+ #:bootstrap-scripts ,bootstrap-scripts
#:configure-flags ,configure-flags
#:make-flags ,make-flags
#:out-of-source? ,out-of-source?
@@ -410,6 +417,7 @@ packages that must not be referenced."
#:patch-shebangs? ,patch-shebangs?
#:strip-binaries? ,strip-binaries?
#:validate-runpath? ,validate-runpath?
+ #:make-dynamic-linker-cache? ,make-dynamic-linker-cache?
#:license-file-regexp ,license-file-regexp
#:strip-flags ,strip-flags
#:strip-directories ,strip-directories)))
@@ -484,6 +492,7 @@ is one of `host' or `target'."
(search-paths '())
(native-search-paths '())
+ (bootstrap-scripts (list 'quote %bootstrap-scripts))
(configure-flags ''())
(make-flags ''())
(out-of-source? #f)
@@ -492,11 +501,12 @@ is one of `host' or `target'."
(parallel-build? #t) (parallel-tests? #t)
(patch-shebangs? #t)
(strip-binaries? #t)
- (strip-flags ''("--strip-debug"
+ (strip-flags ''("--strip-unneeded"
"--enable-deterministic-archives"))
(strip-directories ''("lib" "lib64" "libexec"
"bin" "sbin"))
(validate-runpath? #t)
+ (make-dynamic-linker-cache? #t)
(license-file-regexp %license-file-regexp)
(phases '%standard-phases)
(locale "en_US.utf8")
@@ -567,6 +577,7 @@ platform."
native-search-paths)
#:phases ,phases
#:locale ,locale
+ #:bootstrap-scripts ,bootstrap-scripts
#:configure-flags ,configure-flags
#:make-flags ,make-flags
#:out-of-source? ,out-of-source?
@@ -577,6 +588,7 @@ platform."
#:patch-shebangs? ,patch-shebangs?
#:strip-binaries? ,strip-binaries?
#:validate-runpath? ,validate-runpath?
+ #:make-dynamic-linker-cache? ,make-dynamic-linker-cache?
#:license-file-regexp ,license-file-regexp
#:strip-flags ,strip-flags
#:strip-directories ,strip-directories))))
diff --git a/guix/build/copy-build-system.scm b/guix/build/copy-build-system.scm
index a86f0cde29..ac4a62a074 100644
--- a/guix/build/copy-build-system.scm
+++ b/guix/build/copy-build-system.scm
@@ -58,7 +58,7 @@ In the above, FILTERS are optional.
one of the elements in the list.
- With `#:include-regexp`, install subpaths matching the regexps in the list.
- The `#:exclude*` FILTERS work similarly. Without `#:include*` flags,
- install every subpath but the files matching the `#:exlude*` filters.
+ install every subpath but the files matching the `#:exclude*` filters.
If both `#:include*` and `#:exclude*` are specified, the exclusion is done
on the inclusion list.
diff --git a/guix/build/emacs-build-system.scm b/guix/build/emacs-build-system.scm
index 26ea59bc25..79a1a42c4a 100644
--- a/guix/build/emacs-build-system.scm
+++ b/guix/build/emacs-build-system.scm
@@ -105,23 +105,9 @@ environment variable\n" source-directory)))
"Substitute the absolute \"/bin/\" directory with the right location in the
store in '.el' files."
- (define (file-contains-nul-char? file)
- (call-with-input-file file
- (lambda (in)
- (let loop ((line (read-line in 'concat)))
- (cond
- ((eof-object? line) #f)
- ((string-index line #\nul) #t)
- (else (loop (read-line in 'concat))))))
- #:binary #t))
-
(let* ((out (assoc-ref outputs "out"))
(site-lisp (string-append out %install-dir))
- ;; (ice-9 regex) uses libc's regexp routines, which cannot deal with
- ;; strings containing NULs. Filter out such files. TODO: Remove
- ;; this workaround when <https://bugs.gnu.org/30116> is fixed.
- (el-files (remove file-contains-nul-char?
- (find-files (getcwd) "\\.el$"))))
+ (el-files (find-files (getcwd) "\\.el$")))
(define (substitute-program-names)
(substitute* el-files
(("\"/bin/([^.]\\S*)\"" _ cmd-name)
diff --git a/guix/build/gnu-build-system.scm b/guix/build/gnu-build-system.scm
index 2e7dff2034..e556457db9 100644
--- a/guix/build/gnu-build-system.scm
+++ b/guix/build/gnu-build-system.scm
@@ -1,5 +1,5 @@
;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2018 Mark H Weaver <mhw@netris.org>
;;; Copyright © 2020 Brendan Tildesley <mail@brendan.scot>
;;;
@@ -57,8 +57,7 @@
"Set the 'SOURCE_DATE_EPOCH' environment variable. This is used by tools
that incorporate timestamps as a way to tell them to use a fixed timestamp.
See https://reproducible-builds.org/specs/source-date-epoch/."
- (setenv "SOURCE_DATE_EPOCH" "1")
- #t)
+ (setenv "SOURCE_DATE_EPOCH" "1"))
(define (first-subdirectory directory)
"Return the file name of the first sub-directory of DIRECTORY."
@@ -73,7 +72,9 @@ See https://reproducible-builds.org/specs/source-date-epoch/."
(search-paths '()) (native-search-paths '())
#:allow-other-keys)
(define input-directories
- (match inputs
+ ;; The "source" input can be a directory, but we don't want it for search
+ ;; paths. See <https://issues.guix.gnu.org/44924>.
+ (match (alist-delete "source" inputs)
(((_ . dir) ...)
dir)))
@@ -113,9 +114,7 @@ See https://reproducible-builds.org/specs/source-date-epoch/."
#:separator separator
#:type type
#:pattern pattern)))
- native-search-paths))
-
- #t)
+ native-search-paths)))
(define* (install-locale #:key
(locale "en_US.utf8")
@@ -134,15 +133,13 @@ chance to be set."
(setenv (locale-category->string locale-category) locale)
(format (current-error-port) "using '~a' locale for category ~s~%"
- locale (locale-category->string locale-category))
- #t)
+ locale (locale-category->string locale-category)))
(lambda args
;; This is known to fail for instance in early bootstrap where locales
;; are not available.
(format (current-error-port)
"warning: failed to install '~a' locale: ~a~%"
- locale (strerror (system-error-errno args)))
- #t)))
+ locale (strerror (system-error-errno args))))))
(define* (unpack #:key source #:allow-other-keys)
"Unpack SOURCE in the working directory, and change directory within the
@@ -161,14 +158,9 @@ working directory."
(if (string-suffix? ".zip" source)
(invoke "unzip" source)
(invoke "tar" "xvf" source))
- (chdir (first-subdirectory "."))))
- #t)
-
-(define %bootstrap-scripts
- ;; Typical names of Autotools "bootstrap" scripts.
- '("bootstrap" "bootstrap.sh" "autogen.sh"))
+ (chdir (first-subdirectory ".")))))
-(define* (bootstrap #:key (bootstrap-scripts %bootstrap-scripts)
+(define* (bootstrap #:key bootstrap-scripts
#:allow-other-keys)
"If the code uses Autotools and \"configure\" is missing, run
\"autoreconf\". Otherwise do nothing."
@@ -205,8 +197,7 @@ working directory."
(invoke "autoreconf" "-vif")
(format #t "no 'configure.ac' or anything like that, \
doing nothing~%"))))
- (format #t "GNU build system bootstrapping not needed~%"))
- #t)
+ (format #t "GNU build system bootstrapping not needed~%")))
;; See <http://bugs.gnu.org/17840>.
(define* (patch-usr-bin-file #:key native-inputs inputs
@@ -220,8 +211,7 @@ things like the ABI being used."
(for-each (lambda (file)
(when (executable-file? file)
(patch-/usr/bin/file file)))
- (find-files "." "^configure$")))
- #t)
+ (find-files "." "^configure$"))))
(define* (patch-source-shebangs #:key source #:allow-other-keys)
"Patch shebangs in all source files; this includes non-executable
@@ -233,8 +223,7 @@ $CONFIG_SHELL, but some don't, such as `mkinstalldirs' or Automake's
(lambda (file stat)
;; Filter out symlinks.
(eq? 'regular (stat:type stat)))
- #:stat lstat))
- #t)
+ #:stat lstat)))
(define (patch-generated-file-shebangs . rest)
"Patch shebangs in generated files, including `SHELL' variables in
@@ -249,9 +238,7 @@ makefiles."
#:stat lstat))
;; Patch `SHELL' in generated makefiles.
- (for-each patch-makefile-SHELL (find-files "." "^(GNU)?[mM]akefile$"))
-
- #t)
+ (for-each patch-makefile-SHELL (find-files "." "^(GNU)?[mM]akefile$")))
(define* (configure #:key build target native-inputs inputs outputs
(configure-flags '()) out-of-source?
@@ -381,8 +368,7 @@ makefiles."
`("-j" ,(number->string (parallel-job-count)))
'())
,@make-flags)))
- (format #t "test suite not run~%"))
- #t)
+ (format #t "test suite not run~%")))
(define* (install #:key (make-flags '()) #:allow-other-keys)
(apply invoke "make" "install" make-flags))
@@ -415,8 +401,7 @@ makefiles."
(for-each (lambda (dir)
(let ((files (list-of-files dir)))
(for-each (cut patch-shebang <> path) files)))
- output-bindirs)))
- #t)
+ output-bindirs))))
(define* (strip #:key target outputs (strip-binaries? #t)
(strip-command (if target
@@ -425,7 +410,7 @@ makefiles."
(objcopy-command (if target
(string-append target "-objcopy")
"objcopy"))
- (strip-flags '("--strip-debug"
+ (strip-flags '("--strip-unneeded"
"--enable-deterministic-archives"))
(strip-directories '("lib" "lib64" "libexec"
"bin" "sbin"))
@@ -514,8 +499,7 @@ makefiles."
(let ((sub (string-append dir "/" d)))
(and (directory-exists? sub) sub)))
strip-directories)))
- outputs)))
- #t)
+ outputs))))
(define* (validate-runpath #:key
(validate-runpath? #t)
@@ -560,9 +544,7 @@ phase after stripping."
outputs)))
(unless (every* validate dirs)
(error "RUNPATH validation failed")))
- (format (current-error-port) "skipping RUNPATH validation~%"))
-
- #t)
+ (format (current-error-port) "skipping RUNPATH validation~%")))
(define* (validate-documentation-location #:key outputs
#:allow-other-keys)
@@ -582,8 +564,7 @@ and 'man/'. This phase moves directories to the right place if needed."
(match outputs
(((names . directories) ...)
- (for-each validate-output directories)))
- #t)
+ (for-each validate-output directories))))
(define* (reset-gzip-timestamps #:key outputs #:allow-other-keys)
"Reset embedded timestamps in gzip files found in OUTPUTS."
@@ -599,8 +580,7 @@ and 'man/'. This phase moves directories to the right place if needed."
(match outputs
(((names . directories) ...)
- (for-each process-directory directories)))
- #t)
+ (for-each process-directory directories))))
(define* (compress-documentation #:key outputs
(compress-documentation? #t)
@@ -616,7 +596,7 @@ DOCUMENTATION-COMPRESSOR-FLAGS."
(let ((target (readlink link)))
(delete-file link)
(symlink (string-append target compressed-documentation-extension)
- link)))
+ (string-append link compressed-documentation-extension))))
(define (has-links? file)
;; Return #t if FILE has hard links.
@@ -679,8 +659,7 @@ DOCUMENTATION-COMPRESSOR-FLAGS."
(match outputs
(((names . directories) ...)
(for-each maybe-compress directories)))
- (format #t "not compressing documentation~%"))
- #t)
+ (format #t "not compressing documentation~%")))
(define* (delete-info-dir-file #:key outputs #:allow-other-keys)
"Delete any 'share/info/dir' file from OUTPUTS."
@@ -689,8 +668,7 @@ DOCUMENTATION-COMPRESSOR-FLAGS."
(let ((info-dir-file (string-append directory "/share/info/dir")))
(when (file-exists? info-dir-file)
(delete-file info-dir-file)))))
- outputs)
- #t)
+ outputs))
(define* (patch-dot-desktop-files #:key outputs inputs #:allow-other-keys)
@@ -730,8 +708,74 @@ which cannot be found~%"
(("^TryExec=([^/[:blank:]\r\n]*)(.*)$" _ binary rest)
(string-append "TryExec="
(which binary) rest)))))))))
- outputs)
- #t)
+ outputs))
+
+(define* (make-dynamic-linker-cache #:key outputs
+ (make-dynamic-linker-cache? #t)
+ #:allow-other-keys)
+ "Create a dynamic linker cache under 'etc/ld.so.cache' in each of the
+OUTPUTS. This reduces application startup time by avoiding the 'stat' storm
+that traversing all the RUNPATH entries entails."
+ (define (make-cache-for-output directory)
+ (define bin-directories
+ (filter-map (lambda (sub-directory)
+ (let ((directory (string-append directory "/"
+ sub-directory)))
+ (and (directory-exists? directory)
+ directory)))
+ '("bin" "sbin" "libexec")))
+
+ (define programs
+ ;; Programs that can benefit from the ld.so cache.
+ (append-map (lambda (directory)
+ (if (directory-exists? directory)
+ (find-files directory
+ (lambda (file stat)
+ (and (executable-file? file)
+ (elf-file? file))))
+ '()))
+ bin-directories))
+
+ (define library-path
+ ;; Directories containing libraries that PROGRAMS depend on,
+ ;; recursively.
+ (delete-duplicates
+ (append-map (lambda (program)
+ (map dirname (file-needed/recursive program)))
+ programs)))
+
+ (define cache-file
+ (string-append directory "/etc/ld.so.cache"))
+
+ (define ld.so.conf
+ (string-append (or (getenv "TMPDIR") "/tmp")
+ "/ld.so.conf"))
+
+ (unless (null? library-path)
+ (mkdir-p (dirname cache-file))
+ (guard (c ((invoke-error? c)
+ ;; Do not treat 'ldconfig' failure as an error.
+ (format (current-error-port)
+ "warning: 'ldconfig' failed:~%")
+ (report-invoke-error c (current-error-port))))
+ ;; Create a config file to tell 'ldconfig' where to look for the
+ ;; libraries that PROGRAMS need.
+ (call-with-output-file ld.so.conf
+ (lambda (port)
+ (for-each (lambda (directory)
+ (display directory port)
+ (newline port))
+ library-path)))
+
+ (invoke "ldconfig" "-f" ld.so.conf "-C" cache-file)
+ (format #t "created '~a' from ~a library search path entries~%"
+ cache-file (length library-path)))))
+
+ (if make-dynamic-linker-cache?
+ (match outputs
+ (((_ . directories) ...)
+ (for-each make-cache-for-output directories)))
+ (format #t "ld.so cache not built~%")))
(define %license-file-regexp
;; Regexp matching license files.
@@ -796,8 +840,7 @@ which cannot be found~%"
package))
(map (cut string-append source "/" <>) files)))
(format (current-error-port)
- "failed to find license files~%"))
- #t))
+ "failed to find license files~%"))))
(define %standard-phases
;; Standard build phases, as a list of symbol/procedure pairs.
@@ -813,6 +856,7 @@ which cannot be found~%"
validate-documentation-location
delete-info-dir-file
patch-dot-desktop-files
+ make-dynamic-linker-cache
install-license-files
reset-gzip-timestamps
compress-documentation)))
@@ -840,26 +884,17 @@ in order. Return #t if all the PHASES succeeded, #f otherwise."
(exit 1)))
;; The trick is to #:allow-other-keys everywhere, so that each procedure in
;; PHASES can pick the keyword arguments it's interested in.
- (every (match-lambda
- ((name . proc)
- (let ((start (current-time time-monotonic)))
- (format #t "starting phase `~a'~%" name)
- (let ((result (apply proc args))
- (end (current-time time-monotonic)))
- (format #t "phase `~a' ~:[failed~;succeeded~] after ~,1f seconds~%"
- name result
- (elapsed-time end start))
-
- ;; Issue a warning unless the result is #t.
- (unless (eqv? result #t)
- (format (current-error-port) "\
-## WARNING: phase `~a' returned `~s'. Return values other than #t
-## are deprecated. Please migrate this package so that its phase
-## procedures report errors by raising an exception, and otherwise
-## always return #t.~%"
- name result))
-
- ;; Dump the environment variables as a shell script, for handy debugging.
- (system "export > $NIX_BUILD_TOP/environment-variables")
- result))))
- phases)))
+ (for-each (match-lambda
+ ((name . proc)
+ (let ((start (current-time time-monotonic)))
+ (format #t "starting phase `~a'~%" name)
+ (let ((result (apply proc args))
+ (end (current-time time-monotonic)))
+ (format #t "phase `~a' ~:[failed~;succeeded~] after ~,1f seconds~%"
+ name result
+ (elapsed-time end start))
+
+ ;; Dump the environment variables as a shell script, for handy debugging.
+ (system "export > $NIX_BUILD_TOP/environment-variables")
+ result))))
+ phases)))
diff --git a/guix/build/gremlin.scm b/guix/build/gremlin.scm
index e8ea66dfb3..a2d2169ddc 100644
--- a/guix/build/gremlin.scm
+++ b/guix/build/gremlin.scm
@@ -1,5 +1,5 @@
;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2015, 2018 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2015, 2018, 2020 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -41,6 +41,17 @@
elf-dynamic-info-runpath
expand-origin
+ file-dynamic-info
+ file-runpath
+ file-needed
+ file-needed/recursive
+
+ missing-runpath-error?
+ missing-runpath-error-file
+ runpath-too-long-error?
+ runpath-too-long-error-file
+ set-file-runpath
+
validate-needed-in-runpath
strip-runpath))
@@ -232,6 +243,63 @@ string table if the type is a string."
dynamic-entry-value))
'()))))))
+(define (file-dynamic-info file)
+ "Return the <elf-dynamic-info> record of FILE, or #f if FILE lacks dynamic
+info."
+ (call-with-input-file file
+ (lambda (port)
+ (elf-dynamic-info (parse-elf (get-bytevector-all port))))))
+
+(define (file-runpath file)
+ "Return the DT_RUNPATH dynamic entry of FILE as a list of strings, or #f if
+FILE lacks dynamic info."
+ (and=> (file-dynamic-info file) elf-dynamic-info-runpath))
+
+(define (file-needed file)
+ "Return the list of DT_NEEDED dynamic entries of FILE, or #f if FILE lacks
+dynamic info."
+ (and=> (file-dynamic-info file) elf-dynamic-info-needed))
+
+(define (file-needed/recursive file)
+ "Return two values: the list of absolute .so file names FILE depends on,
+recursively, and the list of .so file names that could not be found. File
+names are resolved by searching the RUNPATH of the file that NEEDs them.
+
+This is similar to the info returned by the 'ldd' command."
+ (let loop ((files (list file))
+ (result '())
+ (not-found '()))
+ (match files
+ (()
+ (values (reverse result)
+ (reverse (delete-duplicates not-found))))
+ ((file . rest)
+ (match (file-dynamic-info file)
+ (#f
+ (loop rest result not-found))
+ (info
+ (let ((runpath (elf-dynamic-info-runpath info))
+ (needed (elf-dynamic-info-needed info)))
+ (if (and runpath needed)
+ (let* ((runpath (map (cute expand-origin <> (dirname file))
+ runpath))
+ (resolved (map (cut search-path runpath <>)
+ needed))
+ (failed (filter-map (lambda (needed resolved)
+ (and (not resolved)
+ (not (libc-library? needed))
+ needed))
+ needed resolved))
+ (needed (remove (lambda (value)
+ (or (not value)
+ ;; XXX: quadratic
+ (member value result)))
+ resolved)))
+ (loop (append rest needed)
+ (append needed result)
+ (append failed not-found)))
+ (loop rest result not-found)))))))))
+
(define %libc-libraries
;; List of libraries as of glibc 2.21 (there are more but those are
;; typically mean to be LD_PRELOADed and thus do not appear as NEEDED.)
@@ -364,4 +432,49 @@ according to DT_NEEDED."
(false-if-exception (close-port port))
(apply throw key args))))
-;;; gremlin.scm ends here
+
+(define-condition-type &missing-runpath-error &elf-error
+ missing-runpath-error?
+ (file missing-runpath-error-file))
+
+(define-condition-type &runpath-too-long-error &elf-error
+ runpath-too-long-error?
+ (file runpath-too-long-error-file))
+
+(define (set-file-runpath file path)
+ "Set the value of the DT_RUNPATH dynamic entry of FILE, which must name an
+ELF file, to PATH, a list of strings. Raise a &missing-runpath-error or
+&runpath-too-long-error when appropriate."
+ (define (call-with-input+output-file file proc)
+ (let ((port (open-file file "r+b")))
+ (guard (c (#t (close-port port) (raise c)))
+ (proc port)
+ (close-port port))))
+
+ (call-with-input+output-file file
+ (lambda (port)
+ (let* ((elf (parse-elf (get-bytevector-all port)))
+ (entries (dynamic-entries elf (dynamic-link-segment elf)))
+ (runpath (find (lambda (entry)
+ (= DT_RUNPATH (dynamic-entry-type entry)))
+ entries))
+ (path (string->utf8 (string-join path ":"))))
+ (unless runpath
+ (raise (condition (&missing-runpath-error (elf elf)
+ (file file)))))
+
+ ;; There might be padding left beyond RUNPATH in the string table, but
+ ;; we don't know, so assume there's no padding.
+ (unless (<= (bytevector-length path)
+ (bytevector-length
+ (string->utf8 (dynamic-entry-value runpath))))
+ (raise (condition (&runpath-too-long-error (elf #f #;elf)
+ (file file)))))
+
+ (seek port (dynamic-entry-offset runpath) SEEK_SET)
+ (put-bytevector port path)
+ (put-u8 port 0)))))
+
+;;; Local Variables:
+;;; eval: (put 'call-with-input+output-file 'scheme-indent-function 1)
+;;; End:
diff --git a/guix/build/lisp-utils.scm b/guix/build/lisp-utils.scm
index 8a02cb68dd..17d2637f87 100644
--- a/guix/build/lisp-utils.scm
+++ b/guix/build/lisp-utils.scm
@@ -281,7 +281,7 @@ DEPENDENCY-PREFIXES to ensure references to those libraries are retained."
type
compress?
#:allow-other-keys)
- "Generate an executable by using asdf operation TYPE, containing whithin the
+ "Generate an executable by using asdf operation TYPE, containing within the
image all DEPENDENCIES, and running ENTRY-PROGRAM in the case of an
executable. Link in any asd files from DEPENDENCY-PREFIXES to ensure
references to those libraries are retained."
diff --git a/guix/build/maven/pom.scm b/guix/build/maven/pom.scm
index c92d409d2b..dd61f659c2 100644
--- a/guix/build/maven/pom.scm
+++ b/guix/build/maven/pom.scm
@@ -243,7 +243,7 @@ to re-declare the namespaces in the top-level element."
(define* (fix-pom-dependencies pom-file inputs
#:key with-plugins? with-build-dependencies?
(excludes '()) (local-packages '()))
- "Open @var{pom-file}, and override its content, rewritting its dependencies
+ "Open @var{pom-file}, and override its content, rewriting its dependencies
to set their version to the latest version available in the @var{inputs}.
@var{#:with-plugins?} controls whether plugins are also overridden.
diff --git a/guix/build/meson-build-system.scm b/guix/build/meson-build-system.scm
index 8043a84abb..cc2ba83889 100644
--- a/guix/build/meson-build-system.scm
+++ b/guix/build/meson-build-system.scm
@@ -100,7 +100,7 @@ for example libraries only needed for the tests."
(find-files dir elf-pred))
existing-elf-dirs))))
(for-each strip-runpath elf-list)))))
- (for-each handle-output outputs)
+ (for-each handle-output (alist-delete "debug" outputs))
#t)
(define %standard-phases
diff --git a/guix/build/minify-build-system.scm b/guix/build/minify-build-system.scm
index 92158a033f..f7f51af301 100644
--- a/guix/build/minify-build-system.scm
+++ b/guix/build/minify-build-system.scm
@@ -23,6 +23,7 @@
#:use-module (guix build utils)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-26)
+ #:use-module (ice-9 match)
#:use-module (ice-9 popen)
#:export (%standard-phases
minify-build
@@ -42,14 +43,17 @@
(minified (open-pipe* OPEN_READ "uglify-js" file)))
(call-with-output-file installed
(cut dump-port minified <>))
- #t))
+ (match (close-pipe minified)
+ (0 #t)
+ (status
+ (error "uglify-js failed" status)))))
(define* (build #:key javascript-files
#:allow-other-keys)
(let ((files (or javascript-files
(find-files "src" "\\.js$"))))
(mkdir-p "guix/build")
- (every (cut minify <> #:directory "guix/build/") files)))
+ (for-each (cut minify <> #:directory "guix/build/") files)))
(define* (install #:key outputs #:allow-other-keys)
(let* ((out (assoc-ref outputs "out"))
@@ -60,8 +64,7 @@
(if (not (zero? (stat:size (stat file))))
(install-file file js)
(error "File is empty: " file)))
- (find-files "guix/build" "\\.min\\.js$")))
- #t)
+ (find-files "guix/build" "\\.min\\.js$"))))
(define %standard-phases
(modify-phases gnu:%standard-phases
diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm
index 09bd8465c8..aad86258c7 100644
--- a/guix/build/python-build-system.scm
+++ b/guix/build/python-build-system.scm
@@ -6,6 +6,9 @@
;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com>
;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net>
+;;; Copyright © 2019, 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com>
+;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net>
+;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -27,6 +30,7 @@
#:use-module (guix build utils)
#:use-module (ice-9 match)
#:use-module (ice-9 ftw)
+ #:use-module (ice-9 format)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-26)
#:export (%standard-phases
@@ -154,9 +158,14 @@
(major+minor (take components 2)))
(string-join major+minor ".")))
+(define (python-output outputs)
+ "Return the path of the python output, if there is one, or fall-back to out."
+ (or (assoc-ref outputs "python")
+ (assoc-ref outputs "out")))
+
(define (site-packages inputs outputs)
"Return the path of the current output's Python site-package."
- (let* ((out (assoc-ref outputs "out"))
+ (let* ((out (python-output outputs))
(python (assoc-ref inputs "python")))
(string-append out "/lib/python"
(python-version python)
@@ -172,18 +181,31 @@ when running checks after installing the package."
(if old-path (string-append ":" old-path) "")))
#t))
-(define* (install #:key outputs (configure-flags '()) use-setuptools?
+(define* (install #:key inputs outputs (configure-flags '()) use-setuptools?
#:allow-other-keys)
"Install a given Python package."
- (let* ((out (assoc-ref outputs "out"))
- (params (append (list (string-append "--prefix=" out))
+ (let* ((out (python-output outputs))
+ (python (assoc-ref inputs "python"))
+ (major-minor (map string->number
+ (take (string-split (python-version python) #\.) 2)))
+ (<3.7? (match major-minor
+ ((major minor)
+ (or (< major 3) (and (= major 3) (< minor 7))))))
+ (params (append (list (string-append "--prefix=" out)
+ "--no-compile")
(if use-setuptools?
;; distutils does not accept these flags
(list "--single-version-externally-managed"
- "--root=/")
+ "--root=/")
'())
configure-flags)))
(call-setuppy "install" params use-setuptools?)
+ ;; Rather than produce potentially non-reproducible .pyc files on Pythons
+ ;; older than 3.7, whose 'compileall' module lacks the
+ ;; '--invalidation-mode' option, do not generate any.
+ (unless <3.7?
+ (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash"
+ out))
#t))
(define* (wrap #:key inputs outputs #:allow-other-keys)
@@ -199,12 +221,8 @@ when running checks after installing the package."
(string-append dir "/sbin"))))
outputs))
- (let* ((out (assoc-ref outputs "out"))
- (python (assoc-ref inputs "python"))
- (var `("PYTHONPATH" prefix
- ,(cons (string-append out "/lib/python"
- (python-version python)
- "/site-packages")
+ (let* ((var `("PYTHONPATH" prefix
+ ,(cons (site-packages inputs outputs)
(search-path-as-string->list
(or (getenv "PYTHONPATH") ""))))))
(for-each (lambda (dir)
@@ -220,11 +238,7 @@ installed with setuptools."
;; Even if the "easy-install.pth" is not longer created, we kept this phase.
;; There still may be packages creating an "easy-install.pth" manually for
;; some good reason.
- (let* ((out (assoc-ref outputs "out"))
- (python (assoc-ref inputs "python"))
- (site-packages (string-append out "/lib/python"
- (python-version python)
- "/site-packages"))
+ (let* ((site-packages (site-packages inputs outputs))
(easy-install-pth (string-append site-packages "/easy-install.pth"))
(new-pth (string-append site-packages "/" name ".pth")))
(when (file-exists? easy-install-pth)
@@ -248,18 +262,34 @@ installed with setuptools."
"Improve determinism of pyc files."
;; Use deterministic hashes for strings, bytes, and datetime objects.
(setenv "PYTHONHASHSEED" "0")
+ ;; Prevent Python from creating .pyc files when loading modules (such as
+ ;; when running a test suite).
+ (setenv "PYTHONDONTWRITEBYTECODE" "1")
+ #t)
+
+(define* (ensure-no-cythonized-files #:rest _)
+ "Check the source code for @code{.c} files which may have been pre-generated
+by Cython."
+ (for-each
+ (lambda (file)
+ (let ((generated-file
+ (string-append (string-drop-right file 3) "c")))
+ (when (file-exists? generated-file)
+ (format #t "Possible Cythonized file found: ~a~%" generated-file))))
+ (find-files "." "\\.pyx$"))
#t)
(define %standard-phases
;; The build phase only builds C extensions and copies the Python sources,
- ;; while the install phase byte-compiles and copies them to the prefix
- ;; directory. The tests are run after the install phase because otherwise
- ;; the cached .pyc generated during the tests execution seem to interfere
- ;; with the byte compilation of the install phase.
+ ;; while the install phase copies then byte-compiles the sources to the
+ ;; prefix directory. The check phase is moved after the installation phase
+ ;; to ease testing the built package.
(modify-phases gnu:%standard-phases
(add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980)
(add-after 'ensure-no-mtimes-pre-1980 'enable-bytecode-determinism
enable-bytecode-determinism)
+ (add-after 'enable-bytecode-determinism 'ensure-no-cythonized-files
+ ensure-no-cythonized-files)
(delete 'bootstrap)
(delete 'configure) ;not needed
(replace 'build build)
diff --git a/guix/build/rpath.scm b/guix/build/rpath.scm
deleted file mode 100644
index 75a1fef5ef..0000000000
--- a/guix/build/rpath.scm
+++ /dev/null
@@ -1,59 +0,0 @@
-;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2013 Ludovic Courtès <ludo@gnu.org>
-;;;
-;;; This file is part of GNU Guix.
-;;;
-;;; GNU Guix is free software; you can redistribute it and/or modify it
-;;; under the terms of the GNU General Public License as published by
-;;; the Free Software Foundation; either version 3 of the License, or (at
-;;; your option) any later version.
-;;;
-;;; GNU Guix is distributed in the hope that it will be useful, but
-;;; WITHOUT ANY WARRANTY; without even the implied warranty of
-;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-;;; GNU General Public License for more details.
-;;;
-;;; You should have received a copy of the GNU General Public License
-;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
-
-(define-module (guix build rpath)
- #:use-module (ice-9 popen)
- #:use-module (ice-9 rdelim)
- #:export (%patchelf
- file-rpath
- augment-rpath))
-
-;;; Commentary:
-;;;
-;;; Tools to manipulate the RPATH and RUNPATH of ELF binaries. Currently they
-;;; rely on PatchELF.
-;;;
-;;; Code:
-
-(define %patchelf
- ;; The `patchelf' command.
- (make-parameter "patchelf"))
-
-(define %not-colon
- (char-set-complement (char-set #\:)))
-
-(define (file-rpath file)
- "Return the RPATH (or RUNPATH) of FILE as a list of directory names, or #f
-on failure."
- (let* ((p (open-pipe* OPEN_READ (%patchelf) "--print-rpath" file))
- (l (read-line p)))
- (and (zero? (close-pipe p))
- (string-tokenize l %not-colon))))
-
-(define (augment-rpath file dir)
- "Add DIR to the front of the RPATH and RUNPATH of FILE. Return the new
-RPATH as a list, or #f on failure."
- (let* ((rpath (or (file-rpath file) '()))
- (rpath* (cons dir rpath)))
- (format #t "~a: changing RPATH from ~s to ~s~%"
- file rpath rpath*)
- (and (zero? (system* (%patchelf) "--set-rpath"
- (string-join rpath* ":") file))
- rpath*)))
-
-;;; rpath.scm ends here
diff --git a/guix/build/ruby-build-system.scm b/guix/build/ruby-build-system.scm
index c957a61115..9aceb187a4 100644
--- a/guix/build/ruby-build-system.scm
+++ b/guix/build/ruby-build-system.scm
@@ -2,6 +2,7 @@
;;; Copyright © 2015 David Thompson <davet@gnu.org>
;;; Copyright © 2015 Pjotr Prins <pjotr.public01@thebird.nl>
;;; Copyright © 2015, 2016 Ben Woodcroft <donttrustben@gmail.com>
+;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -73,13 +74,19 @@ directory."
(define* (replace-git-ls-files #:key source #:allow-other-keys)
"Many gemspec files downloaded from outside rubygems.org use `git ls-files`
-to list of the files to be included in the built gem. However, since this
+to list the files to be included in the built gem. However, since this
operation is not deterministic, we replace it with `find`."
- (when (not (gem-archive? source))
+ (unless (gem-archive? source)
(let ((gemspec (first-gemspec)))
+ ;; Do not include the freshly built .gem itself as it causes problems.
+ ;; Strip the first 2 characters ("./") to more exactly match the output
+ ;; given by 'git ls-files'. This is useful to prevent breaking regexps
+ ;; that could be used to filter the list of files.
(substitute* gemspec
- (("`git ls-files`") "`find . -type f |sort`")
- (("`git ls-files -z`") "`find . -type f -print0 |sort -z`"))))
+ (("`git ls-files`")
+ "`find . -type f -not -regex '.*\\.gem$' | sort | cut -c3-`")
+ (("`git ls-files -z`")
+ "`find . -type f -not -regex '.*\\.gem$' -print0 | sort -z | cut -zc3-`"))))
#t)
(define* (extract-gemspec #:key source #:allow-other-keys)
@@ -129,11 +136,7 @@ is #f."
#:allow-other-keys)
"Install the gem archive SOURCE to the output store item. Additional
GEM-FLAGS are passed to the 'gem' invocation, if present."
- (let* ((ruby-version
- (match:substring (string-match "ruby-(.*)\\.[0-9]$"
- (assoc-ref inputs "ruby"))
- 1))
- (out (assoc-ref outputs "out"))
+ (let* ((out (assoc-ref outputs "out"))
(vendor-dir (string-append out "/lib/ruby/vendor_ruby"))
(gem-file (first-matching-file "\\.gem$"))
(gem-file-basename (basename gem-file))
@@ -144,8 +147,8 @@ GEM-FLAGS are passed to the 'gem' invocation, if present."
(setenv "GEM_VENDOR" vendor-dir)
(or (zero?
- ;; 'zero? system*' allows the custom error handling to function as
- ;; expected, while 'invoke' raises its own exception.
+ ;; 'zero? system*' allows the custom error handling to function as
+ ;; expected, while 'invoke' raises its own exception.
(apply system* "gem" "install" gem-file
"--verbose"
"--local" "--ignore-dependencies" "--vendor"
diff --git a/guix/build/texlive-build-system.scm b/guix/build/texlive-build-system.scm
index 841c631dae..a4c81f07cd 100644
--- a/guix/build/texlive-build-system.scm
+++ b/guix/build/texlive-build-system.scm
@@ -66,13 +66,12 @@
(setenv "error_line" "254") ; must be less than 255
(setenv "half_error_line" "238") ; must be less than error_line - 15
(setenv "max_print_line" "1000"))
- (mkdir "build")
- #t)
+ (mkdir "build"))
(define* (build #:key inputs build-targets tex-format #:allow-other-keys)
- (every (cut compile-with-latex tex-format <>)
- (if build-targets build-targets
- (scandir "." (cut string-suffix? ".ins" <>)))))
+ (for-each (cut compile-with-latex tex-format <>)
+ (if build-targets build-targets
+ (scandir "." (cut string-suffix? ".ins" <>)))))
(define* (install #:key outputs tex-directory #:allow-other-keys)
(let* ((out (assoc-ref outputs "out"))
@@ -81,8 +80,7 @@
(mkdir-p target)
(for-each delete-file (find-files "." "\\.(log|aux)$"))
(for-each (cut install-file <> target)
- (find-files "build" ".*"))
- #t))
+ (find-files "build" ".*"))))
(define %standard-phases
(modify-phases gnu:%standard-phases
diff --git a/guix/build/utils.scm b/guix/build/utils.scm
index 419c10195b..76180e67e0 100644
--- a/guix/build/utils.scm
+++ b/guix/build/utils.scm
@@ -1,10 +1,12 @@
;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2013 Andreas Enge <andreas@enge.fr>
;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org>
-;;; Copyright © 2015, 2018 Mark H Weaver <mhw@netris.org>
+;;; Copyright © 2015, 2018, 2021 Mark H Weaver <mhw@netris.org>
;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net>
;;; Copyright © 2018, 2019 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il>
+;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -52,6 +54,7 @@
directory-exists?
executable-file?
symbolic-link?
+ call-with-temporary-output-file
call-with-ascii-input-file
elf-file?
ar-file?
@@ -110,7 +113,9 @@
make-desktop-entry-file
- locale-category->string))
+ locale-category->string
+
+ %xz-parallel-args))
;;;
@@ -197,6 +202,22 @@ introduce the version part."
"Return #t if FILE is a symbolic link (aka. \"symlink\".)"
(eq? (stat:type (lstat file)) 'symlink))
+(define (call-with-temporary-output-file proc)
+ "Call PROC with a name of a temporary file and open output port to that
+file; close the file and delete it when leaving the dynamic extent of this
+call."
+ (let* ((directory (or (getenv "TMPDIR") "/tmp"))
+ (template (string-append directory "/guix-file.XXXXXX"))
+ (out (mkstemp! template)))
+ (dynamic-wind
+ (lambda ()
+ #t)
+ (lambda ()
+ (proc template out))
+ (lambda ()
+ (false-if-exception (close out))
+ (false-if-exception (delete-file template))))))
+
(define (call-with-ascii-input-file file proc)
"Open FILE as an ASCII or binary file, and pass the resulting port to
PROC. FILE is closed when PROC's dynamic extent is left. Return the
@@ -322,11 +343,13 @@ name."
#:key
(log (current-output-port))
(follow-symlinks? #f)
- keep-mtime?)
+ (copy-file copy-file)
+ keep-mtime? keep-permissions?)
"Copy SOURCE directory to DESTINATION. Follow symlinks if FOLLOW-SYMLINKS?
-is true; otherwise, just preserve them. When KEEP-MTIME? is true, keep the
-modification time of the files in SOURCE on those of DESTINATION. Write
-verbose output to the LOG port."
+is true; otherwise, just preserve them. Call COPY-FILE to copy regular files.
+When KEEP-MTIME? is true, keep the modification time of the files in SOURCE on
+those of DESTINATION. When KEEP-PERMISSIONS? is true, preserve file
+permissions. Write verbose output to the LOG port."
(define strip-source
(let ((len (string-length source)))
(lambda (file)
@@ -343,16 +366,21 @@ verbose output to the LOG port."
(symlink target dest)))
(else
(copy-file file dest)
- (when keep-mtime?
- (set-file-time dest stat))))))
+ (when keep-permissions?
+ (chmod dest (stat:perms stat)))))
+ (when keep-mtime?
+ (set-file-time dest stat))))
(lambda (dir stat result) ; down
(let ((target (string-append destination
(strip-source dir))))
- (mkdir-p target)
- (when keep-mtime?
- (set-file-time target stat))))
+ (mkdir-p target)))
(lambda (dir stat result) ; up
- result)
+ (let ((target (string-append destination
+ (strip-source dir))))
+ (when keep-mtime?
+ (set-file-time target stat))
+ (when keep-permissions?
+ (chmod target (stat:perms stat)))))
(const #t) ; skip
(lambda (file stat errno result)
(format (current-error-port) "i/o error: ~a: ~a~%"
@@ -365,6 +393,16 @@ verbose output to the LOG port."
stat
lstat)))
+(define-syntax-rule (warn-on-error expr file)
+ (catch 'system-error
+ (lambda ()
+ expr)
+ (lambda args
+ (format (current-error-port)
+ "warning: failed to delete ~a: ~a~%"
+ file (strerror
+ (system-error-errno args))))))
+
(define* (delete-file-recursively dir
#:key follow-mounts?)
"Delete DIR recursively, like `rm -rf', without following symlinks. Don't
@@ -375,10 +413,10 @@ errors."
(or follow-mounts?
(= dev (stat:dev stat))))
(lambda (file stat result) ; leaf
- (delete-file file))
+ (warn-on-error (delete-file file) file))
(const #t) ; down
(lambda (dir stat result) ; up
- (rmdir dir))
+ (warn-on-error (rmdir dir) dir))
(const #t) ; skip
(lambda (file stat errno result)
(format (current-error-port)
@@ -746,6 +784,31 @@ PROC's result is returned."
(lambda (key . args)
(false-if-exception (delete-file template))))))
+(define (unused-private-use-code-point s)
+ "Find a code point within a Unicode Private Use Area that is not
+present in S, and return the corresponding character object. If one
+cannot be found, return false."
+ (define (scan lo hi)
+ (and (<= lo hi)
+ (let ((c (integer->char lo)))
+ (if (string-index s c)
+ (scan (+ lo 1) hi)
+ c))))
+ (or (scan #xE000 #xF8FF)
+ (scan #xF0000 #xFFFFD)
+ (scan #x100000 #x10FFFD)))
+
+(define (replace-char c1 c2 s)
+ "Return a string which is equal to S except with all instances of C1
+replaced by C2. If C1 and C2 are equal, return S."
+ (if (char=? c1 c2)
+ s
+ (string-map (lambda (c)
+ (if (char=? c c1)
+ c2
+ c))
+ s)))
+
(define (substitute file pattern+procs)
"PATTERN+PROCS is a list of regexp/two-argument-procedure pairs. For each
line of FILE, and for each PATTERN that it matches, call the corresponding
@@ -764,16 +827,26 @@ end of a line; by itself it won't match the terminating newline of a line."
(let loop ((line (read-line in 'concat)))
(if (eof-object? line)
#t
- (let ((line (fold (lambda (r+p line)
- (match r+p
- ((regexp . proc)
- (match (list-matches regexp line)
- ((and m+ (_ _ ...))
- (proc line m+))
- (_ line)))))
- line
- rx+proc)))
- (display line out)
+ ;; Work around the fact that Guile's regexp-exec does not handle
+ ;; NUL characters (a limitation of the underlying GNU libc's
+ ;; regexec) by temporarily replacing them by an unused private
+ ;; Unicode code point.
+ ;; TODO: Use SRFI-115 instead, once available in Guile.
+ (let* ((nul* (or (and (string-index line #\nul)
+ (unused-private-use-code-point line))
+ #\nul))
+ (line* (replace-char #\nul nul* line))
+ (line1* (fold (lambda (r+p line)
+ (match r+p
+ ((regexp . proc)
+ (match (list-matches regexp line)
+ ((and m+ (_ _ ...))
+ (proc line m+))
+ (_ line)))))
+ line*
+ rx+proc))
+ (line1 (replace-char nul* #\nul line1*)))
+ (display line1 out)
(loop (read-line in 'concat)))))))))
@@ -800,7 +873,7 @@ sub-expression. For example:
((\"hello\")
\"good morning\\n\")
((\"foo([a-z]+)bar(.*)$\" all letters end)
- (string-append \"baz\" letter end)))
+ (string-append \"baz\" letters end)))
Here, anytime a line of FILE contains \"hello\", it is replaced by \"good
morning\". Anytime a line of FILE matches the second regexp, ALL is bound to
@@ -853,29 +926,45 @@ match the terminating newline of a line."
;;;
(define* (dump-port in out
+ #:optional len
#:key (buffer-size 16384)
(progress (lambda (t k) (k))))
- "Read as much data as possible from IN and write it to OUT, using chunks of
-BUFFER-SIZE bytes. Call PROGRESS at the beginning and after each successful
-transfer of BUFFER-SIZE bytes or less, passing it the total number of bytes
-transferred and the continuation of the transfer as a thunk."
+ "Read LEN bytes from IN or as much data as possible if LEN is #f, and write
+it to OUT, using chunks of BUFFER-SIZE bytes. Call PROGRESS at the beginning
+and after each successful transfer of BUFFER-SIZE bytes or less, passing it
+the total number of bytes transferred and the continuation of the transfer as
+a thunk."
(define buffer
(make-bytevector buffer-size))
(define (loop total bytes)
(or (eof-object? bytes)
+ (and len (= total len))
(let ((total (+ total bytes)))
(put-bytevector out buffer 0 bytes)
(progress total
(lambda ()
(loop total
- (get-bytevector-n! in buffer 0 buffer-size)))))))
+ (get-bytevector-n! in buffer 0
+ (if len
+ (min (- len total) buffer-size)
+ buffer-size))))))))
;; Make sure PROGRESS is called when we start so that it can measure
;; throughput.
(progress 0
(lambda ()
- (loop 0 (get-bytevector-n! in buffer 0 buffer-size)))))
+ (loop 0 (get-bytevector-n! in buffer 0
+ (if len
+ (min len buffer-size)
+ buffer-size))))))
+
+(define AT_SYMLINK_NOFOLLOW
+ ;; Guile 2.0 did not define this constant, hence this hack.
+ (let ((variable (module-variable the-root-module 'AT_SYMLINK_NOFOLLOW)))
+ (if variable
+ (variable-ref variable)
+ 256))) ;for GNU/Linux
(define (set-file-time file stat)
"Set the atime/mtime of FILE to that specified by STAT."
@@ -883,7 +972,8 @@ transferred and the continuation of the transfer as a thunk."
(stat:atime stat)
(stat:mtime stat)
(stat:atimensec stat)
- (stat:mtimensec stat)))
+ (stat:mtimensec stat)
+ AT_SYMLINK_NOFOLLOW))
(define (get-char* p)
;; We call it `get-char', but that's really a binary version
@@ -1307,7 +1397,7 @@ not supported."
(lambda ()
(call-with-ascii-input-file prog
(lambda (p)
- (format out header)
+ (display header out)
(dump-port p out)
(close out)
(chmod template mode)
@@ -1446,6 +1536,17 @@ returned."
LC_NAME LC_NUMERIC LC_PAPER LC_TELEPHONE
LC_TIME)))
+
+;;;
+;;; Others.
+;;;
+
+(define (%xz-parallel-args)
+ "The xz arguments required to enable bit-reproducible, multi-threaded
+compression."
+ (list "--memlimit=50%"
+ (format #f "--threads=~a" (max 2 (parallel-job-count)))))
+
;;; Local Variables:
;;; eval: (put 'call-with-output-file/atomic 'scheme-indent-function 1)
;;; eval: (put 'call-with-ascii-input-file 'scheme-indent-function 1)
diff --git a/guix/gexp.scm b/guix/gexp.scm
index 764c89a187..2735d25d0c 100644
--- a/guix/gexp.scm
+++ b/guix/gexp.scm
@@ -1559,7 +1559,8 @@ last one is created from the given <scheme-file> object."
(guile (%guile-for-build))
(module-path %load-path)
(extensions '())
- (deprecation-warnings #f))
+ (deprecation-warnings #f)
+ (optimization-level 1))
"Return a derivation that builds a tree containing the `.go' files
corresponding to MODULES. All the MODULES are built in a context where
they can refer to each other. When TARGET is true, cross-compile MODULES for
@@ -1583,6 +1584,13 @@ TARGET, a GNU triplet."
(system base target)
(system base compile))
+ (define optimizations-for-level
+ (or (and=> (false-if-exception
+ (resolve-interface '(system base optimize)))
+ (lambda (iface)
+ (module-ref iface 'optimizations-for-level))) ;Guile 3.0
+ (const '())))
+
(define (regular? file)
(not (member file '("." ".."))))
@@ -1598,17 +1606,14 @@ TARGET, a GNU triplet."
(ungexp (* total 2))
entry)
- (ungexp-splicing
- (if target
- (gexp ((with-target (ungexp target)
- (lambda ()
- (compile-file entry
- #:output-file output
- #:opts
- %auto-compilation-options)))))
- (gexp ((compile-file entry
- #:output-file output
- #:opts %auto-compilation-options)))))
+ (with-target (ungexp (or target (gexp %host-type)))
+ (lambda ()
+ (compile-file entry
+ #:output-file output
+ #:opts
+ `(,@%auto-compilation-options
+ ,@(optimizations-for-level
+ (ungexp optimization-level))))))
(+ 1 processed))))
diff --git a/guix/packages.scm b/guix/packages.scm
index 6fa761f569..4caaa9cb79 100644
--- a/guix/packages.scm
+++ b/guix/packages.scm
@@ -5,6 +5,7 @@
;;; Copyright © 2016 Alex Kost <alezost@gmail.com>
;;; Copyright © 2017, 2019, 2020 Efraim Flashner <efraim@flashner.co.il>
;;; Copyright © 2019 Marius Bakke <mbakke@fastmail.com>
+;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -167,6 +168,25 @@
;;;
;;; Code:
+(define-syntax-rule (define-compile-time-decoder name string->bytevector)
+ "Define NAME as a macro that runs STRING->BYTEVECTOR at macro expansion time
+if possible."
+ (define-syntax name
+ (lambda (s)
+ "Return the bytevector corresponding to the given textual
+representation."
+ (syntax-case s ()
+ ((_ str)
+ (string? (syntax->datum #'str))
+ ;; A literal string: do the conversion at expansion time.
+ (with-syntax ((bv (string->bytevector (syntax->datum #'str))))
+ #''bv))
+ ((_ str)
+ #'(string->bytevector str))))))
+
+(define-compile-time-decoder base32 nix-base32-string->bytevector)
+(define-compile-time-decoder base64 base64-decode)
+
;; Crytographic content hash.
(define-immutable-record-type <content-hash>
(%content-hash algorithm value)
@@ -302,25 +322,6 @@ specifications to 'hash'."
(set-record-type-printer! <origin> print-origin)
-(define-syntax-rule (define-compile-time-decoder name string->bytevector)
- "Define NAME as a macro that runs STRING->BYTEVECTOR at macro expansion time
-if possible."
- (define-syntax name
- (lambda (s)
- "Return the bytevector corresponding to the given textual
-representation."
- (syntax-case s ()
- ((_ str)
- (string? (syntax->datum #'str))
- ;; A literal string: do the conversion at expansion time.
- (with-syntax ((bv (string->bytevector (syntax->datum #'str))))
- #''bv))
- ((_ str)
- #'(string->bytevector str))))))
-
-(define-compile-time-decoder base32 nix-base32-string->bytevector)
-(define-compile-time-decoder base64 base64-decode)
-
(define (origin-actual-file-name origin)
"Return the file name of ORIGIN, either its 'file-name' field or the file
name of its URI."
@@ -704,6 +705,8 @@ specifies modules in scope when evaluating SNIPPET."
(setenv "PATH" (string-append #+xz "/bin" ":"
#+decomp "/bin"))
+ (setenv "XZ_DEFAULTS" (string-join (%xz-parallel-args)))
+
;; SOURCE may be either a directory or a tarball.
(if (file-is-directory? #+source)
(let* ((store (%store-directory))
@@ -725,26 +728,17 @@ specifies modules in scope when evaluating SNIPPET."
(for-each apply-patch '#+patches)
- (let ((result #+(if snippet
- #~(let ((module (make-fresh-user-module)))
- (module-use-interfaces!
- module
- (map resolve-interface '#+modules))
- ((@ (system base compile) compile)
- '#+snippet
- #:to 'value
- #:opts %auto-compilation-options
- #:env module))
- #~#t)))
- ;; Issue a warning unless the result is #t.
- (unless (eqv? result #t)
- (format (current-error-port) "\
-## WARNING: the snippet returned `~s'. Return values other than #t
-## are deprecated. Please migrate this package so that its snippet
-## reports errors by raising an exception, and otherwise returns #t.~%"
- result))
- (unless result
- (error "snippet returned false")))
+ #+(if snippet
+ #~(let ((module (make-fresh-user-module)))
+ (module-use-interfaces!
+ module
+ (map resolve-interface '#+modules))
+ ((@ (system base compile) compile)
+ '#+snippet
+ #:to 'value
+ #:opts %auto-compilation-options
+ #:env module))
+ #~#t)
(chdir "..")
@@ -1408,6 +1402,22 @@ TARGET."
(bag (package->bag package system target)))
(bag-grafts store bag)))
+(define-inlinable (derivation=? drv1 drv2)
+ "Return true if DRV1 and DRV2 are equal."
+ (or (eq? drv1 drv2)
+ (string=? (derivation-file-name drv1)
+ (derivation-file-name drv2))))
+
+(define (input=? input1 input2)
+ "Return true if INPUT1 and INPUT2 are equivalent."
+ (match input1
+ ((label1 drv1 . outputs1)
+ (match input2
+ ((label2 drv2 . outputs2)
+ (and (string=? label1 label2)
+ (equal? outputs1 outputs2)
+ (derivation=? drv1 drv2)))))))
+
(define* (bag->derivation store bag
#:optional context)
"Return the derivation to build BAG for SYSTEM. Optionally, CONTEXT can be
@@ -1426,9 +1436,12 @@ error reporting."
p))
(_ '()))
inputs))))
-
+ ;; It's possible that INPUTS contains packages that are not 'eq?' but
+ ;; that lead to the same derivation. Delete those duplicates to avoid
+ ;; issues down the road, such as duplicate entries in '%build-inputs'.
(apply (bag-build bag)
- store (bag-name bag) input-drvs
+ store (bag-name bag)
+ (delete-duplicates input-drvs input=?)
#:search-paths paths
#:outputs (bag-outputs bag) #:system system
(bag-arguments bag)))))
@@ -1466,8 +1479,9 @@ This is an internal procedure."
(apply (bag-build bag)
store (bag-name bag)
- #:native-drvs build-drvs
- #:target-drvs (append host-drvs target-drvs)
+ #:native-drvs (delete-duplicates build-drvs input=?)
+ #:target-drvs (delete-duplicates (append host-drvs target-drvs)
+ input=?)
#:search-paths paths
#:native-search-paths npaths
#:outputs (bag-outputs bag)
diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm
index 8ecdcb823f..169cbc2500 100644
--- a/guix/scripts/pack.scm
+++ b/guix/scripts/pack.scm
@@ -5,6 +5,7 @@
;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com>
;;; Copyright © 2018 Efraim Flashner <efraim@flashner.co.il>
;;; Copyright © 2020 Tobias Geerinckx-Rice <me@tobias.gr>
+;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;; Copyright © 2020 Eric Bavier <bavier@posteo.net>
;;;
;;; This file is part of GNU Guix.
@@ -26,6 +27,7 @@
#:use-module (guix scripts)
#:use-module (guix ui)
#:use-module (guix gexp)
+ #:use-module ((guix build utils) #:select (%xz-parallel-args))
#:use-module (guix utils)
#:use-module (guix store)
#:use-module ((guix status) #:select (with-status-verbosity))
@@ -77,29 +79,34 @@
compressor?
(name compressor-name) ;string (e.g., "gzip")
(extension compressor-extension) ;string (e.g., ".lz")
- (command compressor-command)) ;gexp (e.g., #~("/gnu/store/…/gzip" "-9n"))
+ (command compressor-command)) ;gexp (e.g., #~(list "/gnu/store/…/gzip"
+ ; "-9n" ))
(define %compressors
;; Available compression tools.
(list (compressor "gzip" ".gz"
- #~(#+(file-append gzip "/bin/gzip") "-9n"))
+ #~(list #+(file-append gzip "/bin/gzip") "-9n"))
(compressor "lzip" ".lz"
- #~(#+(file-append lzip "/bin/lzip") "-9"))
+ #~(list #+(file-append lzip "/bin/lzip") "-9"))
(compressor "xz" ".xz"
- #~(#+(file-append xz "/bin/xz") "-e"))
+ #~(append (list #+(file-append xz "/bin/xz")
+ "-e")
+ (%xz-parallel-args)))
(compressor "bzip2" ".bz2"
- #~(#+(file-append bzip2 "/bin/bzip2") "-9"))
+ #~(list #+(file-append bzip2 "/bin/bzip2") "-9"))
(compressor "zstd" ".zst"
;; The default level 3 compresses better than gzip in a
;; fraction of the time, while the highest level 19
;; (de)compresses more slowly and worse than xz.
- #~(#+(file-append zstd "/bin/zstd") "-3"))
+ #~(list #+(file-append zstd "/bin/zstd") "-3"))
(compressor "none" "" #f)))
;; This one is only for use in this module, so don't put it in %compressors.
(define bootstrap-xz
(compressor "bootstrap-xz" ".xz"
- #~(#+(file-append %bootstrap-coreutils&co "/bin/xz") "-e")))
+ #~(append (list #+(file-append %bootstrap-coreutils&co "/bin/xz")
+ "-e")
+ (%xz-parallel-args))))
(define (lookup-compressor name)
"Return the compressor object called NAME. Error out if it could not be
@@ -283,7 +290,7 @@ added to the pack."
#+@(if (compressor-command compressor)
#~("-I"
(string-join
- '#+(compressor-command compressor)))
+ #+(compressor-command compressor)))
#~())
"--format=gnu"
@@ -556,11 +563,13 @@ the image."
,@(source-module-closure
`((guix docker)
(guix build store-copy)
+ (guix build utils) ;for %xz-parallel-args
(guix profiles)
(guix search-paths))
#:select? not-config?))
#~(begin
(use-modules (guix docker) (guix build store-copy)
+ (guix build utils)
(guix profiles) (guix search-paths)
(srfi srfi-1) (srfi srfi-19)
(ice-9 match))
@@ -617,7 +626,7 @@ the image."
#~(list (string-append #$profile "/"
#$entry-point)))
#:extra-files directives
- #:compressor '#+(compressor-command compressor)
+ #:compressor #+(compressor-command compressor)
#:creation-time (make-time time-utc 0 1))))))
(gexp->derivation (string-append name ".tar"
diff --git a/guix/store/roots.scm b/guix/store/roots.scm
index 58653507f8..222f69c5c0 100644
--- a/guix/store/roots.scm
+++ b/guix/store/roots.scm
@@ -50,7 +50,7 @@
(define (gc-roots)
"Return the list of garbage collector roots (\"GC roots\"). This includes
-\"regular\" roots fount in %GC-ROOTS-DIRECTORY as well as indirect roots that
+\"regular\" roots found in %GC-ROOTS-DIRECTORY as well as indirect roots that
are user-controlled symlinks stored anywhere on the file system."
(define (regular? file)
(match file
diff --git a/guix/utils.scm b/guix/utils.scm
index 0df46f1062..678954dbfa 100644
--- a/guix/utils.scm
+++ b/guix/utils.scm
@@ -8,6 +8,7 @@
;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com>
;;; Copyright © 2018, 2020 Marius Bakke <marius@gnu.org>
;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il>
+;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -35,7 +36,9 @@
#:use-module (rnrs io ports) ;need 'port-position' etc.
#:use-module ((rnrs bytevectors) #:select (bytevector-u8-set!))
#:use-module (guix memoization)
- #:use-module ((guix build utils) #:select (dump-port mkdir-p delete-file-recursively))
+ #:use-module ((guix build utils)
+ #:select (dump-port mkdir-p delete-file-recursively
+ call-with-temporary-output-file %xz-parallel-args))
#:use-module ((guix build syscalls) #:select (mkdtemp! fdatasync))
#:use-module (guix diagnostics) ;<location>, &error-location, etc.
#:use-module (ice-9 format)
@@ -59,7 +62,9 @@
&fix-hint
fix-hint?
- condition-fix-hint)
+ condition-fix-hint
+
+ call-with-temporary-output-file)
#:export (strip-keyword-arguments
default-keyword-arguments
substitute-keyword-arguments
@@ -96,7 +101,6 @@
tarball-sans-extension
compressed-file?
switch-symlinks
- call-with-temporary-output-file
call-with-temporary-directory
with-atomic-file-output
@@ -219,7 +223,7 @@ a symbol such as 'xz."
(match compression
((or #f 'none) (values input '()))
('bzip2 (filtered-port `(,%bzip2 "-dc") input))
- ('xz (filtered-port `(,%xz "-dc") input))
+ ('xz (filtered-port `(,%xz "-dc" ,@(%xz-parallel-args)) input))
('gzip (filtered-port `(,%gzip "-dc") input))
('lzip (values (lzip-port 'make-lzip-input-port input)
'()))
@@ -231,7 +235,7 @@ a symbol such as 'xz."
(match compression
((or #f 'none) (values input '()))
('bzip2 (filtered-port `(,%bzip2 "-c") input))
- ('xz (filtered-port `(,%xz "-c") input))
+ ('xz (filtered-port `(,%xz "-c" ,@(%xz-parallel-args)) input))
('gzip (filtered-port `(,%gzip "-c") input))
('lzip (values (lzip-port 'make-lzip-input-port/compressed input)
'()))
@@ -290,7 +294,8 @@ program--e.g., '(\"--fast\")."
(match compression
((or #f 'none) (values output '()))
('bzip2 (filtered-output-port `(,%bzip2 "-c" ,@options) output))
- ('xz (filtered-output-port `(,%xz "-c" ,@options) output))
+ ('xz (filtered-output-port `(,%xz "-c" ,@(%xz-parallel-args)
+ ,@options) output))
('gzip (filtered-output-port `(,%gzip "-c" ,@options) output))
('lzip (values (lzip-port 'make-lzip-output-port output)
'()))
@@ -689,22 +694,6 @@ REPLACEMENT."
(substring str start index)
pieces))))))))
-(define (call-with-temporary-output-file proc)
- "Call PROC with a name of a temporary file and open output port to that
-file; close the file and delete it when leaving the dynamic extent of this
-call."
- (let* ((directory (or (getenv "TMPDIR") "/tmp"))
- (template (string-append directory "/guix-file.XXXXXX"))
- (out (mkstemp! template)))
- (dynamic-wind
- (lambda ()
- #t)
- (lambda ()
- (proc template out))
- (lambda ()
- (false-if-exception (close out))
- (false-if-exception (delete-file template))))))
-
(define (call-with-temporary-directory proc)
"Call PROC with a name of a temporary directory; close the directory and
delete it when leaving the dynamic extent of this call."
generated by cgit v1.2.3 (git 2.45.2) at 2024-07-06 08:36:57 +0000