diff options
Diffstat (limited to 'guix/build')
-rw-r--r-- | guix/build/copy-build-system.scm | 2 | ||||
-rw-r--r-- | guix/build/emacs-build-system.scm | 16 | ||||
-rw-r--r-- | guix/build/gnu-build-system.scm | 217 | ||||
-rw-r--r-- | guix/build/gremlin.scm | 117 | ||||
-rw-r--r-- | guix/build/lisp-utils.scm | 2 | ||||
-rw-r--r-- | guix/build/maven/pom.scm | 2 | ||||
-rw-r--r-- | guix/build/meson-build-system.scm | 2 | ||||
-rw-r--r-- | guix/build/minify-build-system.scm | 11 | ||||
-rw-r--r-- | guix/build/python-build-system.scm | 133 | ||||
-rw-r--r-- | guix/build/rpath.scm | 59 | ||||
-rw-r--r-- | guix/build/ruby-build-system.scm | 25 | ||||
-rw-r--r-- | guix/build/texlive-build-system.scm | 42 | ||||
-rw-r--r-- | guix/build/utils.scm | 187 |
13 files changed, 531 insertions, 284 deletions
diff --git a/guix/build/copy-build-system.scm b/guix/build/copy-build-system.scm index a86f0cde29..ac4a62a074 100644 --- a/guix/build/copy-build-system.scm +++ b/guix/build/copy-build-system.scm @@ -58,7 +58,7 @@ In the above, FILTERS are optional. one of the elements in the list. - With `#:include-regexp`, install subpaths matching the regexps in the list. - The `#:exclude*` FILTERS work similarly. Without `#:include*` flags, - install every subpath but the files matching the `#:exlude*` filters. + install every subpath but the files matching the `#:exclude*` filters. If both `#:include*` and `#:exclude*` are specified, the exclusion is done on the inclusion list. diff --git a/guix/build/emacs-build-system.scm b/guix/build/emacs-build-system.scm index 26ea59bc25..79a1a42c4a 100644 --- a/guix/build/emacs-build-system.scm +++ b/guix/build/emacs-build-system.scm @@ -105,23 +105,9 @@ environment variable\n" source-directory))) "Substitute the absolute \"/bin/\" directory with the right location in the store in '.el' files." - (define (file-contains-nul-char? file) - (call-with-input-file file - (lambda (in) - (let loop ((line (read-line in 'concat))) - (cond - ((eof-object? line) #f) - ((string-index line #\nul) #t) - (else (loop (read-line in 'concat)))))) - #:binary #t)) - (let* ((out (assoc-ref outputs "out")) (site-lisp (string-append out %install-dir)) - ;; (ice-9 regex) uses libc's regexp routines, which cannot deal with - ;; strings containing NULs. Filter out such files. TODO: Remove - ;; this workaround when <https://bugs.gnu.org/30116> is fixed. - (el-files (remove file-contains-nul-char? - (find-files (getcwd) "\\.el$")))) + (el-files (find-files (getcwd) "\\.el$"))) (define (substitute-program-names) (substitute* el-files (("\"/bin/([^.]\\S*)\"" _ cmd-name) diff --git a/guix/build/gnu-build-system.scm b/guix/build/gnu-build-system.scm index 2e7dff2034..28c719d9ca 100644 --- a/guix/build/gnu-build-system.scm +++ b/guix/build/gnu-build-system.scm @@ -1,7 +1,8 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2020 Brendan Tildesley <mail@brendan.scot> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -35,6 +36,7 @@ #:use-module (rnrs io ports) #:export (%standard-phases %license-file-regexp + %bootstrap-scripts dump-file-contents gnu-build)) @@ -57,23 +59,26 @@ "Set the 'SOURCE_DATE_EPOCH' environment variable. This is used by tools that incorporate timestamps as a way to tell them to use a fixed timestamp. See https://reproducible-builds.org/specs/source-date-epoch/." - (setenv "SOURCE_DATE_EPOCH" "1") - #t) + (setenv "SOURCE_DATE_EPOCH" "1")) (define (first-subdirectory directory) - "Return the file name of the first sub-directory of DIRECTORY." + "Return the file name of the first sub-directory of DIRECTORY or false, when +there are none." (match (scandir directory (lambda (file) (and (not (member file '("." ".."))) (file-is-directory? (string-append directory "/" file))))) - ((first . _) first))) + ((first . _) first) + (_ #f))) (define* (set-paths #:key target inputs native-inputs (search-paths '()) (native-search-paths '()) #:allow-other-keys) (define input-directories - (match inputs + ;; The "source" input can be a directory, but we don't want it for search + ;; paths. See <https://issues.guix.gnu.org/44924>. + (match (alist-delete "source" inputs) (((_ . dir) ...) dir))) @@ -113,9 +118,7 @@ See https://reproducible-builds.org/specs/source-date-epoch/." #:separator separator #:type type #:pattern pattern))) - native-search-paths)) - - #t) + native-search-paths))) (define* (install-locale #:key (locale "en_US.utf8") @@ -134,15 +137,13 @@ chance to be set." (setenv (locale-category->string locale-category) locale) (format (current-error-port) "using '~a' locale for category ~s~%" - locale (locale-category->string locale-category)) - #t) + locale (locale-category->string locale-category))) (lambda args ;; This is known to fail for instance in early bootstrap where locales ;; are not available. (format (current-error-port) "warning: failed to install '~a' locale: ~a~%" - locale (strerror (system-error-errno args))) - #t))) + locale (strerror (system-error-errno args)))))) (define* (unpack #:key source #:allow-other-keys) "Unpack SOURCE in the working directory, and change directory within the @@ -156,13 +157,25 @@ working directory." ;; Preserve timestamps (set to the Epoch) on the copied tree so that ;; things work deterministically. (copy-recursively source "." - #:keep-mtime? #t)) + #:keep-mtime? #t) + ;; Make the source checkout files writable, for convenience. + (for-each (lambda (f) + (false-if-exception (make-file-writable f))) + (find-files "."))) (begin - (if (string-suffix? ".zip" source) - (invoke "unzip" source) - (invoke "tar" "xvf" source)) - (chdir (first-subdirectory ".")))) - #t) + (cond + ((string-suffix? ".zip" source) + (invoke "unzip" source)) + ((tarball? source) + (invoke "tar" "xvf" source)) + (else + (let ((name (strip-store-file-name source)) + (command (compressor source))) + (copy-file source name) + (when command + (invoke command "--decompress" name))))) + ;; Attempt to change into child directory. + (and=> (first-subdirectory ".") chdir)))) (define %bootstrap-scripts ;; Typical names of Autotools "bootstrap" scripts. @@ -205,8 +218,7 @@ working directory." (invoke "autoreconf" "-vif") (format #t "no 'configure.ac' or anything like that, \ doing nothing~%")))) - (format #t "GNU build system bootstrapping not needed~%")) - #t) + (format #t "GNU build system bootstrapping not needed~%"))) ;; See <http://bugs.gnu.org/17840>. (define* (patch-usr-bin-file #:key native-inputs inputs @@ -220,8 +232,7 @@ things like the ABI being used." (for-each (lambda (file) (when (executable-file? file) (patch-/usr/bin/file file))) - (find-files "." "^configure$"))) - #t) + (find-files "." "^configure$")))) (define* (patch-source-shebangs #:key source #:allow-other-keys) "Patch shebangs in all source files; this includes non-executable @@ -233,8 +244,7 @@ $CONFIG_SHELL, but some don't, such as `mkinstalldirs' or Automake's (lambda (file stat) ;; Filter out symlinks. (eq? 'regular (stat:type stat))) - #:stat lstat)) - #t) + #:stat lstat))) (define (patch-generated-file-shebangs . rest) "Patch shebangs in generated files, including `SHELL' variables in @@ -249,9 +259,7 @@ makefiles." #:stat lstat)) ;; Patch `SHELL' in generated makefiles. - (for-each patch-makefile-SHELL (find-files "." "^(GNU)?[mM]akefile$")) - - #t) + (for-each patch-makefile-SHELL (find-files "." "^(GNU)?[mM]akefile$"))) (define* (configure #:key build target native-inputs inputs outputs (configure-flags '()) out-of-source? @@ -381,8 +389,7 @@ makefiles." `("-j" ,(number->string (parallel-job-count))) '()) ,@make-flags))) - (format #t "test suite not run~%")) - #t) + (format #t "test suite not run~%"))) (define* (install #:key (make-flags '()) #:allow-other-keys) (apply invoke "make" "install" make-flags)) @@ -415,8 +422,7 @@ makefiles." (for-each (lambda (dir) (let ((files (list-of-files dir))) (for-each (cut patch-shebang <> path) files))) - output-bindirs))) - #t) + output-bindirs)))) (define* (strip #:key target outputs (strip-binaries? #t) (strip-command (if target @@ -425,7 +431,7 @@ makefiles." (objcopy-command (if target (string-append target "-objcopy") "objcopy")) - (strip-flags '("--strip-debug" + (strip-flags '("--strip-unneeded" "--enable-deterministic-archives")) (strip-directories '("lib" "lib64" "libexec" "bin" "sbin")) @@ -514,8 +520,7 @@ makefiles." (let ((sub (string-append dir "/" d))) (and (directory-exists? sub) sub))) strip-directories))) - outputs))) - #t) + outputs)))) (define* (validate-runpath #:key (validate-runpath? #t) @@ -560,9 +565,7 @@ phase after stripping." outputs))) (unless (every* validate dirs) (error "RUNPATH validation failed"))) - (format (current-error-port) "skipping RUNPATH validation~%")) - - #t) + (format (current-error-port) "skipping RUNPATH validation~%"))) (define* (validate-documentation-location #:key outputs #:allow-other-keys) @@ -582,8 +585,7 @@ and 'man/'. This phase moves directories to the right place if needed." (match outputs (((names . directories) ...) - (for-each validate-output directories))) - #t) + (for-each validate-output directories)))) (define* (reset-gzip-timestamps #:key outputs #:allow-other-keys) "Reset embedded timestamps in gzip files found in OUTPUTS." @@ -599,8 +601,7 @@ and 'man/'. This phase moves directories to the right place if needed." (match outputs (((names . directories) ...) - (for-each process-directory directories))) - #t) + (for-each process-directory directories)))) (define* (compress-documentation #:key outputs (compress-documentation? #t) @@ -616,7 +617,7 @@ DOCUMENTATION-COMPRESSOR-FLAGS." (let ((target (readlink link))) (delete-file link) (symlink (string-append target compressed-documentation-extension) - link))) + (string-append link compressed-documentation-extension)))) (define (has-links? file) ;; Return #t if FILE has hard links. @@ -679,8 +680,7 @@ DOCUMENTATION-COMPRESSOR-FLAGS." (match outputs (((names . directories) ...) (for-each maybe-compress directories))) - (format #t "not compressing documentation~%")) - #t) + (format #t "not compressing documentation~%"))) (define* (delete-info-dir-file #:key outputs #:allow-other-keys) "Delete any 'share/info/dir' file from OUTPUTS." @@ -689,8 +689,7 @@ DOCUMENTATION-COMPRESSOR-FLAGS." (let ((info-dir-file (string-append directory "/share/info/dir"))) (when (file-exists? info-dir-file) (delete-file info-dir-file))))) - outputs) - #t) + outputs)) (define* (patch-dot-desktop-files #:key outputs inputs #:allow-other-keys) @@ -730,8 +729,74 @@ which cannot be found~%" (("^TryExec=([^/[:blank:]\r\n]*)(.*)$" _ binary rest) (string-append "TryExec=" (which binary) rest))))))))) - outputs) - #t) + outputs)) + +(define* (make-dynamic-linker-cache #:key outputs + (make-dynamic-linker-cache? #t) + #:allow-other-keys) + "Create a dynamic linker cache under 'etc/ld.so.cache' in each of the +OUTPUTS. This reduces application startup time by avoiding the 'stat' storm +that traversing all the RUNPATH entries entails." + (define (make-cache-for-output directory) + (define bin-directories + (filter-map (lambda (sub-directory) + (let ((directory (string-append directory "/" + sub-directory))) + (and (directory-exists? directory) + directory))) + '("bin" "sbin" "libexec"))) + + (define programs + ;; Programs that can benefit from the ld.so cache. + (append-map (lambda (directory) + (if (directory-exists? directory) + (find-files directory + (lambda (file stat) + (and (executable-file? file) + (elf-file? file)))) + '())) + bin-directories)) + + (define library-path + ;; Directories containing libraries that PROGRAMS depend on, + ;; recursively. + (delete-duplicates + (append-map (lambda (program) + (map dirname (file-needed/recursive program))) + programs))) + + (define cache-file + (string-append directory "/etc/ld.so.cache")) + + (define ld.so.conf + (string-append (or (getenv "TMPDIR") "/tmp") + "/ld.so.conf")) + + (unless (null? library-path) + (mkdir-p (dirname cache-file)) + (guard (c ((invoke-error? c) + ;; Do not treat 'ldconfig' failure as an error. + (format (current-error-port) + "warning: 'ldconfig' failed:~%") + (report-invoke-error c (current-error-port)))) + ;; Create a config file to tell 'ldconfig' where to look for the + ;; libraries that PROGRAMS need. + (call-with-output-file ld.so.conf + (lambda (port) + (for-each (lambda (directory) + (display directory port) + (newline port)) + library-path))) + + (invoke "ldconfig" "-f" ld.so.conf "-C" cache-file) + (format #t "created '~a' from ~a library search path entries~%" + cache-file (length library-path))))) + + (if make-dynamic-linker-cache? + (match outputs + (((_ . directories) ...) + (for-each make-cache-for-output directories))) + (format #t "ld.so cache not built~%"))) (define %license-file-regexp ;; Regexp matching license files. @@ -796,8 +861,7 @@ which cannot be found~%" package)) (map (cut string-append source "/" <>) files))) (format (current-error-port) - "failed to find license files~%")) - #t)) + "failed to find license files~%")))) (define %standard-phases ;; Standard build phases, as a list of symbol/procedure pairs. @@ -813,6 +877,7 @@ which cannot be found~%" validate-documentation-location delete-info-dir-file patch-dot-desktop-files + make-dynamic-linker-cache install-license-files reset-gzip-timestamps compress-documentation))) @@ -840,26 +905,30 @@ in order. Return #t if all the PHASES succeeded, #f otherwise." (exit 1))) ;; The trick is to #:allow-other-keys everywhere, so that each procedure in ;; PHASES can pick the keyword arguments it's interested in. - (every (match-lambda - ((name . proc) - (let ((start (current-time time-monotonic))) - (format #t "starting phase `~a'~%" name) - (let ((result (apply proc args)) - (end (current-time time-monotonic))) - (format #t "phase `~a' ~:[failed~;succeeded~] after ~,1f seconds~%" - name result - (elapsed-time end start)) - - ;; Issue a warning unless the result is #t. - (unless (eqv? result #t) - (format (current-error-port) "\ -## WARNING: phase `~a' returned `~s'. Return values other than #t -## are deprecated. Please migrate this package so that its phase -## procedures report errors by raising an exception, and otherwise -## always return #t.~%" - name result)) - - ;; Dump the environment variables as a shell script, for handy debugging. - (system "export > $NIX_BUILD_TOP/environment-variables") - result)))) - phases))) + (for-each (match-lambda + ((name . proc) + (let ((start (current-time time-monotonic))) + (define (end-of-phase success?) + (let ((end (current-time time-monotonic))) + (format #t "phase `~a' ~:[failed~;succeeded~] after ~,1f seconds~%" + name success? + (elapsed-time end start)) + + ;; Dump the environment variables as a shell script, + ;; for handy debugging. + (system "export > $NIX_BUILD_TOP/environment-variables"))) + + (format #t "starting phase `~a'~%" name) + (with-throw-handler #t + (lambda () + (apply proc args) + (end-of-phase #t)) + (lambda args + ;; This handler executes before the stack is unwound. + ;; The exception is automatically re-thrown from here, + ;; and we should get a proper backtrace. + (format (current-error-port) + "error: in phase '~a': uncaught exception: +~{~s ~}~%" name args) + (end-of-phase #f)))))) + phases))) diff --git a/guix/build/gremlin.scm b/guix/build/gremlin.scm index e8ea66dfb3..a2d2169ddc 100644 --- a/guix/build/gremlin.scm +++ b/guix/build/gremlin.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2018 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2018, 2020 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -41,6 +41,17 @@ elf-dynamic-info-runpath expand-origin + file-dynamic-info + file-runpath + file-needed + file-needed/recursive + + missing-runpath-error? + missing-runpath-error-file + runpath-too-long-error? + runpath-too-long-error-file + set-file-runpath + validate-needed-in-runpath strip-runpath)) @@ -232,6 +243,63 @@ string table if the type is a string." dynamic-entry-value)) '())))))) +(define (file-dynamic-info file) + "Return the <elf-dynamic-info> record of FILE, or #f if FILE lacks dynamic +info." + (call-with-input-file file + (lambda (port) + (elf-dynamic-info (parse-elf (get-bytevector-all port)))))) + +(define (file-runpath file) + "Return the DT_RUNPATH dynamic entry of FILE as a list of strings, or #f if +FILE lacks dynamic info." + (and=> (file-dynamic-info file) elf-dynamic-info-runpath)) + +(define (file-needed file) + "Return the list of DT_NEEDED dynamic entries of FILE, or #f if FILE lacks +dynamic info." + (and=> (file-dynamic-info file) elf-dynamic-info-needed)) + +(define (file-needed/recursive file) + "Return two values: the list of absolute .so file names FILE depends on, +recursively, and the list of .so file names that could not be found. File +names are resolved by searching the RUNPATH of the file that NEEDs them. + +This is similar to the info returned by the 'ldd' command." + (let loop ((files (list file)) + (result '()) + (not-found '())) + (match files + (() + (values (reverse result) + (reverse (delete-duplicates not-found)))) + ((file . rest) + (match (file-dynamic-info file) + (#f + (loop rest result not-found)) + (info + (let ((runpath (elf-dynamic-info-runpath info)) + (needed (elf-dynamic-info-needed info))) + (if (and runpath needed) + (let* ((runpath (map (cute expand-origin <> (dirname file)) + runpath)) + (resolved (map (cut search-path runpath <>) + needed)) + (failed (filter-map (lambda (needed resolved) + (and (not resolved) + (not (libc-library? needed)) + needed)) + needed resolved)) + (needed (remove (lambda (value) + (or (not value) + ;; XXX: quadratic + (member value result))) + resolved))) + (loop (append rest needed) + (append needed result) + (append failed not-found))) + (loop rest result not-found))))))))) + (define %libc-libraries ;; List of libraries as of glibc 2.21 (there are more but those are ;; typically mean to be LD_PRELOADed and thus do not appear as NEEDED.) @@ -364,4 +432,49 @@ according to DT_NEEDED." (false-if-exception (close-port port)) (apply throw key args)))) -;;; gremlin.scm ends here + +(define-condition-type &missing-runpath-error &elf-error + missing-runpath-error? + (file missing-runpath-error-file)) + +(define-condition-type &runpath-too-long-error &elf-error + runpath-too-long-error? + (file runpath-too-long-error-file)) + +(define (set-file-runpath file path) + "Set the value of the DT_RUNPATH dynamic entry of FILE, which must name an +ELF file, to PATH, a list of strings. Raise a &missing-runpath-error or +&runpath-too-long-error when appropriate." + (define (call-with-input+output-file file proc) + (let ((port (open-file file "r+b"))) + (guard (c (#t (close-port port) (raise c))) + (proc port) + (close-port port)))) + + (call-with-input+output-file file + (lambda (port) + (let* ((elf (parse-elf (get-bytevector-all port))) + (entries (dynamic-entries elf (dynamic-link-segment elf))) + (runpath (find (lambda (entry) + (= DT_RUNPATH (dynamic-entry-type entry))) + entries)) + (path (string->utf8 (string-join path ":")))) + (unless runpath + (raise (condition (&missing-runpath-error (elf elf) + (file file))))) + + ;; There might be padding left beyond RUNPATH in the string table, but + ;; we don't know, so assume there's no padding. + (unless (<= (bytevector-length path) + (bytevector-length + (string->utf8 (dynamic-entry-value runpath)))) + (raise (condition (&runpath-too-long-error (elf #f #;elf) + (file file))))) + + (seek port (dynamic-entry-offset runpath) SEEK_SET) + (put-bytevector port path) + (put-u8 port 0))))) + +;;; Local Variables: +;;; eval: (put 'call-with-input+output-file 'scheme-indent-function 1) +;;; End: diff --git a/guix/build/lisp-utils.scm b/guix/build/lisp-utils.scm index 8a02cb68dd..17d2637f87 100644 --- a/guix/build/lisp-utils.scm +++ b/guix/build/lisp-utils.scm @@ -281,7 +281,7 @@ DEPENDENCY-PREFIXES to ensure references to those libraries are retained." type compress? #:allow-other-keys) - "Generate an executable by using asdf operation TYPE, containing whithin the + "Generate an executable by using asdf operation TYPE, containing within the image all DEPENDENCIES, and running ENTRY-PROGRAM in the case of an executable. Link in any asd files from DEPENDENCY-PREFIXES to ensure references to those libraries are retained." diff --git a/guix/build/maven/pom.scm b/guix/build/maven/pom.scm index c92d409d2b..dd61f659c2 100644 --- a/guix/build/maven/pom.scm +++ b/guix/build/maven/pom.scm @@ -243,7 +243,7 @@ to re-declare the namespaces in the top-level element." (define* (fix-pom-dependencies pom-file inputs #:key with-plugins? with-build-dependencies? (excludes '()) (local-packages '())) - "Open @var{pom-file}, and override its content, rewritting its dependencies + "Open @var{pom-file}, and override its content, rewriting its dependencies to set their version to the latest version available in the @var{inputs}. @var{#:with-plugins?} controls whether plugins are also overridden. diff --git a/guix/build/meson-build-system.scm b/guix/build/meson-build-system.scm index 8043a84abb..cc2ba83889 100644 --- a/guix/build/meson-build-system.scm +++ b/guix/build/meson-build-system.scm @@ -100,7 +100,7 @@ for example libraries only needed for the tests." (find-files dir elf-pred)) existing-elf-dirs)))) (for-each strip-runpath elf-list))))) - (for-each handle-output outputs) + (for-each handle-output (alist-delete "debug" outputs)) #t) (define %standard-phases diff --git a/guix/build/minify-build-system.scm b/guix/build/minify-build-system.scm index 92158a033f..f7f51af301 100644 --- a/guix/build/minify-build-system.scm +++ b/guix/build/minify-build-system.scm @@ -23,6 +23,7 @@ #:use-module (guix build utils) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) + #:use-module (ice-9 match) #:use-module (ice-9 popen) #:export (%standard-phases minify-build @@ -42,14 +43,17 @@ (minified (open-pipe* OPEN_READ "uglify-js" file))) (call-with-output-file installed (cut dump-port minified <>)) - #t)) + (match (close-pipe minified) + (0 #t) + (status + (error "uglify-js failed" status))))) (define* (build #:key javascript-files #:allow-other-keys) (let ((files (or javascript-files (find-files "src" "\\.js$")))) (mkdir-p "guix/build") - (every (cut minify <> #:directory "guix/build/") files))) + (for-each (cut minify <> #:directory "guix/build/") files))) (define* (install #:key outputs #:allow-other-keys) (let* ((out (assoc-ref outputs "out")) @@ -60,8 +64,7 @@ (if (not (zero? (stat:size (stat file)))) (install-file file js) (error "File is empty: " file))) - (find-files "guix/build" "\\.min\\.js$"))) - #t) + (find-files "guix/build" "\\.min\\.js$")))) (define %standard-phases (modify-phases gnu:%standard-phases diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 09bd8465c8..8ade1d5911 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -6,6 +6,10 @@ ;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2019, 2020, 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net> +;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> +;;; Copyright © 2021 Lars-Dominik Braun <lars@6xq.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -27,6 +31,7 @@ #:use-module (guix build utils) #:use-module (ice-9 match) #:use-module (ice-9 ftw) + #:use-module (ice-9 format) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:export (%standard-phases @@ -128,6 +133,15 @@ (apply invoke "python" "./setup.py" command params))) (error "no setup.py found"))) +(define* (sanity-check #:key tests? inputs outputs #:allow-other-keys) + "Ensure packages depending on this package via setuptools work properly, +their advertised endpoints work and their top level modules are importable +without errors." + (let ((sanity-check.py (assoc-ref inputs "sanity-check.py"))) + ;; Make sure the working directory is empty (i.e. no Python modules in it) + (with-directory-excursion "/tmp" + (invoke "python" sanity-check.py (site-packages inputs outputs))))) + (define* (build #:key use-setuptools? #:allow-other-keys) "Build a given Python package." (call-setuppy "build" '() use-setuptools?) @@ -154,37 +168,58 @@ (major+minor (take components 2))) (string-join major+minor "."))) +(define (python-output outputs) + "Return the path of the python output, if there is one, or fall-back to out." + (or (assoc-ref outputs "python") + (assoc-ref outputs "out"))) + (define (site-packages inputs outputs) "Return the path of the current output's Python site-package." - (let* ((out (assoc-ref outputs "out")) + (let* ((out (python-output outputs)) (python (assoc-ref inputs "python"))) - (string-append out "/lib/python" - (python-version python) - "/site-packages/"))) + (string-append out "/lib/python" (python-version python) "/site-packages"))) (define (add-installed-pythonpath inputs outputs) - "Prepend the Python site-package of OUTPUT to PYTHONPATH. This is useful -when running checks after installing the package." - (let ((old-path (getenv "PYTHONPATH")) - (add-path (site-packages inputs outputs))) - (setenv "PYTHONPATH" - (string-append add-path - (if old-path (string-append ":" old-path) ""))) - #t)) + "Prepend the site-package of OUTPUT to GUIX_PYTHONPATH. This is useful when +running checks after installing the package." + (setenv "GUIX_PYTHONPATH" (string-append (site-packages inputs outputs) ":" + (getenv "GUIX_PYTHONPATH")))) + +(define* (add-install-to-pythonpath #:key inputs outputs #:allow-other-keys) + "A phase that just wraps the 'add-installed-pythonpath' procedure." + (add-installed-pythonpath inputs outputs)) -(define* (install #:key outputs (configure-flags '()) use-setuptools? +(define* (add-install-to-path #:key outputs #:allow-other-keys) + "Adding Python scripts to PATH is also often useful in tests." + (setenv "PATH" (string-append (assoc-ref outputs "out") + "/bin:" + (getenv "PATH")))) + +(define* (install #:key inputs outputs (configure-flags '()) use-setuptools? #:allow-other-keys) "Install a given Python package." - (let* ((out (assoc-ref outputs "out")) - (params (append (list (string-append "--prefix=" out)) + (let* ((out (python-output outputs)) + (python (assoc-ref inputs "python")) + (major-minor (map string->number + (take (string-split (python-version python) #\.) 2))) + (<3.7? (match major-minor + ((major minor) + (or (< major 3) (and (= major 3) (< minor 7)))))) + (params (append (list (string-append "--prefix=" out) + "--no-compile") (if use-setuptools? ;; distutils does not accept these flags (list "--single-version-externally-managed" - "--root=/") + "--root=/") '()) configure-flags))) (call-setuppy "install" params use-setuptools?) - #t)) + ;; Rather than produce potentially non-reproducible .pyc files on Pythons + ;; older than 3.7, whose 'compileall' module lacks the + ;; '--invalidation-mode' option, do not generate any. + (unless <3.7? + (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash" + out)))) (define* (wrap #:key inputs outputs #:allow-other-keys) (define (list-of-files dir) @@ -194,25 +229,19 @@ when running checks after installing the package." (define bindirs (append-map (match-lambda - ((_ . dir) - (list (string-append dir "/bin") - (string-append dir "/sbin")))) + ((_ . dir) + (list (string-append dir "/bin") + (string-append dir "/sbin")))) outputs)) - (let* ((out (assoc-ref outputs "out")) - (python (assoc-ref inputs "python")) - (var `("PYTHONPATH" prefix - ,(cons (string-append out "/lib/python" - (python-version python) - "/site-packages") - (search-path-as-string->list - (or (getenv "PYTHONPATH") "")))))) + (let* ((var `("GUIX_PYTHONPATH" prefix + ,(search-path-as-string->list + (or (getenv "GUIX_PYTHONPATH") ""))))) (for-each (lambda (dir) (let ((files (list-of-files dir))) (for-each (cut wrap-program <> var) files))) - bindirs) - #t)) + bindirs))) (define* (rename-pth-file #:key name inputs outputs #:allow-other-keys) "Rename easy-install.pth to NAME.pth to avoid conflicts between packages @@ -220,16 +249,11 @@ installed with setuptools." ;; Even if the "easy-install.pth" is not longer created, we kept this phase. ;; There still may be packages creating an "easy-install.pth" manually for ;; some good reason. - (let* ((out (assoc-ref outputs "out")) - (python (assoc-ref inputs "python")) - (site-packages (string-append out "/lib/python" - (python-version python) - "/site-packages")) + (let* ((site-packages (site-packages inputs outputs)) (easy-install-pth (string-append site-packages "/easy-install.pth")) (new-pth (string-append site-packages "/" name ".pth"))) (when (file-exists? easy-install-pth) - (rename-file easy-install-pth new-pth)) - #t)) + (rename-file easy-install-pth new-pth)))) (define* (ensure-no-mtimes-pre-1980 #:rest _) "Ensure that there are no mtimes before 1980-01-02 in the source tree." @@ -241,32 +265,49 @@ installed with setuptools." (ftw "." (lambda (file stat flag) (unless (<= early-1980 (stat:mtime stat)) (utime file early-1980 early-1980)) - #t)) - #t)) + #t)))) (define* (enable-bytecode-determinism #:rest _) "Improve determinism of pyc files." ;; Use deterministic hashes for strings, bytes, and datetime objects. (setenv "PYTHONHASHSEED" "0") - #t) + ;; Prevent Python from creating .pyc files when loading modules (such as + ;; when running a test suite). + (setenv "PYTHONDONTWRITEBYTECODE" "1")) + +(define* (ensure-no-cythonized-files #:rest _) + "Check the source code for @code{.c} files which may have been pre-generated +by Cython." + (for-each + (lambda (file) + (let ((generated-file + (string-append (string-drop-right file 3) "c"))) + (when (file-exists? generated-file) + (format #t "Possible Cythonized file found: ~a~%" generated-file)))) + (find-files "." "\\.pyx$"))) (define %standard-phases ;; The build phase only builds C extensions and copies the Python sources, - ;; while the install phase byte-compiles and copies them to the prefix - ;; directory. The tests are run after the install phase because otherwise - ;; the cached .pyc generated during the tests execution seem to interfere - ;; with the byte compilation of the install phase. + ;; while the install phase copies then byte-compiles the sources to the + ;; prefix directory. The check phase is moved after the installation phase + ;; to ease testing the built package. (modify-phases gnu:%standard-phases (add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980) (add-after 'ensure-no-mtimes-pre-1980 'enable-bytecode-determinism enable-bytecode-determinism) + (add-after 'enable-bytecode-determinism 'ensure-no-cythonized-files + ensure-no-cythonized-files) (delete 'bootstrap) (delete 'configure) ;not needed (replace 'build build) (delete 'check) ;moved after the install phase (replace 'install install) - (add-after 'install 'check check) - (add-after 'install 'wrap wrap) + (add-after 'install 'add-install-to-pythonpath add-install-to-pythonpath) + (add-after 'add-install-to-pythonpath 'add-install-to-path + add-install-to-path) + (add-after 'add-install-to-path 'wrap wrap) + (add-after 'wrap 'check check) + (add-after 'check 'sanity-check sanity-check) (add-before 'strip 'rename-pth-file rename-pth-file))) (define* (python-build #:key inputs (phases %standard-phases) diff --git a/guix/build/rpath.scm b/guix/build/rpath.scm deleted file mode 100644 index 75a1fef5ef..0000000000 --- a/guix/build/rpath.scm +++ /dev/null @@ -1,59 +0,0 @@ -;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013 Ludovic Courtès <ludo@gnu.org> -;;; -;;; This file is part of GNU Guix. -;;; -;;; GNU Guix is free software; you can redistribute it and/or modify it -;;; under the terms of the GNU General Public License as published by -;;; the Free Software Foundation; either version 3 of the License, or (at -;;; your option) any later version. -;;; -;;; GNU Guix is distributed in the hope that it will be useful, but -;;; WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;; GNU General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. - -(define-module (guix build rpath) - #:use-module (ice-9 popen) - #:use-module (ice-9 rdelim) - #:export (%patchelf - file-rpath - augment-rpath)) - -;;; Commentary: -;;; -;;; Tools to manipulate the RPATH and RUNPATH of ELF binaries. Currently they -;;; rely on PatchELF. -;;; -;;; Code: - -(define %patchelf - ;; The `patchelf' command. - (make-parameter "patchelf")) - -(define %not-colon - (char-set-complement (char-set #\:))) - -(define (file-rpath file) - "Return the RPATH (or RUNPATH) of FILE as a list of directory names, or #f -on failure." - (let* ((p (open-pipe* OPEN_READ (%patchelf) "--print-rpath" file)) - (l (read-line p))) - (and (zero? (close-pipe p)) - (string-tokenize l %not-colon)))) - -(define (augment-rpath file dir) - "Add DIR to the front of the RPATH and RUNPATH of FILE. Return the new -RPATH as a list, or #f on failure." - (let* ((rpath (or (file-rpath file) '())) - (rpath* (cons dir rpath))) - (format #t "~a: changing RPATH from ~s to ~s~%" - file rpath rpath*) - (and (zero? (system* (%patchelf) "--set-rpath" - (string-join rpath* ":") file)) - rpath*))) - -;;; rpath.scm ends here diff --git a/guix/build/ruby-build-system.scm b/guix/build/ruby-build-system.scm index c957a61115..9aceb187a4 100644 --- a/guix/build/ruby-build-system.scm +++ b/guix/build/ruby-build-system.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2015 David Thompson <davet@gnu.org> ;;; Copyright © 2015 Pjotr Prins <pjotr.public01@thebird.nl> ;;; Copyright © 2015, 2016 Ben Woodcroft <donttrustben@gmail.com> +;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -73,13 +74,19 @@ directory." (define* (replace-git-ls-files #:key source #:allow-other-keys) "Many gemspec files downloaded from outside rubygems.org use `git ls-files` -to list of the files to be included in the built gem. However, since this +to list the files to be included in the built gem. However, since this operation is not deterministic, we replace it with `find`." - (when (not (gem-archive? source)) + (unless (gem-archive? source) (let ((gemspec (first-gemspec))) + ;; Do not include the freshly built .gem itself as it causes problems. + ;; Strip the first 2 characters ("./") to more exactly match the output + ;; given by 'git ls-files'. This is useful to prevent breaking regexps + ;; that could be used to filter the list of files. (substitute* gemspec - (("`git ls-files`") "`find . -type f |sort`") - (("`git ls-files -z`") "`find . -type f -print0 |sort -z`")))) + (("`git ls-files`") + "`find . -type f -not -regex '.*\\.gem$' | sort | cut -c3-`") + (("`git ls-files -z`") + "`find . -type f -not -regex '.*\\.gem$' -print0 | sort -z | cut -zc3-`")))) #t) (define* (extract-gemspec #:key source #:allow-other-keys) @@ -129,11 +136,7 @@ is #f." #:allow-other-keys) "Install the gem archive SOURCE to the output store item. Additional GEM-FLAGS are passed to the 'gem' invocation, if present." - (let* ((ruby-version - (match:substring (string-match "ruby-(.*)\\.[0-9]$" - (assoc-ref inputs "ruby")) - 1)) - (out (assoc-ref outputs "out")) + (let* ((out (assoc-ref outputs "out")) (vendor-dir (string-append out "/lib/ruby/vendor_ruby")) (gem-file (first-matching-file "\\.gem$")) (gem-file-basename (basename gem-file)) @@ -144,8 +147,8 @@ GEM-FLAGS are passed to the 'gem' invocation, if present." (setenv "GEM_VENDOR" vendor-dir) (or (zero? - ;; 'zero? system*' allows the custom error handling to function as - ;; expected, while 'invoke' raises its own exception. + ;; 'zero? system*' allows the custom error handling to function as + ;; expected, while 'invoke' raises its own exception. (apply system* "gem" "install" gem-file "--verbose" "--local" "--ignore-dependencies" "--vendor" diff --git a/guix/build/texlive-build-system.scm b/guix/build/texlive-build-system.scm index 841c631dae..4c255700bb 100644 --- a/guix/build/texlive-build-system.scm +++ b/guix/build/texlive-build-system.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -40,39 +41,11 @@ (string-append "&" format) file)) -(define* (configure #:key inputs #:allow-other-keys) - (let* ((out (string-append (getcwd) "/.texlive-union")) - (texmf.cnf (string-append out "/share/texmf-dist/web2c/texmf.cnf"))) - ;; Build a modifiable union of all inputs (but exclude bash) - (match inputs - (((names . directories) ...) - (union-build out (filter directory-exists? directories) - #:create-all-directories? #t - #:log-port (%make-void-port "w")))) - - ;; The configuration file "texmf.cnf" is provided by the - ;; "texlive-bin" package. We take it and override only the - ;; setting for TEXMFROOT and TEXMF. This file won't be consulted - ;; by default, though, so we still need to set TEXMFCNF. - (substitute* texmf.cnf - (("^TEXMFROOT = .*") - (string-append "TEXMFROOT = " out "/share\n")) - (("^TEXMF = .*") - "TEXMF = $TEXMFROOT/share/texmf-dist\n")) - (setenv "TEXMFCNF" (dirname texmf.cnf)) - (setenv "TEXMF" (string-append out "/share/texmf-dist")) - - ;; Don't truncate lines. - (setenv "error_line" "254") ; must be less than 255 - (setenv "half_error_line" "238") ; must be less than error_line - 15 - (setenv "max_print_line" "1000")) - (mkdir "build") - #t) - (define* (build #:key inputs build-targets tex-format #:allow-other-keys) - (every (cut compile-with-latex tex-format <>) - (if build-targets build-targets - (scandir "." (cut string-suffix? ".ins" <>))))) + (mkdir "build") + (for-each (cut compile-with-latex tex-format <>) + (if build-targets build-targets + (scandir "." (cut string-suffix? ".ins" <>))))) (define* (install #:key outputs tex-directory #:allow-other-keys) (let* ((out (assoc-ref outputs "out")) @@ -81,13 +54,12 @@ (mkdir-p target) (for-each delete-file (find-files "." "\\.(log|aux)$")) (for-each (cut install-file <> target) - (find-files "build" ".*")) - #t)) + (find-files "build" ".*")))) (define %standard-phases (modify-phases gnu:%standard-phases (delete 'bootstrap) - (replace 'configure configure) + (delete 'configure) (replace 'build build) (delete 'check) (replace 'install install))) diff --git a/guix/build/utils.scm b/guix/build/utils.scm index 419c10195b..6c37021673 100644 --- a/guix/build/utils.scm +++ b/guix/build/utils.scm @@ -1,10 +1,12 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Andreas Enge <andreas@enge.fr> ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org> -;;; Copyright © 2015, 2018 Mark H Weaver <mhw@netris.org> +;;; Copyright © 2015, 2018, 2021 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2018, 2019 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> +;;; Copyright © 2020, 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -49,9 +51,14 @@ package-name->name+version parallel-job-count + compressor + tarball? + %xz-parallel-args + directory-exists? executable-file? symbolic-link? + call-with-temporary-output-file call-with-ascii-input-file elf-file? ar-file? @@ -134,12 +141,39 @@ ;;; +;;; Compression helpers. +;;; + +(define (compressor file-name) + "Return the name of the compressor package/binary used to compress or +decompress FILE-NAME, based on its file extension, else false." + (cond ((string-suffix? "gz" file-name) "gzip") + ((string-suffix? "Z" file-name) "gzip") + ((string-suffix? "bz2" file-name) "bzip2") + ((string-suffix? "lz" file-name) "lzip") + ((string-suffix? "zip" file-name) "unzip") + ((string-suffix? "xz" file-name) "xz") + (else #f))) ;no compression used/unknown file extension + +(define (tarball? file-name) + "True when FILE-NAME has a tar file extension." + (string-match "\\.(tar(\\..*)?|tgz|tbz)$" file-name)) + +(define (%xz-parallel-args) + "The xz arguments required to enable bit-reproducible, multi-threaded +compression." + (list "--memlimit=50%" + (format #f "--threads=~a" (max 2 (parallel-job-count))))) + + +;;; ;;; Directories. ;;; (define (%store-directory) "Return the directory name of the store." - (or (getenv "NIX_STORE") + (or (getenv "NIX_STORE_DIR") ;outside of builder + (getenv "NIX_STORE") ;inside builder, set by the daemon "/gnu/store")) (define (store-file-name? file) @@ -197,6 +231,22 @@ introduce the version part." "Return #t if FILE is a symbolic link (aka. \"symlink\".)" (eq? (stat:type (lstat file)) 'symlink)) +(define (call-with-temporary-output-file proc) + "Call PROC with a name of a temporary file and open output port to that +file; close the file and delete it when leaving the dynamic extent of this +call." + (let* ((directory (or (getenv "TMPDIR") "/tmp")) + (template (string-append directory "/guix-file.XXXXXX")) + (out (mkstemp! template))) + (dynamic-wind + (lambda () + #t) + (lambda () + (proc template out)) + (lambda () + (false-if-exception (close out)) + (false-if-exception (delete-file template)))))) + (define (call-with-ascii-input-file file proc) "Open FILE as an ASCII or binary file, and pass the resulting port to PROC. FILE is closed when PROC's dynamic extent is left. Return the @@ -322,11 +372,13 @@ name." #:key (log (current-output-port)) (follow-symlinks? #f) - keep-mtime?) + (copy-file copy-file) + keep-mtime? keep-permissions?) "Copy SOURCE directory to DESTINATION. Follow symlinks if FOLLOW-SYMLINKS? -is true; otherwise, just preserve them. When KEEP-MTIME? is true, keep the -modification time of the files in SOURCE on those of DESTINATION. Write -verbose output to the LOG port." +is true; otherwise, just preserve them. Call COPY-FILE to copy regular files. +When KEEP-MTIME? is true, keep the modification time of the files in SOURCE on +those of DESTINATION. When KEEP-PERMISSIONS? is true, preserve file +permissions. Write verbose output to the LOG port." (define strip-source (let ((len (string-length source))) (lambda (file) @@ -343,16 +395,21 @@ verbose output to the LOG port." (symlink target dest))) (else (copy-file file dest) - (when keep-mtime? - (set-file-time dest stat)))))) + (when keep-permissions? + (chmod dest (stat:perms stat))))) + (when keep-mtime? + (set-file-time dest stat)))) (lambda (dir stat result) ; down (let ((target (string-append destination (strip-source dir)))) - (mkdir-p target) - (when keep-mtime? - (set-file-time target stat)))) + (mkdir-p target))) (lambda (dir stat result) ; up - result) + (let ((target (string-append destination + (strip-source dir)))) + (when keep-mtime? + (set-file-time target stat)) + (when keep-permissions? + (chmod target (stat:perms stat))))) (const #t) ; skip (lambda (file stat errno result) (format (current-error-port) "i/o error: ~a: ~a~%" @@ -365,6 +422,16 @@ verbose output to the LOG port." stat lstat))) +(define-syntax-rule (warn-on-error expr file) + (catch 'system-error + (lambda () + expr) + (lambda args + (format (current-error-port) + "warning: failed to delete ~a: ~a~%" + file (strerror + (system-error-errno args)))))) + (define* (delete-file-recursively dir #:key follow-mounts?) "Delete DIR recursively, like `rm -rf', without following symlinks. Don't @@ -375,10 +442,10 @@ errors." (or follow-mounts? (= dev (stat:dev stat)))) (lambda (file stat result) ; leaf - (delete-file file)) + (warn-on-error (delete-file file) file)) (const #t) ; down (lambda (dir stat result) ; up - (rmdir dir)) + (warn-on-error (rmdir dir) dir)) (const #t) ; skip (lambda (file stat errno result) (format (current-error-port) @@ -746,6 +813,31 @@ PROC's result is returned." (lambda (key . args) (false-if-exception (delete-file template)))))) +(define (unused-private-use-code-point s) + "Find a code point within a Unicode Private Use Area that is not +present in S, and return the corresponding character object. If one +cannot be found, return false." + (define (scan lo hi) + (and (<= lo hi) + (let ((c (integer->char lo))) + (if (string-index s c) + (scan (+ lo 1) hi) + c)))) + (or (scan #xE000 #xF8FF) + (scan #xF0000 #xFFFFD) + (scan #x100000 #x10FFFD))) + +(define (replace-char c1 c2 s) + "Return a string which is equal to S except with all instances of C1 +replaced by C2. If C1 and C2 are equal, return S." + (if (char=? c1 c2) + s + (string-map (lambda (c) + (if (char=? c c1) + c2 + c)) + s))) + (define (substitute file pattern+procs) "PATTERN+PROCS is a list of regexp/two-argument-procedure pairs. For each line of FILE, and for each PATTERN that it matches, call the corresponding @@ -764,16 +856,26 @@ end of a line; by itself it won't match the terminating newline of a line." (let loop ((line (read-line in 'concat))) (if (eof-object? line) #t - (let ((line (fold (lambda (r+p line) - (match r+p - ((regexp . proc) - (match (list-matches regexp line) - ((and m+ (_ _ ...)) - (proc line m+)) - (_ line))))) - line - rx+proc))) - (display line out) + ;; Work around the fact that Guile's regexp-exec does not handle + ;; NUL characters (a limitation of the underlying GNU libc's + ;; regexec) by temporarily replacing them by an unused private + ;; Unicode code point. + ;; TODO: Use SRFI-115 instead, once available in Guile. + (let* ((nul* (or (and (string-index line #\nul) + (unused-private-use-code-point line)) + #\nul)) + (line* (replace-char #\nul nul* line)) + (line1* (fold (lambda (r+p line) + (match r+p + ((regexp . proc) + (match (list-matches regexp line) + ((and m+ (_ _ ...)) + (proc line m+)) + (_ line))))) + line* + rx+proc)) + (line1 (replace-char nul* #\nul line1*))) + (display line1 out) (loop (read-line in 'concat))))))))) @@ -800,7 +902,7 @@ sub-expression. For example: ((\"hello\") \"good morning\\n\") ((\"foo([a-z]+)bar(.*)$\" all letters end) - (string-append \"baz\" letter end))) + (string-append \"baz\" letters end))) Here, anytime a line of FILE contains \"hello\", it is replaced by \"good morning\". Anytime a line of FILE matches the second regexp, ALL is bound to @@ -853,29 +955,45 @@ match the terminating newline of a line." ;;; (define* (dump-port in out + #:optional len #:key (buffer-size 16384) (progress (lambda (t k) (k)))) - "Read as much data as possible from IN and write it to OUT, using chunks of -BUFFER-SIZE bytes. Call PROGRESS at the beginning and after each successful -transfer of BUFFER-SIZE bytes or less, passing it the total number of bytes -transferred and the continuation of the transfer as a thunk." + "Read LEN bytes from IN or as much data as possible if LEN is #f, and write +it to OUT, using chunks of BUFFER-SIZE bytes. Call PROGRESS at the beginning +and after each successful transfer of BUFFER-SIZE bytes or less, passing it +the total number of bytes transferred and the continuation of the transfer as +a thunk." (define buffer (make-bytevector buffer-size)) (define (loop total bytes) (or (eof-object? bytes) + (and len (= total len)) (let ((total (+ total bytes))) (put-bytevector out buffer 0 bytes) (progress total (lambda () (loop total - (get-bytevector-n! in buffer 0 buffer-size))))))) + (get-bytevector-n! in buffer 0 + (if len + (min (- len total) buffer-size) + buffer-size)))))))) ;; Make sure PROGRESS is called when we start so that it can measure ;; throughput. (progress 0 (lambda () - (loop 0 (get-bytevector-n! in buffer 0 buffer-size))))) + (loop 0 (get-bytevector-n! in buffer 0 + (if len + (min len buffer-size) + buffer-size)))))) + +(define AT_SYMLINK_NOFOLLOW + ;; Guile 2.0 did not define this constant, hence this hack. + (let ((variable (module-variable the-root-module 'AT_SYMLINK_NOFOLLOW))) + (if variable + (variable-ref variable) + 256))) ;for GNU/Linux (define (set-file-time file stat) "Set the atime/mtime of FILE to that specified by STAT." @@ -883,7 +1001,8 @@ transferred and the continuation of the transfer as a thunk." (stat:atime stat) (stat:mtime stat) (stat:atimensec stat) - (stat:mtimensec stat))) + (stat:mtimensec stat) + AT_SYMLINK_NOFOLLOW)) (define (get-char* p) ;; We call it `get-char', but that's really a binary version @@ -1307,7 +1426,7 @@ not supported." (lambda () (call-with-ascii-input-file prog (lambda (p) - (format out header) + (display header out) (dump-port p out) (close out) (chmod template mode) |