diff options
| author | Efraim Flashner <efraim@flashner.co.il> | 2021-04-16 14:39:48 +0300 |
|---|---|---|
| committer | Efraim Flashner <efraim@flashner.co.il> | 2021-04-16 14:39:48 +0300 |
| commit | fcc39864dba82e14895afbe841091091366c96bc (patch) | |
| tree | 6e0f05495fd6512051224dc85fd3ab495cbf1a24 /guix | |
| parent | 76fc36d0a7215979bb74c05840f5a4de4ab5ea93 (diff) | |
| parent | 44f9432705d04c069a8acf9e37e3ad856ac0bf82 (diff) | |
| download | guix-patches-fcc39864dba82e14895afbe841091091366c96bc.tar guix-patches-fcc39864dba82e14895afbe841091091366c96bc.tar.gz | |
Merge remote-tracking branch 'origin/master' into core-updates
Conflicts:
gnu/local.mk
gnu/packages/boost.scm
gnu/packages/chez.scm
gnu/packages/compression.scm
gnu/packages/crates-io.scm
gnu/packages/docbook.scm
gnu/packages/engineering.scm
gnu/packages/gcc.scm
gnu/packages/gl.scm
gnu/packages/gtk.scm
gnu/packages/nettle.scm
gnu/packages/python-check.scm
gnu/packages/python-xyz.scm
gnu/packages/radio.scm
gnu/packages/rust.scm
gnu/packages/sqlite.scm
guix/build-system/node.scm
Diffstat (limited to 'guix')
35 files changed, 1316 insertions, 747 deletions
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm index 392f2d9b7b..100d1db4b6 100644 --- a/guix/build-system/go.scm +++ b/guix/build-system/go.scm @@ -34,6 +34,7 @@ go-build go-build-system + go-pseudo-version? go-version->git-ref)) ;; Commentary: @@ -43,17 +44,19 @@ ;; ;; Code: -(define %go-version-rx +(define %go-pseudo-version-rx + ;; Match only the end of the version string; this is so that matching the + ;; more complex leading semantic version pattern is not required. (make-regexp (string-append - "(v?[0-9]\\.[0-9]\\.[0-9])" ;"v" prefix can be omitted in version prefix - "(-|-pre\\.0\\.|-0\\.)" ;separator - "([0-9]{14})-" ;timestamp - "([0-9A-Fa-f]{12})"))) ;commit hash + "([0-9]{14}-)" ;timestamp + "([0-9A-Fa-f]{12})" ;commit hash + "(\\+incompatible)?$"))) ;optional +incompatible tag (define (go-version->git-ref version) "Parse VERSION, a \"pseudo-version\" as defined at <https://golang.org/ref/mod#pseudo-versions>, and extract the commit hash from -it, defaulting to full VERSION if a pseudo-version pattern is not recognized." +it, defaulting to full VERSION (stripped from the \"+incompatible\" suffix if +present) if a pseudo-version pattern is not recognized." ;; A module version like v1.2.3 is introduced by tagging a revision in the ;; underlying source repository. Untagged revisions can be referred to ;; using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef, where @@ -68,11 +71,16 @@ it, defaulting to full VERSION if a pseudo-version pattern is not recognized." (if (string-suffix? "+incompatible" version) (string-drop-right version 13) version)) - (match (regexp-exec %go-version-rx version))) + (match (regexp-exec %go-pseudo-version-rx version))) (if match - (match:substring match 4) + (match:substring match 2) version))) +(define (go-pseudo-version? version) + "True if VERSION is a Go pseudo-version, i.e., a version string made of a +commit hash and its date rather than a proper release tag." + (regexp-exec %go-pseudo-version-rx version)) + (define %go-build-system-modules ;; Build-side modules imported and used by default. `((guix build go-build-system) diff --git a/guix/build-system/node.scm b/guix/build-system/node.scm index 73a6f152dd..ae799d1f4e 100644 --- a/guix/build-system/node.scm +++ b/guix/build-system/node.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016 Jelle Licht <jlicht@fsfe.org> +;;; Copyright © 2019 Timothy Sample <samplet@ngyro.com> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. @@ -18,7 +19,6 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix build-system node) - #:use-module (guix store) #:use-module (guix utils) #:use-module (guix packages) #:use-module (guix gexp) @@ -27,28 +27,21 @@ #:use-module (guix build-system) #:use-module (guix build-system gnu) #:use-module (ice-9 match) - #:export (npm-meta-uri - %node-build-system-modules + #:export (%node-build-system-modules node-build node-build-system)) -(define (npm-meta-uri name) - "Return a URI string for the metadata of node module NAME found in the npm -registry." - (string-append "https://registry.npmjs.org/" name)) - (define %node-build-system-modules ;; Build-side modules imported by default. `((guix build node-build-system) (guix build json) - (guix build union) - ,@%gnu-build-system-modules)) ;; TODO: Might be not needed + ,@%gnu-build-system-modules)) (define (default-node) "Return the default Node package." ;; Lazily resolve the binding to avoid a circular dependency. (let ((node (resolve-interface '(gnu packages node)))) - (module-ref node 'node))) + (module-ref node 'node-lts))) (define* (lower name #:key source inputs native-inputs outputs system target @@ -80,6 +73,7 @@ registry." #:key source (npm-flags ''()) + (test-target "test") (tests? #t) (phases '%standard-phases) (outputs '("out")) @@ -88,8 +82,6 @@ registry." (guile #f) (imported-modules %node-build-system-modules) (modules '((guix build node-build-system) - (guix build json) - (guix build union) (guix build utils)))) "Build SOURCE using NODE and INPUTS." (define builder @@ -100,6 +92,7 @@ registry." #:source #+source #:system #$system #:npm-flags #$npm-flags + #:test-target #$test-target #:tests? #$tests? #:phases #$phases #:outputs #$(outputs->gexp outputs) @@ -117,5 +110,5 @@ registry." (define node-build-system (build-system (name 'node) - (description "The standard Node build system") + (description "The Node build system") (lower lower))) diff --git a/guix/build/graft.scm b/guix/build/graft.scm index c119ee71d1..f04c35fa74 100644 --- a/guix/build/graft.scm +++ b/guix/build/graft.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2015, 2016, 2018 Ludovic Courtès <ludo@gnu.org> -;;; Copyright © 2016 Mark H Weaver <mhw@netris.org> +;;; Copyright © 2016, 2021 Mark H Weaver <mhw@netris.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -55,6 +55,52 @@ (string->char-set "0123456789abcdfghijklmnpqrsvwxyz") <>)) +(define (nix-base32-char-or-nul? c) + "Return true if C is a nix-base32 character or NUL, otherwise return false." + (or (nix-base32-char? c) + (char=? c #\nul))) + +(define (possible-utf16-hash? buffer i w) + "Return true if (I - W) is large enough to hold a UTF-16 encoded +nix-base32 hash and if BUFFER contains NULs in all positions where NULs +are to be expected in a UTF-16 encoded hash+dash pattern whose dash is +found at position I. Otherwise, return false." + (and (<= (* 2 hash-length) (- i w)) + (let loop ((j (+ 1 (- i (* 2 hash-length))))) + (or (>= j i) + (and (zero? (bytevector-u8-ref buffer j)) + (loop (+ j 2))))))) + +(define (possible-utf32-hash? buffer i w) + "Return true if (I - W) is large enough to hold a UTF-32 encoded +nix-base32 hash and if BUFFER contains NULs in all positions where NULs +are to be expected in a UTF-32 encoded hash+dash pattern whose dash is +found at position I. Otherwise, return false." + (and (<= (* 4 hash-length) (- i w)) + (let loop ((j (+ 1 (- i (* 4 hash-length))))) + (or (>= j i) + (and (zero? (bytevector-u8-ref buffer j)) + (zero? (bytevector-u8-ref buffer (+ j 1))) + (zero? (bytevector-u8-ref buffer (+ j 2))) + (loop (+ j 4))))))) + +(define (insert-nuls char-size bv) + "Given a bytevector BV, return a bytevector containing the same bytes but +with (CHAR-SIZE - 1) NULs inserted between every two adjacent bytes from BV. +For example, (insert-nuls 4 #u8(1 2 3)) => #u8(1 0 0 0 2 0 0 0 3)." + (if (= char-size 1) + bv + (let* ((len (bytevector-length bv)) + (bv* (make-bytevector (+ 1 (* char-size + (- len 1))) + 0))) + (let loop ((i 0)) + (when (< i len) + (bytevector-u8-set! bv* (* i char-size) + (bytevector-u8-ref bv i)) + (loop (+ i 1)))) + bv*))) + (define* (replace-store-references input output replacement-table #:optional (store (%store-directory))) "Read data from INPUT, replacing store references according to @@ -76,9 +122,9 @@ bytevectors to the same value." (list->vector (map pred (iota 256))) <>)) - (define nix-base32-byte? + (define nix-base32-byte-or-nul? (optimize-u8-predicate - (compose nix-base32-char? + (compose nix-base32-char-or-nul? integer->char))) (define (dash? byte) (= byte 45)) @@ -86,100 +132,153 @@ bytevectors to the same value." (define request-size (expt 2 20)) ; 1 MiB ;; We scan the file for the following 33-byte pattern: 32 bytes of - ;; nix-base32 characters followed by a dash. To accommodate large files, - ;; we do not read the entire file, but instead work on buffers of up to - ;; 'request-size' bytes. To ensure that every 33-byte sequence appears - ;; entirely within exactly one buffer, adjacent buffers must overlap, - ;; i.e. they must share 32 byte positions. We accomplish this by - ;; "ungetting" the last 32 bytes of each buffer before reading the next - ;; buffer, unless we know that we've reached the end-of-file. + ;; nix-base32 characters followed by a dash. When we find such a pattern + ;; whose hash is in REPLACEMENT-TABLE, we perform the required rewrite and + ;; continue scanning. + ;; + ;; To support UTF-16 and UTF-32 store references, the 33 bytes comprising + ;; this hash+dash pattern may optionally be interspersed by extra NUL bytes. + ;; This simple approach works because the characters we are looking for are + ;; restricted to ASCII. UTF-16 hashes are interspersed with single NUL + ;; bytes ("\0"), and UTF-32 hashes are interspersed with triplets of NULs + ;; ("\0\0\0"). Note that we require NULs to be present only *between* the + ;; other bytes, and not at either end, in order to be insensitive to byte + ;; order. + ;; + ;; To accommodate large files, we do not read the entire file at once, but + ;; instead work on buffers of up to REQUEST-SIZE bytes. To ensure that + ;; every hash+dash pattern appears in its entirety in at least one buffer, + ;; adjacent buffers must overlap by one byte less than the maximum size of a + ;; hash+dash pattern. We accomplish this by "ungetting" a suffix of each + ;; buffer before reading the next buffer, unless we know that we've reached + ;; the end-of-file. (let ((buffer (make-bytevector request-size))) - (let loop () - ;; Note: We avoid 'get-bytevector-n' to work around - ;; <http://bugs.gnu.org/17466>. + (define-syntax-rule (byte-at i) + (bytevector-u8-ref buffer i)) + (let outer-loop () (match (get-bytevector-n! input buffer 0 request-size) ((? eof-object?) 'done) (end - ;; We scan the buffer for dashes that might be preceded by a - ;; nix-base32 hash. The key optimization here is that whenever we - ;; find a NON-nix-base32 character at position 'i', we know that it - ;; cannot be part of a hash, so the earliest position where the next - ;; hash could start is i+1 with the following dash at position i+33. - ;; - ;; Since nix-base32 characters comprise only 1/8 of the 256 possible - ;; byte values, and exclude some of the most common letters in - ;; English text (e t o u), in practice we can advance by 33 positions - ;; most of the time. - (let scan-from ((i hash-length) (written 0)) - ;; 'i' is the first position where we look for a dash. 'written' - ;; is the number of bytes in the buffer that have already been - ;; written. + (define (scan-from i w) + ;; Scan the buffer for dashes that might be preceded by nix hashes, + ;; where I is the minimum position where such a dash might be + ;; found, and W is the number of bytes in the buffer that have been + ;; written so far. We assume that I - W >= HASH-LENGTH. + ;; + ;; The key optimization here is that whenever we find a byte at + ;; position I that cannot occur within a nix hash (because it's + ;; neither a nix-base32 character nor NUL), we can infer that the + ;; earliest position where the next hash could start is at I + 1, + ;; and therefore the earliest position for the following dash is + ;; (+ I 1 HASH-LENGTH), which is I + 33. + ;; + ;; Since nix-base32-or-nul characters comprise only about 1/8 of + ;; the 256 possible byte values, and exclude some of the most + ;; common letters in English text (e t o u), we can advance 33 + ;; positions much of the time. (if (< i end) - (let ((byte (bytevector-u8-ref buffer i))) - (cond ((and (dash? byte) - ;; We've found a dash. Note that we do not know - ;; whether the preceeding 32 bytes are nix-base32 - ;; characters, but we do not need to know. If - ;; they are not, the following lookup will fail. - (lookup-replacement - (string-tabulate (lambda (j) - (integer->char - (bytevector-u8-ref buffer - (+ j (- i hash-length))))) - hash-length))) - => (lambda (replacement) - ;; We've found a hash that needs to be replaced. - ;; First, write out all bytes preceding the hash - ;; that have not yet been written. - (put-bytevector output buffer written - (- i hash-length written)) - ;; Now write the replacement string. - (put-bytevector output replacement) - ;; Since the byte at position 'i' is a dash, - ;; which is not a nix-base32 char, the earliest - ;; position where the next hash might start is - ;; i+1, and the earliest position where the - ;; following dash might start is (+ i 1 - ;; hash-length). Also, increase the write - ;; position to account for REPLACEMENT. - (let ((len (bytevector-length replacement))) - (scan-from (+ i 1 len) - (+ i (- len hash-length)))))) - ;; If the byte at position 'i' is a nix-base32 char, - ;; then the dash we're looking for might be as early as - ;; the following byte, so we can only advance by 1. - ((nix-base32-byte? byte) - (scan-from (+ i 1) written)) - ;; If the byte at position 'i' is NOT a nix-base32 - ;; char, then the earliest position where the next hash - ;; might start is i+1, with the following dash at - ;; position (+ i 1 hash-length). + (let ((byte (byte-at i))) + (cond ((dash? byte) + (found-dash i w)) + ((nix-base32-byte-or-nul? byte) + (scan-from (+ i 1) w)) (else - (scan-from (+ i 1 hash-length) written)))) - - ;; We have finished scanning the buffer. Now we determine how - ;; many bytes have not yet been written, and how many bytes to - ;; "unget". If 'end' is less than 'request-size' then we read - ;; less than we asked for, which indicates that we are at EOF, - ;; so we needn't unget anything. Otherwise, we unget up to - ;; 'hash-length' bytes (32 bytes). However, we must be careful - ;; not to unget bytes that have already been written, because - ;; that would cause them to be written again from the next - ;; buffer. In practice, this case occurs when a replacement is - ;; made near or beyond the end of the buffer. When REPLACEMENT - ;; went beyond END, we consume the extra bytes from INPUT. - (begin - (if (> written end) - (get-bytevector-n! input buffer 0 (- written end)) - (let* ((unwritten (- end written)) - (unget-size (if (= end request-size) - (min hash-length unwritten) - 0)) - (write-size (- unwritten unget-size))) - (put-bytevector output buffer written write-size) - (unget-bytevector input buffer (+ written write-size) - unget-size))) - (loop))))))))) + (not-part-of-hash i w)))) + (finish-buffer i w))) + + (define (not-part-of-hash i w) + ;; Position I is known to not be within a nix hash that we must + ;; rewrite. Therefore, the earliest position where the next hash + ;; might start is I + 1, and therefore the earliest position of + ;; the following dash is (+ I 1 HASH-LENGTH). + (scan-from (+ i 1 hash-length) w)) + + (define (found-dash i w) + ;; We know that there is a dash '-' at position I, and that + ;; I - W >= HASH-LENGTH. The immediately preceding bytes *might* + ;; contain a nix-base32 hash, but that is not yet known. Here, + ;; we rule out all but one possible encoding (ASCII, UTF-16, + ;; UTF-32) by counting how many NULs precede the dash. + (cond ((not (zero? (byte-at (- i 1)))) + ;; The dash is *not* preceded by a NUL, therefore it + ;; cannot possibly be a UTF-16 or UTF-32 hash. Proceed + ;; to check for an ASCII hash. + (found-possible-hash 1 i w)) + + ((not (zero? (byte-at (- i 2)))) + ;; The dash is preceded by exactly one NUL, therefore it + ;; cannot be an ASCII or UTF-32 hash. Proceed to check + ;; for a UTF-16 hash. + (if (possible-utf16-hash? buffer i w) + (found-possible-hash 2 i w) + (not-part-of-hash i w))) + + (else + ;; The dash is preceded by at least two NULs, therefore + ;; it cannot be an ASCII or UTF-16 hash. Proceed to + ;; check for a UTF-32 hash. + (if (possible-utf32-hash? buffer i w) + (found-possible-hash 4 i w) + (not-part-of-hash i w))))) + + (define (found-possible-hash char-size i w) + ;; We know that there is a dash '-' at position I, that + ;; I - W >= CHAR-SIZE * HASH-LENGTH, and that the only + ;; possible encoding for the preceding hash is as indicated by + ;; CHAR-SIZE. Here we check to see if the given hash is in + ;; REPLACEMENT-TABLE, and if so, we perform the required + ;; rewrite. + (let* ((hash (string-tabulate + (lambda (j) + (integer->char + (byte-at (- i (* char-size + (- hash-length j)))))) + hash-length)) + (replacement* (lookup-replacement hash)) + (replacement (and replacement* + (insert-nuls char-size replacement*)))) + (cond + ((not replacement) + (not-part-of-hash i w)) + (else + ;; We've found a hash that needs to be replaced. + ;; First, write out all bytes preceding the hash + ;; that have not yet been written. + (put-bytevector output buffer w + (- i (* char-size hash-length) w)) + ;; Now write the replacement string. + (put-bytevector output replacement) + ;; Now compute the new values of W and I and continue. + (let ((w (+ (- i (* char-size hash-length)) + (bytevector-length replacement)))) + (scan-from (+ w hash-length) w)))))) + + (define (finish-buffer i w) + ;; We have finished scanning the buffer. Now we determine how many + ;; bytes have not yet been written, and how many bytes to "unget". + ;; If END is less than REQUEST-SIZE then we read less than we asked + ;; for, which indicates that we are at EOF, so we needn't unget + ;; anything. Otherwise, we unget up to (* 4 HASH-LENGTH) bytes. + ;; However, we must be careful not to unget bytes that have already + ;; been written, because that would cause them to be written again + ;; from the next buffer. In practice, this case occurs when a + ;; replacement is made near or beyond the end of the buffer. When + ;; REPLACEMENT went beyond END, we consume the extra bytes from + ;; INPUT. + (if (> w end) + (get-bytevector-n! input buffer 0 (- w end)) + (let* ((unwritten (- end w)) + (unget-size (if (= end request-size) + (min (* 4 hash-length) + unwritten) + 0)) + (write-size (- unwritten unget-size))) + (put-bytevector output buffer w write-size) + (unget-bytevector input buffer (+ w write-size) + unget-size))) + (outer-loop)) + + (scan-from hash-length 0)))))) (define (rename-matching-files directory mapping) "Apply MAPPING to the names of all the files in DIRECTORY, where MAPPING is diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm index 8f57045a8c..d74acf2a05 100644 --- a/guix/build/julia-build-system.scm +++ b/guix/build/julia-build-system.scm @@ -101,7 +101,7 @@ Project.toml)." (or (getenv "JULIA_LOAD_PATH") ""))) (setenv "HOME" "/tmp") - (invoke "julia" + (invoke "julia" "--depwarn=yes" (string-append builddir "packages/" package "/test/runtests.jl")))) #t) diff --git a/guix/build/node-build-system.scm b/guix/build/node-build-system.scm index 7799f03595..a55cab237c 100644 --- a/guix/build/node-build-system.scm +++ b/guix/build/node-build-system.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 David Thompson <davet@gnu.org> -;;; Copyright © 2016 Jelle Licht <jlicht@fsfe.org> +;;; Copyright © 2016, 2020 Jelle Licht <jlicht@fsfe.org> +;;; Copyright © 2019, 2021 Timothy Sample <samplet@ngyro.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -19,144 +20,130 @@ (define-module (guix build node-build-system) #:use-module ((guix build gnu-build-system) #:prefix gnu:) - #:use-module (guix build json) - #:use-module (guix build union) #:use-module (guix build utils) + #:use-module (guix build json) + #:use-module (ice-9 ftw) #:use-module (ice-9 match) - #:use-module (ice-9 popen) - #:use-module (ice-9 regex) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-26) #:export (%standard-phases node-build)) ;; Commentary: ;; -;; Builder-side code of the standard Node/npm package build procedure. +;; Builder-side code of the standard Node/NPM package install procedure. ;; ;; Code: -(define* (read-package-data #:key (filename "package.json")) - (call-with-input-file filename - (lambda (port) - (read-json port)))) +(define (set-home . _) + (with-directory-excursion ".." + (let loop ((i 0)) + (let ((dir (string-append "npm-home-" (number->string i)))) + (if (directory-exists? dir) + (loop (1+ i)) + (begin + (mkdir dir) + (setenv "HOME" (string-append (getcwd) "/" dir)) + (format #t "set HOME to ~s~%" (getenv "HOME"))))))) + #t) -(define* (build #:key inputs #:allow-other-keys) - (define (build-from-package-json? package-file) - (let* ((package-data (read-package-data #:filename package-file)) - (scripts (assoc-ref package-data "scripts"))) - (assoc-ref scripts "build"))) - "Build a new node module using the appropriate build system." - ;; XXX: Develop a more robust heuristic, allow override - (cond ((file-exists? "gulpfile.js") - (invoke "gulp")) - ((file-exists? "gruntfile.js") - (invoke "grunt")) - ((file-exists? "Makefile") - (invoke "make")) - ((and (file-exists? "package.json") - (build-from-package-json? "package.json")) - (invoke "npm" "run" "build"))) +(define (module-name module) + (let* ((package.json (string-append module "/package.json")) + (package-meta (call-with-input-file package.json read-json))) + (assoc-ref package-meta "name"))) + +(define (index-modules input-paths) + (define (list-modules directory) + (append-map (lambda (x) + (if (string-prefix? "@" x) + (list-modules (string-append directory "/" x)) + (list (string-append directory "/" x)))) + (filter (lambda (x) + (not (member x '("." "..")))) + (or (scandir directory) '())))) + (let ((index (make-hash-table (* 2 (length input-paths))))) + (for-each (lambda (dir) + (let ((nm (string-append dir "/lib/node_modules"))) + (for-each (lambda (module) + (hash-set! index (module-name module) module)) + (list-modules nm)))) + input-paths) + index)) + +(define* (patch-dependencies #:key inputs #:allow-other-keys) + + (define index (index-modules (map cdr inputs))) + + (define (resolve-dependencies package-meta meta-key) + (fold (lambda (key+value acc) + (match key+value + ('@ acc) + ((key . value) (acons key (hash-ref index key value) acc)))) + '() + (or (assoc-ref package-meta meta-key) '()))) + + (with-atomic-file-replacement "package.json" + (lambda (in out) + (let ((package-meta (read-json in))) + (assoc-set! package-meta "dependencies" + (append + '(@) + (resolve-dependencies package-meta "dependencies") + (resolve-dependencies package-meta "peerDependencies"))) + (assoc-set! package-meta "devDependencies" + (append + '(@) + (resolve-dependencies package-meta "devDependencies"))) + (write-json package-meta out)))) #t) -(define* (link-npm-dependencies #:key inputs #:allow-other-keys) - (define (inputs->node-inputs inputs) - "Filter the directory part from INPUTS." - (filter (lambda (input) - (match input - ((name . _) (node-package? name)))) - inputs)) - (define (inputs->directories inputs) - "Extract the directory part from INPUTS." - (match inputs - (((names . directories) ...) - directories))) - (define (make-node-path root) - (string-append root "/lib/node_modules/")) - - (let ((input-node-directories (inputs->directories - (inputs->node-inputs inputs)))) - (union-build "node_modules" - (map make-node-path input-node-directories)) +(define* (configure #:key outputs inputs #:allow-other-keys) + (let ((npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "--offline" "--ignore-scripts" "install") #t)) -(define configure link-npm-dependencies) +(define* (build #:key inputs #:allow-other-keys) + (let ((package-meta (call-with-input-file "package.json" read-json))) + (if (and=> (assoc-ref package-meta "scripts") + (lambda (scripts) + (assoc-ref scripts "build"))) + (let ((npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "run" "build")) + (format #t "there is no build script to run~%")) + #t)) -(define* (check #:key tests? #:allow-other-keys) +(define* (check #:key tests? inputs #:allow-other-keys) "Run 'npm test' if TESTS?" (if tests? - ;; Should only be enabled once we know that there are tests - (invoke "npm" "test")) + (let ((npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "test")) + (format #t "test suite not run~%")) #t) -(define (node-package? name) - "Check if NAME correspond to the name of an Node package." - (string-prefix? "node-" name)) +(define* (repack #:key inputs #:allow-other-keys) + (invoke "tar" "-czf" "../package.tgz" ".") + #t) (define* (install #:key outputs inputs #:allow-other-keys) - "Install the node module to the output store item. The module itself is -installed in a subdirectory of @file{node_modules} and its runtime dependencies -as defined by @file{package.json} are symlinked into a @file{node_modules} -subdirectory of the module's directory. Additionally, binaries are installed in -the @file{bin} directory." - (let* ((out (assoc-ref outputs "out")) - (target (string-append out "/lib")) - (binaries (string-append out "/bin")) - (data (read-package-data)) - (modulename (assoc-ref data "name")) - (binary-configuration (match (assoc-ref data "bin") - (('@ configuration ...) configuration) - ((? string? configuration) configuration) - (#f #f))) - (dependencies (match (assoc-ref data "dependencies") - (('@ deps ...) deps) - (#f #f)))) - (mkdir-p target) - (copy-recursively "." (string-append target "/node_modules/" modulename)) - ;; Remove references to dependencies - (delete-file-recursively - (string-append target "/node_modules/" modulename "/node_modules")) - (cond - ((string? binary-configuration) - (begin - (mkdir-p binaries) - (symlink (string-append target "/node_modules/" modulename "/" - binary-configuration) - (string-append binaries "/" modulename)))) - ((list? binary-configuration) - (for-each - (lambda (conf) - (match conf - ((key . value) - (begin - (mkdir-p (dirname (string-append binaries "/" key))) - (symlink (string-append target "/node_modules/" modulename "/" - value) - (string-append binaries "/" key)))))) - binary-configuration))) - (when dependencies - (mkdir-p - (string-append target "/node_modules/" modulename "/node_modules")) - (for-each - (lambda (dependency) - (let ((dependency (car dependency))) - (symlink - (string-append (assoc-ref inputs (string-append "node-" dependency)) - "/lib/node_modules/" dependency) - (string-append target "/node_modules/" modulename - "/node_modules/" dependency)))) - dependencies)) + "Install the node module to the output store item." + (let ((out (assoc-ref outputs "out")) + (npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "--prefix" out + "--global" + "--offline" + "--loglevel" "info" + "--production" + "install" "../package.tgz") #t)) - (define %standard-phases (modify-phases gnu:%standard-phases + (add-after 'unpack 'set-home set-home) + (add-before 'configure 'patch-dependencies patch-dependencies) (replace 'configure configure) (replace 'build build) - (replace 'install install) - (delete 'check) - (add-after 'install 'check check) - (delete 'strip))) + (replace 'check check) + (add-before 'install 'repack repack) + (replace 'install install))) (define* (node-build #:key inputs (phases %standard-phases) #:allow-other-keys #:rest args) diff --git a/guix/build/qt-build-system.scm b/guix/build/qt-build-system.scm index 005157b0a4..f59b0c420f 100644 --- a/guix/build/qt-build-system.scm +++ b/guix/build/qt-build-system.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 Federico Beffa <beffa@fbengineering.ch> -;;; Copyright © 2014, 2015 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2019, 2020 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; @@ -49,25 +49,53 @@ (define (variables-for-wrapping base-directories) - (define (collect-sub-dirs base-directories subdirectory) - (filter-map - (lambda (dir) - (let ((directory (string-append dir subdirectory))) - (if (directory-exists? directory) directory #f))) - base-directories)) - - (filter - (lambda (var-to-wrap) (not (null? (last var-to-wrap)))) - (map - (lambda (var-spec) - `(,(first var-spec) = ,(collect-sub-dirs base-directories (last var-spec)))) - (list - ;; these shall match the search-path-specification for Qt and KDE - ;; libraries - '("XDG_DATA_DIRS" "/share") - '("XDG_CONFIG_DIRS" "/etc/xdg") - '("QT_PLUGIN_PATH" "/lib/qt5/plugins") - '("QML2_IMPORT_PATH" "/lib/qt5/qml"))))) + (define (collect-sub-dirs base-directories file-type subdirectory + selectors) + ;; Append SUBDIRECTORY and each of BASE-DIRECTORIES, and return the subset + ;; that exists and has at least one of the SELECTORS sub-directories, + ;; unless SELECTORS is the empty list. FILE-TYPE should by 'directory or + ;; 'regular file. For the later, it allows searching for plain files + ;; rather than directories. + (define exists? (match file-type + ('directory directory-exists?) + ('regular file-exists?))) + + (filter-map (lambda (dir) + (let ((directory (string-append dir subdirectory))) + (and (exists? directory) + (or (null? selectors) + (any (lambda (selector) + (exists? + (string-append directory selector))) + selectors)) + directory))) + base-directories)) + + (filter-map + (match-lambda + ((variable file-type directory selectors ...) + (match (collect-sub-dirs base-directories file-type directory + selectors) + (() + #f) + (directories + `(,variable = ,directories))))) + + ;; These shall match the search-path-specification for Qt and KDE + ;; libraries. + (list '("XDG_DATA_DIRS" directory "/share" + + ;; These are "selectors": consider /share if and only if at least + ;; one of these sub-directories exist. This avoids adding + ;; irrelevant packages to XDG_DATA_DIRS just because they have a + ;; /share sub-directory. + "/glib-2.0/schemas" "/sounds" "/themes" + "/cursors" "/wallpapers" "/icons" "/mime") + '("XDG_CONFIG_DIRS" directory "/etc/xdg") + '("QT_PLUGIN_PATH" directory "/lib/qt5/plugins") + '("QML2_IMPORT_PATH" directory "/lib/qt5/qml") + '("QTWEBENGINEPROCESS_PATH" regular + "/lib/qt5/libexec/QtWebEngineProcess")))) (define* (wrap-all-programs #:key inputs outputs (qt-wrap-excluded-outputs '()) diff --git a/guix/channels.scm b/guix/channels.scm index b812c1b6e5..c40fc0c507 100644 --- a/guix/channels.scm +++ b/guix/channels.scm @@ -597,9 +597,24 @@ to '%package-module-path'." (string-append #$output "/share/guile/site/" (effective-version))) + (define optimizations-for-level + ;; Guile 3.0 provides this procedure but Guile 2.2 didn't. + ;; Since this code may be executed by either version, we can't + ;; rely on its availability. + (or (and=> (false-if-exception + (resolve-interface '(system base optimize))) + (lambda (iface) + (module-ref iface 'optimizations-for-level))) + (const '()))) + + (define -O1 + ;; Optimize for package module compilation speed. + (optimizations-for-level 1)) + (let* ((subdir #$directory) (source (string-append #$source subdir))) - (compile-files source go (find-files source "\\.scm$")) + (compile-files source go (find-files source "\\.scm$") + #:optimization-options (const -O1)) (mkdir-p (dirname scm)) (symlink (string-append #$source subdir) scm)) diff --git a/guix/git.scm b/guix/git.scm index a5103547d3..57fa2ca1ee 100644 --- a/guix/git.scm +++ b/guix/git.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017, 2020 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Kyle Meyer <kyle@kyleam.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -185,10 +186,9 @@ make sure no empty directory is left behind." (lambda () (mkdir-p directory) - (let ((auth-method (%make-auth-ssh-agent))) - (clone url directory - (make-clone-options - #:fetch-options (make-default-fetch-options))))) + (clone url directory + (make-clone-options + #:fetch-options (make-default-fetch-options)))) (lambda _ (false-if-exception (rmdir directory))))) @@ -210,6 +210,9 @@ corresponding Git object." (let ((oid (reference-target (branch-lookup repository branch BRANCH-REMOTE)))) (object-lookup repository oid))) + (('symref . symref) + (let ((oid (reference-name->oid repository symref))) + (object-lookup repository oid))) (('commit . commit) (let ((len (string-length commit))) ;; 'object-lookup-prefix' appeared in Guile-Git in Mar. 2018, so we @@ -280,13 +283,15 @@ dynamic extent of EXP." (report-git-error err)))) (define* (update-submodules repository - #:key (log-port (current-error-port))) + #:key (log-port (current-error-port)) + (fetch-options #f)) "Update the submodules of REPOSITORY, a Git repository object." (for-each (lambda (name) (let ((submodule (submodule-lookup repository name))) (format log-port (G_ "updating submodule '~a'...~%") name) - (submodule-update submodule) + (submodule-update submodule + #:fetch-options fetch-options) ;; Recurse in SUBMODULE. (let ((directory (string-append @@ -294,6 +299,7 @@ dynamic extent of EXP." "/" (submodule-path submodule)))) (with-repository directory repository (update-submodules repository + #:fetch-options fetch-options #:log-port log-port))))) (repository-submodules repository))) @@ -341,7 +347,7 @@ definitely available in REPOSITORY, false otherwise." (define* (update-cached-checkout url #:key - (ref '(branch . "master")) + (ref '()) recursive? (check-out? #t) starting-commit @@ -357,6 +363,7 @@ provided) as returned by 'commit-relation'. REF is pair whose key is [branch | commit | tag | tag-or-commit ] and value the associated data: [<branch name> | <sha1> | <tag name> | <string>]. +If REF is the empty list, the remote HEAD is used. When RECURSIVE? is true, check out submodules as well, if any. @@ -375,6 +382,7 @@ it unchanged." ;; made little sense since the cache should be transparent to them. So ;; here we append "origin/" if it's missing and otherwise keep it. (match ref + (() '(symref . "refs/remotes/origin/HEAD")) (('branch . branch) `(branch . ,(if (string-prefix? "origin/" branch) branch @@ -389,11 +397,11 @@ it unchanged." ;; Only fetch remote if it has not been cloned just before. (when (and cache-exists? (not (reference-available? repository ref))) - (let ((auth-method (%make-auth-ssh-agent))) - (remote-fetch (remote-lookup repository "origin") - #:fetch-options (make-default-fetch-options)))) + (remote-fetch (remote-lookup repository "origin") + #:fetch-options (make-default-fetch-options))) (when recursive? - (update-submodules repository #:log-port log-port)) + (update-submodules repository #:log-port log-port + #:fetch-options (make-default-fetch-options))) ;; Note: call 'commit-relation' from here because it's more efficient ;; than letting users re-open the checkout later on. @@ -435,12 +443,13 @@ it unchanged." (log-port (%make-void-port "w")) (cache-directory (%repository-cache-directory)) - (ref '(branch . "master"))) + (ref '())) "Return two values: the content of the git repository at URL copied into a store directory and the sha1 of the top level commit in this directory. The reference to be checkout, once the repository is fetched, is specified by REF. REF is pair whose key is [branch | commit | tag] and value the associated -data, respectively [<branch name> | <sha1> | <tag name>]. +data, respectively [<branch name> | <sha1> | <tag name>]. If REF is the empty +list, the remote HEAD is used. When RECURSIVE? is true, check out submodules as well, if any. @@ -550,7 +559,7 @@ objects: 'ancestor (meaning that OLD is an ancestor of NEW), 'descendant, or git-checkout make-git-checkout git-checkout? (url git-checkout-url) - (branch git-checkout-branch (default "master")) + (branch git-checkout-branch (default #f)) (commit git-checkout-commit (default #f)) ;#f | tag | commit (recursive? git-checkout-recursive? (default #f))) @@ -589,9 +598,11 @@ objects: 'ancestor (meaning that OLD is an ancestor of NEW), 'descendant, or (match checkout (($ <git-checkout> url branch commit recursive?) (latest-repository-commit* url - #:ref (if commit - `(tag-or-commit . ,commit) - `(branch . ,branch)) + #:ref (cond (commit + `(tag-or-commit . ,commit)) + (branch + `(branch . ,branch)) + (else '())) #:recursive? recursive? #:log-port (current-error-port))))) diff --git a/guix/gnu-maintenance.scm b/guix/gnu-maintenance.scm index 031a899a6c..fece84b341 100644 --- a/guix/gnu-maintenance.scm +++ b/guix/gnu-maintenance.scm @@ -31,7 +31,7 @@ #:use-module (srfi srfi-34) #:use-module (rnrs io ports) #:use-module (system foreign) - #:use-module (guix http-client) + #:use-module ((guix http-client) #:hide (open-socket-for-uri)) #:use-module (guix ftp-client) #:use-module (guix utils) #:use-module (guix memoization) @@ -66,6 +66,7 @@ %gnu-updater %gnu-ftp-updater %savannah-updater + %sourceforge-updater %xorg-updater %kernel.org-updater %generic-html-updater)) @@ -242,7 +243,7 @@ network to check in GNU's database." ;; The "-src" pattern is for "TeXmacs-1.0.7.9-src.tar.gz". ;; The "-gnu[0-9]" pattern is for "icecat-38.4.0-gnu1.tar.bz2". ;; Accept underscores as in "PKG_1.2.tar.gz" for some non-GNU packages. - (make-regexp "^([^.]+)[-_]([0-9]|[^-])+(-(src|gnu[0-9]))?\\.(tar\\.|zip$)")) + (make-regexp "^([^.]+)[-_]([0-9]|[^-])+(-(src|[sS]ource|gnu[0-9]))?\\.(tar\\.|zip$)")) (define %alpha-tarball-rx (make-regexp "^.*-.*[0-9](-|~)?(alpha|beta|rc|RC|cvs|svn|git)-?[0-9\\.]*\\.tar\\.")) @@ -595,7 +596,7 @@ list available from %GNU-FILE-LIST-URI over HTTP(S)." (define %package-name-rx ;; Regexp for a package name, e.g., "foo-X.Y". Since TeXmacs uses ;; "TeXmacs-X.Y-src", the `-src' suffix is allowed. - (make-regexp "^(.*)[-_](([0-9]|\\.)+)(-src)?")) + (make-regexp "^(.*)[-_](([0-9]|\\.)+)(-src|\\.src|\\.orig)?")) (define (gnu-package-name->name+version name+version) "Return the package name and version number extracted from NAME+VERSION." @@ -637,9 +638,6 @@ GNOME packages; EMMS is included though, because its releases are on gnu.org." (lambda (urls) (map rewrite-url urls)))))) -(define savannah-package? - (url-prefix-predicate "mirror://savannah/")) - (define %savannah-base ;; One of the Savannah mirrors listed at ;; <http://download0.savannah.gnu.org/mirmon/savannah/> that serves valid @@ -663,6 +661,59 @@ GNOME packages; EMMS is included though, because its releases are on gnu.org." #:directory directory) (cut adjusted-upstream-source <> rewrite)))) +(define (latest-sourceforge-release package) + "Return the latest release of PACKAGE." + (define (uri-append uri extension) + ;; Return URI with EXTENSION appended. + (build-uri (uri-scheme uri) + #:host (uri-host uri) + #:path (string-append (uri-path uri) extension))) + + (define (valid-uri? uri port) + ;; Return true if URI is reachable. + (false-if-exception + (case (response-code (http-head uri #:port port #:keep-alive? #t)) + ((200 302) #t) + (else #f)))) + + (let* ((name (package-upstream-name package)) + (base (string-append "https://sourceforge.net/projects/" + name "/files")) + (url (string-append base "/latest/download")) + (uri (string->uri url)) + (port (false-if-exception (open-socket-for-uri uri))) + (response (and port + (http-head uri #:port port #:keep-alive? #t)))) + (dynamic-wind + (const #t) + (lambda () + (and response + (= 302 (response-code response)) + (response-location response) + (match (string-tokenize (uri-path (response-location response)) + (char-set-complement (char-set #\/))) + ((_ components ...) + (let* ((path (string-join components "/")) + (url (string-append "mirror://sourceforge/" path))) + (and (release-file? name (basename path)) + + ;; Take the heavy-handed approach of probing 3 additional + ;; URLs. XXX: Would be nicer if this could be avoided. + (let* ((loc (response-location response)) + (sig (any (lambda (extension) + (let ((uri (uri-append loc extension))) + (and (valid-uri? uri port) + (string-append url extension)))) + '(".asc" ".sig" ".sign")))) + (upstream-source + (package name) + (version (tarball->version (basename path))) + (urls (list url)) + (signature-urls (and sig (list sig))))))))))) + (lambda () + (when port + (close-port port)))))) + (define (latest-xorg-release package) "Return the latest release of PACKAGE." (let ((uri (string->uri (origin-uri (package-source package))))) @@ -706,14 +757,19 @@ GNOME packages; EMMS is included though, because its releases are on gnu.org." "ftp.gnu.org" "download.savannah.gnu.org" "pypi.org" "crates.io" "rubygems.org" "bioconductor.org"))) - (url-predicate (lambda (url) - (match (string->uri url) - (#f #f) - (uri - (let ((scheme (uri-scheme uri)) - (host (uri-host uri))) - (and (memq scheme '(http https)) - (not (member host hosting-sites)))))))))) + (define http-url? + (url-predicate (lambda (url) + (match (string->uri url) + (#f #f) + (uri + (let ((scheme (uri-scheme uri)) + (host (uri-host uri))) + (and (memq scheme '(http https)) + (not (member host hosting-sites))))))))) + + (lambda (package) + (or (assoc-ref (package-properties package) 'release-monitoring-url) + (http-url? package))))) (define (latest-html-updatable-release package) "Return the latest release of PACKAGE. Do that by crawling the HTML page of @@ -772,6 +828,13 @@ the directory containing its source tarball." (pred (url-prefix-predicate "mirror://savannah/")) (latest latest-savannah-release))) +(define %sourceforge-updater + (upstream-updater + (name 'sourceforge) + (description "Updater for packages hosted on sourceforge.net") + (pred (url-prefix-predicate "mirror://sourceforge/")) + (latest latest-sourceforge-release))) + (define %xorg-updater (upstream-updater (name 'xorg) diff --git a/guix/http-client.scm b/guix/http-client.scm index 4b4c14ed0b..a2e11a1b73 100644 --- a/guix/http-client.scm +++ b/guix/http-client.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2012, 2015 Free Software Foundation, Inc. ;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr> @@ -147,6 +147,28 @@ Raise an '&http-get-error' condition if downloading fails." (uri->string uri) code (response-reason-phrase resp)))))))))))) +(define-syntax-rule (false-if-networking-error exp) + "Return #f if EXP triggers a network related exception as can occur when +reusing stale cached connections." + ;; FIXME: Duplicated from 'with-cached-connection'. + (catch #t + (lambda () + exp) + (lambda (key . args) + ;; If PORT was cached and the server closed the connection in the + ;; meantime, we get EPIPE. In that case, open a fresh connection and + ;; retry. We might also get 'bad-response or a similar exception from + ;; (web response) later on, once we've sent the request, or a + ;; ERROR/INVALID-SESSION from GnuTLS. + (if (or (and (eq? key 'system-error) + (= EPIPE (system-error-errno `(,key ,@args)))) + (and (eq? key 'gnutls-error) + (eq? (first args) error/invalid-session)) + (memq key + '(bad-response bad-header bad-header-component))) + #f + (apply throw key args))))) + (define* (http-multiple-get base-uri proc seed requests #:key port (verify-certificate? #t) (open-connection guix:open-connection-for-uri) @@ -185,25 +207,15 @@ returning." ;; Inherit the HTTP proxying property from P. (set-http-proxy-port?! buffer (http-proxy-port? p)) - (catch #t - (lambda () - (for-each (cut write-request <> buffer) - batch) - (put-bytevector p (get)) - (force-output p)) - (lambda (key . args) - ;; If PORT becomes unusable, open a fresh connection and - ;; retry. - (if (or (and (eq? key 'system-error) - (= EPIPE (system-error-errno `(,key ,@args)))) - (and (eq? key 'gnutls-error) - (eq? (first args) error/invalid-session))) - (begin - (close-port p) ; close the broken port - (connect #f - requests - result)) - (apply throw key args))))) + (unless (false-if-networking-error + (begin + (for-each (cut write-request <> buffer) batch) + (put-bytevector p (get)) + (force-output p) + #t)) + ;; If PORT becomes unusable, open a fresh connection and retry. + (close-port p) ; close the broken port + (connect #f requests result))) ;; Now start processing responses. (let loop ((sent batch) @@ -219,42 +231,27 @@ returning." (remainder (connect p remainder result)))) ((head tail ...) - (catch #t - (lambda () - (let* ((resp (read-response p)) - (body (response-body-port resp)) - (result (proc head resp body result))) - ;; The server can choose to stop responding at any time, - ;; in which case we have to try again. Check whether - ;; that is the case. Note that even upon "Connection: - ;; close", we can read from BODY. - (match (assq 'connection (response-headers resp)) - (('connection 'close) - (close-port p) - (connect #f ;try again - (drop requests (+ 1 processed)) - result)) - (_ - (loop tail (+ 1 processed) result))))) ;keep going - (lambda (key . args) - ;; If PORT was cached and the server closed the connection - ;; in the meantime, we get EPIPE. In that case, open a - ;; fresh connection and retry. We might also get - ;; 'bad-response or a similar exception from (web response) - ;; later on, once we've sent the request, or a - ;; ERROR/INVALID-SESSION from GnuTLS. - (if (or (and (eq? key 'system-error) - (= EPIPE (system-error-errno `(,key ,@args)))) - (and (eq? key 'gnutls-error) - (eq? (first args) error/invalid-session)) - (memq key - '(bad-response bad-header bad-header-component))) - (begin - (close-port p) - (connect #f ; try again - (drop requests (+ 1 processed)) - result)) - (apply throw key args)))))))))) + (match (false-if-networking-error (read-response p)) + ((? response? resp) + (let* ((body (response-body-port resp)) + (result (proc head resp body result))) + ;; The server can choose to stop responding at any time, + ;; in which case we have to try again. Check whether + ;; that is the case. Note that even upon "Connection: + ;; close", we can read from BODY. + (match (assq 'connection (response-headers resp)) + (('connection 'close) + (close-port p) + (connect #f ;try again + (drop requests (+ 1 processed)) + result)) + (_ + (loop tail (+ 1 processed) result))))) + (#f + (close-port p) + (connect #f ; try again + (drop requests processed) + result))))))))) ;;; diff --git a/guix/import/go.scm b/guix/import/go.scm index 7452b4c903..bc53f8f558 100644 --- a/guix/import/go.scm +++ b/guix/import/go.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com> ;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -32,7 +33,7 @@ #:use-module (guix http-client) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix memoization) - #:autoload (htmlprag) (html->sxml) ;from Guile-Lib + #:use-module (htmlprag) ;from Guile-Lib #:autoload (guix git) (update-cached-checkout) #:autoload (gcrypt hash) (open-hash-port hash-algorithm sha256) #:autoload (guix serialization) (write-file) @@ -42,20 +43,29 @@ #:use-module (ice-9 rdelim) #:use-module (ice-9 receive) #:use-module (ice-9 regex) + #:use-module (ice-9 textual-ports) #:use-module ((rnrs io ports) #:select (call-with-port)) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-2) #:use-module (srfi srfi-9) #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) - #:use-module (sxml xpath) + #:use-module (srfi srfi-34) + #:use-module (sxml match) + #:use-module ((sxml xpath) #:renamer (lambda (s) + (if (eq? 'filter s) + 'xfilter + s))) #:use-module (web client) #:use-module (web response) #:use-module (web uri) - #:export (go-path-escape - go-module->guix-package + #:export (go-module->guix-package go-module-recursive-import)) +;;; Parameterize htmlprag to parse valid HTML more reliably. +(%strict-tokenizer? #t) + ;;; Commentary: ;;; ;;; (guix import go) attempts to make it easier to create Guix package @@ -83,12 +93,18 @@ ;;; assumption that there will be no collision. ;;; TODO list -;;; - get correct hash in vcs->origin -;;; - print partial result during recursive imports (need to catch -;;; exceptions) +;;; - get correct hash in vcs->origin for Mercurial and Subversion ;;; Code: +(define http-fetch* + ;; Like http-fetch, but memoized and returning the body as a string. + (memoize (lambda args + (call-with-port (apply http-fetch args) get-string-all)))) + +(define json-fetch* + (memoize json-fetch)) + (define (go-path-escape path) "Escape a module path by replacing every uppercase letter with an exclamation mark followed with its lowercase equivalent, as per the module @@ -98,54 +114,87 @@ https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths)." (string-append "!" (string-downcase (match:substring occurrence)))) (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post)) -(define (go-module-latest-version goproxy-url module-path) - "Fetch the version number of the latest version for MODULE-PATH from the -given GOPROXY-URL server." - (assoc-ref (json-fetch (format #f "~a/~a/@latest" goproxy-url - (go-path-escape module-path))) - "Version")) +;; Prevent inlining of this procedure, which is accessed by unit tests. +(set! go-path-escape go-path-escape) +(define (go.pkg.dev-info name) + (http-fetch* (string-append "https://pkg.go.dev/" name))) + +(define* (go-module-version-string goproxy name #:key version) + "Fetch the version string of the latest version for NAME from the given +GOPROXY server, or for VERSION when specified." + (let ((file (if version + (string-append "@v/" version ".info") + "@latest"))) + (assoc-ref (json-fetch* (format #f "~a/~a/~a" + goproxy (go-path-escape name) file)) + "Version"))) + +(define* (go-module-available-versions goproxy name) + "Retrieve the available versions for a given module from the module proxy. +Versions are being returned **unordered** and may contain different versioning +styles for the same package." + (let* ((url (string-append goproxy "/" (go-path-escape name) "/@v/list")) + (body (http-fetch* url)) + (versions (remove string-null? (string-split body #\newline)))) + (if (null? versions) + (list (go-module-version-string goproxy name)) ;latest version + versions))) (define (go-package-licenses name) "Retrieve the list of licenses that apply to NAME, a Go package or module -name (e.g. \"github.com/golang/protobuf/proto\"). The data is scraped from -the https://pkg.go.dev/ web site." - (let*-values (((url) (string-append "https://pkg.go.dev/" name - "?tab=licenses")) - ((response body) (http-get url)) - ;; Extract the text contained in a h2 child node of any - ;; element marked with a "License" class attribute. - ((select) (sxpath `(// (* (@ (equal? (class "License")))) - h2 // *text*)))) - (and (eq? (response-code response) 200) - (match (select (html->sxml body)) - (() #f) ;nothing selected - (licenses licenses))))) - -(define (go.pkg.dev-info name) - (http-get (string-append "https://pkg.go.dev/" name))) -(define go.pkg.dev-info* - (memoize go.pkg.dev-info)) +name (e.g. \"github.com/golang/protobuf/proto\")." + (let* ((body (go.pkg.dev-info (string-append name "?tab=licenses"))) + ;; Extract the text contained in a h2 child node of any + ;; element marked with a "License" class attribute. + (select (sxpath `(// (* (@ (equal? (class "License")))) + h2 // *text*)))) + (select (html->sxml body)))) + +(define (sxml->texi sxml-node) + "A very basic SXML to Texinfo converter which attempts to preserve HTML +formatting and links as text." + (sxml-match sxml-node + ((strong ,text) + (format #f "@strong{~a}" text)) + ((a (@ (href ,url)) ,text) + (format #f "@url{~a,~a}" url text)) + ((code ,text) + (format #f "@code{~a}" text)) + (,something-else something-else))) (define (go-package-description name) "Retrieve a short description for NAME, a Go package name, -e.g. \"google.golang.org/protobuf/proto\". The data is scraped from the -https://pkg.go.dev/ web site." - (let*-values (((response body) (go.pkg.dev-info* name)) - ;; Extract the text contained in a h2 child node of any - ;; element marked with a "License" class attribute. - ((select) (sxpath - `(// (section - (@ (equal? (class "Documentation-overview")))) - (p 1))))) - (and (eq? (response-code response) 200) - (match (select (html->sxml body)) - (() #f) ;nothing selected - (((p . strings)) - ;; The paragraph text is returned as a list of strings embedding - ;; newline characters. Join them and strip the newline - ;; characters. - (string-delete #\newline (string-join strings))))))) +e.g. \"google.golang.org/protobuf/proto\"." + (let* ((body (go.pkg.dev-info name)) + (sxml (html->sxml body)) + (overview ((sxpath + `(// + (* (@ (equal? (class "Documentation-overview")))) + (p 1))) sxml)) + ;; Sometimes, the first paragraph just contains images/links that + ;; has only "\n" for text. The following filter is designed to + ;; omit it. + (contains-text? (lambda (node) + (remove string-null? + (map string-trim-both + (filter (node-typeof? '*text*) + (cdr node)))))) + (select-content (sxpath + `(// + (* (@ (equal? (class "UnitReadme-content")))) + div // p ,(xfilter contains-text?)))) + ;; Fall-back to use content; this is less desirable as it is more + ;; verbose, but not every page has an overview. + (description (if (not (null? overview)) + overview + (select-content sxml))) + (description* (and (not (null? description)) + (first description)))) + (match description* + (() #f) ;nothing selected + ((p elements ...) + (apply string-append (filter string? (map sxml->texi elements))))))) (define (go-package-synopsis module-name) "Retrieve a short synopsis for a Go module named MODULE-NAME, @@ -153,17 +202,17 @@ e.g. \"google.golang.org/protobuf\". The data is scraped from the https://pkg.go.dev/ web site." ;; Note: Only the *module* (rather than package) page has the README title ;; used as a synopsis on the https://pkg.go.dev web site. - (let*-values (((response body) (go.pkg.dev-info* module-name)) - ;; Extract the text contained in a h2 child node of any - ;; element marked with a "License" class attribute. - ((select) (sxpath - `(// (div (@ (equal? (class "UnitReadme-content")))) - // h3 *text*)))) - (and (eq? (response-code response) 200) - (match (select (html->sxml body)) - (() #f) ;nothing selected - ((title more ...) ;title is the first string of the list - (string-trim-both title)))))) + (let* ((url (string-append "https://pkg.go.dev/" module-name)) + (body (http-fetch* url)) + ;; Extract the text contained in a h2 child node of any + ;; element marked with a "License" class attribute. + (select-title (sxpath + `(// (div (@ (equal? (class "UnitReadme-content")))) + // h3 *text*)))) + (match (select-title (html->sxml body)) + (() #f) ;nothing selected + ((title more ...) ;title is the first string of the list + (string-trim-both title))))) (define (list->licenses licenses) "Given a list of LICENSES mostly following the SPDX conventions, return the @@ -188,13 +237,13 @@ corresponding Guix license or 'unknown-license!" 'unknown-license!))) licenses)) -(define (fetch-go.mod goproxy-url module-path version) - "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH -and VERSION." - (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url +(define (fetch-go.mod goproxy module-path version) + "Fetch go.mod from the given GOPROXY server for the given MODULE-PATH +and VERSION and return an input port." + (let ((url (format #f "~a/~a/@v/~a.mod" goproxy (go-path-escape module-path) (go-path-escape version)))) - (http-fetch url))) + (http-fetch* url))) (define %go.mod-require-directive-rx ;; A line in a require directive is composed of a module path and @@ -202,118 +251,119 @@ and VERSION." ;; the end. (make-regexp (string-append - "^[[:blank:]]*" - "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)" - "([[:blank:]]+//.*)?"))) + "^[[:blank:]]*([^[:blank:]]+)[[:blank:]]+" ;the module path + "([^[:blank:]]+)" ;the version + "([[:blank:]]+//.*)?"))) ;an optional comment (define %go.mod-replace-directive-rx ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline ;; | ModulePath [ Version ] "=>" ModulePath Version newline . (make-regexp (string-append - "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?" - "[[:blank:]]+" "=>" "[[:blank:]]+" - "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"))) - -(define (parse-go.mod port) - "Parse the go.mod file accessible via the input PORT, returning a list of -requirements." - (define-record-type <results> - (make-results requirements replacements) - results? - (requirements results-requirements) - (replacements results-replacements)) + "([^[:blank:]]+)" ;the module path + "([[:blank:]]+([^[:blank:]]+))?" ;optional version + "[[:blank:]]+=>[[:blank:]]+" + "([^[:blank:]]+)" ;the file or module path + "([[:blank:]]+([^[:blank:]]+))?"))) ;the version (if a module path) + +(define (parse-go.mod content) + "Parse the go.mod file CONTENT, returning a list of requirements." ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar ;; which we think necessary for our use case. - (define (toplevel results) - "Main parser, RESULTS is a pair of alist serving as accumulator for - all encountered requirements and replacements." - (let ((line (read-line port))) + (define (toplevel requirements replaced) + "This is the main parser. The results are accumulated in THE REQUIREMENTS +and REPLACED lists." + (let ((line (read-line))) (cond ((eof-object? line) ;; parsing ended, give back the result - results) + (values requirements replaced)) ((string=? line "require (") ;; a require block begins, delegate parsing to IN-REQUIRE - (in-require results)) + (in-require requirements replaced)) ((string=? line "replace (") ;; a replace block begins, delegate parsing to IN-REPLACE - (in-replace results)) + (in-replace requirements replaced)) ((string-prefix? "require " line) - ;; a standalone require directive - (let* ((stripped-line (string-drop line 8)) - (new-results (require-directive results stripped-line))) - (toplevel new-results))) + ;; a require directive by itself + (let* ((stripped-line (string-drop line 8))) + (call-with-values + (lambda () + (require-directive requirements replaced stripped-line)) + toplevel))) ((string-prefix? "replace " line) - ;; a standalone replace directive - (let* ((stripped-line (string-drop line 8)) - (new-results (replace-directive results stripped-line))) - (toplevel new-results))) + ;; a replace directive by itself + (let* ((stripped-line (string-drop line 8))) + (call-with-values + (lambda () + (replace-directive requirements replaced stripped-line)) + toplevel))) (#t ;; unrecognised line, ignore silently - (toplevel results))))) + (toplevel requirements replaced))))) - (define (in-require results) - (let ((line (read-line port))) + (define (in-require requirements replaced) + (let ((line (read-line))) (cond ((eof-object? line) ;; this should never happen here but we ignore silently - results) + (values requirements replaced)) ((string=? line ")") ;; end of block, coming back to toplevel - (toplevel results)) + (toplevel requirements replaced)) (#t - (in-require (require-directive results line)))))) + (call-with-values (lambda () + (require-directive requirements replaced line)) + in-require))))) - (define (in-replace results) - (let ((line (read-line port))) + (define (in-replace requirements replaced) + (let ((line (read-line))) (cond ((eof-object? line) ;; this should never happen here but we ignore silently - results) + (values requirements replaced)) ((string=? line ")") ;; end of block, coming back to toplevel - (toplevel results)) + (toplevel requirements replaced)) (#t - (in-replace (replace-directive results line)))))) - - (define (replace-directive results line) - "Extract replaced modules and new requirements from replace directive - in LINE and add to RESULTS." - (match results - (($ <results> requirements replaced) - (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line)) - (module-path (match:substring rx-match 1)) - (version (match:substring rx-match 3)) - (new-module-path (match:substring rx-match 4)) - (new-version (match:substring rx-match 6)) - (new-replaced (alist-cons module-path version replaced)) - (new-requirements - (if (string-match "^\\.?\\./" new-module-path) - requirements - (alist-cons new-module-path new-version requirements)))) - (make-results new-requirements new-replaced))))) - (define (require-directive results line) - "Extract requirement from LINE and add it to RESULTS." + (call-with-values (lambda () + (replace-directive requirements replaced line)) + in-replace))))) + + (define (replace-directive requirements replaced line) + "Extract replaced modules and new requirements from the replace directive +in LINE and add them to the REQUIREMENTS and REPLACED lists." + (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line)) + (module-path (match:substring rx-match 1)) + (version (match:substring rx-match 3)) + (new-module-path (match:substring rx-match 4)) + (new-version (match:substring rx-match 6)) + (new-replaced (cons (list module-path version) replaced)) + (new-requirements + (if (string-match "^\\.?\\./" new-module-path) + requirements + (cons (list new-module-path new-version) requirements)))) + (values new-requirements new-replaced))) + + (define (require-directive requirements replaced line) + "Extract requirement from LINE and augment the REQUIREMENTS and REPLACED +lists." (let* ((rx-match (regexp-exec %go.mod-require-directive-rx line)) (module-path (match:substring rx-match 1)) - ;; we saw double-quoted string in the wild without escape - ;; sequences so we just trim the quotes + ;; Double-quoted strings were seen in the wild without escape + ;; sequences; trim the quotes to be on the safe side. (module-path (string-trim-both module-path #\")) (version (match:substring rx-match 2))) - (match results - (($ <results> requirements replaced) - (make-results (alist-cons module-path version requirements) replaced))))) - - (let ((results (toplevel (make-results '() '())))) - (match results - (($ <results> requirements replaced) - ;; At last we remove replaced modules from the requirements list - (fold - (lambda (replacedelem requirements) - (alist-delete! (car replacedelem) requirements)) - requirements - replaced))))) + (values (cons (list module-path version) requirements) replaced))) + + (with-input-from-string content + (lambda () + (receive (requirements replaced) + (toplevel '() '()) + ;; At last remove the replaced modules from the requirements list. + (remove (lambda (r) + (assoc (car r) replaced)) + requirements))))) ;; Prevent inlining of this procedure, which is accessed by unit tests. (set! parse-go.mod parse-go.mod) @@ -324,8 +374,10 @@ requirements." (url-prefix vcs-url-prefix) (root-regex vcs-root-regex) (type vcs-type)) + (define (make-vcs prefix regexp type) - (%make-vcs prefix (make-regexp regexp) type)) + (%make-vcs prefix (make-regexp regexp) type)) + (define known-vcs ;; See the following URL for the official Go equivalent: ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087 @@ -376,13 +428,27 @@ hence the need to derive this information." (vcs-qualified-module-path->root-repo-url module-path) module-path)) -(define (go-module->guix-package-name module-path) - "Converts a module's path to the canonical Guix format for Go packages." - (string-downcase (string-append "go-" (string-replace-substring - (string-replace-substring - module-path - "." "-") - "/" "-")))) +(define* (go-module->guix-package-name module-path #:optional version) + "Converts a module's path to the canonical Guix format for Go packages. +Optionally include a VERSION string to append to the name." + ;; Map dot, slash and underscore characters to hyphens. + (let ((module-path* (string-map (lambda (c) + (if (member c '(#\. #\/ #\_)) + #\- + c)) + module-path))) + (string-downcase (string-append "go-" module-path* + (if version + (string-append "-" version) + ""))))) + +(define (strip-.git-suffix/maybe repo-url) + "Strip a repository URL '.git' suffix from REPO-URL if hosted at GitHub." + (match repo-url + ((and (? (cut string-prefix? "https://github.com" <>)) + (? (cut string-suffix? ".git" <>))) + (string-drop-right repo-url 4)) + (_ repo-url))) (define-record-type <module-meta> (make-module-meta import-prefix vcs repo-root) @@ -396,21 +462,22 @@ hence the need to derive this information." because goproxy servers don't currently provide all the information needed to build a package." ;; <meta name="go-import" content="import-prefix vcs repo-root"> - (let* ((port (http-fetch (format #f "https://~a?go-get=1" module-path))) + (let* ((meta-data (http-fetch* (format #f "https://~a?go-get=1" module-path))) (select (sxpath `(// head (meta (@ (equal? (name "go-import")))) // content)))) - (match (select (call-with-port port html->sxml)) - (() #f) ;nothing selected + (match (select (html->sxml meta-data)) + (() #f) ;nothing selected (((content content-text)) (match (string-split content-text #\space) ((root-path vcs repo-url) - (make-module-meta root-path (string->symbol vcs) repo-url))))))) + (make-module-meta root-path (string->symbol vcs) + (strip-.git-suffix/maybe repo-url)))))))) -(define (module-meta-data-repo-url meta-data goproxy-url) +(define (module-meta-data-repo-url meta-data goproxy) "Return the URL where the fetcher which will be used can download the source." (if (member (module-meta-vcs meta-data) '(fossil mod)) - goproxy-url + goproxy (module-meta-repo-root meta-data))) ;; XXX: Copied from (guix scripts hash). @@ -463,6 +530,9 @@ control system is being used." (method git-fetch) (uri (git-reference (url ,vcs-repo-url) + ;; This is done because the version field of the package, + ;; which the generated quoted expression refers to, has been + ;; stripped of any 'v' prefixed. (commit ,(if (and plain-version? v-prefixed?) '(string-append "v" version) '(go-version->git-ref version))))) @@ -500,48 +570,95 @@ control system is being used." vcs-type vcs-repo-url))))) (define* (go-module->guix-package module-path #:key - (goproxy-url "https://proxy.golang.org")) - (let* ((latest-version (go-module-latest-version goproxy-url module-path)) - (port (fetch-go.mod goproxy-url module-path latest-version)) - (dependencies (map car (call-with-port port parse-go.mod))) + (goproxy "https://proxy.golang.org") + version + pin-versions?) + "Return the package S-expression corresponding to MODULE-PATH at VERSION, a Go package. +The meta-data is fetched from the GOPROXY server and https://pkg.go.dev/. +When VERSION is unspecified, the latest version available is used." + (let* ((available-versions (go-module-available-versions goproxy module-path)) + (version* (or version + (go-module-version-string goproxy module-path))) ;latest + ;; Elide the "v" prefix Go uses. + (strip-v-prefix (cut string-trim <> #\v)) + ;; Pseudo-versions do not appear in the versions list; skip the + ;; following check. + (_ (unless (or (go-pseudo-version? version*) + (member version* available-versions)) + (error (format #f "error: version ~s is not available +hint: use one of the following available versions ~a\n" + version* available-versions)))) + (content (fetch-go.mod goproxy module-path version*)) + (dependencies+versions (parse-go.mod content)) + (dependencies (if pin-versions? + dependencies+versions + (map car dependencies+versions))) (guix-name (go-module->guix-package-name module-path)) (root-module-path (module-path->repository-root module-path)) ;; The VCS type and URL are not included in goproxy information. For ;; this we need to fetch it from the official module page. (meta-data (fetch-module-meta-data root-module-path)) (vcs-type (module-meta-vcs meta-data)) - (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url)) + (vcs-repo-url (module-meta-data-repo-url meta-data goproxy)) (synopsis (go-package-synopsis root-module-path)) (description (go-package-description module-path)) (licenses (go-package-licenses module-path))) (values `(package (name ,guix-name) - ;; Elide the "v" prefix Go uses - (version ,(string-trim latest-version #\v)) + (version ,(strip-v-prefix version*)) (source - ,(vcs->origin vcs-type vcs-repo-url latest-version)) + ,(vcs->origin vcs-type vcs-repo-url version*)) (build-system go-build-system) (arguments '(#:import-path ,root-module-path)) - ,@(maybe-inputs (map go-module->guix-package-name dependencies)) + ,@(maybe-propagated-inputs + (map (match-lambda + ((name version) + (go-module->guix-package-name name (strip-v-prefix version))) + (name + (go-module->guix-package-name name))) + dependencies)) (home-page ,(format #f "https://~a" root-module-path)) (synopsis ,synopsis) - (description ,description) - (license ,(match (and=> licenses list->licenses) - ((license) license) - ((licenses ...) `(list ,@licenses)) - (x x)))) - dependencies))) + (description ,(and=> description beautify-description)) + (license ,(match (list->licenses licenses) + (() #f) ;unknown license + ((license) ;a single license + license) + ((license ...) ;a list of licenses + `(list ,@license))))) + (if pin-versions? + dependencies+versions + dependencies)))) (define go-module->guix-package* (memoize go-module->guix-package)) (define* (go-module-recursive-import package-name - #:key (goproxy-url "https://proxy.golang.org")) + #:key (goproxy "https://proxy.golang.org") + version + pin-versions?) + (recursive-import package-name - #:repo->guix-package (lambda* (name . _) - (go-module->guix-package* - name - #:goproxy-url goproxy-url)) - #:guix-name go-module->guix-package-name)) + #:repo->guix-package + (lambda* (name #:key version repo) + ;; Disable output buffering so that the following warning gets printed + ;; consistently. + (setvbuf (current-error-port) 'none) + (guard (c ((http-get-error? c) + (warning (G_ "Failed to import package ~s. +reason: ~s could not be fetched: HTTP error ~a (~s). +This package and its dependencies won't be imported.~%") + name + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c)) + (values '() '()))) + (receive (package-sexp dependencies) + (go-module->guix-package* name #:goproxy goproxy + #:version version + #:pin-versions? pin-versions?) + (values package-sexp dependencies)))) + #:guix-name go-module->guix-package-name + #:version version)) diff --git a/guix/import/print.scm b/guix/import/print.scm index a2ab810a5c..dcc38abc70 100644 --- a/guix/import/print.scm +++ b/guix/import/print.scm @@ -79,7 +79,16 @@ when evaluated." (file-name (origin-file-name source)) (patches (origin-patches source))) `(origin - (method ,(procedure-name method)) + ;; Since 'procedure-name' returns the procedure name within the + ;; module where it's defined, not its public name. Thus, try hard to + ;; find its public name and use 'procedure-name' as a last resort. + (method ,(or (any (lambda (module) + (variable-name method module)) + '((guix download) + (guix git-download) + (guix hg-download) + (guix svn-download))) + (procedure-name method))) (uri (string-append ,@(match (factorize-uri uri version) ((? string? uri) (list uri)) (factorized factorized)))) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 64d1385164..d817318a91 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -7,6 +7,7 @@ ;;; Copyright © 2019 Robert Vollmert <rob@vllmrt.net> ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -57,6 +58,7 @@ package-names->package-inputs maybe-inputs maybe-native-inputs + maybe-propagated-inputs package->definition spdx-string->license @@ -169,6 +171,7 @@ of the string VERSION is replaced by the symbol 'version." ("Imlib2" 'license:imlib2) ("IPA" 'license:ipa) ("IPL-1.0" 'license:ibmpl1.0) + ("LAL-1.3" 'license:lal1.3) ("LGPL-2.0" 'license:lgpl2.0) ("LGPL-2.0+" 'license:lgpl2.0+) ("LGPL-2.1" 'license:lgpl2.1) @@ -246,27 +249,34 @@ use in an 'inputs' field of a package definition." (input (make-input input #f))) names)) -(define* (maybe-inputs package-names #:optional (output #f)) +(define* (maybe-inputs package-names #:optional (output #f) + #:key (type #f)) "Given a list of PACKAGE-NAMES, tries to generate the 'inputs' field of a -package definition." - (match (package-names->package-inputs package-names output) - (() - '()) - ((package-inputs ...) - `((inputs (,'quasiquote ,package-inputs)))))) +package definition. TYPE can be used to specify the type of the inputs; +either the 'native or 'propagated symbols are accepted. Left unspecified, the +snippet generated is for regular inputs." + (let ((field-name (match type + ('native 'native-inputs) + ('propagated 'propagated-inputs) + (_ 'inputs)))) + (match (package-names->package-inputs package-names output) + (() + '()) + ((package-inputs ...) + `((,field-name (,'quasiquote ,package-inputs))))))) (define* (maybe-native-inputs package-names #:optional (output #f)) - "Given a list of PACKAGE-NAMES, tries to generate the 'inputs' field of a -package definition." - (match (package-names->package-inputs package-names output) - (() - '()) - ((package-inputs ...) - `((native-inputs (,'quasiquote ,package-inputs)))))) + "Same as MAYBE-INPUTS, but for native inputs." + (maybe-inputs package-names output #:type 'native)) + +(define* (maybe-propagated-inputs package-names #:optional (output #f)) + "Same as MAYBE-INPUTS, but for propagated inputs." + (maybe-inputs package-names output #:type 'propagated)) (define* (package->definition guix-package #:optional append-version?/string) - "If APPEND-VERSION?/STRING is #t, append the package's major+minor -version. If APPEND-VERSION?/string is a string, append this string." + "If APPEND-VERSION?/STRING is #t, append the package's major+minor version. +If it is the symbol 'full, append the package's complete version. If +APPEND-VERSION?/string is a string, append this string." (match guix-package ((or ('package ('name name) ('version version) . rest) @@ -278,6 +288,8 @@ version. If APPEND-VERSION?/string is a string, append this string." (string-append name "-" append-version?/string)) ((eq? append-version?/string #t) (string-append name "-" (version-major+minor version))) + ((eq? 'full append-version?/string) + (string-append name "-" version)) (else name))) ,guix-package)))) @@ -437,8 +449,8 @@ obtain a node's uniquely identifying \"key\"." "Return a list of package expressions for PACKAGE-NAME and all its dependencies, sorted in topological order. For each package, call (REPO->GUIX-PACKAGE NAME :KEYS version repo), which should return a -package expression and a list of dependencies; call (GUIX-NAME NAME) to -obtain the Guix package name corresponding to the upstream name." +package expression and a list of dependencies; call (GUIX-NAME PACKAGE-NAME) +to obtain the Guix package name corresponding to the upstream name." (define-record-type <node> (make-node name version package dependencies) node? diff --git a/guix/ipfs.scm b/guix/ipfs.scm new file mode 100644 index 0000000000..31a89888a7 --- /dev/null +++ b/guix/ipfs.scm @@ -0,0 +1,183 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix ipfs) + #:use-module (json) + #:use-module (guix base64) + #:use-module ((guix build utils) #:select (dump-port)) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) + #:use-module (rnrs io ports) + #:use-module (rnrs bytevectors) + #:use-module (ice-9 match) + #:use-module (ice-9 ftw) + #:use-module (web uri) + #:use-module (web client) + #:use-module (web response) + #:export (%ipfs-base-url + add-data + add-file + + content? + content-name + content-hash + content-size + + add-empty-directory + add-to-directory + read-contents + publish-name)) + +;;; Commentary: +;;; +;;; This module implements bindings for the HTTP interface of the IPFS +;;; gateway, documented here: <https://docs.ipfs.io/reference/api/http/>. It +;;; allows you to add and retrieve files over IPFS, and a few other things. +;;; +;;; Code: + +(define %ipfs-base-url + ;; URL of the IPFS gateway. + (make-parameter "http://localhost:5001")) + +(define* (call url decode #:optional (method http-post) + #:key body (false-if-404? #t) (headers '())) + "Invoke the endpoint at URL using METHOD. Decode the resulting JSON body +using DECODE, a one-argument procedure that takes an input port; when DECODE +is false, return the input port. When FALSE-IF-404? is true, return #f upon +404 responses." + (let*-values (((response port) + (method url #:streaming? #t + #:body body + + ;; Always pass "Connection: close". + #:keep-alive? #f + #:headers `((connection close) + ,@headers)))) + (cond ((= 200 (response-code response)) + (if decode + (let ((result (decode port))) + (close-port port) + result) + port)) + ((and false-if-404? + (= 404 (response-code response))) + (close-port port) + #f) + (else + (close-port port) + (throw 'ipfs-error url response))))) + +;; Result of a file addition. +(define-json-mapping <content> make-content content? + json->content + (name content-name "Name") + (hash content-hash "Hash") + (bytes content-bytes "Bytes") + (size content-size "Size" string->number)) + +;; Result of a 'patch/add-link' operation. +(define-json-mapping <directory> make-directory directory? + json->directory + (hash directory-hash "Hash") + (links directory-links "Links" json->links)) + +;; A "link". +(define-json-mapping <link> make-link link? + json->link + (name link-name "Name") + (hash link-hash "Hash") + (size link-size "Size" string->number)) + +;; A "binding", also known as a "name". +(define-json-mapping <binding> make-binding binding? + json->binding + (name binding-name "Name") + (value binding-value "Value")) + +(define (json->links json) + (match json + (#f '()) + (links (map json->link links)))) + +(define %multipart-boundary + ;; XXX: We might want to find a more reliable boundary. + (string-append (make-string 24 #\-) "2698127afd7425a6")) + +(define (bytevector->form-data bv port) + "Write to PORT a 'multipart/form-data' representation of BV." + (display (string-append "--" %multipart-boundary "\r\n" + "Content-Disposition: form-data\r\n" + "Content-Type: application/octet-stream\r\n\r\n") + port) + (put-bytevector port bv) + (display (string-append "\r\n--" %multipart-boundary "--\r\n") + port)) + +(define* (add-data data #:key (name "file.txt") recursive?) + "Add DATA, a bytevector, to IPFS. Return a content object representing it." + (call (string-append (%ipfs-base-url) + "/api/v0/add?arg=" (uri-encode name) + "&recursive=" + (if recursive? "true" "false")) + json->content + #:headers + `((content-type + . (multipart/form-data + (boundary . ,%multipart-boundary)))) + #:body + (call-with-bytevector-output-port + (lambda (port) + (bytevector->form-data data port))))) + +(define (not-dot? entry) + (not (member entry '("." "..")))) + +(define* (add-file file #:key (name (basename file))) + "Add FILE under NAME to the IPFS and return a content object for it." + (add-data (match (call-with-input-file file get-bytevector-all) + ((? eof-object?) #vu8()) + (bv bv)) + #:name name)) + +(define* (add-empty-directory #:key (name "directory")) + "Return a content object for an empty directory." + (add-data #vu8() #:recursive? #t #:name name)) + +(define* (add-to-directory directory file name) + "Add FILE to DIRECTORY under NAME, and return the resulting directory. +DIRECTORY and FILE must be hashes identifying objects in the IPFS store." + (call (string-append (%ipfs-base-url) + "/api/v0/object/patch/add-link?arg=" + (uri-encode directory) + "&arg=" (uri-encode name) "&arg=" (uri-encode file) + "&create=true") + json->directory)) + +(define* (read-contents object #:key offset length) + "Return an input port to read the content of OBJECT from." + (call (string-append (%ipfs-base-url) + "/api/v0/cat?arg=" object) + #f)) + +(define* (publish-name object) + "Publish OBJECT under the current peer ID." + (call (string-append (%ipfs-base-url) + "/api/v0/name/publish?arg=" object) + json->binding)) diff --git a/guix/licenses.scm b/guix/licenses.scm index 0a36067387..4718ccf83f 100644 --- a/guix/licenses.scm +++ b/guix/licenses.scm @@ -16,6 +16,7 @@ ;;; Copyright © 2017 Rutger Helling <rhelling@mykolab.com> ;;; Copyright © 2020 André Batista <nandre@riseup.net> ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> +;;; Copyright © 2021 Felix Gruber <felgru@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -68,6 +69,7 @@ imlib2 ipa knuth + lal1.3 lgpl2.0 lgpl2.0+ lgpl2.1 lgpl2.1+ lgpl3 lgpl3+ llgpl lppl lppl1.0+ lppl1.1+ lppl1.2 lppl1.2+ lppl1.3 lppl1.3+ @@ -411,6 +413,11 @@ at URI, which may be a file:// URI pointing the package's tree." "http://www.ctan.org/license/knuth" "Modification are only permitted under a different name.")) +(define lal1.3 + (license "Free Art License 1.3" + "http://artlibre.org/licence/lal/en/" + "https://www.gnu.org/licenses/license-list#FreeArt")) + (define lgpl2.0 (license "LGPL 2.0" "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.html" diff --git a/guix/lint.scm b/guix/lint.scm index 311bc94cc3..a7d6bbba4f 100644 --- a/guix/lint.scm +++ b/guix/lint.scm @@ -6,7 +6,7 @@ ;;; Copyright © 2016 Danny Milosavljevic <dannym+a@scratchpost.org> ;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; Copyright © 2017 Alex Kost <alezost@gmail.com> -;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr> +;;; Copyright © 2017, 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; Copyright © 2017, 2018, 2020 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2018, 2019 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2020 Chris Marusich <cmmarusich@gmail.com> @@ -170,6 +170,18 @@ (requires-store? lint-checker-requires-store? (default #f))) +(define (check-name package) + "Check whether PACKAGE's name matches our guidelines." + (let ((name (package-name package))) + ;; Currently checks only whether the name is too short. + (if (and (<= (string-length name) 1) + (not (string=? name "r"))) ; common-sense exception + (list + (make-warning package + (G_ "name should be longer than a single character") + #:field 'name)) + '()))) + (define (properly-starts-sentence? s) (string-match "^[(\"'`[:upper:][:digit:]]" s)) @@ -1179,21 +1191,32 @@ vulnerability records for PACKAGE by calling PACKAGE-VULNERABILITIES." (define (check-for-updates package) "Check if there is an update available for PACKAGE." - (match (with-networking-fail-safe - (format #f (G_ "while retrieving upstream info for '~a'") - (package-name package)) - #f - (package-latest-release* package)) - ((? upstream-source? source) - (if (version>? (upstream-source-version source) - (package-version package)) - (list - (make-warning package - (G_ "can be upgraded to ~a") - (list (upstream-source-version source)) - #:field 'version)) - '())) - (#f '()))) ; cannot find newer upstream release + (match (lookup-updater package) + (#f + (list (make-warning package (G_ "no updater for ~a") + (list (package-name package)) + #:field 'source))) + ((? upstream-updater? updater) + (match (with-networking-fail-safe + (format #f (G_ "while retrieving upstream info for '~a'") + (package-name package)) + #f + (package-latest-release package)) + ((? upstream-source? source) + (if (version>? (upstream-source-version source) + (package-version package)) + (list + (make-warning package + (G_ "can be upgraded to ~a") + (list (upstream-source-version source)) + #:field 'version)) + '())) + (#f ;cannot find upstream release + (list (make-warning package + (G_ "updater '~a' failed to find \ +upstream releases") + (list (upstream-updater-name updater)) + #:field 'source))))))) (define (check-archival package) @@ -1264,7 +1287,8 @@ try again later") ((? origin? origin) ;; Since "save" origins are not supported for non-VCS source, all ;; we can do is tell whether a given tarball is available or not. - (if (origin-hash origin) ;XXX: for ungoogled-chromium + (if (and=> (origin-hash origin) ;XXX: for ungoogled-chromium + content-hash-value) ;& icecat (let ((hash (origin-hash origin))) (match (lookup-content (content-hash-value hash) (symbol->string @@ -1446,6 +1470,10 @@ them for PACKAGE." (define %local-checkers (list (lint-checker + (name 'name) + (description "Validate package names") + (check check-name)) + (lint-checker (name 'description) (description "Validate package descriptions") (check check-description-style)) diff --git a/guix/scripts/archive.scm b/guix/scripts/archive.scm index 91be1b02e1..ceac640432 100644 --- a/guix/scripts/archive.scm +++ b/guix/scripts/archive.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. @@ -63,7 +63,7 @@ (print-build-trace? . #t) (print-extended-build-trace? . #t) (multiplexed-build-output? . #t) - (verbosity . 2) + (verbosity . 3) (debug . 0))) (define (show-help) diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index fa1bbf867d..2decdb45ed 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -333,7 +333,7 @@ use '--no-offload' instead~%"))) (print-build-trace? . #t) (print-extended-build-trace? . #t) (multiplexed-build-output? . #t) - (verbosity . 2) + (verbosity . 3) (debug . 0))) (define (show-help) diff --git a/guix/scripts/copy.scm b/guix/scripts/copy.scm index 2780d4fbe9..52b476db54 100644 --- a/guix/scripts/copy.scm +++ b/guix/scripts/copy.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2016, 2017, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -163,7 +163,7 @@ Copy ITEMS to or from the specified host over SSH.\n")) (print-extended-build-trace? . #t) (multiplexed-build-output? . #t) (debug . 0) - (verbosity . 2))) + (verbosity . 3))) ;;; diff --git a/guix/scripts/describe.scm b/guix/scripts/describe.scm index be2279d254..b5f6249176 100644 --- a/guix/scripts/describe.scm +++ b/guix/scripts/describe.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> ;;; Copyright © 2020 Ekaitz Zarraga <ekaitz@elenq.tech> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -286,12 +287,9 @@ text. The hyperlink links to a web view of COMMIT, when available." (define-command (guix-describe . args) (synopsis "describe the channel revisions currently used") - (let* ((opts (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") - name)) - cons - %default-options)) + (let* ((opts (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler cons)) (format (assq-ref opts 'format)) (profile (or (assq-ref opts 'profile) (current-profile)))) (with-error-handling diff --git a/guix/scripts/discover.scm b/guix/scripts/discover.scm index 6aade81ed1..be1eaa6e95 100644 --- a/guix/scripts/discover.scm +++ b/guix/scripts/discover.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2020 Mathieu Othacehe <othacehe@gnu.org> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -127,12 +128,11 @@ to synchronize with the writer." (synopsis "discover Guix related services using Avahi") (with-error-handling - (let* ((opts (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (leave (G_ "~A: extraneous argument~%") arg)) - %default-options)) + (let* ((opts (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler + (lambda (arg result) + (leave (G_ "~A: extraneous argument~%") arg)))) (cache (assoc-ref opts 'cache)) (publish-file (publish-file cache))) (parameterize ((%publish-file publish-file)) diff --git a/guix/scripts/download.scm b/guix/scripts/download.scm index ce8dd8b02c..5a91390358 100644 --- a/guix/scripts/download.scm +++ b/guix/scripts/download.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2012, 2013, 2015, 2016, 2017, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -162,15 +163,13 @@ and 'base16' ('hex' and 'hexadecimal' can be used as well).\n")) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (when (assq 'argument result) - (leave (G_ "~A: extraneous argument~%") arg)) - - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler + (lambda (arg result) + (when (assq 'argument result) + (leave (G_ "~A: extraneous argument~%") arg)) + (alist-cons 'argument arg result)))) (with-error-handling (let* ((opts (parse-options)) diff --git a/guix/scripts/edit.scm b/guix/scripts/edit.scm index 49c9d945b6..b4c0507591 100644 --- a/guix/scripts/edit.scm +++ b/guix/scripts/edit.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015, 2016, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mathieu Lirzin <mthl@gnu.org> -;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2020, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -84,11 +84,9 @@ line." (define (parse-arguments) ;; Return the list of package names. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - cons - '())) + (parse-command-line args %options (list (list)) + #:build-options? #f + #:argument-handler cons)) (with-error-handling (let* ((specs (reverse (parse-arguments))) diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm index 1d2b45d942..98554ef79b 100644 --- a/guix/scripts/import.scm +++ b/guix/scripts/import.scm @@ -119,7 +119,8 @@ Run IMPORTER with ARGS.\n")) (current-output-port)))))) (match (apply (resolve-importer importer) args) ((and expr (or ('package _ ...) - ('let _ ...))) + ('let _ ...) + ('define-public _ ...))) (print expr)) ((? list? expressions) (for-each (lambda (expr) diff --git a/guix/scripts/import/cran.scm b/guix/scripts/import/cran.scm index 4767bc082d..aa3ef324e0 100644 --- a/guix/scripts/import/cran.scm +++ b/guix/scripts/import/cran.scm @@ -50,6 +50,8 @@ Import and convert the CRAN package for PACKAGE-NAME.\n")) (display (G_ " -r, --recursive import packages recursively")) (display (G_ " + -s, --style=STYLE choose output style, either specification or variable")) + (display (G_ " -V, --version display version information and exit")) (newline) (show-bug-report-information)) diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm index afdba4e8f1..04b07f80cc 100644 --- a/guix/scripts/import/go.scm +++ b/guix/scripts/import/go.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com> +;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,34 +22,38 @@ #:use-module (guix utils) #:use-module (guix scripts) #:use-module (guix import go) + #:use-module (guix import utils) #:use-module (guix scripts import) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) #:use-module (srfi srfi-37) #:use-module (ice-9 match) #:use-module (ice-9 format) + #:use-module (ice-9 receive) #:export (guix-import-go)) - + ;;; ;;; Command-line options. ;;; (define %default-options - '()) + '((goproxy . "https://proxy.golang.org"))) (define (show-help) - (display (G_ "Usage: guix import go PACKAGE-PATH -Import and convert the Go module for PACKAGE-PATH.\n")) + (display (G_ "Usage: guix import go PACKAGE-PATH[@VERSION] +Import and convert the Go module for PACKAGE-PATH. Optionally, a version +can be specified after the arobas (@) character.\n")) (display (G_ " -h, --help display this help and exit")) (display (G_ " - -V, --version display version information and exit")) - (display (G_ " - -r, --recursive generate package expressions for all Go modules\ - that are not yet in Guix")) + -r, --recursive generate package expressions for all Go modules +that are not yet in Guix")) (display (G_ " -p, --goproxy=GOPROXY specify which goproxy server to use")) + (display (G_ " + --pin-versions use the exact versions of a module's dependencies")) (newline) (show-bug-report-information)) @@ -58,9 +63,6 @@ Import and convert the Go module for PACKAGE-PATH.\n")) (lambda args (show-help) (exit 0))) - (option '(#\V "version") #f #f - (lambda args - (show-version-and-exit "guix import go"))) (option '(#\r "recursive") #f #f (lambda (opt name arg result) (alist-cons 'recursive #t result))) @@ -69,9 +71,12 @@ Import and convert the Go module for PACKAGE-PATH.\n")) (alist-cons 'goproxy (string->symbol arg) (alist-delete 'goproxy result)))) + (option '("pin-versions") #f #f + (lambda (opt name arg result) + (alist-cons 'pin-versions? #t result))) %standard-import-options)) - + ;;; ;;; Entry point. ;;; @@ -91,27 +96,31 @@ Import and convert the Go module for PACKAGE-PATH.\n")) (('argument . value) value) (_ #f)) - (reverse opts)))) + (reverse opts))) + ;; Append the full version to the package symbol name when using + ;; pinned versions. + (package->definition* (if (assoc-ref opts 'pin-versions?) + (cut package->definition <> 'full) + package->definition))) (match args - ((module-name) - (if (assoc-ref opts 'recursive) - (map (match-lambda - ((and ('package ('name name) . rest) pkg) - `(define-public ,(string->symbol name) - ,pkg)) - (_ #f)) - (go-module-recursive-import module-name - #:goproxy-url - (or (assoc-ref opts 'goproxy) - "https://proxy.golang.org"))) - (let ((sexp (go-module->guix-package module-name - #:goproxy-url - (or (assoc-ref opts 'goproxy) - "https://proxy.golang.org")))) - (unless sexp - (leave (G_ "failed to download meta-data for module '~a'~%") - module-name)) - sexp))) + ((spec) ;e.g., github.com/golang/protobuf@v1.3.1 + (receive (name version) + (package-name->name+version spec) + (let ((arguments (list name + #:goproxy (assoc-ref opts 'goproxy) + #:version version + #:pin-versions? + (assoc-ref opts 'pin-versions?)))) + (if (assoc-ref opts 'recursive) + ;; Recursive import. + (map package->definition* + (apply go-module-recursive-import arguments)) + ;; Single import. + (let ((sexp (apply go-module->guix-package arguments))) + (unless sexp + (leave (G_ "failed to download meta-data for module '~a'~%") + module-name)) + (package->definition* sexp)))))) (() (leave (G_ "too few arguments~%"))) ((many ...) diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index fa85088ed0..39bb224cad 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2020 by Amar M. Singh <nly@disroot.org> ;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -1117,12 +1118,11 @@ methods, return the applicable compression." (synopsis "publish build results over HTTP") (with-error-handling - (let* ((opts (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (leave (G_ "~A: extraneous argument~%") arg)) - %default-options)) + (let* ((opts (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler + (lambda (arg result) + (leave (G_ "~A: extraneous argument~%") arg)))) (advertise? (assoc-ref opts 'advertise?)) (user (assoc-ref opts 'user)) (port (assoc-ref opts 'port)) diff --git a/guix/scripts/repl.scm b/guix/scripts/repl.scm index 9f20803efc..50d18c7760 100644 --- a/guix/scripts/repl.scm +++ b/guix/scripts/repl.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> -;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2020, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; Copyright © 2020 Konrad Hinsen <konrad.hinsen@fastmail.net> ;;; ;;; This file is part of GNU Guix. @@ -143,14 +143,13 @@ call THUNK." (synopsis "read-eval-print loop (REPL) for interactive programming") (define opts - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) + (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler (lambda (arg result) (append `((script . ,arg) (ignore-dot-guile? . #t)) - result)) - %default-options)) + result)))) (define user-config (and=> (getenv "HOME") diff --git a/guix/scripts/search.scm b/guix/scripts/search.scm index 0c9e6af07b..27b9da5278 100644 --- a/guix/scripts/search.scm +++ b/guix/scripts/search.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -66,11 +67,9 @@ This is an alias for 'guix package -s'.\n")) result)) (define opts - (args-fold* args %options - (lambda (opt name arg . rest) - (leave (G_ "~A: unrecognized option~%") name)) - handle-argument - '())) + (parse-command-line args %options (list (list)) + #:build-options? #f + #:argument-handler handle-argument)) (unless (assoc-ref opts 'query) (leave (G_ "missing arguments: no regular expressions to search for~%"))) diff --git a/guix/scripts/show.scm b/guix/scripts/show.scm index 535d03c1a6..c747eedd21 100644 --- a/guix/scripts/show.scm +++ b/guix/scripts/show.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2019 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2019, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -66,11 +66,9 @@ This is an alias for 'guix package --show='.\n")) result)) (define opts - (args-fold* args %options - (lambda (opt name arg . rest) - (leave (G_ "~A: unrecognized option~%") name)) - handle-argument - '())) + (parse-command-line args %options (list (list)) + #:build-options? #f + #:argument-handler handle-argument)) (unless (assoc-ref opts 'query) (leave (G_ "missing arguments: no package to show~%"))) diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 46323c7216..48309f9b3a 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -63,7 +63,7 @@ #:use-module (web uri) #:use-module (guix http-client) #:export (%allow-unauthenticated-substitutes? - %error-to-file-descriptor-4? + %reply-file-descriptor substitute-urls guix-substitute)) @@ -279,29 +279,29 @@ Internal tool to substitute a pre-built binary to a local build.\n")) "Evaluate EXP... Return its CPU usage as a fraction between 0 and 1." (call-with-cpu-usage-monitoring (lambda () exp ...))) -(define (display-narinfo-data narinfo) - "Write to the current output port the contents of NARINFO in the format -expected by the daemon." - (format #t "~a\n~a\n~a\n" +(define (display-narinfo-data port narinfo) + "Write to PORT the contents of NARINFO in the format expected by the +daemon." + (format port "~a\n~a\n~a\n" (narinfo-path narinfo) (or (and=> (narinfo-deriver narinfo) (cute string-append (%store-prefix) "/" <>)) "") (length (narinfo-references narinfo))) - (for-each (cute format #t "~a/~a~%" (%store-prefix) <>) + (for-each (cute format port "~a/~a~%" (%store-prefix) <>) (narinfo-references narinfo)) (let-values (((uri compression file-size) (narinfo-best-uri narinfo #:fast-decompression? %prefer-fast-decompression?))) - (format #t "~a\n~a\n" + (format port "~a\n~a\n" (or file-size 0) (or (narinfo-size narinfo) 0)))) -(define* (process-query command +(define* (process-query port command #:key cache-urls acl) - "Reply to COMMAND, a query as written by the daemon to this process's + "Reply on PORT to COMMAND, a query as written by the daemon to this process's standard input. Use ACL as the access-control list against which to check authorized substitutes." (define valid? @@ -338,17 +338,17 @@ authorized substitutes." #:open-connection open-connection-for-uri/cached #:make-progress-reporter make-progress-reporter))) (for-each (lambda (narinfo) - (format #t "~a~%" (narinfo-path narinfo))) + (format port "~a~%" (narinfo-path narinfo))) substitutable) - (newline))) + (newline port))) (("info" paths ..1) ;; Reply info about PATHS if it's in CACHE-URLS. (let ((substitutable (lookup-narinfos/diverse cache-urls paths valid? #:open-connection open-connection-for-uri/cached #:make-progress-reporter make-progress-reporter))) - (for-each display-narinfo-data substitutable) - (newline))) + (for-each (cut display-narinfo-data port <>) substitutable) + (newline port))) (wtf (error "unknown `--query' command" wtf)))) @@ -428,14 +428,14 @@ server certificates." "Bind PORT with EXP... to a socket connected to URI." (call-with-cached-connection uri (lambda (port) exp ...))) -(define* (process-substitution store-item destination +(define* (process-substitution port store-item destination #:key cache-urls acl deduplicate? print-build-trace?) "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to DESTINATION as a nar file. Verify the substitute against ACL, and verify its hash against what appears in the narinfo. When DEDUPLICATE? is true, and if -DESTINATION is in the store, deduplicate its files. Print a status line on -the current output port." +DESTINATION is in the store, deduplicate its files. Print a status line to +PORT." (define narinfo (lookup-narinfo cache-urls store-item (if (%allow-unauthenticated-substitutes?) @@ -555,17 +555,20 @@ the current output port." (every (compose zero? cdr waitpid) pids) ;; Skip a line after what 'progress-reporter/file' printed, and another - ;; one to visually separate substitutions. - (display "\n\n" (current-error-port)) + ;; one to visually separate substitutions. When PRINT-BUILD-TRACE? is + ;; true, leave it up to (guix status) to prettify things. + (newline (current-error-port)) + (unless print-build-trace? + (newline (current-error-port))) ;; Check whether we got the data announced in NARINFO. (let ((actual (get-hash))) (if (bytevector=? actual expected) ;; Tell the daemon that we're done. - (format (current-output-port) "success ~a ~a~%" + (format port "success ~a ~a~%" (narinfo-hash narinfo) (narinfo-size narinfo)) ;; The actual data has a different hash than that in NARINFO. - (format (current-output-port) "hash-mismatch ~a ~a ~a~%" + (format port "hash-mismatch ~a ~a ~a~%" (hash-algorithm-name algorithm) (bytevector->nix-base32-string expected) (bytevector->nix-base32-string actual))))))) @@ -655,7 +658,7 @@ is shorter than MAX elements, then it is directly returned." ;; If the following option is passed to the daemon, use the substitutes list ;; provided by "guix discover" process. (let* ((option (find-daemon-option "discover")) - (discover? (and option (string=? option "yes")))) + (discover? (and option (string=? option "true")))) (if discover? (randomize-substitute-urls (read-substitute-urls)) '()))) @@ -679,28 +682,10 @@ default value." (unless (string->uri uri) (leave (G_ "~a: invalid URI~%") uri))) -(define %error-to-file-descriptor-4? - ;; Whether to direct 'current-error-port' to file descriptor 4 like - ;; 'guix-daemon' expects. - (make-parameter #t)) - -;; The daemon's agent code opens file descriptor 4 for us and this is where -;; stderr should go. -(define-syntax-rule (with-redirected-error-port exp ...) - "Evaluate EXP... with the current error port redirected to file descriptor 4 -if needed, as expected by the daemon's agent." - (let ((thunk (lambda () exp ...))) - (if (%error-to-file-descriptor-4?) - (parameterize ((current-error-port (fdopen 4 "wl"))) - ;; Redirect diagnostics to file descriptor 4 as well. - (guix-warning-port (current-error-port)) - - ;; 'with-continuation-barrier' captures the initial value of - ;; 'current-error-port' to report backtraces in case of uncaught - ;; exceptions. Without it, backtraces would be printed to FD 2, - ;; thereby confusing the daemon. - (with-continuation-barrier thunk)) - (thunk)))) +(define %reply-file-descriptor + ;; The file descriptor where replies to the daemon must be sent, or #f to + ;; use the current output port instead. + (make-parameter 4)) (define-command (guix-substitute . args) (category internal) @@ -716,68 +701,73 @@ if needed, as expected by the daemon's agent." (define deduplicate? (find-daemon-option "deduplicate")) - (with-redirected-error-port - (mkdir-p %narinfo-cache-directory) - (maybe-remove-expired-cache-entries %narinfo-cache-directory - cached-narinfo-files - #:entry-expiration - cached-narinfo-expiration-time - #:cleanup-period - %narinfo-expired-cache-entry-removal-delay) - (check-acl-initialized) - - ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error - ;; message. - (for-each validate-uri (substitute-urls)) - - ;; Attempt to install the client's locale so that messages are suitably - ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default - ;; so don't change it. - (match (or (find-daemon-option "untrusted-locale") - (find-daemon-option "locale")) - (#f #f) - (locale (false-if-exception (setlocale LC_MESSAGES locale)))) - - (catch 'system-error - (lambda () - (set-thread-name "guix substitute")) - (const #t)) ;GNU/Hurd lacks 'prctl' - - (with-networking - (with-error-handling ; for signature errors - (match args - (("--query") - (let ((acl (current-acl))) - (let loop ((command (read-line))) - (or (eof-object? command) - (begin - (process-query command - #:cache-urls (substitute-urls) - #:acl acl) - (loop (read-line))))))) - (("--substitute") - ;; Download STORE-PATH and store it as a Nar in file DESTINATION. - ;; Specify the number of columns of the terminal so the progress - ;; report displays nicely. - (parameterize ((current-terminal-columns (client-terminal-columns))) - (let loop () - (match (read-line) - ((? eof-object?) - #t) - ((= string-tokenize ("substitute" store-path destination)) - (process-substitution store-path destination - #:cache-urls (substitute-urls) - #:acl (current-acl) - #:deduplicate? deduplicate? - #:print-build-trace? - print-build-trace?) - (loop)))))) - ((or ("-V") ("--version")) - (show-version-and-exit "guix substitute")) - (("--help") - (show-help)) - (opts - (leave (G_ "~a: unrecognized options~%") opts))))))) + (define reply-port + ;; Port used to reply to the daemon. + (if (%reply-file-descriptor) + (fdopen (%reply-file-descriptor) "wl") + (current-output-port))) + + (mkdir-p %narinfo-cache-directory) + (maybe-remove-expired-cache-entries %narinfo-cache-directory + cached-narinfo-files + #:entry-expiration + cached-narinfo-expiration-time + #:cleanup-period + %narinfo-expired-cache-entry-removal-delay) + (check-acl-initialized) + + ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error + ;; message. + (for-each validate-uri (substitute-urls)) + + ;; Attempt to install the client's locale so that messages are suitably + ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default + ;; so don't change it. + (match (or (find-daemon-option "untrusted-locale") + (find-daemon-option "locale")) + (#f #f) + (locale (false-if-exception (setlocale LC_MESSAGES locale)))) + + (catch 'system-error + (lambda () + (set-thread-name "guix substitute")) + (const #t)) ;GNU/Hurd lacks 'prctl' + + (with-networking + (with-error-handling ; for signature errors + (match args + (("--query") + (let ((acl (current-acl))) + (let loop ((command (read-line))) + (or (eof-object? command) + (begin + (process-query reply-port command + #:cache-urls (substitute-urls) + #:acl acl) + (loop (read-line))))))) + (("--substitute") + ;; Download STORE-PATH and store it as a Nar in file DESTINATION. + ;; Specify the number of columns of the terminal so the progress + ;; report displays nicely. + (parameterize ((current-terminal-columns (client-terminal-columns))) + (let loop () + (match (read-line) + ((? eof-object?) + #t) + ((= string-tokenize ("substitute" store-path destination)) + (process-substitution reply-port store-path destination + #:cache-urls (substitute-urls) + #:acl (current-acl) + #:deduplicate? deduplicate? + #:print-build-trace? + print-build-trace?) + (loop)))))) + ((or ("-V") ("--version")) + (show-version-and-exit "guix substitute")) + (("--help") + (show-help)) + (opts + (leave (G_ "~a: unrecognized options~%") opts)))))) ;;; Local Variables: ;;; eval: (put 'with-timeout 'scheme-indent-function 1) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index c226f08371..0a051ee4e3 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -1145,7 +1145,7 @@ Some ACTIONS support additional ARGS.\n")) "Return the verbosity level based on OPTS, the alist of parsed options." (or (assoc-ref opts 'verbosity) (if (eq? (assoc-ref opts 'action) 'build) - 2 1))) + 3 1))) ;;; diff --git a/guix/status.scm b/guix/status.scm index d47bf1700c..362ae2882c 100644 --- a/guix/status.scm +++ b/guix/status.scm @@ -403,10 +403,12 @@ the current build phase." #:optional (port (current-error-port)) #:key (colorize? (color-output? port)) + (print-urls? #t) (print-log? #t)) "Print information about EVENT and STATUS to PORT. When COLORIZE? is true, produce colorful output. When PRINT-LOG? is true, display the build log in -addition to build events." +addition to build events. When PRINT-URLS? is true, display the URL of +substitutes being downloaded." (define info (if colorize? (cute colorize-string <> (color BOLD)) @@ -526,9 +528,10 @@ addition to build events." (format port (info (G_ "substituting ~a...")) item) (newline port))) (('download-started item uri _ ...) - (erase-current-line*) - (format port (info (G_ "downloading from ~a ...")) uri) - (newline port)) + (when print-urls? + (erase-current-line*) + (format port (info (G_ "downloading from ~a ...")) uri) + (newline port))) (('download-progress item uri (= string->number size) (= string->number transferred)) @@ -602,6 +605,17 @@ addition to build events." (colorize? (color-output? port))) (print-build-event event old-status status port #:colorize? colorize? + #:print-urls? #f + #:print-log? #f)) + +(define* (print-build-event/quiet-with-urls event old-status status + #:optional + (port (current-error-port)) + #:key + (colorize? (color-output? port))) + (print-build-event event old-status status port + #:colorize? colorize? + #:print-urls? #t ;show download URLs #:print-log? #f)) (define* (build-status-updater #:optional (on-change (const #t))) @@ -787,6 +801,7 @@ evaluate EXP... in that context." "Return the logging procedure that corresponds to LEVEL." (cond ((<= level 0) (const #t)) ((= level 1) print-build-event/quiet) + ((= level 2) print-build-event/quiet-with-urls) (else print-build-event))) (define (call-with-status-verbosity level thunk) diff --git a/guix/upstream.scm b/guix/upstream.scm index accd8967d8..632e9ebc4f 100644 --- a/guix/upstream.scm +++ b/guix/upstream.scm @@ -264,12 +264,15 @@ them matches." #:optional (updaters (force %updaters))) "Return an upstream source to update PACKAGE, a <package> object, or #f if -none of UPDATERS matches PACKAGE. It is the caller's responsibility to ensure -that the returned source is newer than the current one." - (match (lookup-updater package updaters) - ((? upstream-updater? updater) - ((upstream-updater-latest updater) package)) - (_ #f))) +none of UPDATERS matches PACKAGE. When several updaters match PACKAGE, try +them until one of them returns an upstream source. It is the caller's +responsibility to ensure that the returned source is newer than the current +one." + (any (match-lambda + (($ <upstream-updater> name description pred latest) + (and (pred package) + (latest package)))) + updaters)) (define* (package-latest-release* package #:optional diff --git a/guix/utils.scm b/guix/utils.scm index 7db9f52ff6..19990ceb8a 100644 --- a/guix/utils.scm +++ b/guix/utils.scm @@ -712,6 +712,7 @@ VERSIONS. For example: (define (tarball-sans-extension tarball) "Return TARBALL without its .tar.* or .zip extension." (let ((end (or (string-contains tarball ".tar") + (string-contains tarball ".tgz") (string-contains tarball ".zip")))) (substring tarball 0 end))) |