From 022288ba53bd44c9311c6cffef5e8224a2e74598 Mon Sep 17 00:00:00 2001 From: Ivan Petkov Date: Sun, 30 Jun 2019 07:18:20 -0700 Subject: guix: import: crate: fix redundant inputs list nesting * guix/import/crate.scm (maybe-cargo-inputs): Remove one level of lists. * guix/import/crate.scm (maybe-cargo-development-inputs): Same. * tests/crate.scm: (crate->guix-package)[package]<#:arguments>: Remove one level of list nesting. --- tests/crate.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/crate.scm b/tests/crate.scm index a4a328d507..72c3a13350 100644 --- a/tests/crate.scm +++ b/tests/crate.scm @@ -91,7 +91,7 @@ ('build-system 'cargo-build-system) ('arguments ('quasiquote - (('#:cargo-inputs (("rust-bar" ('unquote rust-bar))))))) + ('#:cargo-inputs (("rust-bar" ('unquote rust-bar)))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") -- cgit v1.2.3 From 01589acc5e114e46e314777bc0d60ebfa620e4f8 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:00 -0400 Subject: import: pypi: Do not consider requirements.txt files. PyPI packages are mandated to have a setup.py file, which contains a listing of the required dependencies. The setuptools/distutils machinery embed metadata in the archives they produce, which contains this information. There is no need nor gain to collect the requirements from a "requirements.txt" file, as it is not the true record of dependencies for PyPI packages and may contain extraneous requirements or not exist at all. * guix/import/pypi.scm (guess-requirements): Update comment. [guess-requirements-from-source]: Do not attempt to parse the file requirements.txt. Streamline logic. * tests/pypi.scm (test-requires.txt): Rename from test-requirements, to hint at the file being tested. ("pypi->guix-package"): Adapt so that the fake package contains a requires.txt file rather than a requirements.txt file. ("pypi->guix-package, wheels"): Likewise. --- guix/import/pypi.scm | 35 +++++++++++++---------------------- tests/pypi.scm | 23 ++++++++++++----------- 2 files changed, 25 insertions(+), 33 deletions(-) (limited to 'tests') diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 3a20fc4b9b..8269aa61d7 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -206,35 +206,26 @@ cannot determine package dependencies")) (call-with-temporary-directory (lambda (dir) (let* ((pypi-name (string-take dirname (string-rindex dirname #\-))) - (req-files (list (string-append dirname "/requirements.txt") - (string-append dirname "/" pypi-name ".egg-info" - "/requires.txt"))) - (exit-codes (map (lambda (file-name) - (parameterize ((current-error-port (%make-void-port "rw+")) - (current-output-port (%make-void-port "rw+"))) - (system* "tar" "xf" tarball "-C" dir file-name))) - req-files))) - ;; Only one of these files needs to exist. - (if (any zero? exit-codes) - (match (find-files dir) - ((file . _) - (read-requirements file)) - (() - (warning (G_ "No requirements file found.\n")))) + (requires.txt (string-append dirname "/" pypi-name + ".egg-info" "/requires.txt")) + (exit-code (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (system* "tar" "xf" tarball "-C" dir requires.txt)))) + (if (zero? exit-code) + (read-requirements (string-append dir "/" requires.txt)) (begin - (warning (G_ "Failed to extract requirements files\n")) + (warning + (G_ "Failed to extract file: ~a from source.~%") + requires.txt) '()))))) '()))) - ;; First, try to compute the requirements using the wheel, since that is the - ;; most reliable option. If a wheel is not provided for this package, try - ;; getting them by reading either the "requirements.txt" file or the - ;; "requires.txt" from the egg-info directory from the source tarball. Note - ;; that "requirements.txt" is not mandatory, so this is likely to fail. + ;; First, try to compute the requirements using the wheel, else, fallback to + ;; reading the "requires.txt" from the egg-info directory from the source + ;; tarball. (or (guess-requirements-from-wheel) (guess-requirements-from-source))) - (define (compute-inputs source-url wheel-url tarball) "Given the SOURCE-URL of an already downloaded TARBALL, return a list of name/variable pairs describing the required inputs of this package. Also diff --git a/tests/pypi.scm b/tests/pypi.scm index 6daa44a6e7..a0271fffad 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -23,7 +23,7 @@ #:use-module (gcrypt hash) #:use-module (guix tests) #:use-module (guix build-system python) - #:use-module ((guix build utils) #:select (delete-file-recursively which)) + #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) #:use-module (srfi srfi-64) #:use-module (ice-9 match)) @@ -55,11 +55,12 @@ (define test-source-hash "") -(define test-requirements -"# A comment +(define test-requires.txt "\ +# A comment # A comment after a space bar -baz > 13.37") +baz > 13.37 +") (define test-metadata "{ @@ -107,10 +108,10 @@ baz > 13.37") (match url ("https://example.com/foo-1.0.0.tar.gz" (begin - (mkdir "foo-1.0.0") - (with-output-to-file "foo-1.0.0/requirements.txt" + (mkdir-p "foo-1.0.0/foo.egg-info/") + (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requirements))) + (display test-requires.txt))) (system* "tar" "czvf" file-name "foo-1.0.0/") (delete-file-recursively "foo-1.0.0") (set! test-source-hash @@ -157,11 +158,11 @@ baz > 13.37") (lambda (url file-name) (match url ("https://example.com/foo-1.0.0.tar.gz" - (begin - (mkdir "foo-1.0.0") - (with-output-to-file "foo-1.0.0/requirements.txt" + (begin + (mkdir-p "foo-1.0.0/foo.egg-info/") + (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requirements))) + (display test-requires.txt))) (system* "tar" "czvf" file-name "foo-1.0.0/") (delete-file-recursively "foo-1.0.0") (set! test-source-hash -- cgit v1.2.3 From a853acebe1ccdba51ae509cd8a838a954c7c8bd2 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Mon, 10 Jun 2019 12:20:08 +0900 Subject: tests: pypi: Mute the output of tar. The output of tar when creating archives for the purpose of tests is not useful, so we mute it. * tests/pypi.scm ("pypi->guix-package"): Mute the output of tar. ("pypi->guix-package, wheels"): Likewise. --- tests/pypi.scm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/pypi.scm b/tests/pypi.scm index a0271fffad..6df69073dc 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -112,7 +112,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) @@ -163,7 +164,8 @@ baz > 13.37 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () (display test-requires.txt))) - (system* "tar" "czvf" file-name "foo-1.0.0/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") (set! test-source-hash (call-with-input-file file-name port-sha256)))) -- cgit v1.2.3 From c4797121beea74ae93e3ce17677b9e72b8df920d Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:00 -0400 Subject: import: pypi: Do not parse optional requirements from source. * guix/import/pypi.scm: Export PARSE-REQUIRES.TXT. (clean-requirement): Move procedure to the top level. (guess-requirements): Move the READ-REQUIREMENTS procedure to the top level, and rename it to PARSE-REQUIRES.TXT. Move the CLEAN-REQUIREMENT procedure to the top level. Move the COMMENT? functions inside the PARSE-REQUIRES.TXT procedure. (parse-requires.txt): Add a SECTION-HEADER? predicate, and use it to prevent parsing optional requirements. * tests/pypi.scm (test-requires-with-sections): New variable. ("parse-requires.txt, with sections"): New test. --- guix/import/pypi.scm | 74 +++++++++++++++++++++++++++++++--------------------- tests/pypi.scm | 14 ++++++++++ 2 files changed, 58 insertions(+), 30 deletions(-) (limited to 'tests') diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 8269aa61d7..d9db876222 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -47,7 +47,8 @@ #:use-module (guix upstream) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) - #:export (guix-package->pypi-name + #:export (parse-requires.txt + guix-package->pypi-name pypi-recursive-import pypi->guix-package %pypi-updater)) @@ -117,6 +118,47 @@ package definition." ((package-inputs ...) `((propagated-inputs (,'quasiquote ,package-inputs)))))) +(define (clean-requirement s) + ;; Given a requirement LINE, as can be found in a setuptools requires.txt + ;; file, remove everything other than the actual name of the required + ;; package, and return it. + (cond + ((string-index s (char-set #\space #\> #\= #\<)) => (cut string-take s <>)) + (else s))) + +(define (parse-requires.txt requires.txt) + "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of +requirement names." + ;; This is a very incomplete parser, whose job is to select the non-optional + ;; dependencies and strip them out of any version information. + ;; Alternatively, we could implement a PEG parser with the (ice-9 peg) + ;; library and the requirements grammar defined by PEP-0508 + ;; (https://www.python.org/dev/peps/pep-0508/). + + (define (comment? line) + ;; Return #t if the given LINE is a comment, #f otherwise. + (string-prefix? "#" (string-trim line))) + + (define (section-header? line) + ;; Return #t if the given LINE is a section header, #f otherwise. + (string-prefix? "[" (string-trim line))) + + (call-with-input-file requires.txt + (lambda (port) + (let loop ((result '())) + (let ((line (read-line port))) + ;; Stop when a section is encountered, as sections contain optional + ;; (extra) requirements. Non-optional requirements must appear + ;; before any section is defined. + (if (or (eof-object? line) (section-header? line)) + (reverse result) + (cond + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (clean-requirement line) + result)))))))))) + (define (guess-requirements source-url wheel-url tarball) "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of the required packages specified in the requirements.txt file. TARBALL will @@ -139,34 +181,6 @@ be extracted in a temporary directory." cannot determine package dependencies")) #f))))) - (define (clean-requirement s) - ;; Given a requirement LINE, as can be found in a Python requirements.txt - ;; file, remove everything other than the actual name of the required - ;; package, and return it. - (string-take s - (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<)))) - (string-length s)))) - - (define (comment? line) - ;; Return #t if the given LINE is a comment, #f otherwise. - (eq? (string-ref (string-trim line) 0) #\#)) - - (define (read-requirements requirements-file) - ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list - ;; of name/variable pairs describing the requirements. - (call-with-input-file requirements-file - (lambda (port) - (let loop ((result '())) - (let ((line (read-line port))) - (if (eof-object? line) - result - (cond - ((or (string-null? line) (comment? line)) - (loop result)) - (else - (loop (cons (clean-requirement line) - result)))))))))) - (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's ;; requirements. @@ -212,7 +226,7 @@ cannot determine package dependencies")) (current-output-port (%make-void-port "rw+"))) (system* "tar" "xf" tarball "-C" dir requires.txt)))) (if (zero? exit-code) - (read-requirements (string-append dir "/" requires.txt)) + (parse-requires.txt (string-append dir "/" requires.txt)) (begin (warning (G_ "Failed to extract file: ~a from source.~%") diff --git a/tests/pypi.scm b/tests/pypi.scm index 6df69073dc..03455ba6be 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -62,6 +62,14 @@ bar baz > 13.37 ") +(define test-requires-with-sections "\ +foo ~= 3 +bar != 2 + +[test] +pytest (>=2.5.0) +") + (define test-metadata "{ \"run_requires\": [ @@ -101,6 +109,12 @@ baz > 13.37 (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" (pypi-uri "cram" "0.7")))))))) +(test-equal "parse-requires.txt, with sections" + '("foo" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-requires.txt test-requires-with-sections))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch -- cgit v1.2.3 From 803fb336d62ea65990e263ce58d8552e04c9c038 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:01 -0400 Subject: import: pypi: Improve parsing of requirement specifications. The previous solution was fragile and could leave unwanted characters in a requirement name, such as '[' or ']'. Partially fixes . * guix/import/pypi.scm (use-modules): Export SPECIFICATION->REQUIREMENT-NAME (%requirement-name-regexp): New variable. (clean-requirement): Rename to... (specification->requirement-name): this, which now uses %requirement-name-regexp to select the requirement name from the requirement specification. (parse-requires.txt): Adapt. --- guix/import/pypi.scm | 54 ++++++++++++++++++++++++++++++++++++++-------------- tests/pypi.scm | 12 ++++++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) (limited to 'tests') diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index d9db876222..6a881bda12 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -48,6 +48,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (parse-requires.txt + specification->requirement-name guix-package->pypi-name pypi-recursive-import pypi->guix-package @@ -118,22 +119,47 @@ package definition." ((package-inputs ...) `((propagated-inputs (,'quasiquote ,package-inputs)))))) -(define (clean-requirement s) - ;; Given a requirement LINE, as can be found in a setuptools requires.txt - ;; file, remove everything other than the actual name of the required - ;; package, and return it. - (cond - ((string-index s (char-set #\space #\> #\= #\<)) => (cut string-take s <>)) - (else s))) +(define %requirement-name-regexp + ;; Regexp to match the requirement name in a requirement specification. + + ;; Some grammar, taken from PEP-0508 (see: + ;; https://www.python.org/dev/peps/pep-0508/). + + ;; Using this grammar makes the PEP-0508 regexp easier to understand for + ;; humans. The use of a regexp is preferred to more primitive string + ;; manipulations because we can more directly match what upstream uses + ;; (again, per PEP-0508). The regexp approach is also easier to extend, + ;; should we want to implement more completely the grammar of PEP-0508. + + ;; The unified rule can be expressed as: + ;; specification = wsp* ( url_req | name_req ) wsp* + + ;; where url_req is: + ;; url_req = name wsp* extras? wsp* urlspec wsp+ quoted_marker? + + ;; and where name_req is: + ;; name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker? + + ;; Thus, we need only matching NAME, which is expressed as: + ;; identifer_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit) + ;; identifier = letterOrDigit identifier_end* + ;; name = identifier + (let* ((letter-or-digit "[A-Za-z0-9]") + (identifier-end (string-append "(" letter-or-digit "|" + "[-_.]*" letter-or-digit ")")) + (identifier (string-append "^" letter-or-digit identifier-end "*")) + (name identifier)) + (make-regexp name))) + +(define (specification->requirement-name spec) + "Given a specification SPEC, return the requirement name." + (match:substring + (or (regexp-exec %requirement-name-regexp spec) + (error (G_ "Could not extract requirement name in spec:") spec)))) (define (parse-requires.txt requires.txt) "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of requirement names." - ;; This is a very incomplete parser, whose job is to select the non-optional - ;; dependencies and strip them out of any version information. - ;; Alternatively, we could implement a PEG parser with the (ice-9 peg) - ;; library and the requirements grammar defined by PEP-0508 - ;; (https://www.python.org/dev/peps/pep-0508/). (define (comment? line) ;; Return #t if the given LINE is a comment, #f otherwise. @@ -156,7 +182,7 @@ requirement names." ((or (string-null? line) (comment? line)) (loop result)) (else - (loop (cons (clean-requirement line) + (loop (cons (specification->requirement-name line) result)))))))))) (define (guess-requirements source-url wheel-url tarball) @@ -198,7 +224,7 @@ cannot determine package dependencies")) (hash-ref (list-ref run_requires 0) "requires") '()))) - (map clean-requirement requirements))))) + (map specification->requirement-name requirements))))) (lambda () (delete-file json-file) (rmdir dirname)))))) diff --git a/tests/pypi.scm b/tests/pypi.scm index 03455ba6be..c40be6c21d 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -55,6 +55,14 @@ (define test-source-hash "") +(define test-specifications + '("Fizzy [foo, bar]" + "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1" + "SomethingWithMarker[foo]>1.0;python_version<\"2.7\"" + "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\"" + "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\ +sha1=da9234ee9982d4bbb3c72346a6de940a148ea686")) + (define test-requires.txt "\ # A comment # A comment after a space @@ -109,6 +117,10 @@ pytest (>=2.5.0) (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" (pypi-uri "cram" "0.7")))))))) +(test-equal "specification->requirement-name" + '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip") + (map specification->requirement-name test-specifications)) + (test-equal "parse-requires.txt, with sections" '("foo" "bar") (mock ((ice-9 ports) call-with-input-file -- cgit v1.2.3 From c799ad7276dcc771912f4fc7edca400ea48b8ec2 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:02 -0400 Subject: import: pypi: Support more types of archives. This change enables the PyPI importer to look for requirements in a source archive of a different type than "tar.gz" or "tar.bz2". Also, scan the source archive to find a requires.txt file. * guix/import/pypi.scm: (guess-requirements)[tarball-directory]: Remove procedure. [guess-requirements-from-source]: Use COMRESSED-FILE? to determine if an archive type is supported, and some file extension logic that chooses either "tar" or "unzip" as the extractor. Search for the requires.txt file in the archive instead of using a static, expected location. (guess-requirements): Rename the TARBALL argument to ARCHIVE, to denote the archive format is no longer bound specifically to the Tar format. (compute-inputs): Likewise. * tests/pypi.scm ("pypi->guix-package, no wheel"): Mock the requires.txt at a non-standard location. ("pypi->guix-package, no usable requirement file."): New test. --- guix/import/pypi.scm | 77 ++++++++++++++++++++++------------------------------ tests/pypi.scm | 52 +++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 46 deletions(-) (limited to 'tests') diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index ad59a8b731..a6106ab4ec 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -39,7 +39,8 @@ #:use-module ((guix build utils) #:select ((package-name->name+version . hyphen-package-name->name+version) - find-files)) + find-files + invoke)) #:use-module (guix import utils) #:use-module ((guix download) #:prefix download:) #:use-module (guix import json) @@ -189,28 +190,11 @@ requirement names." (loop (cons (specification->requirement-name line) result)))))))))) -(define (guess-requirements source-url wheel-url tarball) - "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list -of the required packages specified in the requirements.txt file. TARBALL will +(define (guess-requirements source-url wheel-url archive) + "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list +of the required packages specified in the requirements.txt file. ARCHIVE will be extracted in a temporary directory." - (define (tarball-directory url) - ;; Given the URL of the package's tarball, return the name of the directory - ;; that will be created upon decompressing it. If the filetype is not - ;; supported, return #f. - ;; TODO: Support more archive formats. - (let ((basename (substring url (+ 1 (string-rindex url #\/))))) - (cond - ((string-suffix? ".tar.gz" basename) - (string-drop-right basename 7)) - ((string-suffix? ".tar.bz2" basename) - (string-drop-right basename 8)) - (else - (begin - (warning (G_ "Unsupported archive format: \ -cannot determine package dependencies")) - #f))))) - (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's ;; requirements. @@ -239,29 +223,34 @@ cannot determine package dependencies")) (call-with-temporary-output-file (lambda (temp port) (if wheel-url - (and (url-fetch wheel-url temp) - (read-wheel-metadata temp)) - #f)))) + (and (url-fetch wheel-url temp) + (read-wheel-metadata temp)) + #f)))) (define (guess-requirements-from-source) ;; Return the package's requirements by guessing them from the source. - (let ((dirname (tarball-directory source-url))) - (if (string? dirname) - (call-with-temporary-directory - (lambda (dir) - (let* ((pypi-name (string-take dirname (string-rindex dirname #\-))) - (requires.txt (string-append dirname "/" pypi-name - ".egg-info" "/requires.txt")) - (exit-code (parameterize ((current-error-port (%make-void-port "rw+")) - (current-output-port (%make-void-port "rw+"))) - (system* "tar" "xf" tarball "-C" dir requires.txt)))) - (if (zero? exit-code) - (parse-requires.txt (string-append dir "/" requires.txt)) - (begin - (warning - (G_ "Failed to extract file: ~a from source.~%") - requires.txt) - '()))))) + (if (compressed-file? source-url) + (call-with-temporary-directory + (lambda (dir) + (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (if (string=? "zip" (file-extension source-url)) + (invoke "unzip" archive "-d" dir) + (invoke "tar" "xf" archive "-C" dir))) + (let ((requires.txt-files + (find-files dir (lambda (abs-file-name _) + (string-match "\\.egg-info/requires.txt$" + abs-file-name))))) + (match requires.txt-files + (() + (warning (G_ "Cannot guess requirements from source archive:\ + no requires.txt file found.~%")) + '()) + (else (parse-requires.txt (first requires.txt-files))))))) + (begin + (warning (G_ "Unsupported archive format; \ +cannot determine package dependencies from source archive: ~a~%") + (basename source-url)) '()))) ;; First, try to compute the requirements using the wheel, else, fallback to @@ -270,13 +259,13 @@ cannot determine package dependencies")) (or (guess-requirements-from-wheel) (guess-requirements-from-source))) -(define (compute-inputs source-url wheel-url tarball) - "Given the SOURCE-URL of an already downloaded TARBALL, return a list of +(define (compute-inputs source-url wheel-url archive) + "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of name/variable pairs describing the required inputs of this package. Also return the unaltered list of upstream dependency names." (let ((dependencies (remove (cut string=? "argparse" <>) - (guess-requirements source-url wheel-url tarball)))) + (guess-requirements source-url wheel-url archive)))) (values (sort (map (lambda (input) (let ((guix-name (python->package-name input))) diff --git a/tests/pypi.scm b/tests/pypi.scm index c40be6c21d..b45d2c9d2f 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -20,6 +20,7 @@ (define-module (test-pypi) #:use-module (guix import pypi) #:use-module (guix base32) + #:use-module (guix memoization) #:use-module (gcrypt hash) #:use-module (guix tests) #:use-module (guix build-system python) @@ -134,8 +135,9 @@ pytest (>=2.5.0) (match url ("https://example.com/foo-1.0.0.tar.gz" (begin - (mkdir-p "foo-1.0.0/foo.egg-info/") - (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" + ;; Unusual requires.txt location should still be found. + (mkdir-p "foo-1.0.0/src/bizarre.egg-info") + (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt" (lambda () (display test-requires.txt))) (parameterize ((current-output-port (%make-void-port "rw+"))) @@ -241,4 +243,50 @@ pytest (>=2.5.0) (x (pk 'fail x #f)))))) +(test-assert "pypi->guix-package, no usable requirement file." + ;; Replace network resources with sample data. + (mock ((guix import utils) url-fetch + (lambda (url file-name) + (match url + ("https://example.com/foo-1.0.0.tar.gz" + (mkdir-p "foo-1.0.0/foo.egg-info/") + (parameterize ((current-output-port (%make-void-port "rw+"))) + (system* "tar" "czvf" file-name "foo-1.0.0/")) + (delete-file-recursively "foo-1.0.0") + (set! test-source-hash + (call-with-input-file file-name port-sha256))) + ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) + (_ (error "Unexpected URL: " url))))) + (mock ((guix http-client) http-fetch + (lambda (url . rest) + (match url + ("https://pypi.org/pypi/foo/json" + (values (open-input-string test-json) + (string-length test-json))) + ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) + (_ (error "Unexpected URL: " url))))) + ;; Not clearing the memoization cache here would mean returning the value + ;; computed in the previous test. + (invalidate-memoization! pypi->guix-package) + (match (pypi->guix-package "foo") + (('package + ('name "python-foo") + ('version "1.0.0") + ('source ('origin + ('method 'url-fetch) + ('uri ('pypi-uri "foo" 'version)) + ('sha256 + ('base32 + (? string? hash))))) + ('build-system 'python-build-system) + ('home-page "http://example.com") + ('synopsis "summary") + ('description "summary") + ('license 'license:lgpl2.0)) + (string=? (bytevector->nix-base32-string + test-source-hash) + hash)) + (x + (pk 'fail x #f)))))) + (test-end "pypi") -- cgit v1.2.3 From f0190a5dcdc1882e5a058dc7f12bc7ccb166c63d Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 00:26:03 -0400 Subject: import: pypi: Parse wheel METADATA instead of metadata.json. With newer Wheel releases, there is no more metadata.json file; the METADATA file should be used instead (see: https://github.com/pypa/wheel/issues/195). This change updates our PyPI importer so that it uses the latter. * guix/import/pypi.scm (define-module): Remove unnecessary modules and export the PARSE-WHEEL-METADATA procedure. (parse-wheel-metadata): Add procedure. (guess-requirements): Use it. * tests/pypi.scm (test-metadata): Test it. --- guix/import/pypi.scm | 90 ++++++++++++++++++++++++++++++++-------------------- tests/pypi.scm | 60 +++++++++++++++++++++++++++-------- 2 files changed, 101 insertions(+), 49 deletions(-) (limited to 'tests') diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index a6106ab4ec..7cf1e92101 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -21,9 +21,7 @@ ;;; along with GNU Guix. If not, see . (define-module (guix import pypi) - #:use-module (ice-9 binary-ports) #:use-module (ice-9 match) - #:use-module (ice-9 pretty-print) #:use-module (ice-9 regex) #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) @@ -31,9 +29,6 @@ #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) - #:use-module (rnrs bytevectors) - #:use-module (json) - #:use-module (web uri) #:use-module (guix ui) #:use-module (guix utils) #:use-module ((guix build utils) @@ -49,6 +44,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (parse-requires.txt + parse-wheel-metadata specification->requirement-name guix-package->pypi-name pypi-recursive-import @@ -177,18 +173,49 @@ requirement names." ;; Stop when a section is encountered, as sections contain optional ;; (extra) requirements. Non-optional requirements must appear ;; before any section is defined. - (if (or (eof-object? line) (section-header? line)) - ;; Duplicates can occur, since the same requirement can be - ;; listed multiple times with different conditional markers, e.g. - ;; pytest >= 3 ; python_version >= "3.3" - ;; pytest < 3 ; python_version < "3.3" - (reverse (delete-duplicates result)) - (cond - ((or (string-null? line) (comment? line)) - (loop result)) - (else - (loop (cons (specification->requirement-name line) - result)))))))))) + (cond + ((or (eof-object? line) (section-header? line)) + ;; Duplicates can occur, since the same requirement can be + ;; listed multiple times with different conditional markers, e.g. + ;; pytest >= 3 ; python_version >= "3.3" + ;; pytest < 3 ; python_version < "3.3" + (reverse (delete-duplicates result))) + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (specification->requirement-name line) + result))))))))) + +(define (parse-wheel-metadata metadata) + "Given METADATA, a Wheel metadata file, return a list of requirement names." + ;; METADATA is a RFC-2822-like, header based file. + + (define (requires-dist-header? line) + ;; Return #t if the given LINE is a Requires-Dist header. + (string-match "^Requires-Dist: " line)) + + (define (requires-dist-value line) + (string-drop line (string-length "Requires-Dist: "))) + + (define (extra? line) + ;; Return #t if the given LINE is an "extra" requirement. + (string-match "extra == '(.*)'" line)) + + (call-with-input-file metadata + (lambda (port) + (let loop ((requirements '())) + (let ((line (read-line port))) + ;; Stop at the first 'Provides-Extra' section: the non-optional + ;; requirements appear before the optional ones. + (cond + ((eof-object? line) + (reverse (delete-duplicates requirements))) + ((and (requires-dist-header? line) (not (extra? line))) + (loop (cons (specification->requirement-name + (requires-dist-value line)) + requirements))) + (else + (loop requirements)))))))) (define (guess-requirements source-url wheel-url archive) "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list @@ -197,25 +224,18 @@ be extracted in a temporary directory." (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's - ;; requirements. + ;; requirements, or #f if the metadata file contained therein couldn't be + ;; extracted. (let* ((dirname (wheel-url->extracted-directory wheel-url)) - (json-file (string-append dirname "/metadata.json"))) - (and (zero? (system* "unzip" "-q" wheel-archive json-file)) - (dynamic-wind - (const #t) - (lambda () - (call-with-input-file json-file - (lambda (port) - (let* ((metadata (json->scm port)) - (run_requires (hash-ref metadata "run_requires")) - (requirements (if run_requires - (hash-ref (list-ref run_requires 0) - "requires") - '()))) - (map specification->requirement-name requirements))))) - (lambda () - (delete-file json-file) - (rmdir dirname)))))) + (metadata (string-append dirname "/METADATA"))) + (call-with-temporary-directory + (lambda (dir) + (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata)) + (parse-wheel-metadata (string-append dir "/" metadata)) + (begin + (warning + (G_ "Failed to extract file: ~a from wheel.~%") metadata) + #f)))))) (define (guess-requirements-from-wheel) ;; Return the package's requirements using the wheel, or #f if an error diff --git a/tests/pypi.scm b/tests/pypi.scm index b45d2c9d2f..8b42c2f071 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -22,6 +22,7 @@ #:use-module (guix base32) #:use-module (guix memoization) #:use-module (gcrypt hash) + #:use-module (guix memoization) #:use-module (guix tests) #:use-module (guix build-system python) #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) @@ -79,17 +80,33 @@ bar != 2 pytest (>=2.5.0) ") -(define test-metadata - "{ - \"run_requires\": [ - { - \"requires\": [ - \"bar\", - \"baz (>13.37)\" - ] - } - ] -}") +(define test-metadata "\ +Classifier: Programming Language :: Python :: 3.7 +Requires-Dist: baz ~= 3 +Requires-Dist: bar != 2 +Provides-Extra: test +pytest (>=2.5.0) +") + +(define test-metadata-with-extras " +Classifier: Programming Language :: Python :: 3.7 +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Dist: wrapt (<2,>=1) +Requires-Dist: bar + +Provides-Extra: dev +Requires-Dist: tox ; extra == 'dev' +Requires-Dist: bumpversion (<1) ; extra == 'dev' +") + +;;; Provides-Extra can appear before Requires-Dist. +(define test-metadata-with-extras-jedi "\ +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Provides-Extra: testing +Requires-Dist: parso (>=0.3.0) +Provides-Extra: testing +Requires-Dist: pytest (>=3.1.0); extra == 'testing' +") (test-begin "pypi") @@ -128,6 +145,18 @@ pytest (>=2.5.0) call-with-input-string) (parse-requires.txt test-requires-with-sections))) +(test-equal "parse-wheel-metadata, with extras" + '("wrapt" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras))) + +(test-equal "parse-wheel-metadata, with extras - Jedi" + '("parso") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras-jedi))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch @@ -191,7 +220,7 @@ pytest (>=2.5.0) (mkdir-p "foo-1.0.0/foo.egg-info/") (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requires.txt))) + (display "wrong data to make sure we're testing wheels "))) (parameterize ((current-output-port (%make-void-port "rw+"))) (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") @@ -200,13 +229,13 @@ pytest (>=2.5.0) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" (begin (mkdir "foo-1.0.0.dist-info") - (with-output-to-file "foo-1.0.0.dist-info/metadata.json" + (with-output-to-file "foo-1.0.0.dist-info/METADATA" (lambda () (display test-metadata))) (let ((zip-file (string-append file-name ".zip"))) ;; zip always adds a "zip" extension to the file it creates, ;; so we need to rename it. - (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json") + (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA") (rename-file zip-file file-name)) (delete-file-recursively "foo-1.0.0.dist-info"))) (_ (error "Unexpected URL: " url))))) @@ -218,6 +247,9 @@ pytest (>=2.5.0) (string-length test-json))) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) (_ (error "Unexpected URL: " url))))) + ;; Not clearing the memoization cache here would mean returning the value + ;; computed in the previous test. + (invalidate-memoization! pypi->guix-package) (match (pypi->guix-package "foo") (('package ('name "python-foo") -- cgit v1.2.3 From d514276b93ccf168438bbd2892fbc64d5661106c Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 28 Mar 2019 23:12:26 -0400 Subject: import: pypi: Include optional test inputs as native-inputs. * guix/import/pypi.scm (maybe-inputs): Add INPUT-TYPE argument, and use it. (test-section?): New predicate. (parse-requires.txt): Collect the optional test inputs, and return them as the second element of the returned list. (parse-wheel-metadata): Likewise. (guess-requirements): Adapt. (make-pypi-sexp): Likewise, and include the test inputs requirements as native inputs in the returned package expression. * tests/pypi.scm (test-requires.txt): Include a test section in the test-requires.txt data. (test-requires.txt-beaker): New variable. ("parse-requires.txt"): Adapt. ("parse-requires.txt - Beaker"): New test. ("parse-wheel-metadata, with extras"): Adapt. ("parse-wheel-metadata, with extras - Jedi"): Adapt. ("pypi->guix-package, no wheel"): Re-indent, and add the expected native-inputs. ("pypi->guix-package, wheels"): Likewise. --- guix/import/pypi.scm | 177 +++++++++++++++++++++++++++++++++------------------ tests/pypi.scm | 79 ++++++++++++++--------- 2 files changed, 166 insertions(+), 90 deletions(-) (limited to 'tests') diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 23a1e69061..537431dd69 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2015, 2016, 2017 Ludovic Courtès ;;; Copyright © 2017 Mathieu Othacehe ;;; Copyright © 2018 Ricardo Wurmus +;;; Copyright © 2019 Maxim Cournoyer ;;; ;;; This file is part of GNU Guix. ;;; @@ -26,6 +27,7 @@ #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) @@ -107,14 +109,15 @@ package on PyPI." ((name version _ ...) (string-append name "-" version ".dist-info")))) -(define (maybe-inputs package-inputs) +(define (maybe-inputs package-inputs input-type) "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a -package definition." +package definition. INPUT-TYPE, a symbol, is used to populate the name of +the input field." (match package-inputs (() '()) ((package-inputs ...) - `((propagated-inputs (,'quasiquote ,package-inputs)))))) + `((,input-type (,'quasiquote ,package-inputs)))))) (define %requirement-name-regexp ;; Regexp to match the requirement name in a requirement specification. @@ -154,9 +157,19 @@ package definition." (or (regexp-exec %requirement-name-regexp spec) (error (G_ "Could not extract requirement name in spec:") spec)))) +(define (test-section? name) + "Return #t if the section name contains 'test' or 'dev'." + (any (cut string-contains-ci name <>) + '("test" "dev"))) + (define (parse-requires.txt requires.txt) - "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of -requirement names." + "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of lists +of requirements. + +The first list contains the required dependencies while the second the +optional test dependencies. Note that currently, optional, non-test +dependencies are omitted since these can be difficult or expensive to +satisfy." (define (comment? line) ;; Return #t if the given LINE is a comment, #f otherwise. @@ -168,26 +181,49 @@ requirement names." (call-with-input-file requires.txt (lambda (port) - (let loop ((result '())) + (let loop ((required-deps '()) + (test-deps '()) + (inside-test-section? #f) + (optional? #f)) (let ((line (read-line port))) - ;; Stop when a section is encountered, as sections contain optional - ;; (extra) requirements. Non-optional requirements must appear - ;; before any section is defined. (cond - ((or (eof-object? line) (section-header? line)) + ((eof-object? line) ;; Duplicates can occur, since the same requirement can be ;; listed multiple times with different conditional markers, e.g. ;; pytest >= 3 ; python_version >= "3.3" ;; pytest < 3 ; python_version < "3.3" - (reverse (delete-duplicates result))) + (map (compose reverse delete-duplicates) + (list required-deps test-deps))) ((or (string-null? line) (comment? line)) - (loop result)) - (else + (loop required-deps test-deps inside-test-section? optional?)) + ((section-header? line) + ;; Encountering a section means that all the requirements + ;; listed below are optional. Since we want to pick only the + ;; test dependencies from the optional dependencies, we must + ;; track those separately. + (loop required-deps test-deps (test-section? line) #t)) + (inside-test-section? + (loop required-deps + (cons (specification->requirement-name line) + test-deps) + inside-test-section? optional?)) + ((not optional?) (loop (cons (specification->requirement-name line) - result))))))))) + required-deps) + test-deps inside-test-section? optional?)) + (optional? + ;; Skip optional items. + (loop required-deps test-deps inside-test-section? optional?)) + (else + (warning (G_ "parse-requires.txt reached an unexpected \ +condition on line ~a~%") line)))))))) (define (parse-wheel-metadata metadata) - "Given METADATA, a Wheel metadata file, return a list of requirement names." + "Given METADATA, a Wheel metadata file, return a list of lists of +requirements. + +Refer to the documentation of PARSE-REQUIRES.TXT for a description of the +returned value." ;; METADATA is a RFC-2822-like, header based file. (define (requires-dist-header? line) @@ -201,21 +237,29 @@ requirement names." ;; Return #t if the given LINE is an "extra" requirement. (string-match "extra == '(.*)'" line)) + (define (test-requirement? line) + (and=> (match:substring (extra? line) 1) test-section?)) + (call-with-input-file metadata (lambda (port) - (let loop ((requirements '())) + (let loop ((required-deps '()) + (test-deps '())) (let ((line (read-line port))) - ;; Stop at the first 'Provides-Extra' section: the non-optional - ;; requirements appear before the optional ones. (cond ((eof-object? line) - (reverse (delete-duplicates requirements))) + (map (compose reverse delete-duplicates) + (list required-deps test-deps))) ((and (requires-dist-header? line) (not (extra? line))) (loop (cons (specification->requirement-name (requires-dist-value line)) - requirements))) + required-deps) + test-deps)) + ((and (requires-dist-header? line) (test-requirement? line)) + (loop required-deps + (cons (specification->requirement-name (requires-dist-value line)) + test-deps))) (else - (loop requirements)))))))) + (loop required-deps test-deps)))))))) ;skip line (define (guess-requirements source-url wheel-url archive) "Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list @@ -268,37 +312,46 @@ be extracted in a temporary directory." (() (warning (G_ "Cannot guess requirements from source archive:\ no requires.txt file found.~%")) - '()) + (list '() '())) (else (parse-requires.txt (first requires.txt-files))))))) (begin (warning (G_ "Unsupported archive format; \ cannot determine package dependencies from source archive: ~a~%") (basename source-url)) - '()))) + (list '() '())))) ;; First, try to compute the requirements using the wheel, else, fallback to ;; reading the "requires.txt" from the egg-info directory from the source - ;; tarball. + ;; archive. (or (guess-requirements-from-wheel) (guess-requirements-from-source))) (define (compute-inputs source-url wheel-url archive) - "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of -name/variable pairs describing the required inputs of this package. Also + "Given the SOURCE-URL and WHEEL-URL of an already downloaded ARCHIVE, return +a pair of lists, each consisting of a list of name/variable pairs, for the +propagated inputs and the native inputs, respectively. Also return the unaltered list of upstream dependency names." - (let ((dependencies - (remove (cut string=? "argparse" <>) - (guess-requirements source-url wheel-url archive)))) - (values (sort - (map (lambda (input) - (let ((guix-name (python->package-name input))) - (list guix-name (list 'unquote (string->symbol guix-name))))) - dependencies) - (lambda args - (match args - (((a _ ...) (b _ ...)) - (string-ci) deps)) + + (define (requirement->package-name/sort deps) + (sort + (map (lambda (input) + (let ((guix-name (python->package-name input))) + (list guix-name (list 'unquote (string->symbol guix-name))))) + deps) + (lambda args + (match args + (((a _ ...) (b _ ...)) + (string-cipackage-name/sort strip-argparse)) + + (let ((dependencies (guess-requirements source-url wheel-url archive))) + (values (map process-requirements dependencies) + (concatenate dependencies)))) (define (make-pypi-sexp name version source-url wheel-url home-page synopsis description license) @@ -307,29 +360,31 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." (call-with-temporary-output-file (lambda (temp port) (and (url-fetch source-url temp) - (receive (input-package-names upstream-dependency-names) + (receive (guix-dependencies upstream-dependencies) (compute-inputs source-url wheel-url temp) - (values - `(package - (name ,(python->package-name name)) - (version ,version) - (source (origin - (method url-fetch) - - ;; Sometimes 'pypi-uri' doesn't quite work due to mixed - ;; cases in NAME, for instance, as is the case with - ;; "uwsgi". In that case, fall back to a full URL. - (uri (pypi-uri ,(string-downcase name) version)) - (sha256 - (base32 - ,(guix-hash-url temp))))) - (build-system python-build-system) - ,@(maybe-inputs input-package-names) - (home-page ,home-page) - (synopsis ,synopsis) - (description ,description) - (license ,(license->symbol license))) - upstream-dependency-names)))))) + (match guix-dependencies + ((required-inputs test-inputs) + (values + `(package + (name ,(python->package-name name)) + (version ,version) + (source (origin + (method url-fetch) + ;; Sometimes 'pypi-uri' doesn't quite work due to mixed + ;; cases in NAME, for instance, as is the case with + ;; "uwsgi". In that case, fall back to a full URL. + (uri (pypi-uri ,(string-downcase name) version)) + (sha256 + (base32 + ,(guix-hash-url temp))))) + (build-system python-build-system) + ,@(maybe-inputs required-inputs 'propagated-inputs) + ,@(maybe-inputs test-inputs 'native-inputs) + (home-page ,home-page) + (synopsis ,synopsis) + (description ,description) + (license ,(license->symbol license))) + upstream-dependencies)))))))) (define pypi->guix-package (memoize diff --git a/tests/pypi.scm b/tests/pypi.scm index 8b42c2f071..43d45f1dd8 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson ;;; Copyright © 2016 Ricardo Wurmus +;;; Copyright © 2019 Maxim Cournoyer ;;; ;;; This file is part of GNU Guix. ;;; @@ -68,11 +69,6 @@ sha1=da9234ee9982d4bbb3c72346a6de940a148ea686")) (define test-requires.txt "\ # A comment # A comment after a space -bar -baz > 13.37 -") - -(define test-requires-with-sections "\ foo ~= 3 bar != 2 @@ -80,12 +76,25 @@ bar != 2 pytest (>=2.5.0) ") +;; Beaker contains only optional dependencies. +(define test-requires.txt-beaker "\ +[crypto] +pycryptopp>=0.5.12 + +[cryptography] +cryptography + +[testsuite] +Mock +coverage +") + (define test-metadata "\ Classifier: Programming Language :: Python :: 3.7 Requires-Dist: baz ~= 3 Requires-Dist: bar != 2 Provides-Extra: test -pytest (>=2.5.0) +Requires-Dist: pytest (>=2.5.0) ; extra == 'test' ") (define test-metadata-with-extras " @@ -139,25 +148,31 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip") (map specification->requirement-name test-specifications)) -(test-equal "parse-requires.txt, with sections" - '("foo" "bar") +(test-equal "parse-requires.txt" + (list '("foo" "bar") '("pytest")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) - (parse-requires.txt test-requires-with-sections))) + (parse-requires.txt test-requires.txt))) + +(test-equal "parse-requires.txt - Beaker" + (list '() '("Mock" "coverage")) + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-requires.txt test-requires.txt-beaker))) (test-equal "parse-wheel-metadata, with extras" - '("wrapt" "bar") + (list '("wrapt" "bar") '("tox" "bumpversion")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) (parse-wheel-metadata test-metadata-with-extras))) (test-equal "parse-wheel-metadata, with extras - Jedi" - '("parso") + (list '("parso") '("pytest")) (mock ((ice-9 ports) call-with-input-file call-with-input-string) (parse-wheel-metadata test-metadata-with-extras-jedi))) -(test-assert "pypi->guix-package" +(test-assert "pypi->guix-package, no wheel" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch (lambda (url file-name) @@ -198,7 +213,10 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' ('propagated-inputs ('quasiquote (("python-bar" ('unquote 'python-bar)) - ("python-baz" ('unquote 'python-baz))))) + ("python-foo" ('unquote 'python-foo))))) + ('native-inputs + ('quasiquote + (("python-pytest" ('unquote 'python-pytest))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") @@ -219,25 +237,25 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' (begin (mkdir-p "foo-1.0.0/foo.egg-info/") (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" - (lambda () - (display "wrong data to make sure we're testing wheels "))) + (lambda () + (display "wrong data to make sure we're testing wheels "))) (parameterize ((current-output-port (%make-void-port "rw+"))) (system* "tar" "czvf" file-name "foo-1.0.0/")) - (delete-file-recursively "foo-1.0.0") - (set! test-source-hash - (call-with-input-file file-name port-sha256)))) + (delete-file-recursively "foo-1.0.0") + (set! test-source-hash + (call-with-input-file file-name port-sha256)))) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" - (begin - (mkdir "foo-1.0.0.dist-info") - (with-output-to-file "foo-1.0.0.dist-info/METADATA" - (lambda () - (display test-metadata))) - (let ((zip-file (string-append file-name ".zip"))) - ;; zip always adds a "zip" extension to the file it creates, - ;; so we need to rename it. - (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA") - (rename-file zip-file file-name)) - (delete-file-recursively "foo-1.0.0.dist-info"))) + (begin + (mkdir "foo-1.0.0.dist-info") + (with-output-to-file "foo-1.0.0.dist-info/METADATA" + (lambda () + (display test-metadata))) + (let ((zip-file (string-append file-name ".zip"))) + ;; zip always adds a "zip" extension to the file it creates, + ;; so we need to rename it. + (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA") + (rename-file zip-file file-name)) + (delete-file-recursively "foo-1.0.0.dist-info"))) (_ (error "Unexpected URL: " url))))) (mock ((guix http-client) http-fetch (lambda (url . rest) @@ -265,6 +283,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing' ('quasiquote (("python-bar" ('unquote 'python-bar)) ("python-baz" ('unquote 'python-baz))))) + ('native-inputs + ('quasiquote + (("python-pytest" ('unquote 'python-pytest))))) ('home-page "http://example.com") ('synopsis "summary") ('description "summary") -- cgit v1.2.3