07/12: import: pypi: Parse wheel METADATA instead of metadata.json.

[guix-commits]

apteryx pushed a commit to branch master
in repository guix.

commit f0190a5dcdc1882e5a058dc7f12bc7ccb166c63d
Author: Maxim Cournoyer <address@hidden>
Date:   Thu Mar 28 00:26:03 2019 -0400

    import: pypi: Parse wheel METADATA instead of metadata.json.
    
    With newer Wheel releases, there is no more metadata.json file; the METADATA
    file should be used instead (see: https://github.com/pypa/wheel/issues/195).
    
    This change updates our PyPI importer so that it uses the latter.
    
    * guix/import/pypi.scm (define-module): Remove unnecessary modules and 
export
    the PARSE-WHEEL-METADATA procedure.
    (parse-wheel-metadata): Add procedure.
    (guess-requirements): Use it.
    * tests/pypi.scm (test-metadata): Test it.
---
 guix/import/pypi.scm | 90 ++++++++++++++++++++++++++++++++--------------------
 tests/pypi.scm       | 60 +++++++++++++++++++++++++++--------
 2 files changed, 101 insertions(+), 49 deletions(-)

diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index a6106ab..7cf1e92 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,9 +21,7 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (guix import pypi)
-  #:use-module (ice-9 binary-ports)
   #:use-module (ice-9 match)
-  #:use-module (ice-9 pretty-print)
   #:use-module (ice-9 regex)
   #:use-module (ice-9 receive)
   #:use-module ((ice-9 rdelim) #:select (read-line))
@@ -31,9 +29,6 @@
   #:use-module (srfi srfi-26)
   #:use-module (srfi srfi-34)
   #:use-module (srfi srfi-35)
-  #:use-module (rnrs bytevectors)
-  #:use-module (json)
-  #:use-module (web uri)
   #:use-module (guix ui)
   #:use-module (guix utils)
   #:use-module ((guix build utils)
@@ -49,6 +44,7 @@
   #:use-module ((guix licenses) #:prefix license:)
   #:use-module (guix build-system python)
   #:export (parse-requires.txt
+            parse-wheel-metadata
             specification->requirement-name
             guix-package->pypi-name
             pypi-recursive-import
@@ -177,18 +173,49 @@ requirement names."
           ;; Stop when a section is encountered, as sections contain optional
           ;; (extra) requirements.  Non-optional requirements must appear
           ;; before any section is defined.
-          (if (or (eof-object? line) (section-header? line))
-              ;; Duplicates can occur, since the same requirement can be
-              ;; listed multiple times with different conditional markers, e.g.
-              ;; pytest >= 3 ; python_version >= "3.3"
-              ;; pytest < 3 ; python_version < "3.3"
-              (reverse (delete-duplicates result))
-              (cond
-               ((or (string-null? line) (comment? line))
-                (loop result))
-               (else
-                (loop (cons (specification->requirement-name line)
-                            result))))))))))
+          (cond
+           ((or (eof-object? line) (section-header? line))
+            ;; Duplicates can occur, since the same requirement can be
+            ;; listed multiple times with different conditional markers, e.g.
+            ;; pytest >= 3 ; python_version >= "3.3"
+            ;; pytest < 3 ; python_version < "3.3"
+            (reverse (delete-duplicates result)))
+           ((or (string-null? line) (comment? line))
+            (loop result))
+           (else
+            (loop (cons (specification->requirement-name line)
+                        result)))))))))
+
+(define (parse-wheel-metadata metadata)
+  "Given METADATA, a Wheel metadata file, return a list of requirement names."
+  ;; METADATA is a RFC-2822-like, header based file.
+
+  (define (requires-dist-header? line)
+    ;; Return #t if the given LINE is a Requires-Dist header.
+    (string-match "^Requires-Dist: " line))
+
+  (define (requires-dist-value line)
+    (string-drop line (string-length "Requires-Dist: ")))
+
+  (define (extra? line)
+    ;; Return #t if the given LINE is an "extra" requirement.
+    (string-match "extra == '(.*)'" line))
+
+  (call-with-input-file metadata
+    (lambda (port)
+      (let loop ((requirements '()))
+        (let ((line (read-line port)))
+          ;; Stop at the first 'Provides-Extra' section: the non-optional
+          ;; requirements appear before the optional ones.
+          (cond
+           ((eof-object? line)
+            (reverse (delete-duplicates requirements)))
+           ((and (requires-dist-header? line) (not (extra? line)))
+            (loop (cons (specification->requirement-name
+                         (requires-dist-value line))
+                        requirements)))
+           (else
+            (loop requirements))))))))
 
 (define (guess-requirements source-url wheel-url archive)
   "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
@@ -197,25 +224,18 @@ be extracted in a temporary directory."
 
   (define (read-wheel-metadata wheel-archive)
     ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
-    ;; requirements.
+    ;; requirements, or #f if the metadata file contained therein couldn't be
+    ;; extracted.
     (let* ((dirname (wheel-url->extracted-directory wheel-url))
-           (json-file (string-append dirname "/metadata.json")))
-      (and (zero? (system* "unzip" "-q" wheel-archive json-file))
-           (dynamic-wind
-             (const #t)
-             (lambda ()
-               (call-with-input-file json-file
-                 (lambda (port)
-                   (let* ((metadata (json->scm port))
-                          (run_requires (hash-ref metadata "run_requires"))
-                          (requirements (if run_requires
-                                            (hash-ref (list-ref run_requires 0)
-                                                       "requires")
-                                            '())))
-                     (map specification->requirement-name requirements)))))
-             (lambda ()
-               (delete-file json-file)
-               (rmdir dirname))))))
+           (metadata (string-append dirname "/METADATA")))
+      (call-with-temporary-directory
+       (lambda (dir)
+         (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata))
+             (parse-wheel-metadata (string-append dir "/" metadata))
+             (begin
+               (warning
+                (G_ "Failed to extract file: ~a from wheel.~%") metadata)
+               #f))))))
 
   (define (guess-requirements-from-wheel)
     ;; Return the package's requirements using the wheel, or #f if an error
diff --git a/tests/pypi.scm b/tests/pypi.scm
index b45d2c9..8b42c2f 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -22,6 +22,7 @@
   #:use-module (guix base32)
   #:use-module (guix memoization)
   #:use-module (gcrypt hash)
+  #:use-module (guix memoization)
   #:use-module (guix tests)
   #:use-module (guix build-system python)
   #:use-module ((guix build utils) #:select (delete-file-recursively which 
mkdir-p))
@@ -79,17 +80,33 @@ bar != 2
 pytest (>=2.5.0)
 ")
 
-(define test-metadata
-  "{
-  \"run_requires\": [
-    {
-      \"requires\": [
-        \"bar\",
-        \"baz (>13.37)\"
-      ]
-    }
-  ]
-}")
+(define test-metadata "\
+Classifier: Programming Language :: Python :: 3.7
+Requires-Dist: baz ~= 3
+Requires-Dist: bar != 2
+Provides-Extra: test
+pytest (>=2.5.0)
+")
+
+(define test-metadata-with-extras "
+Classifier: Programming Language :: Python :: 3.7
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: wrapt (<2,>=1)
+Requires-Dist: bar
+
+Provides-Extra: dev
+Requires-Dist: tox ; extra == 'dev'
+Requires-Dist: bumpversion (<1) ; extra == 'dev'
+")
+
+;;; Provides-Extra can appear before Requires-Dist.
+(define test-metadata-with-extras-jedi "\
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Provides-Extra: testing
+Requires-Dist: parso (>=0.3.0)
+Provides-Extra: testing
+Requires-Dist: pytest (>=3.1.0); extra == 'testing'
+")
 
 (test-begin "pypi")
 
@@ -128,6 +145,18 @@ pytest (>=2.5.0)
          call-with-input-string)
         (parse-requires.txt test-requires-with-sections)))
 
+(test-equal "parse-wheel-metadata, with extras"
+  '("wrapt" "bar")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras)))
+
+(test-equal "parse-wheel-metadata, with extras - Jedi"
+  '("parso")
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras-jedi)))
+
 (test-assert "pypi->guix-package"
   ;; Replace network resources with sample data.
     (mock ((guix import utils) url-fetch
@@ -191,7 +220,7 @@ pytest (>=2.5.0)
                 (mkdir-p "foo-1.0.0/foo.egg-info/")
                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
                    (lambda ()
-                     (display test-requires.txt)))
+                     (display "wrong data to make sure we're testing wheels 
")))
                 (parameterize ((current-output-port (%make-void-port "rw+")))
                   (system* "tar" "czvf" file-name "foo-1.0.0/"))
                  (delete-file-recursively "foo-1.0.0")
@@ -200,13 +229,13 @@ pytest (>=2.5.0)
              ("https://example.com/foo-1.0.0-py2.py3-none-any.whl";
                (begin
                  (mkdir "foo-1.0.0.dist-info")
-                 (with-output-to-file "foo-1.0.0.dist-info/metadata.json"
+                 (with-output-to-file "foo-1.0.0.dist-info/METADATA"
                    (lambda ()
                      (display test-metadata)))
                  (let ((zip-file (string-append file-name ".zip")))
                    ;; zip always adds a "zip" extension to the file it creates,
                    ;; so we need to rename it.
-                   (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
+                   (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
                    (rename-file zip-file file-name))
                  (delete-file-recursively "foo-1.0.0.dist-info")))
              (_ (error "Unexpected URL: " url)))))
@@ -218,6 +247,9 @@ pytest (>=2.5.0)
                             (string-length test-json)))
                    ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"; #f)
                    (_ (error "Unexpected URL: " url)))))
+              ;; Not clearing the memoization cache here would mean returning 
the value
+              ;; computed in the previous test.
+              (invalidate-memoization! pypi->guix-package)
               (match (pypi->guix-package "foo")
                 (('package
                    ('name "python-foo")