155/242: gnu: python-numpy: Update to 1.21.6 and parallelize build/tests.

[guix-commits]

apteryx pushed a commit to branch wip-ipython-polyglossia
in repository guix.

commit 9f11570a9518acb5bc9c65a93312d959c9b02baf
Author: Maxim Cournoyer <maxim.cournoyer@gmail.com>
AuthorDate: Mon Apr 18 14:28:31 2022 -0400

    gnu: python-numpy: Update to 1.21.6 and parallelize build/tests.
    
    * gnu/packages/python-xyz.scm (python-numpy): Update to 1.21.6.  Move inputs
    and native-inputs fields after arguments.  Use gexps.
    [modules]: New argument.
    [phases]{parallelize-build}: New phase.
    {configure-blas}: Format via (ice-9 format).
    {fix-executable-paths}: Use search-file-input.  Remove obsolete 
system_info.py
    substitution.
    {check}: Remove extraneous add-installed-pythonpath and setenv calls.  
Provide
    the '-n' pytest-xdist option.
    [native-inputs]: Use python-hypothesis-next.
    (python-numpy-next): Inherit from python-numpy.
    (python2-numpy): Adjust for gexps.
    [phases]{delete-failing-test}: Rename to...
    {delete-failing-tests}: ... this.
    (python2-numpy-1.8): Likewise, and delete 'delete-failing-tests' phase.
    Disable tests.
    [native-inputs]: Define as the empty list.
---
 gnu/packages/python-xyz.scm | 194 +++++++++++++++++++++-----------------------
 1 file changed, 93 insertions(+), 101 deletions(-)

diff --git a/gnu/packages/python-xyz.scm b/gnu/packages/python-xyz.scm
index b53ceb2522..7d3c220c7d 100644
--- a/gnu/packages/python-xyz.scm
+++ b/gnu/packages/python-xyz.scm
@@ -5693,10 +5693,13 @@ writing C extensions for Python as easy as Python 
itself.")
                 "17fqacrpis05w1rpi7d7sbimrk20xf8h6d3vrz5nf6ix3899abil"))))
     (properties '())))
 
-(define-public python-numpy-next
+;; NOTE: when upgrading numpy please make sure that python-numba,
+;; python-pandas and python-scipy still build, as these three packages are
+;; often used together.
+(define-public python-numpy
   (package
-    (name "python-numpy-next")
-    (version "1.22.3")
+    (name "python-numpy")
+    (version "1.21.6")
     (source
      (origin
        (method url-fetch)
@@ -5705,50 +5708,57 @@ writing C extensions for Python as easy as Python 
itself.")
              version "/numpy-" version ".tar.gz"))
        (sha256
         (base32
-         "19dw91pqbqcniw2z57kiyqs1qp56g7kqy1bdyv664g8s62sc01m9"))))
+         "0b0c5y35rd3mvwfk5is1d5ppfw9nl4d2rgx9xkwh1p0w394wdvyl"))))
     (build-system python-build-system)
-    (inputs
-     (list openblas))
-    (native-inputs
-     (list python-cython python-hypothesis-6.23 python-pytest
-           python-pytest-xdist gfortran))
     (arguments
-     `(#:phases
-       (modify-phases %standard-phases
-         (add-before 'build 'configure-blas
-           (lambda* (#:key inputs #:allow-other-keys)
-             (call-with-output-file "site.cfg"
-               (lambda (port)
-                 (format port
-                         "\
+     (list
+      #:modules '((guix build utils)
+                  (guix build python-build-system)
+                  (ice-9 format))
+      #:phases
+      #~(modify-phases %standard-phases
+          (add-before 'build 'parallelize-build
+            (lambda _
+              (setenv "NPY_NUM_BUILD_JOBS"
+                      (number->string (parallel-job-count)))))
+          (add-before 'build 'configure-blas
+            (lambda* (#:key inputs #:allow-other-keys)
+              (call-with-output-file "site.cfg"
+                (lambda (port)
+                  (format port
+                          "\
 [openblas]
 libraries = openblas
 library_dirs = ~a/lib
-include_dirs = ~a/include
-"
-                         (assoc-ref inputs "openblas")
-                         (assoc-ref inputs "openblas"))))))
-         (add-before 'build 'fix-executable-paths
-           (lambda* (#:key inputs #:allow-other-keys)
-             ;; Make /gnu/store/...-bash-.../bin/sh the default shell,
-             ;; instead of /bin/sh.
-             (substitute* "numpy/distutils/exec_command.py"
-               (("(os.environ.get\\('SHELL', ')(/bin/sh'\\))" match 
match-start match-end)
-                (string-append match-start (assoc-ref inputs "bash") 
match-end)))
-             ;; Use "gcc" executable, not "cc".
-             (substitute* "numpy/distutils/system_info.py"
-               (("c = distutils\\.ccompiler\\.new_compiler\\(\\)")
-                "c = distutils.ccompiler.new_compiler(); 
c.set_executables(compiler='gcc',compiler_so='gcc',linker_exe='gcc',linker_so='gcc
 -shared')"))))
-         (replace 'check
-           (lambda* (#:key tests? outputs inputs #:allow-other-keys)
-             (when tests?
-               ;; Make installed package available for running the tests.
-               (add-installed-pythonpath inputs outputs)
-               ;; Make sure "f2py" etc is found.
-               (setenv "PATH" (string-append (assoc-ref outputs "out") "/bin"
-                                             ":" (getenv "PATH")))
-               (invoke "./runtests.py"
-                       "-j" (number->string (parallel-job-count)))))))))
+include_dirs = ~:*~a/include~%" #$(this-package-input "openblas"))))))
+          (add-before 'build 'fix-executable-paths
+            (lambda* (#:key inputs #:allow-other-keys)
+              ;; Make /gnu/store/...-bash-.../bin/sh the default shell,
+              ;; instead of /bin/sh.
+              (substitute* "numpy/distutils/exec_command.py"
+                (("'/bin/sh'")
+                 (format #f "~s" (search-input-file inputs "bin/bash"))))))
+          (replace 'check
+            (lambda* (#:key tests? outputs inputs #:allow-other-keys)
+              (when tests?
+                (invoke "./runtests.py" "-vv" "--no-build" "--mode=fast"
+                        "-j" (number->string (parallel-job-count))
+                        ;; Contrary to scipy, the runtests.py script of numpy
+                        ;; does *not* automatically provide -n when -j is used
+                        ;; (see: https://github.com/numpy/numpy/issues/21359).
+                        "--" "-n" (number->string (parallel-job-count))
+                        "-k" (string-append
+                              ;; These tests may fail on 32-bit systems (see:
+                              ;; https://github.com/numpy/numpy/issues/18387).
+                              "not test_float_remainder_overflow "
+                              "and not test_pareto"))))))))
+    (native-inputs
+     (list python-cython
+           python-hypothesis-next
+           python-pytest
+           python-pytest-xdist
+           gfortran))
+    (inputs (list bash openblas))
     (home-page "https://numpy.org";)
     (synopsis "Fundamental package for scientific computing with Python")
     (description "NumPy is the fundamental package for scientific computing
@@ -5759,10 +5769,26 @@ capabilities.")
     (properties `((python2-variant . ,(delay python2-numpy))))
     (license license:bsd-3)))
 
+(define-public python-numpy-next
+  (package
+    (inherit python-numpy)
+    (name "python-numpy-next")
+    (version "1.22.3")
+    (source
+     (origin
+       (inherit (package-source python-numpy))
+       (method url-fetch)
+       (uri (string-append
+             "https://github.com/numpy/numpy/releases/download/v";
+             version "/numpy-" version ".tar.gz"))
+       (sha256
+        (base32
+         "19dw91pqbqcniw2z57kiyqs1qp56g7kqy1bdyv664g8s62sc01m9"))))))
+
 ;; Numpy 1.16.x are the last versions that support Python 2.
 (define-public python2-numpy
   (let ((numpy (package-with-python2
-                (strip-python2-variant python-numpy-next))))
+                (strip-python2-variant python-numpy))))
     (package
       (inherit numpy)
       (name "python-numpy")
@@ -5778,80 +5804,46 @@ capabilities.")
       (arguments
        (substitute-keyword-arguments (package-arguments numpy)
          ((#:phases phases)
-          `(modify-phases ,phases
-             (add-after 'unpack 'delete-failing-test
-               (lambda _
-                 ;; There's just one failing test here.
-                 (delete-file "numpy/linalg/tests/test_linalg.py")
-                 ;; ...and this one depends on the previous one.
-                 (delete-file 
"numpy/matrixlib/tests/test_matrix_linalg.py")))))))
+          #~(modify-phases #$phases
+              (add-after 'unpack 'delete-failing-tests
+                (lambda _
+                  ;; There's just one failing test here.
+                  (delete-file "numpy/linalg/tests/test_linalg.py")
+                  ;; ...and this one depends on the previous one.
+                  (delete-file "numpy/matrixlib/tests/test_matrix_linalg.py")))
+              (replace 'check
+                ;; Older versions don't cope well with the extra Pytest
+                ;; options, so remove them.
+                (lambda* (#:key tests? outputs inputs #:allow-other-keys)
+                  (when tests?
+                    (invoke "./runtests.py" "-vv" "--no-build" "--mode=fast"
+                            "-j" (number->string (parallel-job-count))))))))))
       (native-inputs
        (list python2-cython python2-pytest gfortran)))))
 
-;; Needed by python-numba, see https://github.com/numba/numba/issues/7176
-(define-public python-numpy-1.20
-  (package
-    (inherit python-numpy-next)
-    (name "python-numpy")
-    (version "1.20.3")
-    (source (origin
-              (method url-fetch)
-              (uri (string-append
-                    "https://github.com/numpy/numpy/releases/download/v";
-                    version "/numpy-" version ".tar.gz"))
-              (sha256
-               (base32
-                "140zq9snx0di4id4g97vaw9zz8x2rfla5lp3a70j666f5030yd5p"))))
-    ;; python-numpy-next replaced python-hypothesis with
-    ;; python-hypothesis-6.23. We switch it back here, to prevent
-    ;; python-numpy-1.20 and its numerous dependents from being rebuilt.
-    (native-inputs
-     (list python-cython python-hypothesis python-pytest
-           python-pytest-xdist gfortran))
-    ;; 92 tests fail, many of them because parts of the temp file name
-    ;; accidentally ends up in a comparison.
-    (arguments
-     (substitute-keyword-arguments (package-arguments python-numpy-next)
-       ((#:tests? _ #t) #f)))))
-
-;; NOTE: when upgrading numpy please make sure that python-pandas and
-;; python-scipy still build, as these three packages are often used together.
-(define-public python-numpy python-numpy-1.20)
-
 ;; NOTE: NumPy 1.8 is packaged only for Python 2 because it is of
 ;; interest only for legacy code going back to NumPy's predecessor
 ;; Numeric.
 (define-public python2-numpy-1.8
-  (package (inherit python2-numpy)
+  (package
+    (inherit python2-numpy)
     (version "1.8.2")
     (source
      (origin
        (method git-fetch)
        (uri (git-reference
-              (url "https://github.com/numpy/numpy";)
-              (commit (string-append "v" version))))
+             (url "https://github.com/numpy/numpy";)
+             (commit (string-append "v" version))))
        (file-name (git-file-name "numpy" version))
        (sha256
         (base32
          "0ikgi15rsqwbkfsjjxrwh40lqyal2wvyp3923y6w6ch3dcr82sfk"))))
-    (arguments
-     (substitute-keyword-arguments (package-arguments python2-numpy)
-       ((#:phases phases)
-        `(modify-phases ,phases
-           (replace 'configure-blas
-             (lambda* (#:key inputs #:allow-other-keys)
-               (call-with-output-file "site.cfg"
-                 (lambda (port)
-                   (format port
-                           "[openblas]
-libraries = openblas
-library_dirs = ~a/lib
-include_dirs = ~a/include
-"
-                           (assoc-ref inputs "openblas")
-                           (assoc-ref inputs "openblas"))))))))))
-    (native-inputs
-     (list python2-nose))
+    (arguments (substitute-keyword-arguments (package-arguments python2-numpy)
+                 ((#:tests? _ #f) #f)   ;disable tests
+                 ((#:phases phases)
+                  #~(modify-phases #$phases
+                      (delete 'delete-failing-tests)))))
+    (native-inputs '())
     (description "NumPy is the fundamental package for scientific computing
 with Python.  It contains among other things: a powerful N-dimensional array
 object, sophisticated (broadcasting) functions, tools for integrating C/C++