From 66e5568885199a734fda7946a6cbe32a09218dee Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Fri, 10 Jul 2020 14:12:36 +0200 Subject: [PATCH] WIP: Proper pypy3 support --- gnu/packages/check.scm | 62 ++++++- .../patches/pypy3-7.3.1-install-paths.patch | 19 ++ .../patches/pypy3-7.3.1-ssl-paths.patch | 41 +++++ gnu/packages/python-science.scm | 3 + gnu/packages/python-xyz.scm | 168 +++++++++++++++++- gnu/packages/python.scm | 103 +++++++---- gnu/packages/xml.scm | 20 ++- guix/build-system/python.scm | 24 ++- guix/build/python-build-system.scm | 18 +- 9 files changed, 403 insertions(+), 55 deletions(-) create mode 100644 gnu/packages/patches/pypy3-7.3.1-install-paths.patch create mode 100644 gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch diff --git a/gnu/packages/check.scm b/gnu/packages/check.scm index 3b946d7482..29ad6e144a 100644 --- a/gnu/packages/check.scm +++ b/gnu/packages/check.scm @@ -726,6 +726,9 @@ have been used.") ("python2-funcsigs" ,python2-funcsigs) ,@(package-propagated-inputs base)))))) +(define-public pypy3-mock + (package-with-pypy3 (strip-pypy3-variant python-mock))) + (define-public python-nose (package (name "python-nose") @@ -749,6 +752,9 @@ have been used.") (define-public python2-nose (package-with-python2 python-nose)) +(define-public pypy3-nose + (package-with-pypy3 python-nose)) + (define-public python-nose2 (package (name "python-nose2") @@ -778,6 +784,9 @@ interfaces and processes.") (define-public python2-nose2 (package-with-python2 python-nose2)) +(define-public pypy3-nose2 + (package-with-pypy3 python-nose2)) + (define-public python-unittest2 (package (name "python-unittest2") @@ -865,7 +874,8 @@ standard library.") and functions, detailed info on failing assert statements, modular fixtures, and many external plugins.") (license license:expat) - (properties `((python2-variant . ,(delay python2-pytest)))))) + (properties `((python2-variant . ,(delay python2-pytest)) + (pypy3-variant . ,(delay pypy3-pytest)))))) ;; Pytest 4.x are the last versions that support Python 2. (define-public python2-pytest @@ -903,13 +913,17 @@ and many external plugins.") ("python-pytest" ,python2-pytest-bootstrap) ("python-setuptools-scm" ,python2-setuptools-scm))))) +(define-public pypy3-pytest + (package-with-pypy3 (strip-pypy3-variant python-pytest))) + (define-public python-pytest-bootstrap (package (inherit (strip-python2-variant python-pytest)) (name "python-pytest-bootstrap") (native-inputs `(("python-setuptools-scm" ,python-setuptools-scm))) (arguments `(#:tests? #f)) - (properties `((python2-variant . ,(delay python2-pytest-bootstrap)))))) + (properties `((python2-variant . ,(delay python2-pytest-bootstrap)) + (pypy3-variant . ,(delay pypy3-pytest-bootstrap)))))) (define-public python2-pytest-bootstrap (hidden-package @@ -933,6 +947,22 @@ and many external plugins.") ("python-py" ,python2-py) ("python-wcwidth" ,python2-wcwidth)))))) +(define-public pypy3-pytest-bootstrap + (let ((base (package-with-pypy3 (strip-pypy3-variant python-pytest-bootstrap)))) + (package/inherit + base + (name "pypy3-pytest-bootstrap") + (native-inputs `(("pypy3-setuptools-scm" ,pypy3-setuptools-scm))) + (propagated-inputs + `(("python-atomicwrites" ,pypy3-atomicwrites) + ("python-attrs" ,pypy3-attrs-bootstrap) + ("python-more-itertools" ,pypy3-more-itertools) + ("python-packaging" ,pypy3-packaging-bootstrap) + ("python-pluggy" ,pypy3-pluggy-bootstrap) + ("python-py" ,pypy3-py) + ("python-six" ,pypy3-six-bootstrap) + ("python-wcwidth" ,pypy3-wcwidth)))))) + (define-public python-pytest-cov (package (name "python-pytest-cov") @@ -1840,6 +1870,9 @@ seamlessly into your existing Python unit testing work flow.") `(("python2-enum34" ,python2-enum34) ,@(package-propagated-inputs hypothesis)))))) +(define-public pypy3-hypothesis + (package-with-pypy3 python-hypothesis)) + (define-public python-lit (package (name "python-lit") @@ -2687,11 +2720,12 @@ grew out of the @dfn{Vc} project.") (modify-phases %standard-phases ;; The default test suite does not run these extra tests. (add-after 'check 'check-pytest-plugin - (lambda _ + (lambda* (#:key tests? #:allow-other-keys) + (if tests? (invoke "python" "-m" "pytest" "pyfakefs/pytest_tests/pytest_plugin_test.py") - #t))))) + #t)))))) (native-inputs `(("python-pytest" ,python-pytest))) (build-system python-build-system) @@ -2710,11 +2744,20 @@ the test. The pyfakefs library provides a solution to problems like this by mocking file system interactions. In other words, it arranges for the code under test to interact with a fake file system instead of the real file system. The code under test requires no modification to work with pyfakefs.") - (license license:asl2.0))) + (license license:asl2.0) + (properties `((pypy3-variant . ,(delay pypy3-pyfakefs)))))) (define-public python2-pyfakefs (package-with-python2 python-pyfakefs)) +(define-public pypy3-pyfakefs + (let ((base (package-with-pypy3 (strip-pypy3-variant python-pyfakefs)))) + (package/inherit + base + (arguments + `(#:tests? #f + ,@(package-arguments base)))))) + ;; This minimal variant is used to avoid a circular dependency between ;; python2-importlib-metadata, which requires pyfakefs for its tests, and ;; python2-pytest, which requires python2-importlib-metadata. @@ -2728,6 +2771,15 @@ system. The code under test requires no modification to work with pyfakefs.") `(#:python ,python-2 #:tests? #f))))) +(define-public pypy3-pyfakefs-bootstrap + (hidden-package + (package + (inherit pypy3-pyfakefs) + (name "pypy3-pyfakefs-bootstrap") + (native-inputs '()) + (arguments + `(#:tests? #f ,@(package-arguments pypy3-pyfakefs)))))) + (define-public python-aiounittest (package (name "python-aiounittest") diff --git a/gnu/packages/patches/pypy3-7.3.1-install-paths.patch b/gnu/packages/patches/pypy3-7.3.1-install-paths.patch new file mode 100644 index 0000000000..bd987bd4bc --- /dev/null +++ b/gnu/packages/patches/pypy3-7.3.1-install-paths.patch @@ -0,0 +1,19 @@ +Move site-packages to /lib/pypyX.X/site-packages, so it matches Python’s +behavior. + +diff -Naur pypy3.6-v7.3.1-src.orig/lib-python/3/distutils/command/install.py pypy3.6-v7.3.1-src/lib-python/3/distutils/command/install.py +--- pypy3.6-v7.3.1-src.orig/lib-python/3/distutils/command/install.py 1970-01-01 01:00:01.000000000 +0100 ++++ pypy3.6-v7.3.1-src/lib-python/3/distutils/command/install.py 2020-07-11 10:18:34.357208155 +0200 +@@ -44,9 +44,9 @@ + }, + 'nt': WINDOWS_SCHEME, + 'pypy': { +- 'purelib': '$base/site-packages', +- 'platlib': '$base/site-packages', +- 'headers': '$base/include/$dist_name', ++ 'purelib': '$base/lib/pypy$py_version_short/site-packages', ++ 'platlib': '$platbase/lib/pypy$py_version_short/site-packages', ++ 'headers': '$base/include/pypy$py_version_short$abiflags/$dist_name', + 'scripts': '$base/bin', + 'data' : '$base', + }, diff --git a/gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch b/gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch new file mode 100644 index 0000000000..d21133b4ae --- /dev/null +++ b/gnu/packages/patches/pypy3-7.3.1-ssl-paths.patch @@ -0,0 +1,41 @@ +Fix default certificate search path, still allowing the user to override it +with environment variables. + +--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py ++++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py +@@ -1679,20 +1679,9 @@ def get_default_verify_paths(): + https://golang.org/src/crypto/x509/root_linux.go (for the files) + ''' + certFiles = [ +- "/etc/ssl/certs/ca-certificates.crt", # Debian/Ubuntu/Gentoo etc. +- "/etc/pki/tls/certs/ca-bundle.crt", # Fedora/RHEL 6 +- "/etc/ssl/ca-bundle.pem", # OpenSUSE +- "/etc/pki/tls/cacert.pem", # OpenELEC +- "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # CentOS/RHEL 7 +- "/etc/ssl/cert.pem", # Alpine Linux + ] + certDirectories = [ +- "/etc/ssl/certs", # SLES10/SLES11 +- "/system/etc/security/cacerts", # Android +- "/usr/local/share/certs", # FreeBSD +- "/etc/pki/tls/certs", # Fedora/RHEL +- "/etc/openssl/certs", # NetBSD +- "/var/ssl/certs", # AIX ++ "@GUIX_CERT_PATH@", + ] + + # optimization: reuse the values from a local varaible +@@ -1707,9 +1696,10 @@ def get_default_verify_paths(): + ofile = _cstr_decode_fs(lib.X509_get_default_cert_file()) + odir = _cstr_decode_fs(lib.X509_get_default_cert_dir()) + +- if os.path.exists(ofile) and os.path.exists(odir): +- get_default_verify_paths.retval = (ofile_env, ofile, odir_env, odir) +- return get_default_verify_paths.retval ++ if not os.path.exists(ofile): ++ ofile = None ++ if not os.path.exists(odir): ++ odir = None + + # OpenSSL didn't supply the goods. Try some other options + for f in certFiles: diff --git a/gnu/packages/python-science.scm b/gnu/packages/python-science.scm index 3d6d4e909e..750f46fb67 100644 --- a/gnu/packages/python-science.scm +++ b/gnu/packages/python-science.scm @@ -355,6 +355,9 @@ doing practical, real world data analysis in Python.") "if 'NULL byte' in msg or 'line contains NUL' in msg:")) #t))))))) +(define-public pypy3-pandas + (package-with-pypy3 python-pandas)) + (define-public python-xarray (package (name "python-xarray") diff --git a/gnu/packages/python-xyz.scm b/gnu/packages/python-xyz.scm index 7079bbda5d..8229db3af9 100644 --- a/gnu/packages/python-xyz.scm +++ b/gnu/packages/python-xyz.scm @@ -1644,6 +1644,9 @@ Python file, so it can be easily copied into your project.") (define-public python2-six-bootstrap (package-with-python2 python-six-bootstrap)) +(define-public pypy3-six-bootstrap + (package-with-pypy3 python-six-bootstrap)) + (define-public python-schedule (package (name "python-schedule") @@ -2245,6 +2248,9 @@ code introspection, and logging.") (define-public python2-py (package-with-python2 python-py)) +(define-public pypy3-py + (package-with-pypy3 python-py)) + ;; Recent versions of python-fixtures and python-testrepository need ;; python-pbr for packaging, which itself needs these two packages for ;; testing. @@ -2961,6 +2967,9 @@ e.g. filters, callbacks and errbacks can all be promises.") (properties `((python2-variant . ,(delay python2-virtualenv)))) (license license:expat))) +(define-public pypy3-virtualenv + (package-with-pypy3 python-virtualenv)) + (define-public python2-virtualenv (let ((base (package-with-python2 (strip-python2-variant python-virtualenv)))) (package @@ -4031,7 +4040,9 @@ provides additional functionality on the produced Mallard documents.") ;; some tests require access to "$HOME/.cython" (lambda _ (setenv "HOME" "/tmp") #t)) (replace 'check - (lambda _ + (lambda* (#:key tests? #:allow-other-keys) + (if tests? + (begin ;; Disable compiler optimizations to greatly reduce the running ;; time of the test suite. (setenv "CFLAGS" "-O0") @@ -4043,14 +4054,16 @@ provides additional functionality on the produced Mallard documents.") ;; . ,@(if (not (target-64bit?)) '("-x" "run.parallel") - '()))))))) + '())) + #t))))))) (home-page "https://cython.org/") (synopsis "C extensions for Python") (description "Cython is an optimising static compiler for both the Python programming language and the extended Cython programming language. It makes writing C extensions for Python as easy as Python itself.") (license license:asl2.0) - (properties `((python2-variant . ,(delay python2-cython)))))) + (properties `((python2-variant . ,(delay python2-cython)) + (pypy3-variant . ,(delay pypy3-cython)))))) (define-public python2-cython (let ((base (package-with-python2 (strip-python2-variant python-cython)))) @@ -4087,6 +4100,12 @@ writing C extensions for Python as easy as Python itself.") "")) #t))))))))) +(define-public pypy3-cython + (let ((pkg (package-with-pypy3 (strip-pypy3-variant python-cython)))) + (package (inherit pkg) + ; Tests get stuck at some point + (arguments `(#:tests? #f ,@(package-arguments pkg)))))) + ;; The RPython toolchain currently does not support Python 3. (define-public python2-rpython (package @@ -4176,6 +4195,7 @@ include_dirs = ~a/include (lambda* (#:key outputs inputs #:allow-other-keys) ;; Make installed package available for running the tests (add-installed-pythonpath inputs outputs) + (system "export > $NIX_BUILD_TOP/environment-variables2") ;; Make sure "f2py" etc is found. (setenv "PATH" (string-append (assoc-ref outputs "out") "/bin" ":" (getenv "PATH"))) @@ -4208,6 +4228,9 @@ capabilities.") (base32 "0lg1cycxzi4rvvrd5zxinpdz0ni792fpx6xjd75z1923zcac8qrb"))))))) +(define-public pypy3-numpy + (package-with-pypy3 python-numpy)) + ;; NOTE: NumPy 1.8 is packaged only for Python 2 because it is of ;; interest only for legacy code going back to NumPy's predecessor ;; Numeric. @@ -5198,7 +5221,16 @@ by pycodestyle.") (description "Distlib is a library which implements low-level functions that relate to packaging and distribution of Python software. It is intended to be used as the basis for third-party packaging tools.") - (license license:psfl))) + (license license:psfl) + (properties `((pypy3-variant . ,(delay pypy3-distlib)))))) + +(define-public pypy3-distlib + (let ((base (package-with-pypy3 (strip-pypy3-variant python-distlib)))) + (package/inherit + base + (arguments + ; tests failb because they try to rmtree a read-only store copy + `(#:tests? #f ,@(package-arguments base)))))) (define-public python-distutils-extra (package @@ -5943,6 +5975,9 @@ them as the version argument or in a SCM managed file.") (define-public python2-setuptools-scm (package-with-python2 python-setuptools-scm)) +(define-public pypy3-setuptools-scm + (package-with-pypy3 python-setuptools-scm)) + (define-public python-sexpdata (package (name "python-sexpdata") @@ -5985,6 +6020,9 @@ all the newest features of the standard @code{pathlib} can be used also on older Python versions.") (license license:expat))) +(define-public pypy3-pathlib2 + (package-with-pypy3 python-pathlib2)) + (define-public python2-importlib-resources (package (name "python2-importlib-resources") @@ -6018,6 +6056,39 @@ older Python versions.") for older versions of Python.") (license license:asl2.0))) +(define-public pypy3-importlib-resources + (package + (name "pypy3-importlib-resources") + (version "1.0.2") + (source (origin + (method url-fetch) + (uri (pypi-uri "importlib_resources" version)) + (sha256 + (base32 + "0y3hg12iby1qyaspnbisz4s4vxax7syikk3skznwqizqyv89y9yk")))) + (build-system python-build-system) + (arguments + `(#:python ,pypy3 + #:phases (modify-phases %standard-phases + ;; The build system tests for python-wheel, but it is + ;; not required for Guix nor the test suite. Just drop + ;; it to make bootstrapping pytest easier. + (add-after 'unpack 'drop-wheel-dependency + (lambda _ + (substitute* "setup.cfg" + (("^[[:blank:]]+wheel") + "")) + #t))))) + (propagated-inputs + `(("python-pathlib2" ,pypy3-pathlib2) + ("python-typing" ,pypy3-typing))) + (home-page "https://gitlab.com/python-devs/importlib_resources") + (synopsis "Backport of @code{importlib.resources} from Python 3.7") + (description + "This package provides an implementation of @code{importlib.resources} +for older versions of Python.") + (license license:asl2.0))) + ;; For importlib-metadata-bootstrap below. (define-public python2-importlib-resources-bootstrap (hidden-package @@ -6028,6 +6099,15 @@ for older versions of Python.") `(("python-pathlib2-bootstrap" ,python2-pathlib2-bootstrap) ("python-typing" ,python2-typing)))))) +(define-public pypy3-importlib-resources-bootstrap + (hidden-package + (package/inherit + pypy3-importlib-resources + (name "pypy3-importlib-resources-bootstrap") + (propagated-inputs + `(("python-pathlib2-bootstrap" ,pypy3-pathlib2-bootstrap) + ("python-typing" ,pypy3-typing)))))) + (define-public python-importlib-metadata (package (name "python-importlib-metadata") @@ -6055,7 +6135,7 @@ its top-level name. This functionality intends to replace most uses of @code{pkg_resources} entry point API and metadata API. Along with @code{importlib.resources} in Python 3.7 and newer, this can eliminate the need to use the older and less efficient @code{pkg_resources} package.") - (properties `((python2-variant . ,(delay python2-importlib-metadata)))) + (properties `((python2-variant . ,(delay python2-importlib-metadata)) (pypy3-variant . ,(delay pypy3-importlib-metadata)))) (license license:asl2.0))) (define-public python2-importlib-metadata @@ -6075,6 +6155,18 @@ need to use the older and less efficient @code{pkg_resources} package.") ("python-pathlib2" ,python2-pathlib2) ,@(package-propagated-inputs base)))))) +(define-public pypy3-importlib-metadata + (let ((base (package-with-pypy3 (strip-pypy3-variant python-importlib-metadata)))) + (package/inherit + base + (name "pypy3-importlib-metadata") + (native-inputs + `(("python-setuptools-scm" ,pypy3-setuptools-scm) + ;; break loop: pytest → pluggy → importlib-metadata + ("python-pyfakefs" ,pypy3-pyfakefs-bootstrap) + ("python-packaging" ,pypy3-packaging-bootstrap) + ("python-importlib-resources" ,pypy3-importlib-resources)))))) + ;; This package is used by python2-pytest, and thus must not depend on it. (define-public python2-importlib-metadata-bootstrap (hidden-package @@ -6091,6 +6183,20 @@ need to use the older and less efficient @code{pkg_resources} package.") ("python-contextlib2" ,python2-contextlib2-bootstrap) ("python-importlib-resources" ,python2-importlib-resources-bootstrap)))))) +(define-public pypy3-importlib-metadata-bootstrap + (let ((base pypy3-importlib-metadata)) + (hidden-package + (package + (inherit base) + (name "pypy3-importlib-metadata-bootstrap") + (arguments `(#:tests? #f ,@(package-arguments pypy3-importlib-metadata))) +(native-inputs + `(("python-setuptools-scm" ,pypy3-setuptools-scm) + ("python-pyfakefs" ,pypy3-pyfakefs-bootstrap) + ("python-packaging" ,pypy3-packaging-bootstrap) + ("python-importlib-resources" ,pypy3-importlib-resources-bootstrap))) + )))) + (define-public python-importmagic (package (name "python-importmagic") @@ -9699,6 +9805,9 @@ for atomic file system operations.") (define-public python2-atomicwrites (package-with-python2 python-atomicwrites)) +(define-public pypy3-atomicwrites + (package-with-pypy3 python-atomicwrites)) + (define-public python-click-threading (package (name "python-click-threading") @@ -10169,7 +10278,8 @@ library as well as on the command line.") (description "Pluggy is an extraction of the plugin manager as used by Pytest but stripped of Pytest specific details.") (home-page "https://pypi.org/project/pluggy/") - (properties `((python2-variant . ,(delay python2-pluggy)))) + (properties `((python2-variant . ,(delay python2-pluggy)) + (pypy3-variant . ,(delay pypy3-pluggy)))) (license license:expat))) (define-public python2-pluggy @@ -10180,6 +10290,14 @@ Pytest but stripped of Pytest specific details.") (propagated-inputs `(("python-importlib-metadata" ,python2-importlib-metadata)))))) +(define-public pypy3-pluggy + (let ((base (package-with-pypy3 (strip-pypy3-variant + python-pluggy)))) + (package/inherit + base + (propagated-inputs + `(("python-importlib-metadata" ,pypy3-importlib-metadata)))))) + ;; This package requires python2-importlib-metadata, but that package ;; ends up needing python2-pluggy via python2-pytest, so we need this ;; variant to solve the circular dependency. @@ -10194,6 +10312,17 @@ Pytest but stripped of Pytest specific details.") (propagated-inputs `(("python-importlib-metadata" ,python2-importlib-metadata-bootstrap)))))) +(define-public pypy3-pluggy-bootstrap + (hidden-package + (package/inherit + pypy3-pluggy + (name "pypy3-pluggy-bootstrap") + (arguments + `(#:tests? #f + ,@(package-arguments pypy3-pluggy))) + (propagated-inputs + `(("python-importlib-metadata" ,pypy3-importlib-metadata-bootstrap)))))) + (define-public python-tox (package (name "python-tox") @@ -10966,6 +11095,15 @@ ambiguities (forward vs. backward slashes, etc.). `(("python2-scandir" ,python2-scandir) ("python2-six" ,python2-six-bootstrap)))))) +(define-public pypy3-pathlib2-bootstrap + (hidden-package + (package + (inherit pypy3-pathlib2) + (name "pypy3-pathlib2-bootstrap") + (propagated-inputs + `(("python-scandir" ,pypy3-scandir) + ("python-six" ,pypy3-six-bootstrap)))))) + (define-public python-jellyfish (package (name "python-jellyfish") @@ -11870,6 +12008,9 @@ specified in POSIX.1-2001 and POSIX.1-2008.") (define-public python2-wcwidth (package-with-python2 python-wcwidth)) +(define-public pypy3-wcwidth + (package-with-pypy3 python-wcwidth)) + (define-public python2-jsonrpclib (package (name "python2-jsonrpclib") @@ -14026,6 +14167,9 @@ protocols.") (define-public python2-attrs-bootstrap (package-with-python2 python-attrs-bootstrap)) +(define-public pypy3-attrs-bootstrap + (package-with-pypy3 python-attrs-bootstrap)) + (define-public python2-cliapp (package (name "python2-cliapp") @@ -14574,6 +14718,9 @@ This package is part of the Python standard library since version 3.5.") (define-public python2-scandir (package-with-python2 python-scandir)) +(define-public pypy3-scandir + (package-with-pypy3 python-scandir)) + (define-public python2-stemming (package (name "python2-stemming") @@ -14735,6 +14882,9 @@ information.") `(#:tests? #f ,@(package-arguments python2-packaging)))))) +(define-public pypy3-packaging-bootstrap + (package-with-pypy3 python-packaging-bootstrap)) + (define-public python-relatorio (package (name "python-relatorio") @@ -16525,6 +16675,9 @@ and other tools.") (define-public python2-typing (package-with-python2 python-typing)) +(define-public pypy3-typing + (package-with-pypy3 python-typing)) + (define-public python-typing-extensions (package (name "python-typing-extensions") @@ -16766,6 +16919,9 @@ working with iterables.") (propagated-inputs `(("python2-six" ,python2-six-bootstrap))))) +(define-public pypy3-more-itertools + (package-with-pypy3 python-more-itertools)) + (define-public python-latexcodec (package (name "python-latexcodec") diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm index e2b254bf6f..fe240eaf5e 100644 --- a/gnu/packages/python.scm +++ b/gnu/packages/python.scm @@ -426,6 +426,18 @@ data types.") (lambda _ (unsetenv "SOURCE_DATE_EPOCH") #t)) (add-after 'check 'reset-SOURCE_DATE_EPOCH (lambda _ (setenv "SOURCE_DATE_EPOCH" "1") #t)) + (add-after 'install 'install-symlinks + (lambda* (#:key outputs #:allow-other-keys) + (let ((bin (string-append (assoc-ref outputs "out") "/bin"))) + (for-each + (lambda (old new) + (let ((oldpath (string-append bin "/" old))) + (when (file-exists? oldpath) + (symlink oldpath + (string-append bin "/" new))))) + `("python3" ,"pydoc3" ,"idle3" ,"pip3" ,"python3-config") + `("python" ,"pydoc" ,"idle" ,"pip" ,"python-config")) + #t))) (replace 'rebuild-bytecode (lambda* (#:key outputs #:allow-other-keys) (let ((out (assoc-ref outputs "out"))) @@ -659,7 +671,9 @@ ease from the desktop to a microcontroller or embedded system.") (sha256 (base32 "10zsk8jby8j6visk5mzikpb1cidvz27qq4pfpa26jv53klic6b0c")) - (patches (search-patches "pypy3-7.3.1-fix-tests.patch")))) + (patches (search-patches "pypy3-7.3.1-fix-tests.patch" + "pypy3-7.3.1-install-paths.patch" + "pypy3-7.3.1-ssl-paths.patch")))) (build-system gnu-build-system) (native-inputs `(("python-2" ,python-2) @@ -667,13 +681,13 @@ ease from the desktop to a microcontroller or embedded system.") ("tar" ,tar) ; Required for package.py ("python2-pycparser" ,python2-pycparser) ("python2-hypothesis" ,python2-hypothesis) - ("nss-certs" ,nss-certs) ; For ssl tests ("gzip" ,gzip))) (inputs `(("libffi" ,libffi) ("zlib" ,zlib) ("ncurses" ,ncurses) ("openssl" ,openssl) + ("nss-certs" ,nss-certs) ; For ssl module ("expat" ,expat) ("bzip2" ,bzip2) ("sqlite" ,sqlite) @@ -684,7 +698,7 @@ ease from the desktop to a microcontroller or embedded system.") ("bash-minimal" ,bash-minimal) ; Used as /bin/sh ("xz" ,xz))) ; liblzma (arguments - `(#:tests? #f ;FIXME: Disabled for now, there are many tests failing. + `(#:tests? #f ;FIXME: Disabled for now, there are many tests failing. #:modules ((ice-9 ftw) (ice-9 match) (guix build utils) (guix build gnu-build-system)) #:phases (modify-phases %standard-phases @@ -713,6 +727,9 @@ ease from the desktop to a microcontroller or embedded system.") (substitute* '("lib_pypy/_curses_build.py") ;; Find curses (("/usr/local") (assoc-ref inputs "ncurses"))) + (substitute* '("lib_pypy/_dbm.py") + ;; Use gdbm compat library, so we don’t need to pull in bdb + (("ctypes.util.find_library\\('db'\\)") (string-append "'" (assoc-ref inputs "gdbm") "/lib/libgdbm_compat.so'"))) (substitute* '("lib_pypy/_sqlite3_build.py") ;; Always use search paths (("sys\\.platform\\.startswith\\('freebsd'\\)") "True") @@ -729,6 +746,14 @@ ease from the desktop to a microcontroller or embedded system.") (substitute* '("lib-python/3/distutils/unixccompiler.py") ;; gcc-toolchain does not provide symlink cc -> gcc (("\"cc\"") "\"gcc\"")) + (substitute* '("lib_pypy/_cffi_ssl/_stdssl/__init__.py") + ;; Add nss-certs to default certificate search path, + ;; otherwise every packages has to specify nss-certs and + ;; openssl as input to set the proper env variables. + ;; Depends on -ssl-paths.patch. + (("@GUIX_CERT_PATH@") + (string-append (assoc-ref inputs "nss-certs") + "/etc/ssl/certs"))) #t)) (add-after 'unpack 'set-source-file-times-to-1980 @@ -748,7 +773,8 @@ ease from the desktop to a microcontroller or embedded system.") (string-append "--make-jobs=" (number->string (parallel-job-count))) "-Ojit" - "targetpypystandalone")) + "targetpypystandalone" + "--allworkingmodules")) ;; Build c modules and package everything, so tests work. (with-directory-excursion "pypy/tool/release" (unsetenv "PYTHONPATH") ; Do not use the system’s python libs: @@ -756,7 +782,9 @@ ease from the desktop to a microcontroller or embedded system.") ; attribute 'IntFlag' (invoke "python2" "package.py" "--archive-name" "pypy-dist" - "--builddir" (getcwd))))) + "--builddir" (getcwd)) + ;; install pip and setuptools into the dist directory + (invoke "pypy-dist/bin/pypy3" "-m" "ensurepip")))) (replace 'check (lambda* (#:key tests? #:allow-other-keys) (if tests? @@ -774,32 +802,45 @@ ease from the desktop to a microcontroller or embedded system.") #t)) (replace 'install (lambda* (#:key inputs outputs #:allow-other-keys) - (with-directory-excursion "pypy/tool/release" - ;; Delete test data. - (for-each - (lambda (x) - (delete-file-recursively (string-append - "pypy-dist/lib-python/3/" x))) - '("tkinter/test" - "test" - "sqlite3/test" - "lib2to3/tests" - "idlelib/idle_test" - "distutils/tests" - "ctypes/test" - "unittest/test")) - ;; Patch shebang referencing python2 - (substitute* '("pypy-dist/lib-python/3/cgi.py" - "pypy-dist/lib-python/3/encodings/rot_13.py") - (("#!.+/bin/python") - (string-append "#!" (assoc-ref outputs "out") "/bin/pypy3"))) - (with-fluids ((%default-port-encoding "ISO-8859-1")) - (substitute* '("pypy-dist/lib_pypy/_md5.py" - "pypy-dist/lib_pypy/_sha1.py") - (("#!.+/bin/python") - (string-append "#!" (assoc-ref outputs "out") "/bin/pypy3")))) - (copy-recursively "pypy-dist" (assoc-ref outputs "out"))) - #t))))) + (let* ((out (assoc-ref outputs "out")) + (bin-pypy3 (string-append out "/bin/pypy3")) + (shebang-match-python "#!.+/bin/python") + (shebang-pypy3 (string-append "#!" bin-pypy3)) + (dist-dir "pypy/tool/release/pypy-dist")) + (with-directory-excursion dist-dir + ;; Delete test data. + (for-each + (lambda (x) + (delete-file-recursively (string-append + "lib-python/3/" x))) + '("tkinter/test" + "test" + "sqlite3/test" + "lib2to3/tests" + "idlelib/idle_test" + "distutils/tests" + "ctypes/test" + "unittest/test" + "ensurepip" ; we already used to to install pip/setuptools + )) + ;; Patch shebang referencing python2 + (substitute* '("lib-python/3/cgi.py" + "lib-python/3/encodings/rot_13.py") + ((shebang-match-python) shebang-pypy3)) + (with-fluids ((%default-port-encoding "ISO-8859-1")) + (substitute* '("lib_pypy/_md5.py" + "lib_pypy/_sha1.py") + ((shebang-match-python) shebang-pypy3)))) + (copy-recursively dist-dir out) + ;; Make sure pypy3 is callable as python/python3, so we + ;; don’t have to patch every single package. + (symlink bin-pypy3 (string-append out "/bin/python")) + (symlink bin-pypy3 (string-append out "/bin/python3")) + #t)))))) + (native-search-paths + (list (search-path-specification + (variable "PYTHONPATH") + (files '("lib/pypy3.6/site-packages"))))) (home-page "https://www.pypy.org/") (synopsis "Python implementation with just-in-time compilation") (description "PyPy is a faster, alternative implementation of the Python diff --git a/gnu/packages/xml.scm b/gnu/packages/xml.scm index 90181a1773..d2dad42a1c 100644 --- a/gnu/packages/xml.scm +++ b/gnu/packages/xml.scm @@ -2083,8 +2083,10 @@ because lxml.etree already has it's own implementation of XPath 1.0.") (arguments `(#:phases (modify-phases %standard-phases (replace 'check - (lambda _ - (invoke "make" "test")))))) + (lambda* (#:key tests? #:allow-other-keys) + (if tests? + (begin + (invoke "make" "test")))))))) (inputs `(("libxml2" ,libxml2) ("libxslt" ,libxslt))) @@ -2093,11 +2095,20 @@ because lxml.etree already has it's own implementation of XPath 1.0.") (description "The lxml XML toolkit is a Pythonic binding for the C libraries libxml2 and libxslt.") - (license license:bsd-3))) ; and a few more, see LICENSES.txt + (license license:bsd-3) ; and a few more, see LICENSES.txt + (properties `((pypy3-variant . ,(delay pypy3-lxml)))))) (define-public python2-lxml (package-with-python2 python-lxml)) +(define-public pypy3-lxml + (let ((pkg (package-with-pypy3 (strip-pypy3-variant python-lxml)))) + (package (inherit pkg) + (arguments `( + ;; some tests hang for unknown reasons + #:tests? #f ,@(package-arguments pkg) + ))))) + (define-public python-xmlschema (package (name "python-xmlschema") @@ -2140,6 +2151,9 @@ finding schema's elements and attributes; and can encode and decode XML data to JSON and other formats.") (license license:expat))) +(define-public pypy3-xmlschema + (package-with-pypy3 python-xmlschema)) + (define-public python-xmltodict (package (name "python-xmltodict") diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm index e39c06528e..59395ef19f 100644 --- a/guix/build-system/python.scm +++ b/guix/build-system/python.scm @@ -32,7 +32,9 @@ #:use-module (srfi srfi-26) #:export (%python-build-system-modules package-with-python2 + package-with-pypy3 strip-python2-variant + strip-pypy3-variant default-python default-python2 python-build @@ -63,13 +65,19 @@ extension, such as '.tar.gz'." "Return the default Python package." ;; Lazily resolve the binding to avoid a circular dependency. (let ((python (resolve-interface '(gnu packages python)))) - (module-ref python 'python-wrapper))) + ;; Must *not* be python-wrapper, which lacks site-libraries directory + (module-ref python 'python))) (define (default-python2) "Return the default Python 2 package." (let ((python (resolve-interface '(gnu packages python)))) (module-ref python 'python-2))) +(define (default-pypy3) + "Return the default pypy3 package." + (let ((python (resolve-interface '(gnu packages python)))) + (module-ref python 'pypy3))) + (define* (package-with-explicit-python python old-prefix new-prefix #:key variant-property) "Return a procedure of one argument, P. The procedure creates a package with @@ -129,12 +137,26 @@ pre-defined variants." "python-" "python2-" #:variant-property 'python2-variant)) +(define package-with-pypy3 + ;; Note: delay call to 'default-pypy3' until after the 'arguments' field + ;; of packages is accessed to avoid a circular dependency when evaluating + ;; the top-level of (gnu packages python). + (package-with-explicit-python (delay (default-pypy3)) + "python-" "pypy3-" + #:variant-property 'pypy3-variant)) + (define (strip-python2-variant p) "Remove the 'python2-variant' property from P." (package (inherit p) (properties (alist-delete 'python2-variant (package-properties p))))) +(define (strip-pypy3-variant p) + "Remove the 'pypy3-variant' property from P." + (package + (inherit p) + (properties (alist-delete 'pypy3-variant (package-properties p))))) + (define* (lower name #:key source inputs native-inputs outputs system target (python (default-python)) diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 09bd8465c8..34e33fd4c1 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -28,6 +28,7 @@ #:use-module (ice-9 match) #:use-module (ice-9 ftw) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:export (%standard-phases add-installed-pythonpath @@ -158,9 +159,12 @@ "Return the path of the current output's Python site-package." (let* ((out (assoc-ref outputs "out")) (python (assoc-ref inputs "python"))) - (string-append out "/lib/python" - (python-version python) - "/site-packages/"))) + (string-append + out "/" + (with-directory-excursion python + (first (filter + file-exists? + (map (lambda (x) (string-append "lib/" x "/site-packages")) (scandir "lib" (lambda (x) (not (member x '("." "..")))))))))))) (define (add-installed-pythonpath inputs outputs) "Prepend the Python site-package of OUTPUT to PYTHONPATH. This is useful @@ -202,9 +206,7 @@ when running checks after installing the package." (let* ((out (assoc-ref outputs "out")) (python (assoc-ref inputs "python")) (var `("PYTHONPATH" prefix - ,(cons (string-append out "/lib/python" - (python-version python) - "/site-packages") + ,(cons (site-packages inputs outputs) (search-path-as-string->list (or (getenv "PYTHONPATH") "")))))) (for-each (lambda (dir) @@ -222,9 +224,7 @@ installed with setuptools." ;; some good reason. (let* ((out (assoc-ref outputs "out")) (python (assoc-ref inputs "python")) - (site-packages (string-append out "/lib/python" - (python-version python) - "/site-packages")) + (site-packages (site-packages inputs outputs)) (easy-install-pth (string-append site-packages "/easy-install.pth")) (new-pth (string-append site-packages "/" name ".pth"))) (when (file-exists? easy-install-pth) -- 2.26.2