[PATCH 1/2] gnu: Add numpy

[PATCH 1/2] gnu: Add numpy

[Federico Beffa]

[-- Attachment #1: Type: text/plain, Size: 59 bytes --]

Here the first packages depending on ATLAS.

Regards,
Fede

[-- Attachment #2: 0001-gnu-Add-numpy.patch --]
[-- Type: text/x-patch, Size: 5578 bytes --]

From 063b33562c4fee2ea4e3fd1b53a27770047e9201 Mon Sep 17 00:00:00 2001
From: Federico Beffa <beffa@fbengineering.ch>
Date: Sun, 26 Oct 2014 15:03:17 +0100
Subject: [PATCH 1/2] gnu: Add numpy

* gnu/packages/python.scm(python-numpy, python2-numpy): New variables.
---
 gnu/packages/python.scm | 112 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)

diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index c995c1b..b39957a 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -37,6 +37,8 @@
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages databases)
   #:use-module (gnu packages zip)
+  #:use-module (gnu packages maths)
+  #:use-module (gnu packages gcc)
   #:use-module (gnu packages multiprecision)
   #:use-module (guix packages)
   #:use-module (guix download)
@@ -1832,3 +1834,113 @@ sources.")
 
 (define-public python2-sphinx
   (package-with-python2 python-sphinx))
+
+(define python-numpy-reference
+  (origin
+    (method url-fetch)
+    (uri (string-append "mirror://sourceforge/numpy/reference.pdf"))
+    (sha256 
+     (base32
+      "0izsmzazhd042ra6p7as2jhwx8713sc5gpn9s5nifwm68biwn0lp"))))
+
+(define python-numpy-userguide
+  (origin
+    (method url-fetch)
+    (uri (string-append "mirror://sourceforge/numpy/userguide.pdf"))
+    (sha256 
+     (base32
+      "1z2xjlxldv01ad2775k9birg1hkqay5wbi5vmgz3rlr6dczvk77k"))))
+
+(define-public python-numpy
+  (package
+    (name "python-numpy")
+    (version "1.9.0")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (string-append "mirror://sourceforge/numpy"
+                           "/numpy-" version ".tar.gz"))
+       (sha256
+        (base32
+         "1xbzsb84wg32zx5ilgvkdp0lwfqrycwhh5a2q9nx5iwyjpgcxrpp"))))
+    (build-system python-build-system)
+    (inputs
+     `(("python" ,python) ; otherwise ld does not find libpython3.3m
+       ("python-numpy-userguide" ,python-numpy-userguide)
+       ("python-numpy-reference" ,python-numpy-reference)
+       ("atlas" ,atlas)))
+    (native-inputs
+     `(("python-nose" ,python-nose)  ; used by tests
+       ("gfortran" ,gfortran-4.8)))
+    (outputs '("out" "doc"))         ; doc is ca. 9MB
+    (arguments
+     `(#:phases
+       (alist-replace
+        'build
+        (lambda* (#:key outputs inputs 
+                  #:allow-other-keys #:rest args)
+          (let* ((build (assoc-ref %standard-phases 'build))
+                 (atlas-threaded
+                  (string-append (assoc-ref inputs "atlas") 
+                                 "/lib/libtatlas.so"))
+                 ;; On single core CPUs only the serial library is created.
+                 (atlas-lib
+                  (if (file-exists? atlas-threaded)
+                      atlas-threaded
+                      (string-append (assoc-ref inputs "atlas") 
+                                     "/lib/libsatlas.so"))))
+            (setenv "ATLAS" atlas-lib)
+            (apply build args)))
+        (alist-cons-after
+         'install 'install-doc
+         (lambda* (#:key outputs inputs #:allow-other-keys)
+           ;; procedure from gnu-build-system.scm
+           (define (package-name)
+             (let* ((out  (assoc-ref outputs "out"))
+                    (base (basename out))
+                    (dash (string-rindex base #\-)))
+               (string-drop (if dash
+                                (substring base 0 dash)
+                                base)
+                            (+ 1 (string-index base #\-)))))
+
+           (let ((doc (string-append (assoc-ref outputs "doc") 
+                                     "/share/doc/" (package-name))))
+             (mkdir-p doc)
+             (and (zero? 
+                   (system* "cp" 
+                            (assoc-ref inputs "python-numpy-reference") 
+                            (string-append doc "/reference.pdf")))
+                  (zero? 
+                   (system* "cp" 
+                            (assoc-ref inputs "python-numpy-userguide")
+                            (string-append doc "/userguide.pdf"))))))
+         ;; Tests can only be run after the library has been installed and not
+         ;; within the source directory.
+         (alist-cons-after
+          'install 'check
+          (lambda _ 
+            (with-directory-excursion "/tmp"
+              (zero? (system* "python" "-c" "import numpy; numpy.test()"))))
+          (alist-delete 
+           'check 
+           %standard-phases))))))
+    (home-page "http://www.numpy.org/")
+    (synopsis "Fundamental package for scientific computing with Python")
+    (description "NumPy is the fundamental package for scientific computing
+with Python. It contains among other things: a powerful N-dimensional array
+object, sophisticated (broadcasting) functions, tools for integrating C/C++
+and Fortran code, useful linear algebra, Fourier transform, and random number
+capabilities.")
+    (license bsd-3)))
+
+(define-public python2-numpy
+  (package (inherit (package-with-python2 python-numpy))
+           ;; If we don't redefine the inputs here, then python (version 3) is
+           ;; also imported and its libraries come first in PYTHONPATH.  This
+           ;; causes the 'check phase to fail.
+           (inputs
+            `(("python-2" ,python-2) ; otherwise ld does not find libpython2.7
+              ("python-numpy-userguide" ,python-numpy-userguide)
+              ("python-numpy-reference" ,python-numpy-reference)
+              ("atlas" ,atlas)))))
-- 
1.8.4

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

[-- Attachment #1: Type: text/plain, Size: 1325 bytes --]

Federico Beffa <beffa@ieee.org> skribis:

> From 063b33562c4fee2ea4e3fd1b53a27770047e9201 Mon Sep 17 00:00:00 2001
> From: Federico Beffa <beffa@fbengineering.ch>
> Date: Sun, 26 Oct 2014 15:03:17 +0100
> Subject: [PATCH 1/2] gnu: Add numpy
>
> * gnu/packages/python.scm(python-numpy, python2-numpy): New variables.
                           ^
Missing space.

> +(define python-numpy-reference
> +  (origin
> +    (method url-fetch)
> +    (uri (string-append "mirror://sourceforge/numpy/reference.pdf"))
> +    (sha256 
> +     (base32
> +      "0izsmzazhd042ra6p7as2jhwx8713sc5gpn9s5nifwm68biwn0lp"))))
> +
> +(define python-numpy-userguide
> +  (origin
> +    (method url-fetch)
> +    (uri (string-append "mirror://sourceforge/numpy/userguide.pdf"))
> +    (sha256 
> +     (base32
> +      "1z2xjlxldv01ad2775k9birg1hkqay5wbi5vmgz3rlr6dczvk77k"))))

Can these manuals be built from source?

If they can’t, then that’s non-free documentation.

If they can but the process is tedious, then it’s OK to leave it that
way with a TODO, and also a comment stating what its license is.

> +    (inputs
> +     `(("python" ,python) ; otherwise ld does not find libpython3.3m

This is because Python is not added to ‘LIBRARY_PATH’, right?

I think this is fixed by this patch:


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: Type: text/x-patch, Size: 624 bytes --]

--- a/guix/build-system/python.scm
+++ b/guix/build-system/python.scm
@@ -110,10 +110,11 @@ prepended to the name."
                               '())
                         ,@inputs
 
+                        ("python" ,python)
+
                         ;; Keep the standard inputs of 'gnu-build-system'.
                         ,@(standard-packages)))
-         (build-inputs `(("python" ,python)
-                         ,@native-inputs))
+         (build-inputs native-inputs)
          (outputs outputs)
          (build python-build)
          (arguments (strip-keyword-arguments private-keywords arguments)))))

[-- Attachment #3: Type: text/plain, Size: 2872 bytes --]


Can you confirm?

> +       ("python-numpy-userguide" ,python-numpy-userguide)
> +       ("python-numpy-reference" ,python-numpy-reference)
> +       ("atlas" ,atlas)))

As discussed in the other thread, this should probably be the generic
(unoptimized) ATLAS here.

> +       (alist-replace
> +        'build
> +        (lambda* (#:key outputs inputs 
> +                  #:allow-other-keys #:rest args)
> +          (let* ((build (assoc-ref %standard-phases 'build))

Use ‘alist-cons-before’ instead of calling the original ‘build’ phase (I
think this was discussed before and there was rough consensus on that.)

> +        (alist-cons-after
> +         'install 'install-doc
> +         (lambda* (#:key outputs inputs #:allow-other-keys)
> +           ;; procedure from gnu-build-system.scm
> +           (define (package-name)
> +             (let* ((out  (assoc-ref outputs "out"))
> +                    (base (basename out))
> +                    (dash (string-rindex base #\-)))
> +               (string-drop (if dash
> +                                (substring base 0 dash)
> +                                base)
> +                            (+ 1 (string-index base #\-)))))
> +
> +           (let ((doc (string-append (assoc-ref outputs "doc") 
> +                                     "/share/doc/" (package-name))))

Just use (string-append ,name "-" ,version) so the ‘package-name’
procedure isn’t needed.

> +             (and (zero? 
> +                   (system* "cp" 
> +                            (assoc-ref inputs "python-numpy-reference") 
> +                            (string-append doc "/reference.pdf")))
> +                  (zero? 
> +                   (system* "cp" 
> +                            (assoc-ref inputs "python-numpy-userguide")
> +                            (string-append doc "/userguide.pdf"))))))

Use the ‘copy-file’ procedure instead of calling ‘cp’.

> +         ;; Tests can only be run after the library has been installed and not
> +         ;; within the source directory.
> +         (alist-cons-after
> +          'install 'check
> +          (lambda _ 
> +            (with-directory-excursion "/tmp"
> +              (zero? (system* "python" "-c" "import numpy; numpy.test()"))))
> +          (alist-delete 
> +           'check 
> +           %standard-phases))))))

Just (alist-replace 'check ...) ?

> +(define-public python2-numpy
> +  (package (inherit (package-with-python2 python-numpy))
> +           ;; If we don't redefine the inputs here, then python (version 3) is
> +           ;; also imported and its libraries come first in PYTHONPATH.  This
> +           ;; causes the 'check phase to fail.

Normally this will no longer be needed if the patch above solves the
problem.

Thanks!

Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

On Sun, Oct 26, 2014 at 7:56 PM, Ludovic Courtès <ludo@gnu.org> wrote:
> Can these manuals be built from source?

I get error messages that matplotlib is missing.  I started looking at
matplotlib as well, but I've found that, for the TkAgg back-end, it
needs TKinter which is part of the standard python libraries and it is
built if, during the build phase, tk is available. The current python
package does not build that library (no tk).  In fact, I would like to
add it to be able to move forward with matplotlib :-)

For the other back-ends some packages are missing (python-gtk, Qt, ...)

Another annoying problem is that matplotlib needs numpy, thereby
creating a dependency loop (although only for the documentation).

> If they can but the process is tedious, then it’s OK to leave it that
> way with a TODO, and also a comment stating what its license is.

I've not seen any license statement specific for the documentation.
But, given that the source is distributed with the code, can't we just
assume it is the same license?

>> +    (inputs
>> +     `(("python" ,python) ; otherwise ld does not find libpython3.3m
>
> This is because Python is not added to ‘LIBRARY_PATH’, right?
>
> I think this is fixed by this patch:
>
>
>
> Can you confirm?

Actually it does not.  The "...-python-numpy-1.9.0-guile-builder"
still shows that only the python-wrapper is imported. The wrapper does
not include the libraries.

> As discussed in the other thread, this should probably be the generic
> (unoptimized) ATLAS here.

I think that as a temporary situation this is OK.  However, in my
opinion it does not make much sense to have a separate sub-optimal
package.  I would like to propose to add specific flags to a single
atlas package and, once the new "local-build" flag will be available,
we will remove those flags.

>
>> +       (alist-replace
>> +        'build
>> +        (lambda* (#:key outputs inputs
>> +                  #:allow-other-keys #:rest args)
>> +          (let* ((build (assoc-ref %standard-phases 'build))
>
> Use ‘alist-cons-before’ instead of calling the original ‘build’ phase (I
> think this was discussed before and there was rough consensus on that.)

OK.

> Just use (string-append ,name "-" ,version) so the ‘package-name’
> procedure isn’t needed.

that's much better :-)

> Use the ‘copy-file’ procedure instead of calling ‘cp’.

The reason that made me use 'cp' instead of 'copy-file' is that the
documentation states that the return value of the latter is not
specified and I'm still not familiar with Guile exception handling.
However, I see many places where 'copy-file' is used without any
'catch'.  In principle this should never fail.  If it is OK, I will do
the same.

>
>> +         ;; Tests can only be run after the library has been installed and not
>> +         ;; within the source directory.
>> +         (alist-cons-after
>> +          'install 'check
>> +          (lambda _
>> +            (with-directory-excursion "/tmp"
>> +              (zero? (system* "python" "-c" "import numpy; numpy.test()"))))
>> +          (alist-delete
>> +           'check
>> +           %standard-phases))))))
>
> Just (alist-replace 'check ...) ?

Actually the check phase needs to be moved after the install one (see
comment).  That's the reason for this two step approach.

Thanks for the comments.
Regards,
Fede

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

Federico Beffa <beffa@ieee.org> skribis:

> On Sun, Oct 26, 2014 at 7:56 PM, Ludovic Courtès <ludo@gnu.org> wrote:
>> Can these manuals be built from source?
>
> I get error messages that matplotlib is missing.  I started looking at
> matplotlib as well, but I've found that, for the TkAgg back-end, it
> needs TKinter which is part of the standard python libraries and it is
> built if, during the build phase, tk is available. The current python
> package does not build that library (no tk).  In fact, I would like to
> add it to be able to move forward with matplotlib :-)

Heh, OK.  Then can you just summarize that in a comment with TODO, to
give the reader an incentive to look at it?  :-)

> For the other back-ends some packages are missing (python-gtk, Qt, ...)
>
> Another annoying problem is that matplotlib needs numpy, thereby
> creating a dependency loop (although only for the documentation).

If it’s only to build the documentation, we should be able to handle it
easily (by first building a documentation-less variant of numpy, for
instance.)

>> If they can but the process is tedious, then it’s OK to leave it that
>> way with a TODO, and also a comment stating what its license is.
>
> I've not seen any license statement specific for the documentation.
> But, given that the source is distributed with the code, can't we just
> assume it is the same license?

Yes.

>>> +    (inputs
>>> +     `(("python" ,python) ; otherwise ld does not find libpython3.3m
>>
>> This is because Python is not added to ‘LIBRARY_PATH’, right?
>>
>> I think this is fixed by this patch:
>>
>>
>>
>> Can you confirm?
>
> Actually it does not.  The "...-python-numpy-1.9.0-guile-builder"
> still shows that only the python-wrapper is imported. The wrapper does
> not include the libraries.

Ah right.  And what if you again remove Python from ‘inputs’, and add

  #:python ,python

to the arguments?

That means it will use the actual Python 3.x package, not the wrapper,
so everything will be visible.  The downside is that there will be no
‘python’ command, only ‘python3’.

Perhaps the right fix will be to change ‘python-wrapper’ to symlink the
‘lib’ sub-directory of ‘python’.

>> As discussed in the other thread, this should probably be the generic
>> (unoptimized) ATLAS here.
>
> I think that as a temporary situation this is OK.  However, in my
> opinion it does not make much sense to have a separate sub-optimal
> package.  I would like to propose to add specific flags to a single
> atlas package and, once the new "local-build" flag will be available,
> we will remove those flags.

Yeah.

After some more thought, I’ve finally bit the bullet:

  1. Commit 77b0ac9 adds the #:substitutable? flag to gnu-build-system.

  2. Commit f15615b uses it for ATLAS.

It may still make sense to have the non-optimized version for those who
do not need performance and do not want to build locally maybe, WDYT?

>> Use the ‘copy-file’ procedure instead of calling ‘cp’.
>
> The reason that made me use 'cp' instead of 'copy-file' is that the
> documentation states that the return value of the latter is not
> specified and I'm still not familiar with Guile exception handling.
> However, I see many places where 'copy-file' is used without any
> 'catch'.  In principle this should never fail.  If it is OK, I will do
> the same.

‘copy-file’ will throw an exception if something goes wrong; since the
exception is uncaught, the build process will fail.

That’s usually the behavior that we want, so I think it’s best.

>>> +         ;; Tests can only be run after the library has been installed and not
>>> +         ;; within the source directory.
>>> +         (alist-cons-after
>>> +          'install 'check
>>> +          (lambda _
>>> +            (with-directory-excursion "/tmp"
>>> +              (zero? (system* "python" "-c" "import numpy; numpy.test()"))))
>>> +          (alist-delete
>>> +           'check
>>> +           %standard-phases))))))
>>
>> Just (alist-replace 'check ...) ?
>
> Actually the check phase needs to be moved after the install one (see
> comment).  That's the reason for this two step approach.

Ah OK.

Thanks!

Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

On Tue, Oct 28, 2014 at 10:34 AM, Ludovic Courtès <ludo@gnu.org> wrote:
> Ah right.  And what if you again remove Python from ‘inputs’, and add
>
>   #:python ,python
>
> to the arguments?
>
> That means it will use the actual Python 3.x package, not the wrapper,
> so everything will be visible.  The downside is that there will be no
> ‘python’ command, only ‘python3’.

Well, as you say, it does not find the 'python' command and stops with
an error.  I think that fixing the command name is even worse that
having python as input.

>
> Perhaps the right fix will be to change ‘python-wrapper’ to symlink the
> ‘lib’ sub-directory of ‘python’.

It was also suggested to make python a propagating input of the wrapper.
https://lists.gnu.org/archive/html/guix-devel/2014-10/msg00303.html

In my opinion, one of these two fixes would be desirable.

> After some more thought, I’ve finally bit the bullet:
>
>   1. Commit 77b0ac9 adds the #:substitutable? flag to gnu-build-system.
>
>   2. Commit f15615b uses it for ATLAS.

Wow! That was quick. Thanks!

> It may still make sense to have the non-optimized version for those who
> do not need performance and do not want to build locally maybe, WDYT?

Personally my main use of ATLAS is through numpy/scipy. Therefore I
would like to be able to use a good performing ATLAS with those
packages.  If that needs a manual installation step, that's fine with
me.

In principle we could make a second package and add a suffix to the
version number such that the general binary version takes precedence.
To get a locally built library one would then have to give an explicit
ATLAS version to guix.  Or, do you have other approaches in mind?

Regards,
Fede

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

ludo@gnu.org (Ludovic Courtès) skribis:

> After some more thought, I’ve finally bit the bullet:
>
>   1. Commit 77b0ac9 adds the #:substitutable? flag to gnu-build-system.
>
>   2. Commit f15615b uses it for ATLAS.

There was a related bug: ‘guix build atlas’ and similar would tell you
that atlas will be downloaded, but in fact it would be built locally as
intended.

Commit d2d0514 fixes that.

Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

> Personally my main use of ATLAS is through numpy/scipy. Therefore I
> would like to be able to use a good performing ATLAS with those
> packages.  If that needs a manual installation step, that's fine with
> me.
>
> In principle we could make a second package and add a suffix to the
> version number such that the general binary version takes precedence.
> To get a locally built library one would then have to give an explicit
> ATLAS version to guix.  Or, do you have other approaches in mind?

Actually this is not a good idea because then, every time you update
your system, you get the slow version.

In my opinion the fact that guix makes it easy to build a package on
the user machine is really a key advantage over other package
managers/distributions, for a few, somewhat special packages, ATLAS
being one.  It would really be a pity to give up such nice a feature
to spare some build time (ca. 10 min. on my system from 2007).

Regards,
Fede

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

[-- Attachment #1: Type: text/plain, Size: 30 bytes --]

Updated patch.

Regards,
Fede

[-- Attachment #2: 0001-gnu-Add-numpy.patch --]
[-- Type: text/x-patch, Size: 5589 bytes --]

From 6673a353080fd4b5136553624a7d777d243fc9a2 Mon Sep 17 00:00:00 2001
From: Federico Beffa <beffa@fbengineering.ch>
Date: Wed, 29 Oct 2014 20:44:33 +0100
Subject: [PATCH] gnu: Add numpy.

* gnu/packages/python.scm (python-numpy, python2-numpy): New variables.
  (python-wrapper): Add symlink to python lib directory.
---
 gnu/packages/python.scm | 101 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 99 insertions(+), 2 deletions(-)

diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index c34bdc3..c317ed2 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -227,14 +227,18 @@ data types.")
          (begin
            (use-modules (guix build utils))
            (let ((bin (string-append (assoc-ref %outputs "out") "/bin"))
-                 (python (string-append (assoc-ref %build-inputs "python") "/bin/")))
+                 (python (string-append (assoc-ref %build-inputs "python") "/bin/"))
+                 (lib (string-append (assoc-ref %outputs "out") "/lib"))
+                 (python-lib (string-append 
+                              (assoc-ref %build-inputs "python") "/lib/")))
                 (mkdir-p bin)
                 (for-each
                   (lambda (old new)
                     (symlink (string-append python old)
                              (string-append bin "/" new)))
                   `("python3", "pydoc3", "idle3")
-                  `("python",  "pydoc",  "idle"))))))
+                  `("python",  "pydoc",  "idle"))
+                (symlink python-lib lib)))))
     (description (string-append (package-description python)
      "\n\nThis wrapper package provides symbolic links to the python binaries
       without version suffix."))))
@@ -1832,3 +1836,96 @@ sources.")
 
 (define-public python2-sphinx
   (package-with-python2 python-sphinx))
+
+(define python-numpy-reference
+  (origin
+    (method url-fetch)
+    (uri (string-append "mirror://sourceforge/numpy/reference.pdf"))
+    (sha256 
+     (base32
+      "0izsmzazhd042ra6p7as2jhwx8713sc5gpn9s5nifwm68biwn0lp"))))
+
+(define python-numpy-userguide
+  (origin
+    (method url-fetch)
+    (uri (string-append "mirror://sourceforge/numpy/userguide.pdf"))
+    (sha256 
+     (base32
+      "1z2xjlxldv01ad2775k9birg1hkqay5wbi5vmgz3rlr6dczvk77k"))))
+
+(define-public python-numpy
+  (package
+    (name "python-numpy")
+    (version "1.9.0")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (string-append "mirror://sourceforge/numpy"
+                           "/numpy-" version ".tar.gz"))
+       (sha256
+        (base32
+         "1xbzsb84wg32zx5ilgvkdp0lwfqrycwhh5a2q9nx5iwyjpgcxrpp"))))
+    (build-system python-build-system)
+    (inputs
+     `(("python-numpy-userguide" ,python-numpy-userguide)
+       ("python-numpy-reference" ,python-numpy-reference)
+       ("atlas" ,atlas)))
+    (native-inputs
+     `(("python-nose" ,python-nose)  ; used by tests
+       ("gfortran" ,gfortran-4.8)))
+    (outputs '("out" "doc"))         ; doc is ca. 9MB
+    (arguments
+     `(#:phases
+       (alist-cons-before
+        'build 'set-environment-variables
+        (lambda* (#:key outputs inputs 
+                  #:allow-other-keys #:rest args)
+          (let* ((atlas-threaded
+                  (string-append (assoc-ref inputs "atlas") 
+                                 "/lib/libtatlas.so"))
+                 ;; On single core CPUs only the serial library is created.
+                 (atlas-lib
+                  (if (file-exists? atlas-threaded)
+                      atlas-threaded
+                      (string-append (assoc-ref inputs "atlas") 
+                                     "/lib/libsatlas.so"))))
+            (setenv "ATLAS" atlas-lib)))
+        ;; TODO: generate documentation locally, including info files.  The
+        ;; documentation can be built from the source.  However, it needs
+        ;; matplotlib which we currently do not have.
+        (alist-cons-after
+         'install 'install-doc
+         (lambda* (#:key outputs inputs #:allow-other-keys)
+           (let ((doc (string-append (assoc-ref outputs "doc") 
+                                     "/share/doc/" ,name "-" ,version)))
+             (mkdir-p doc)
+             (copy-file (assoc-ref inputs "python-numpy-reference")
+                        (string-append doc "/reference.pdf"))
+             (copy-file (assoc-ref inputs "python-numpy-userguide")
+                        (string-append doc "/userguide.pdf"))))
+         ;; Tests can only be run after the library has been installed and not
+         ;; within the source directory.
+         (alist-cons-after
+          'install 'check
+          (lambda _ 
+            (with-directory-excursion "/tmp"
+              (zero? (system* "python" "-c" "import numpy; numpy.test()"))))
+          (alist-delete 
+           'check 
+           %standard-phases))))))
+    (home-page "http://www.numpy.org/")
+    (synopsis "Fundamental package for scientific computing with Python")
+    (description "NumPy is the fundamental package for scientific computing
+with Python. It contains among other things: a powerful N-dimensional array
+object, sophisticated (broadcasting) functions, tools for integrating C/C++
+and Fortran code, useful linear algebra, Fourier transform, and random number
+capabilities.")
+    (license bsd-3)))
+
+(define python2-numpy-reference python-numpy-reference)
+
+(define python2-numpy-userguide python-numpy-userguide)
+
+(define-public python2-numpy
+  (package-with-python2 python-numpy))
+
-- 
1.8.4

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

Federico Beffa <beffa@ieee.org> skribis:

> In my opinion the fact that guix makes it easy to build a package on
> the user machine is really a key advantage over other package
> managers/distributions, for a few, somewhat special packages, ATLAS
> being one.  It would really be a pity to give up such nice a feature
> to spare some build time (ca. 10 min. on my system from 2007).

Yeah, that’s right.  Let’s forget about the generic version until
someone has a more compelling argument in favor of it.

Thanks,
Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

Federico Beffa <beffa@ieee.org> skribis:

> On Tue, Oct 28, 2014 at 10:34 AM, Ludovic Courtès <ludo@gnu.org> wrote:

[...]

>> Perhaps the right fix will be to change ‘python-wrapper’ to symlink the
>> ‘lib’ sub-directory of ‘python’.
>
> It was also suggested to make python a propagating input of the wrapper.
> https://lists.gnu.org/archive/html/guix-devel/2014-10/msg00303.html

Yes, that’s even better.

Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

Federico Beffa <beffa@ieee.org> skribis:

> From 6673a353080fd4b5136553624a7d777d243fc9a2 Mon Sep 17 00:00:00 2001
> From: Federico Beffa <beffa@fbengineering.ch>
> Date: Wed, 29 Oct 2014 20:44:33 +0100
> Subject: [PATCH] gnu: Add numpy.
>
> * gnu/packages/python.scm (python-numpy, python2-numpy): New variables.
>   (python-wrapper): Add symlink to python lib directory.

Sorry for the delay and for the extra round trip.

Could you make the python-wrapper change a separate patch?

> +++ b/gnu/packages/python.scm
> @@ -227,14 +227,18 @@ data types.")
>           (begin
>             (use-modules (guix build utils))
>             (let ((bin (string-append (assoc-ref %outputs "out") "/bin"))
> -                 (python (string-append (assoc-ref %build-inputs "python") "/bin/")))
> +                 (python (string-append (assoc-ref %build-inputs "python") "/bin/"))
> +                 (lib (string-append (assoc-ref %outputs "out") "/lib"))
> +                 (python-lib (string-append 
> +                              (assoc-ref %build-inputs "python") "/lib/")))
>                  (mkdir-p bin)
>                  (for-each
>                    (lambda (old new)
>                      (symlink (string-append python old)
>                               (string-append bin "/" new)))
>                    `("python3", "pydoc3", "idle3")
> -                  `("python",  "pydoc",  "idle"))))))
> +                  `("python",  "pydoc",  "idle"))
> +                (symlink python-lib lib)))))

The other option would have been to add ‘python’ to ‘propagated-inputs’,
which would also have taken care of sub-directories other than lib/.

As a side-effect, it would also pull in the ‘python3’, ‘pydoc3’,
etc. commands, but I think it cannot hurt.

WDYT?

> +(define python2-numpy-reference python-numpy-reference)
> +
> +(define python2-numpy-userguide python-numpy-userguide)

These variables are unused and not needed, so they can be removed.

Thanks,
Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Andreas Enge]

On Mon, Oct 27, 2014 at 09:26:41PM +0100, Federico Beffa wrote:
> I get error messages that matplotlib is missing.  I started looking at
> matplotlib as well, but I've found that, for the TkAgg back-end, it
> needs TKinter which is part of the standard python libraries and it is
> built if, during the build phase, tk is available.

So we should add tk as an input to the python package?

Andreas

Re: [PATCH 1/2] gnu: Add numpy

[Andreas Enge]

On Tue, Oct 28, 2014 at 10:34:48AM +0100, Ludovic Courtès wrote:
> Ah right.  And what if you again remove Python from ‘inputs’, and add
>   #:python ,python
> to the arguments?
> That means it will use the actual Python 3.x package, not the wrapper,
> so everything will be visible.  The downside is that there will be no
> ‘python’ command, only ‘python3’.
> 
> Perhaps the right fix will be to change ‘python-wrapper’ to symlink the
> ‘lib’ sub-directory of ‘python’.

As already stated, I think we should add python as a propagated input to
python-wrapper. The symlinks are just needed between the differently named
python and python3.

Should we give this a try?

Andreas

Re: [PATCH 1/2] gnu: Add numpy

[Andreas Enge]

On Wed, Oct 29, 2014 at 08:55:52PM +0100, Federico Beffa wrote:
> Updated patch.

> -                 (python (string-append (assoc-ref %build-inputs "python") "/bin/")))
> +                 (python (string-append (assoc-ref %build-inputs "python") "/bin/"))
> +                 (lib (string-append (assoc-ref %outputs "out") "/lib"))
> +                 (python-lib (string-append 
> +                              (assoc-ref %build-inputs "python") "/lib/")))
> -                  `("python",  "pydoc",  "idle"))))))
> +                  `("python",  "pydoc",  "idle"))
> +                (symlink python-lib lib)))))

I do not think we should go this route and instead add the propagated
input "python".

In any case, these should be two separate patches.

Andreas

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

On Wed, Oct 29, 2014 at 11:20 PM, Andreas Enge <andreas@enge.fr> wrote:
> On Mon, Oct 27, 2014 at 09:26:41PM +0100, Federico Beffa wrote:
>> I get error messages that matplotlib is missing.  I started looking at
>> matplotlib as well, but I've found that, for the TkAgg back-end, it
>> needs TKinter which is part of the standard python libraries and it is
>> built if, during the build phase, tk is available.
>
> So we should add tk as an input to the python package?

That would be great!  In this way the python package should build the
Tkinter library which currently is missing.

Re: [PATCH 1/2] gnu: Add numpy

[Andreas Enge]

On Thu, Oct 30, 2014 at 05:56:53PM +0100, Federico Beffa wrote:
> On Wed, Oct 29, 2014 at 11:20 PM, Andreas Enge <andreas@enge.fr> wrote:
> > So we should add tk as an input to the python package?
> 
> That would be great!  In this way the python package should build the
> Tkinter library which currently is missing.

Okay. I think I should do this together with the changes to python-wrapper
in the python branch to be created. For both python-2 and python-3?

Andreas

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

[-- Attachment #1: Type: text/plain, Size: 670 bytes --]

On Wed, Oct 29, 2014 at 11:11 PM, Ludovic Courtès <ludo@gnu.org> wrote:
>> +(define python2-numpy-reference python-numpy-reference)
>> +
>> +(define python2-numpy-userguide python-numpy-userguide)
>
> These variables are unused and not needed, so they can be removed.

Ah, you are right.  I thought that 'package-with-python2' would change
all input prefixes from 'python' to 'python2'.

Please note that this patch will NOT build until the 'python-wrapper'
will be applied (but I tested it again with my previous
'python-wrapper' patch). It is my understanding that Andreas will look
into the 'python-wrapper' patch at a suitable time.

Regards,
Fede

[-- Attachment #2: 0001-gnu-Add-numpy.patch --]
[-- Type: text/x-patch, Size: 4224 bytes --]

From cab16a360bf478aef6feda8b1977d4318f87eeec Mon Sep 17 00:00:00 2001
From: Federico Beffa <beffa@fbengineering.ch>
Date: Thu, 30 Oct 2014 17:10:35 +0100
Subject: [PATCH] gnu: Add numpy.

* gnu/packages/python.scm (python-numpy, python2-numpy): New variables.
---
 gnu/packages/python.scm | 88 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)

diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index c34bdc3..0e2f567 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -1832,3 +1832,91 @@ sources.")
 
 (define-public python2-sphinx
   (package-with-python2 python-sphinx))
+
+(define python-numpy-reference
+  (origin
+    (method url-fetch)
+    (uri (string-append "mirror://sourceforge/numpy/reference.pdf"))
+    (sha256 
+     (base32
+      "0izsmzazhd042ra6p7as2jhwx8713sc5gpn9s5nifwm68biwn0lp"))))
+
+(define python-numpy-userguide
+  (origin
+    (method url-fetch)
+    (uri (string-append "mirror://sourceforge/numpy/userguide.pdf"))
+    (sha256 
+     (base32
+      "1z2xjlxldv01ad2775k9birg1hkqay5wbi5vmgz3rlr6dczvk77k"))))
+
+(define-public python-numpy
+  (package
+    (name "python-numpy")
+    (version "1.9.0")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (string-append "mirror://sourceforge/numpy"
+                           "/numpy-" version ".tar.gz"))
+       (sha256
+        (base32
+         "1xbzsb84wg32zx5ilgvkdp0lwfqrycwhh5a2q9nx5iwyjpgcxrpp"))))
+    (build-system python-build-system)
+    (inputs
+     `(("python-numpy-userguide" ,python-numpy-userguide)
+       ("python-numpy-reference" ,python-numpy-reference)
+       ("atlas" ,atlas)))
+    (native-inputs
+     `(("python-nose" ,python-nose)  ; used by tests
+       ("gfortran" ,gfortran-4.8)))
+    (outputs '("out" "doc"))         ; doc is ca. 9MB
+    (arguments
+     `(#:phases
+       (alist-cons-before
+        'build 'set-environment-variables
+        (lambda* (#:key inputs #:allow-other-keys)
+          (let* ((atlas-threaded
+                  (string-append (assoc-ref inputs "atlas") 
+                                 "/lib/libtatlas.so"))
+                 ;; On single core CPUs only the serial library is created.
+                 (atlas-lib
+                  (if (file-exists? atlas-threaded)
+                      atlas-threaded
+                      (string-append (assoc-ref inputs "atlas") 
+                                     "/lib/libsatlas.so"))))
+            (setenv "ATLAS" atlas-lib)))
+        ;; TODO: generate documentation locally, including info files.  The
+        ;; documentation can be built from the source.  However, it needs
+        ;; matplotlib which we currently do not have.
+        (alist-cons-after
+         'install 'install-doc
+         (lambda* (#:key outputs inputs #:allow-other-keys)
+           (let ((doc (string-append (assoc-ref outputs "doc") 
+                                     "/share/doc/" ,name "-" ,version)))
+             (mkdir-p doc)
+             (copy-file (assoc-ref inputs "python-numpy-reference")
+                        (string-append doc "/reference.pdf"))
+             (copy-file (assoc-ref inputs "python-numpy-userguide")
+                        (string-append doc "/userguide.pdf"))))
+         ;; Tests can only be run after the library has been installed and not
+         ;; within the source directory.
+         (alist-cons-after
+          'install 'check
+          (lambda _ 
+            (with-directory-excursion "/tmp"
+              (zero? (system* "python" "-c" "import numpy; numpy.test()"))))
+          (alist-delete 
+           'check 
+           %standard-phases))))))
+    (home-page "http://www.numpy.org/")
+    (synopsis "Fundamental package for scientific computing with Python")
+    (description "NumPy is the fundamental package for scientific computing
+with Python. It contains among other things: a powerful N-dimensional array
+object, sophisticated (broadcasting) functions, tools for integrating C/C++
+and Fortran code, useful linear algebra, Fourier transform, and random number
+capabilities.")
+    (license bsd-3)))
+
+(define-public python2-numpy
+  (package-with-python2 python-numpy))
+
-- 
1.8.4

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

On Thu, Oct 30, 2014 at 6:03 PM, Andreas Enge <andreas@enge.fr> wrote:
> Okay. I think I should do this together with the changes to python-wrapper
> in the python branch to be created. For both python-2 and python-3?

Yes, please.  That will allow making a matplotlib package for both
versions of python.

Regards,
Fede

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

ludo@gnu.org (Ludovic Courtès) skribis:

> This is because Python is not added to ‘LIBRARY_PATH’, right?
>
> I think this is fixed by this patch:
>
> --- a/guix/build-system/python.scm
> +++ b/guix/build-system/python.scm
> @@ -110,10 +110,11 @@ prepended to the name."
>                                '())
>                          ,@inputs
>  
> +                        ("python" ,python)
> +
>                          ;; Keep the standard inputs of 'gnu-build-system'.
>                          ,@(standard-packages)))
> -         (build-inputs `(("python" ,python)
> -                         ,@native-inputs))
> +         (build-inputs native-inputs)
>           (outputs outputs)
>           (build python-build)
>           (arguments (strip-keyword-arguments private-keywords arguments)))))

FTR, this patch isn’t needed at all.  What mattered here was to fix
‘python-wrapper’ to have a lib/ sub-directory, which Andreas has just
done in ‘wip-python’.

Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

Federico Beffa <beffa@ieee.org> skribis:

> Please note that this patch will NOT build until the 'python-wrapper'
> will be applied (but I tested it again with my previous
> 'python-wrapper' patch). It is my understanding that Andreas will look
> into the 'python-wrapper' patch at a suitable time.

The ‘python-wrapper’ was updated in 3c0f2329.

> From cab16a360bf478aef6feda8b1977d4318f87eeec Mon Sep 17 00:00:00 2001
> From: Federico Beffa <beffa@fbengineering.ch>
> Date: Thu, 30 Oct 2014 17:10:35 +0100
> Subject: [PATCH] gnu: Add numpy.
>
> * gnu/packages/python.scm (python-numpy, python2-numpy): New variables.

IIUC, this patch can now be applied.  Please push it if there’s no
blocker left.

Thanks!

Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

Sorry for taking this long to answer.

On Wed, Nov 19, 2014 at 10:04 AM, Ludovic Courtès <ludo@gnu.org> wrote:
> IIUC, this patch can now be applied.  Please push it if there’s no
> blocker left.

In the mean time version 1.9.1 was released.  With this release I
noticed that the pdf documentation gets updated and therefore the hash
changes and breaks the package.  For this reason I think that, as you
suggested before, we should build the documentation locally.  As
mentioned before, this requires matplotlib and matplotlib requires
numpy.  So I will make a numpy package without documentation and use
it as input to matplotlib as you suggested.  I'm thinking of calling
this 'numpy-bootstrap' and of not exporting it (use 'define' instead
of 'define-public').  I now got matplotlib building and able to
generate its own documentation, but would like to add the pygtk
backend before supplying a patch for review.

Regards,
Fede

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

Federico Beffa <beffa@ieee.org> skribis:

> In the mean time version 1.9.1 was released.  With this release I
> noticed that the pdf documentation gets updated and therefore the hash
> changes and breaks the package.

Oh, OK.

> For this reason I think that, as you suggested before, we should build
> the documentation locally.  As mentioned before, this requires
> matplotlib and matplotlib requires numpy.  So I will make a numpy
> package without documentation and use it as input to matplotlib as you
> suggested.  I'm thinking of calling this 'numpy-bootstrap' and of not
> exporting it (use 'define' instead of 'define-public').  I now got
> matplotlib building and able to generate its own documentation, but
> would like to add the pygtk backend before supplying a patch for
> review.

Sounds like a good plan.

Thanks,
Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

[-- Attachment #1: Type: text/plain, Size: 283 bytes --]

On Sat, Nov 22, 2014 at 6:21 PM, Ludovic Courtès <ludo@gnu.org> wrote:
> Sounds like a good plan.

Please find attached the first step with initial (bootstrap) numpy
versions.  I've tested that the packages build before changing
"define-public" to "define".

Regards,
Fede

[-- Attachment #2: 0001-gnu-Add-numpy-bootstrap.patch --]
[-- Type: text/x-patch, Size: 3420 bytes --]

From 9be3873522b2c4b642a467e421bd8684068feb40 Mon Sep 17 00:00:00 2001
From: Federico Beffa <beffa@fbengineering.ch>
Date: Mon, 24 Nov 2014 20:51:29 +0100
Subject: [PATCH] gnu: Add numpy-bootstrap.

* gnu/packages/python.scm (python-numpy-bootstrap, python2-numpy-bootstrap):
  New file-local variables.
---
 gnu/packages/python.scm | 59 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index 3786604..7967201 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -35,6 +35,8 @@
   #:use-module (gnu packages readline)
   #:use-module (gnu packages openssl)
   #:use-module (gnu packages elf)
+  #:use-module (gnu packages maths)
+  #:use-module (gnu packages gcc)
   #:use-module (gnu packages pkg-config)
   #:use-module (gnu packages databases)
   #:use-module (gnu packages zip)
@@ -1873,3 +1875,60 @@ writing C extensions for Python as easy as Python itself.")
     (name "python2-cython")
     (inputs
      `(("python-2" ,python-2))))) ; this is not automatically changed
+
+;; This version of numpy is missing the documentation and is only used to
+;; build matplotlib which is required to build numpy's documentation.
+(define python-numpy-bootstrap
+  (package
+    (name "python-numpy-bootstrap")
+    (version "1.9.1")
+    (source
+     (origin
+       (method url-fetch)
+       (uri (string-append "mirror://sourceforge/numpy"
+                           "/numpy-" version ".tar.gz"))
+       (sha256
+        (base32
+         "070ybfvpgfmiz2hs94x445hvkh9dh52nyi0m8jp5kdihgvhbnx80"))))
+    (build-system python-build-system)
+    (inputs
+     `(("python-nose" ,python-nose)
+       ("atlas" ,atlas)))
+    (native-inputs
+     `(("gfortran" ,gfortran-4.8)))
+    (arguments
+     `(#:phases
+       (alist-cons-before
+        'build 'set-environment-variables
+        (lambda* (#:key inputs #:allow-other-keys)
+          (let* ((atlas-threaded
+                  (string-append (assoc-ref inputs "atlas") 
+                                 "/lib/libtatlas.so"))
+                 ;; On single core CPUs only the serial library is created.
+                 (atlas-lib
+                  (if (file-exists? atlas-threaded)
+                      atlas-threaded
+                      (string-append (assoc-ref inputs "atlas") 
+                                     "/lib/libsatlas.so"))))
+            (setenv "ATLAS" atlas-lib)))
+        ;; Tests can only be run after the library has been installed and not
+        ;; within the source directory.
+        (alist-cons-after
+         'install 'check
+         (lambda _ 
+           (with-directory-excursion "/tmp"
+             (zero? (system* "python" "-c" "import numpy; numpy.test()"))))
+         (alist-delete 
+          'check 
+          %standard-phases)))))
+    (home-page "http://www.numpy.org/")
+    (synopsis "Fundamental package for scientific computing with Python")
+    (description "NumPy is the fundamental package for scientific computing
+with Python. It contains among other things: a powerful N-dimensional array
+object, sophisticated (broadcasting) functions, tools for integrating C/C++
+and Fortran code, useful linear algebra, Fourier transform, and random number
+capabilities.")
+    (license bsd-3)))
+
+(define python2-numpy-bootstrap
+  (package-with-python2 python-numpy-bootstrap))
-- 
1.8.4

Re: [PATCH 1/2] gnu: Add numpy

[Ludovic Courtès]

Federico Beffa <beffa@ieee.org> skribis:

> Please find attached the first step with initial (bootstrap) numpy
> versions.  I've tested that the packages build before changing
> "define-public" to "define".

OK.  Note that you can also test once it’s private, with

  guix build -e '(@@ (gnu packages python) python-numpy-bootstrap)'

> From 9be3873522b2c4b642a467e421bd8684068feb40 Mon Sep 17 00:00:00 2001
> From: Federico Beffa <beffa@fbengineering.ch>
> Date: Mon, 24 Nov 2014 20:51:29 +0100
> Subject: [PATCH] gnu: Add numpy-bootstrap.
>
> * gnu/packages/python.scm (python-numpy-bootstrap, python2-numpy-bootstrap):
>   New file-local variables.

LGTM, please push.

Thank you,
Ludo’.

Re: [PATCH 1/2] gnu: Add numpy

[Federico Beffa]

On Mon, Nov 24, 2014 at 10:06 PM, Ludovic Courtès <ludo@gnu.org> wrote:
> OK.  Note that you can also test once it’s private, with
>
>   guix build -e '(@@ (gnu packages python) python-numpy-bootstrap)'
>

OK, thanks. Will keep in mind for the future.

> LGTM, please push.

Pushed.Thanks.
Fede