From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([208.118.235.92]:36448) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gejMI-0006f9-7D for guix-patches@gnu.org; Wed, 02 Jan 2019 11:27:08 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gejME-0005qU-TQ for guix-patches@gnu.org; Wed, 02 Jan 2019 11:27:06 -0500 Received: from debbugs.gnu.org ([208.118.235.43]:47247) by eggs.gnu.org with esmtps (TLS1.0:RSA_AES_128_CBC_SHA1:16) (Exim 4.71) (envelope-from ) id 1gejME-0005qD-OA for guix-patches@gnu.org; Wed, 02 Jan 2019 11:27:02 -0500 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1gejME-0002PQ-Ig for guix-patches@gnu.org; Wed, 02 Jan 2019 11:27:02 -0500 Subject: [bug#33952] [PATCH 14/14] gnu: Add tensorflow. Resent-Message-ID: From: Ricardo Wurmus Date: Wed, 2 Jan 2019 17:25:12 +0100 Message-ID: <20190102162512.28727-5-ricardo.wurmus@mdc-berlin.de> In-Reply-To: <20190102162512.28727-1-ricardo.wurmus@mdc-berlin.de> References: <20190102161807.28297-1-ricardo.wurmus@mdc-berlin.de> <20190102162512.28727-1-ricardo.wurmus@mdc-berlin.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Content-Type: text/plain List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: guix-patches-bounces+kyle=kyleam.com@gnu.org Sender: "Guix-patches" To: 33952@debbugs.gnu.org Cc: Ricardo Wurmus * gnu/packages/machine-learning.scm (tensorflow): New variable. --- gnu/packages/machine-learning.scm | 482 ++++++++++++++++++++++++++++++ 1 file changed, 482 insertions(+) diff --git a/gnu/packages/machine-learning.scm b/gnu/packages/machine-learning.scm index 870ca7c4d..4664b2099 100644 --- a/gnu/packages/machine-learning.scm +++ b/gnu/packages/machine-learning.scm @@ -46,6 +46,7 @@ #:use-module (gnu packages check) #:use-module (gnu packages compression) #:use-module (gnu packages cran) + #:use-module (gnu packages databases) #:use-module (gnu packages dejagnu) #:use-module (gnu packages gcc) #:use-module (gnu packages image) @@ -851,6 +852,487 @@ applications and browsers to backend services.") with the HTTP/2-based RPC framework gRPC.") (license license:asl2.0))) +(define-public tensorflow + (package + (name "tensorflow") + (version "1.9.0") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/tensorflow/tensorflow.git") + (commit (string-append "v" version)))) + (file-name (string-append "tensorflow-" version "-checkout")) + (sha256 + (base32 + "0a9kwha395g3wgxfwln5j8vn9nkspmd75xldrlqdq540w996g8xa")))) + (build-system cmake-build-system) + (arguments + `(#:tests? #f ; no "check" target + #:build-type "Release" + #:configure-flags + (let ((protobuf (assoc-ref %build-inputs "protobuf")) + (snappy (assoc-ref %build-inputs "snappy")) + (sqlite (assoc-ref %build-inputs "sqlite"))) + (list ;; TODO: Use protobuf from Guix + ;; (string-append "-Dprotobuf_STATIC_LIBRARIES=" + ;; protobuf "/lib/libprotobuf.a") + (string-append "-DPROTOBUF_PROTOC_EXECUTABLE=" + protobuf "/bin/protoc") + ;; TODO: Use snappy from Guix + ;; (string-append "-Dsnappy_STATIC_LIBRARIES=" + ;; snappy "/lib/libsnappy.a") + ;; (string-append "-Dsnappy_HEADERS=" + ;; snappy "/include/snappy.h") + ;; TODO: this is ignored. Should be a definition in code. + ;;"-DTF_USE_SNAPPY" + ;; Use sqlite from Guix + (string-append "-Dsqlite_STATIC_LIBRARIES=" + sqlite "/lib/libsqlite.a") + (string-append "-Dsqlite_HEADERS=" + sqlite "/include/sqlite3.h " + sqlite "/include/sqlite3ext.h") + ;; Use system libraries wherever possible. Currently, this + ;; only affects zlib. + "-Dsystemlib_ALL=ON" + + ;;"-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true" + "-Dtensorflow_ENABLE_POSITION_INDEPENDENT_CODE=ON" + "-Dtensorflow_BUILD_SHARED_LIB=ON" + "-Dtensorflow_OPTIMIZE_FOR_NATIVE_ARCH=OFF" + "-Dtensorflow_ENABLE_SSL_SUPPORT=OFF" + "-Dtensorflow_BUILD_CONTRIB_KERNELS=OFF")) + #:make-flags + (list "CC=gcc") + #:modules ((ice-9 ftw) + (guix build utils) + (guix build cmake-build-system)) + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'set-source-file-times-to-1980 + ;; At the end of the tf_python_build_pip_package target, a ZIP + ;; archive should be generated via bdist_wheel, but it fails with + ;; "ZIP does not support timestamps before 1980". Luckily, + ;; SOURCE_DATE_EPOCH is respected, which we set to some time in + ;; 1980. + (lambda _ (setenv "SOURCE_DATE_EPOCH" "315532800") #t)) + ;; See https://github.com/tensorflow/tensorflow/issues/20517#issuecomment-406373913 + (add-after 'unpack 'python3.7-compatibility + (lambda _ + (substitute* '("tensorflow/python/eager/pywrap_tfe_src.cc" + "tensorflow/python/lib/core/ndarray_tensor.cc" + "tensorflow/python/lib/core/py_func.cc") + (("PyUnicode_AsUTF8") "(char *)PyUnicode_AsUTF8")) + (substitute* "tensorflow/c/eager/c_api.h" + (("unsigned char async") + "unsigned char is_async")) + + ;; Remove dependency on tensorboard, a complicated but probably + ;; optional package. + (substitute* "tensorflow/tools/pip_package/setup.py" + ((".*'tensorboard >.*") "")) + #t)) + (add-after 'python3.7-compatibility 'chdir + (lambda _ (chdir "tensorflow/contrib/cmake") #t)) + (add-after 'chdir 'disable-downloads + (lambda* (#:key inputs #:allow-other-keys) + (substitute* (find-files "external" "\\.cmake$") + (("GIT_REPOSITORY.*") "") + (("GIT_TAG.*") "") + (("PREFIX ") + "DOWNLOAD_COMMAND \"\"\nPREFIX ")) + + ;; Use packages from Guix + (let ((grpc (assoc-ref inputs "grpc"))) + (substitute* "CMakeLists.txt" + ;; Sqlite + (("include\\(sqlite\\)") "") + (("\\$\\{sqlite_STATIC_LIBRARIES\\}") + (string-append (assoc-ref inputs "sqlite") + "/lib/libsqlite3.so")) + (("sqlite_copy_headers_to_destination") "") + + ;; PNG + (("include\\(png\\)") "") + (("\\$\\{png_STATIC_LIBRARIES\\}") + (string-append (assoc-ref inputs "libpng") + "/lib/libpng16.so")) + (("png_copy_headers_to_destination") "") + + ;; JPEG + (("include\\(jpeg\\)") "") + (("\\$\\{jpeg_STATIC_LIBRARIES\\}") + (string-append (assoc-ref inputs "libjpeg") + "/lib/libjpeg.so")) + (("jpeg_copy_headers_to_destination") "") + + ;; GIF + (("include\\(gif\\)") "") + (("\\$\\{gif_STATIC_LIBRARIES\\}") + (string-append (assoc-ref inputs "giflib") + "/lib/libgif.so")) + (("gif_copy_headers_to_destination") "") + + ;; lmdb + (("include\\(lmdb\\)") "") + (("\\$\\{lmdb_STATIC_LIBRARIES\\}") + (string-append (assoc-ref inputs "lmdb") + "/lib/liblmdb.so")) + (("lmdb_copy_headers_to_destination") "") + + ;; TODO: the "protobuf" project is depended on by other + ;; projects in the cmake files, so removing it is a little + ;; tricky. + ;;(("include\\(protobuf\\)") "") + ;; (("\\$\\{protobuf_STATIC_LIBRARIES\\}") + ;; (string-append (assoc-ref inputs "protobuf:static") + ;; "/lib/libprotobuf.a")) + ;;(("protobuf_copy_headers_to_destination") "") + ;;(("^ +protobuf$") "") + + ;; gRPC + ;; TODO: the gRPC sources are needed by protobuf + ;; (("include\\(grpc\\)") "") + ;; (("\\$\\{grpc_STATIC_LIBRARIES\\}") + ;; (string-append grpc "/lib/libaddress_sorting.a " + ;; grpc "/lib/libgpr.a " + ;; grpc "/lib/libgrpc++.a " + ;; grpc "/lib/libgrpc.a " + ;; grpc "/lib/libgrpc++_cronet.a " + ;; grpc "/lib/libgrpc_cronet.a " + ;; grpc "/lib/libgrpc++_error_details.a " + ;; grpc "/lib/libgrpc_plugin_support.a " + ;; grpc "/lib/libgrpcpp_channelz.a " + ;; grpc "/lib/libgrpc++_reflection.a " + ;; grpc "/lib/libgrpc++_unsecure.a " + ;; grpc "/lib/libgrpc_unsecure.a ")) + ;; (("list\\(APPEND tensorflow_EXTERNAL_DEPENDENCIES grpc\\)") "") + )) + + ;; ;; Remove dependency on bundled grpc + ;; (substitute* "tf_core_distributed_runtime.cmake" + ;; (("tf_core_cpu grpc") "tf_core_cpu")) + ;; (substitute* "tf_tools.cmake" + ;; (("add_dependencies\\(\\$\\{proto_text\\} grpc\\)") "")) + + (substitute* "external/grpc.cmake" + (("\\$\\{CMAKE_CURRENT_BINARY_DIR\\}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a") + (string-append (assoc-ref inputs "c-ares") + "/lib/libcares.so"))) + #t)) + (add-after 'configure 'unpack-third-party + (lambda* (#:key inputs #:allow-other-keys) + ;; This is needed to configure bundled packages properly. + (setenv "CONFIG_SHELL" (which "bash")) + (for-each + (lambda (name) + (let* ((what (assoc-ref inputs (string-append name "-src"))) + (name* (string-map (lambda (c) + (if (char=? c #\-) + #\_ c)) name)) + (where (string-append "../build/" name* "/src/" name*))) + (cond + ((string-suffix? ".zip" what) + (mkdir-p where) + (with-directory-excursion where + (invoke "unzip" what))) + ((string-suffix? ".tar.gz" what) + (mkdir-p where) + (invoke "tar" "xf" what + "-C" where "--strip-components=1")) + (else + ;; TODO: merge with "where" + (let ((parent (string-append "../build/" name* "/src/"))) + (mkdir-p parent) + (with-directory-excursion parent + (when (file-exists? name*) + (delete-file-recursively name*)) + (copy-recursively what name*) + (map make-file-writable + (find-files name* ".*")))))))) + (list "boringssl" + "cub" + "double-conversion" + "eigen" + "farmhash" + "fft2d" + "grpc" ; TODO: the sources need to be available for protobuf magic + "highwayhash" + "jsoncpp" + "nsync" + "protobuf" + "re2" + "snappy")) + + (rename-file "../build/cub/src/cub/cub-1.8.0/" + "../build/cub/src/cub/cub/") + + ;; gRPC dependencies: use installed packages instead of bundled + ;; sources. + (substitute* "../build/grpc/src/grpc/CMakeLists.txt" + (("set\\(gRPC_ZLIB_PROVIDER \"module\"") + "set(gRPC_ZLIB_PROVIDER \"package\"") + (("set\\(gRPC_CARES_PROVIDER \"module\"") + "set(gRPC_CARES_PROVIDER \"package\"") + (("set\\(gRPC_SSL_PROVIDER \"module\"") + "set(gRPC_SSL_PROVIDER \"package\"") + (("set\\(gRPC_PROTOBUF_PROVIDER \"module\"") + "set(gRPC_PROTOBUF_PROVIDER \"package\"")) + #t)) + (add-after 'unpack 'fix-python-build + (lambda* (#:key outputs #:allow-other-keys) + ;; Ensure that all Python dependencies can be found at build time. + (substitute* "tensorflow/contrib/cmake/tf_python.cmake" + (("PYTHONPATH=\\$\\{CMAKE_CURRENT_BINARY_DIR\\}/tf_python" m) + (string-append m ":" (getenv "PYTHONPATH")))) + + ;; Correct the RUNPATH of ops libraries generated for Python. + ;; TODO: this doesn't work :( + ;; /gnu/store/...-tensorflow-1.9.0/lib/python3.7/site-packages/tensorflow/contrib/seq2seq/python/ops/lib_beam_search_ops.so: + ;; warning: RUNPATH contains bogus entries: ("/tmp/guix-build-tensorflow-1.9.0.drv-0/source/tensorflow/contrib/build") + ;; /gnu/store/...-tensorflow-1.9.0/lib/python3.7/site-packages/tensorflow/contrib/seq2seq/python/ops/lib_beam_search_ops.so: + ;; error: depends on 'libpywrap_tensorflow_internal.so', which + ;; cannot be found in RUNPATH ... + (substitute* "tensorflow/contrib/cmake/tf_cc_ops.cmake" + (("set_target_properties.*") + (string-append "set_target_properties(${_AT_TARGET} PROPERTIES \ +COMPILE_FLAGS ${target_compile_flags} \ +INSTALL_RPATH_USE_LINK_PATH TRUE \ +INSTALL_RPATH " (assoc-ref outputs "out") "/lib)\n"))) + #t)) + (add-after 'unpack 'find-eigen-headers + (lambda* (#:key inputs #:allow-other-keys) + ;; Ensure that Eigen headers can be found + (setenv "CPLUS_INCLUDE_PATH" + (string-append (getenv "CPLUS_INCLUDE_PATH") + ":" + (assoc-ref inputs "eigen") + "/include/eigen3")) + #t)) + (add-after 'build 'build-pip-package + (lambda* (#:key outputs #:allow-other-keys) + (setenv "LDFLAGS" + (string-append "-Wl,-rpath=" + (assoc-ref outputs "out") "/lib")) + (invoke "make" "tf_python_build_pip_package") + #t)) + (add-after 'build-pip-package 'install-python + (lambda* (#:key outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (wheel (car (find-files "../build/tf_python/dist/" "\\.whl$")))) + (invoke "python" "-m" "pip" "install" wheel + (string-append "--prefix=" out)) + + ;; XXX: broken RUNPATH, see fix-python-build phase. + (delete-file (string-append out "/lib/python3.7/site-packages/tensorflow/contrib/seq2seq/python/ops/lib_beam_search_ops.so")) + #t)))))) + (native-inputs + `(("pkg-config" ,pkg-config) + ("protobuf" ,protobuf-next) ; protoc + ;; The commit hashes and URLs for third-party source code are taken + ;; from "tensorflow/workspace.bzl". + ("boringssl-src" + ,(let ((commit "ee7aa02") + (revision "1")) + (origin + (method git-fetch) + (uri (git-reference + (url "https://boringssl.googlesource.com/boringssl") + (commit commit))) + (file-name (string-append "boringssl-0-" revision + (string-take commit 7) + "-checkout")) + (sha256 + (base32 + "1jf693q0nw0adsic6cgmbdx6g7wr4rj4vxa8j1hpn792fqhd8wgw"))))) + ("cub-src" + ,(let ((version "1.8.0")) + (origin + (method url-fetch) + (uri (string-append "https://mirror.bazel.build/github.com/NVlabs/" + "cub/archive/" version ".zip")) + (file-name (string-append "cub-" version ".zip")) + (sha256 + (base32 + "1hsqikqridb90dkxkjr2918dcry6pfh46ccnwrzawl56aamhdykb"))))) + ("double-conversion-src" + ,(let ((commit "5664746") + (revision "1")) + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/google/double-conversion.git") + (commit commit))) + (file-name (string-append "double-conversion-0-" revision + (string-take commit 7) + "-checkout")) + (sha256 + (base32 + "1h5lppqqxcvdg5jq42i5msgwx20ryij3apvmndflngrgdpc04gn1"))))) + ("eigen-src" + ,(let ((version "fd6845384b86")) + (origin + (method url-fetch) + (uri (string-append "https://mirror.bazel.build/bitbucket.org/" + "eigen/eigen/get/" version ".tar.gz")) + (file-name (string-append "eigen-" version ".tar.gz")) + (sha256 + (base32 + "1aan4mvan8i5xsx4ivyqlg8ckbnnaqjw6i9ad8my992gg1fl2mnr"))))) + ("farmhash-src" + ,(let ((commit "816a4ae622e964763ca0862d9dbd19324a1eaf45")) + (origin + (method url-fetch) + (uri (string-append + "https://mirror.bazel.build/github.com/google/farmhash/archive/" + commit ".tar.gz")) + (file-name (string-append "farmhash-0-" + (string-take commit 7) + ".tar.gz")) + (sha256 + (base32 + "185b2xdxl4d4cnsnv6abg8s22gxvx8673jq2yaq85bz4cdy58q35"))))) + ;; The license notice on the home page at + ;; http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html says: + ;; Copyright Takuya OOURA, 1996-2001 + ;; + ;; You may use, copy, modify and distribute this code for any purpose + ;; (include commercial use) and without fee. Please refer to this + ;; package when you modify this code. + ;; + ;; We take the identical tarball from the Bazel mirror, because the URL + ;; at the home page is not versioned and might change. + ("fft2d-src" + ,(origin + (method url-fetch) + (uri "https://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz") + (file-name "fft2d.tar.gz") + (sha256 + (base32 + "15jjkfvhqvl2c0753d2di8hz0pyzn598g74wqy79awdrf1y67fsj")))) + ("grpc-src" + ,(let ((version "d184fa229d75d336aedea0041bd59cb93e7e267f")) + (origin + (method url-fetch) + (uri (string-append + "https://mirror.bazel.build/" + "github.com/grpc/grpc/archive/" + version".tar.gz")) + (file-name (string-append "grpc-" version ".tar.gz")) + (sha256 + (base32 + "0wsn0yvwnc08i9cq76083kgfv9k8q1wafnap6gvn32ki1qqk2nw9"))))) + ("highwayhash-src" + ,(let ((commit "be5edafc2e1a455768e260ccd68ae7317b6690ee") + (revision "1")) + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/google/highwayhash.git") + (commit commit))) + (file-name (string-append "highwayhash-0-" revision + (string-take commit 7) + "-checkout")) + (sha256 + (base32 + "154jwf98cyy54hldr94pgjn85zynly3abpnc1avmb8a18lzwjyb6"))))) + ("jsoncpp-src" + ,(let ((commit "4356d9b") + (revision "1")) + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/open-source-parsers/jsoncpp.git") + (commit commit))) + (file-name (string-append "jsoncpp-0-" revision + (string-take commit 7) + "-checkout")) + (sha256 + (base32 + "1anixxs5nwqknmcdxjd9zii5x0z7jx5qy011narjp2vxid59dzqa"))))) + ("nsync-src" + ,(let ((version "0559ce013feac8db639ee1bf776aca0325d28777") + (revision "1")) + (origin + (method url-fetch) + (uri (string-append "https://mirror.bazel.build/" + "github.com/google/nsync/archive/" + version ".tar.gz")) + (file-name (string-append "nsync-0." revision + "-" (string-take version 7) + ".tar.gz")) + (sha256 + (base32 + "0qdkyqym34x739mmzv97ah5r7ph462v5xkxqxvidmcfqbi64b132"))))) + ("protobuf-src" + ,(let ((version "396336eb961b75f03b25824fe86cf6490fb75e3a")) + (origin + (method url-fetch) + (uri (string-append "https://mirror.bazel.build/" + "github.com/google/protobuf/archive/" + version ".tar.gz")) + (file-name (string-append "protobuf-" version ".tar.gz")) + (sha256 + (base32 + "1qsr5hgmmikshrg3035q6klbpv945lxfz0h8xhry4aj7rxx90vc4"))))) + ("re2-src" + ,(let ((commit "e7efc48") + (revision "1")) + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/google/re2") + (commit commit))) + (file-name (string-append "re2-0-" revision + (string-take commit 7) + "-checkout")) + (sha256 + (base32 + "161g9841rjfsy5pn52fcis0s9hdr7rxvb06pad38j5rppfihvign"))))) + ("snappy-src" + ,(let ((version "1.1.7")) + (origin + (method url-fetch) + (uri (string-append "https://mirror.bazel.build/" + "github.com/google/snappy/archive/" + version ".tar.gz")) + (file-name (string-append "snappy-" version ".tar.gz")) + (sha256 + (base32 + "1m7rcdqzkys5lspj8jcsaah8w33zh28s771bw0ga2lgzfgl05yix"))))) + ("googletest" ,googletest) + ("swig" ,swig) + ("unzip" ,unzip))) + (propagated-inputs + `(("python-absl-py" ,python-absl-py) + ("python-astor" ,python-astor) + ("python-gast" ,python-gast) + ("python-grpcio" ,python-grpcio) + ("python-numpy" ,python-numpy) + ("python-protobuf" ,python-protobuf) + ("python-six" ,python-six) + ("python-termcolo" ,python-termcolor) + ("python-wheel" ,python-wheel))) + (inputs + `(("c-ares" ,c-ares-next) ; for grpc + ("eigen" ,eigen) + ("gemmlowp" ,gemmlowp-for-tensorflow) + ("lmdb" ,lmdb) + ("libjpeg" ,libjpeg) + ("libpng" ,libpng) + ("giflib" ,giflib) + ("sqlite" ,sqlite) + ("python" ,python-wrapper) + ("zlib" ,zlib))) + (home-page "https://tensorflow.org") + (synopsis "Machine learning framework") + (description + "TensorFlow is a software library for high performance numerical +computation. Its flexible architecture allows easy deployment of computation +across a variety of platforms, and from desktops to clusters of servers to +mobile and edge devices.") + (license license:asl2.0))) + (define-public dlib (package (name "dlib") -- 2.19.1