Merge branch 'master' into core-updates-frozen

 Conflicts:
	gnu/packages/algebra.scm
	gnu/packages/games.scm
	gnu/packages/golang.scm
	gnu/packages/kerberos.scm
	gnu/packages/mail.scm
	gnu/packages/python.scm
	gnu/packages/ruby.scm
	gnu/packages/scheme.scm
	gnu/packages/tex.scm
	gnu/packages/tls.scm
	gnu/packages/version-control.scm

1 files changed, 412 insertions, 26 deletions

diff --git a/gnu/packages/machine-learning.scm b/gnu/packages/machine-learning.scm
index a266c70886..7e7bb89943 100644
--- a/gnu/packages/machine-learning.scm
+++ b/gnu/packages/machine-learning.scm
@@ -56,6 +56,7 @@
   #:use-module (gnu packages check)
   #:use-module (gnu packages compression)
   #:use-module (gnu packages cmake)
+  #:use-module (gnu packages cpp)
   #:use-module (gnu packages cran)
   #:use-module (gnu packages databases)
   #:use-module (gnu packages dejagnu)
@@ -64,10 +65,12 @@
   #:use-module (gnu packages graphviz)
   #:use-module (gnu packages gstreamer)
   #:use-module (gnu packages image)
+  #:use-module (gnu packages libffi)
   #:use-module (gnu packages linux)
   #:use-module (gnu packages llvm)
   #:use-module (gnu packages maths)
   #:use-module (gnu packages mpi)
+  #:use-module (gnu packages ninja)
   #:use-module (gnu packages ocaml)
   #:use-module (gnu packages onc-rpc)
   #:use-module (gnu packages parallel)
@@ -86,6 +89,7 @@
   #:use-module (gnu packages statistics)
   #:use-module (gnu packages sqlite)
   #:use-module (gnu packages swig)
+  #:use-module (gnu packages tls)
   #:use-module (gnu packages web)
   #:use-module (gnu packages xml)
   #:use-module (gnu packages xorg)
@@ -660,6 +664,50 @@ standard data types.")
   ;; headers, hence the name change.
   (deprecated-package "python-onnx" onnx))
 
+(define-public onnx-optimizer
+  (package
+    (name "onnx-optimizer")
+    ;; Note: 0.2.x is *more* recent than 1.5.0.
+    (version "0.2.6")
+    (home-page "https://github.com/onnx/optimizer")
+    (source (origin
+              (method git-fetch)
+              (uri (git-reference
+                    (url home-page)
+                    (commit (string-append "v" version))))
+              (sha256
+               (base32
+                "1wkqqdxcxpfbf8zpbdfdd3zz5jkw775g31gyykj11z4y6pp659l6"))
+              (file-name (git-file-name name version))
+              (patches (search-patches "onnx-optimizer-system-library.patch"))
+              (modules '((guix build utils)))
+              (snippet '(delete-file-recursively "third_party"))))
+    (build-system python-build-system)
+    (arguments (package-arguments onnx))          ;reuse build system tweaks
+    (native-inputs
+     `(("cmake" ,cmake)
+       ("python-pytest" ,python-pytest)
+       ("python-pytest-runner" ,python-pytest-runner)
+       ("python-nbval" ,python-nbval)
+       ("python-coverage" ,python-coverage)))
+    (inputs
+     `(("onnx" ,onnx)
+       ("protobuf" ,protobuf)
+       ("pybind11" ,pybind11)))
+    (propagated-inputs
+     `(("python-numpy" ,python-numpy)))
+    (synopsis "Library to optimize ONNX models")
+    (description
+     "This package provides a C++ and Python library for performing arbitrary
+optimizations on ONNX models, as well as a growing list of prepackaged
+optimization passes.
+
+Not all possible optimizations can be directly implemented on ONNX graphs---
+some will need additional backend-specific information---but many can, and the
+aim is to provide all such passes along with ONNX so that they can be re-used
+with a single function call.")
+    (license license:expat)))
+
 (define-public rxcpp
   (package
     (name "rxcpp")
@@ -701,23 +749,22 @@ synchronization, thread-safety, concurrent data structures, and non-blocking
 I/O.")
     (license license:asl2.0)))
 
-(define-public gemmlowp-for-tensorflow
-  ;; The commit hash is taken from "tensorflow/workspace.bzl".
-  (let ((commit "38ebac7b059e84692f53e5938f97a9943c120d98")
-        (revision "2"))
+
+(define-public gemmlowp
+  (let ((commit "f9959600daa42992baace8a49544a00a743ce1b6")
+        (version "0.1")
+        (revision "1"))
     (package
       (name "gemmlowp")
-      (version (git-version "0" revision commit))
+      (version (git-version version revision commit))
+      (home-page "https://github.com/google/gemmlowp")
       (source (origin
-                (method url-fetch)
-                (uri (string-append "https://mirror.bazel.build/"
-                                    "github.com/google/gemmlowp/archive/"
-                                    commit ".zip"))
-                (file-name (string-append "gemmlowp-" version ".zip"))
+                (method git-fetch)
+                (uri (git-reference (url home-page) (commit commit)))
+                (file-name (git-file-name name version))
                 (sha256
                  (base32
-                  "0n56s2g8hrssm4w8qj1v58gfm56a04n9v992ixkmvk6zjiralzxq"))))
-      (build-system cmake-build-system)
+                  "1hzfhlhzcb827aza6a7drydc67dw5fm3qfqilb9ibskan8dsf0c6"))))
       (arguments
        `(#:configure-flags
          (list ,@(match (%current-system)
@@ -737,18 +784,15 @@ I/O.")
                       (inc (string-append out "/include/")))
                  (install-file "../build/libeight_bit_int_gemm.so" lib)
                  (for-each (lambda (dir)
-                             (let ((target (string-append inc "/" dir)))
-                               (mkdir-p target)
+                             (let ((target
+                                    (string-append inc "/gemmlowp/" dir)))
                                (for-each (lambda (h)
                                            (install-file h target))
                                          (find-files (string-append "../" dir)
                                                      "\\.h$"))))
                            '("meta" "profiling" "public" "fixedpoint"
-                             "eight_bit_int_gemm" "internal"))
-                 #t))))))
-      (native-inputs
-       `(("unzip" ,unzip)))
-      (home-page "https://github.com/google/gemmlowp")
+                             "eight_bit_int_gemm" "internal"))))))))
+      (build-system cmake-build-system)
       (synopsis "Small self-contained low-precision GEMM library")
       (description
        "This is a small self-contained low-precision @dfn{general matrix
@@ -758,6 +802,46 @@ at most 8 bits.  To avoid overflow, results are internally accumulated on more
 than 8 bits, and at the end only some significant 8 bits are kept.")
       (license license:asl2.0))))
 
+(define-public gemmlowp-for-tensorflow
+  ;; The commit hash is taken from "tensorflow/workspace.bzl".
+  (let ((commit "38ebac7b059e84692f53e5938f97a9943c120d98")
+        (revision "2"))
+    (package
+      (inherit gemmlowp)
+      (version (git-version "0" revision commit))
+      (source (origin
+                (method url-fetch)
+                (uri (string-append "https://mirror.bazel.build/"
+                                    "github.com/google/gemmlowp/archive/"
+                                    commit ".zip"))
+                (file-name (string-append "gemmlowp-" version ".zip"))
+                (sha256
+                 (base32
+                  "0n56s2g8hrssm4w8qj1v58gfm56a04n9v992ixkmvk6zjiralzxq"))))
+      (arguments
+       (substitute-keyword-arguments (package-arguments gemmlowp)
+         ((#:phases phases)
+          `(modify-phases ,phases
+             (replace 'install
+               (lambda* (#:key outputs #:allow-other-keys)
+                 (let* ((out (assoc-ref outputs "out"))
+                        (lib (string-append out "/lib/"))
+                        (inc (string-append out "/include/")))
+                   (install-file "../build/libeight_bit_int_gemm.so" lib)
+                   (for-each (lambda (dir)
+                               ;; Note: Install headers straight into
+                               ;; $includedir instead of $includedir/gemmlowp.
+                               (let ((target (string-append inc "/" dir)))
+                                 (for-each (lambda (h)
+                                             (install-file h target))
+                                           (find-files (string-append "../" dir)
+                                                       "\\.h$"))))
+                             '("meta" "profiling" "public" "fixedpoint"
+                               "eight_bit_int_gemm" "internal")))))))))
+      (native-inputs
+       `(("unzip" ,unzip)))
+      (properties '((hidden? . #t))))))
+
 (define-public dlib
   (package
     (name "dlib")
@@ -2030,6 +2114,175 @@ together building blocks and a subclassing API with an imperative style for
 advanced research.")
     (license license:asl2.0)))
 
+(define-public tensorflow-lite
+  (package
+    (name "tensorflow-lite")
+    (version "2.5.0")
+    (source
+     (origin
+       (method git-fetch)
+       (uri (git-reference
+             (url "https://github.com/tensorflow/tensorflow")
+             (commit (string-append "v" version))))
+       (file-name (git-file-name name version))
+       (sha256
+        (base32
+         "1jdw2i1rq06zqd6aabh7bbm0avsg4pygnfmd7gviv0blhih9054l"))))
+    (build-system cmake-build-system)
+    (arguments
+     `(#:tests? #false                  ; no "check" target
+       #:build-type "Release"
+       #:configure-flags
+       (list
+        "-DTFLITE_ENABLE_GPU=OFF"
+        "-DTFLITE_ENABLE_RUY=OFF"
+
+        ;; TODO: The build system attempts to build xnnpack from source.  We
+        ;; would like to use our xnnpack package here, but this requires more
+        ;; work.
+        "-DTFLITE_ENABLE_XNNPACK=OFF"
+
+        ;; Pretend we've already fetched abseil.  We won't actually build it
+        ;; but use the existing package.
+        "-Dabseil-cpp_POPULATED=TRUE"
+
+        ;; Don't fetch the sources.  We have already built flatbuffers.
+        "-Dflatbuffers_POPULATED=TRUE"
+
+        "-DFFT2D_SOURCE_DIR=/tmp/fft2d"
+        "-Dneon2sse_SOURCE_DIR=/tmp/neon2sse"
+        "-Dneon2sse_BINARY_DIR=/tmp/neon2sse-bin"
+        "-DFARMHASH_SOURCE_DIR=/tmp/farmhash"
+        "-Dgemmlowp_SOURCE_DIR=/tmp/gemmlowp"
+        (string-append "-DRUY_SOURCE_DIR="
+                       (assoc-ref %build-inputs "ruy-src")))
+       #:phases
+       (modify-phases %standard-phases
+         (add-after 'unpack 'chdir
+           (lambda _ (chdir "tensorflow/lite")))
+         (add-after 'chdir 'copy-sources
+           (lambda* (#:key inputs #:allow-other-keys)
+             ;; Use external cmake finders instead of these stubs that won't
+             ;; find anything but the bundled sources.
+             (delete-file "tools/cmake/modules/Findabsl.cmake")
+             (delete-file "tools/cmake/modules/Findeigen.cmake")
+
+             (substitute* "CMakeLists.txt"
+               (("find_package\\(eigen REQUIRED")
+                "find_package(eigen REQUIRED NAMES Eigen3"))
+             (substitute* "tools/cmake/modules/Findflatbuffers.cmake"
+               (("get_target_property.*")
+                (format #false "set(FLATBUFFERS_INCLUDE_DIRS ~a/include)\n"
+                        (assoc-ref inputs "flatbuffers"))))
+
+             ;; Don't fetch source code; we already have everything we need.
+             (substitute* '("tools/cmake/modules/fft2d.cmake"
+                            "tools/cmake/modules/ruy.cmake"
+                            "tools/cmake/modules/farmhash.cmake"
+                            "tools/cmake/modules/neon2sse.cmake"
+                            "tools/cmake/modules/gemmlowp.cmake")
+               (("OverridableFetchContent_Populate.*") ""))
+
+             (mkdir-p "/tmp/farmhash")
+             (with-directory-excursion "/tmp/farmhash"
+               (invoke "tar" "--strip-components=1"
+                       "-xf" (assoc-ref inputs "farmhash-src")))
+
+             (mkdir-p "/tmp/fft2d")
+             (with-directory-excursion "/tmp/fft2d"
+               (invoke "tar" "--strip-components=1"
+                       "-xf" (assoc-ref inputs "fft2d-src")))
+
+             (copy-recursively (assoc-ref inputs "neon2sse-src")
+                               "/tmp/neon2sse/")
+             (copy-recursively (assoc-ref inputs "gemmlowp-src")
+                               "/tmp/gemmlowp/")))
+         (add-after 'copy-sources 'prepare-shared-library-build
+           (lambda _ (chdir "c")))
+         (replace 'install
+           (lambda* (#:key outputs #:allow-other-keys)
+             (let* ((out (assoc-ref outputs "out"))
+                    (lib (string-append out "/lib"))
+                    (headers (string-append out "/include/tensorflow/lite")))
+               (install-file "../build/libtensorflowlite_c.so" lib)
+               (with-directory-excursion ".."
+                 (for-each
+                  (lambda (file)
+                    (let ((target-dir (string-append headers "/" (dirname file))))
+                      (install-file file target-dir)))
+                  (find-files "." "\\.h$")))))))))
+    (inputs
+     `(("abseil-cpp" ,abseil-cpp)
+       ("eigen" ,eigen-for-tensorflow-lite)
+       ("flatbuffers" ,flatbuffers)
+       ("python" ,python)))
+    (native-inputs
+     `(("pkg-config" ,pkg-config)
+       ("gemmlowp-src"
+        ;; The commit hash is taken from
+        ;; "tensorflow/lite/tools/cmake/modules/gemmlowp.cmake".
+        ,(let ((commit "fda83bdc38b118cc6b56753bd540caa49e570745"))
+           (origin
+             (method git-fetch)
+             (uri (git-reference
+                   (url "https://github.com/google/gemmlowp")
+                   (commit commit)))
+             (file-name (git-file-name "gemmlowp" (string-take commit 8)))
+             (sha256
+              (base32
+               "1sbp8kmr2azwlvfbzryy1frxi99jhsh1nc93bdbxdf8zdgpv0kxl")))))
+       ("neon2sse-src"
+        ,(let ((commit "a1652fd5253afbf3e39357b012974f93511f6108"))
+           (origin
+             (method git-fetch)
+             (uri (git-reference
+                   (url "https://github.com/intel/ARM_NEON_2_x86_SSE")
+                   (commit commit)))
+             (file-name (git-file-name "neon2sse" (string-take commit 8)))
+             (sha256
+              (base32
+               "1q8gkxag9wlnwdwad2pclsrkwzrdjy94hyrkayrsvxyj7szb5y8i")))))
+       ("farmhash-src"
+        ,(let ((commit "816a4ae622e964763ca0862d9dbd19324a1eaf45"))
+           (origin
+             (method url-fetch)
+             (uri (string-append
+                   "https://mirror.bazel.build/github.com/google/farmhash/archive/"
+                   commit ".tar.gz"))
+             (file-name (git-file-name "farmhash" (string-take commit 8)))
+             (sha256
+              (base32
+               "185b2xdxl4d4cnsnv6abg8s22gxvx8673jq2yaq85bz4cdy58q35")))))
+       ("fft2d-src"
+        ,(origin
+           (method url-fetch)
+           (uri (string-append "https://storage.googleapis.com/"
+                               "mirror.tensorflow.org/"
+                               "www.kurims.kyoto-u.ac.jp/~ooura/fft2d.tgz"))
+           (file-name "fft2d.tar.gz")
+           (sha256
+            (base32
+             "1jfflzi74fag9z4qmgwvp90aif4dpbr1657izmxlgvf4hy8fk9xd"))))
+       ("ruy-src"
+        ,(let ((commit "9c56af3fce210a8a103eda19bd6f47c08a9e3d90"))
+           (origin
+             (method git-fetch)
+             (uri (git-reference
+                   (url "https://github.com/google/ruy")
+                   (commit commit)
+                   (recursive? #true)))
+             (file-name (git-file-name "ruy" (string-take commit 8)))
+             (sha256
+              (base32
+               "1cfd5gk6kaj8kbl3h98gx1ap8czd59y6p8qq8nr28fklpyzf5cis")))))))
+    (home-page "https://tensorflow.org")
+    (synopsis "Machine learning framework")
+    (description
+     "TensorFlow is a flexible platform for building and training machine
+learning models.  This package provides the \"lite\" variant for mobile
+devices.")
+    (license license:asl2.0)))
+
 (define-public python-iml
   (package
     (name "python-iml")
@@ -2194,8 +2447,8 @@ that:
 
 (define-public gloo
   (let ((version "0.0.0") ; no proper version tag
-        (commit "ca528e32fea9ca8f2b16053cff17160290fc84ce")
-        (revision "0"))
+        (commit "c22a5cfba94edf8ea4f53a174d38aa0c629d070f")
+        (revision "1"))
     (package
       (name "gloo")
       (version (git-version version revision commit))
@@ -2208,18 +2461,20 @@ that:
          (file-name (git-file-name name version))
          (sha256
           (base32
-           "1q9f80zy75f6njrzrqkmhc0g3qxs4gskr7ns2jdqanxa2ww7a99w"))))
+           "1crmqgybzkgkpbmcx16912gsl5qsj49swa0ikx6mhqgph0chrh11"))))
       (build-system cmake-build-system)
       (native-inputs
        `(("googletest" ,googletest)))
+      (inputs
+       `(("openssl" ,openssl)))
       (arguments
        `(#:configure-flags '("-DBUILD_TEST=1")
          #:phases
          (modify-phases %standard-phases
            (replace 'check
-             (lambda _
-               (invoke "make" "gloo_test")
-               #t)))))
+             (lambda* (#:key tests? #:allow-other-keys)
+               (when tests?
+                 (invoke "make" "gloo_test")))))))
       (synopsis "Collective communications library")
       (description
        "Gloo is a collective communications library.  It comes with a
@@ -2265,7 +2520,7 @@ general non-linear dimension reduction.")
     (package
       (name "xnnpack")
       (version (git-version version revision commit))
-      (home-page "https://github.com/google/XNNPACK")
+      (home-page "https://github.com/google/XNNPACK") ;fork of QNNPACK
       (source (origin
                 (method git-fetch)
                 (uri (git-reference (url home-page) (commit commit)))
@@ -2305,3 +2560,134 @@ high-level machine learning frameworks, such as TensorFlow Lite,
 TensorFlow.js, PyTorch, and MediaPipe.")
       (license license:bsd-3))))
 
+(define-public python-pytorch
+  (package
+    (name "python-pytorch")
+    (version "1.9.0")
+    (source (origin
+              (method git-fetch)
+              (uri (git-reference
+                    (url "https://github.com/pytorch/pytorch")
+                    (commit (string-append "v" version))
+                    (recursive? #t)))
+              (file-name (git-file-name name version))
+              (sha256
+               (base32
+                "0cznsh68hwk5761gv7iijb4g6jgjpvs3bbixwpzzmkbkbn2q96c1"))
+              (patches (search-patches "python-pytorch-system-libraries.patch"
+                                       "python-pytorch-runpath.patch"))
+              (modules '((guix build utils)))
+              (snippet
+               '(begin
+                  ;; XXX: Let's be clear: this package is a bundling fest.  We
+                  ;; delete as much as we can, but there's still a lot left.
+                  (for-each (lambda (directory)
+                              (delete-file-recursively
+                               (string-append "third_party/" directory)))
+                            '("benchmark" "cpuinfo" "eigen"
+
+                              ;; FIXME: QNNPACK (of which XNNPACK is a fork)
+                              ;; needs these.
+                              ;; "FP16" "FXdiv" "gemmlowp" "psimd"
+
+                              "gloo" "googletest" "ios-cmake"
+                              "onnx" "protobuf" "pthreadpool"
+                              "pybind11" "python-enum" "python-peachpy"
+                              "python-six" "tbb" "XNNPACK" "zstd"))
+
+                  ;; Adjust references to the onnx-optimizer headers.
+                  (substitute* "caffe2/onnx/backend.cc"
+                    (("onnx/optimizer/")
+                     "onnxoptimizer/"))))))
+    (build-system python-build-system)
+    (arguments
+     '(#:phases (modify-phases %standard-phases
+                  (add-before 'build 'use-system-libraries
+                    (lambda* (#:key outputs #:allow-other-keys)
+                      ;; Tell 'setup.py' to let 'CMakeLists.txt' know that we
+                      ;; want to use "system libraries" instead of the bundled
+                      ;; ones.
+                      (setenv "USE_SYSTEM_LIBS" "1")
+
+                      ;; XXX: Disable that for simplicity for now.
+                      (setenv "USE_FBGEMM" "0")))
+                  (add-before 'build 'make-things-writable
+                    (lambda _
+                      ;; The 'build_caffe2' function in
+                      ;; 'tools/build_pytorch_libs.py', called from the
+                      ;; top-level 'setup.py', needs write access to this
+                      ;; directory.
+                      (for-each make-file-writable
+                                (find-files "caffe2/proto" "."
+                                            #:directories? #t))))
+                  (replace 'check
+                    (lambda* (#:key inputs outputs tests? #:allow-other-keys)
+                      ;; Run the test suite following the instructions in
+                      ;; 'CONTRIBUTING.md'.  XXX: Unfortunately this doesn't
+                      ;; work, unless you set PYTHONPATH presumably.
+                      (when tests?
+                        (let ((python-site (site-packages inputs outputs)))
+                          (setenv "PYTHONPATH"
+                                  (string-append python-site ":"
+                                                 (getenv "PYTHONPATH")))
+                          (invoke "python" "test/run_test.py")))))
+                  (add-after 'install 'remove-test-executables
+                    (lambda* (#:key inputs outputs #:allow-other-keys)
+                      ;; Remove test executables, but keep other executables
+                      ;; such as 'torch_shm_manager' and and .so files such as
+                      ;; 'libtorch_global_deps.so'.
+                      (let ((python-site (site-packages inputs outputs)))
+                        (for-each delete-file
+                                  (find-files python-site
+                                              "(^test_cpp_rpc|_test)$"))))))
+
+       ;; XXX: Tests attempt to download data such as
+       ;; <https://raw.githubusercontent.com/pytorch/test-infra/master/stats/slow-tests.json>.
+       #:tests? #f))
+    (native-inputs
+     `(("cmake" ,cmake)
+       ("ninja" ,ninja)))
+    (inputs
+     `(("eigen" ,eigen)
+       ;; ("fmt" ,fmt)
+       ("fp16" ,fp16)
+       ("gemmlowp" ,gemmlowp)
+       ("googletest" ,googletest)
+       ("googlebenchmark" ,googlebenchmark)
+       ("gloo" ,gloo)
+       ("openblas" ,openblas)
+       ("openmpi" ,openmpi)
+       ("pthreadpool" ,pthreadpool)
+       ("protobuf" ,protobuf)
+       ("pybind11" ,pybind11)
+       ("sleef" ,sleef)
+       ("xnnpack" ,xnnpack)
+       ("zstd" ,zstd)))
+    (propagated-inputs
+     `(("python-astunparse" ,python-astunparse)
+       ("python-numpy" ,python-numpy)
+       ("python-pyyaml" ,python-pyyaml)
+       ("python-cffi" ,python-cffi)
+       ("python-peachpy" ,python-peachpy)
+       ("python-typing-extensions" ,python-typing-extensions)
+       ("python-future" ,python-future)
+       ("python-six" ,python-six)
+       ("python-requests" ,python-requests)
+       ("onnx" ,onnx)                       ;propagated for its Python modules
+       ("onnx-optimizer" ,onnx-optimizer)
+       ("cpuinfo" ,cpuinfo)))
+    (home-page "https://pytorch.org/")
+    (synopsis "Python library for tensor computation and deep neural networks")
+    (description
+     "PyTorch is a Python package that provides two high-level features:
+
+@itemize
+@item tensor computation (like NumPy) with strong GPU acceleration;
+@item deep neural networks (DNNs) built on a tape-based autograd system.
+@end itemize
+
+You can reuse Python packages such as NumPy, SciPy, and Cython to extend
+PyTorch when needed.
+
+Note: currently this package does not provide GPU support.")
+    (license license:bsd-3)))