Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

[Ludovic Courtès]

[-- Attachment #1: Type: text/plain, Size: 1649 bytes --]

Hello!

The patch below produces a ‘clang’ package that contains
‘clang-tools-extra’ commands¹.

The problem is the size.  Before:

--8<---------------cut here---------------start------------->8---
$ guix size /gnu/store/qxdpxbvfdxfy5dnz4haql4xlxpmb5r6b-clang-10.0.0 |tail -1
total: 995.2 MiB
$ guix size /gnu/store/qxdpxbvfdxfy5dnz4haql4xlxpmb5r6b-clang-10.0.0 |head -2
store item                                                       total    self
/gnu/store/qxdpxbvfdxfy5dnz4haql4xlxpmb5r6b-clang-10.0.0           995.2   456.5  45.9%
--8<---------------cut here---------------end--------------->8---

After:

--8<---------------cut here---------------start------------->8---
$ guix size /gnu/store/5h3xgpg33wip2b8ccri690jp6ikbq16s-clang-10.0.0 |tail -1
total: 1525.4 MiB
$ guix size /gnu/store/5h3xgpg33wip2b8ccri690jp6ikbq16s-clang-10.0.0 |head -2
store item                                                       total    self
/gnu/store/5h3xgpg33wip2b8ccri690jp6ikbq16s-clang-10.0.0          1525.4   986.8  64.7%
--8<---------------cut here---------------end--------------->8---

(How they manage to fill that much disk space, I wonder.)

Any idea how to best package it?

We could of course have a ‘clang-full’ package, but it seems wasteful.
We could also have a separate output for the extra commands, but it’s
inconvenient.  It would be ideal to have a ‘clang-tools-extra’ package
that depends on ‘clang’, but building them separately appears to be
impossible.

Thanks in advance!  :-)

Ludo’.

¹ https://releases.llvm.org/10.0.0/tools/clang/tools/extra/docs/index.html


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: Type: text/x-patch, Size: 3114 bytes --]

diff --git a/gnu/packages/llvm.scm b/gnu/packages/llvm.scm
index d6c519bcbd..a3bcd75190 100644
--- a/gnu/packages/llvm.scm
+++ b/gnu/packages/llvm.scm
@@ -166,7 +166,11 @@ compiler.  In LLVM this library is called \"compiler-rt\".")
     (supported-systems (delete "mips64el-linux" %supported-systems))))
 
 (define* (clang-from-llvm llvm clang-runtime hash
-                          #:key (patches '()))
+                          #:key (patches '()) tools-extra)
+  "Produce Clang with dependencies on LLVM and CLANG-RUNTIME, and applying the
+given PATCHES.  When TOOLS-EXTRA is given, it must point to the
+'clang-tools-extra' tarball, which contains code for 'clang-tidy', 'pp-trace',
+'modularize', and other tools."
   (package
     (name "clang")
     (version (package-version llvm))
@@ -187,7 +191,10 @@ compiler.  In LLVM this library is called \"compiler-rt\".")
     (inputs
      `(("libxml2" ,libxml2)
        ("gcc-lib" ,gcc "lib")
-       ,@(package-inputs llvm)))
+       ,@(package-inputs llvm)
+       ,@(if tools-extra
+             `(("clang-tools-extra" ,tools-extra))
+             '())))
     (propagated-inputs
      `(("llvm" ,llvm)
        ("clang-runtime" ,clang-runtime)))
@@ -208,6 +215,19 @@ compiler.  In LLVM this library is called \"compiler-rt\".")
        #:build-type "Release"
 
        #:phases (modify-phases %standard-phases
+                  ,@(if tools-extra
+                        `((add-after 'unpack 'add-tools-extra
+                            (lambda* (#:key inputs #:allow-other-keys)
+                              (let ((extra (assoc-ref inputs
+                                                      "clang-tools-extra")))
+                                (invoke "tar" "xf" extra)
+                                (rename-file ,(string-append
+                                               "clang-tools-extra-"
+                                               (package-version llvm)
+                                               ".src")
+                                             "tools/extra")
+                                #t))))
+                        '())
                   (add-after 'unpack 'add-missing-triplets
                     (lambda _
                       ;; Clang iterates through known triplets to search for
@@ -376,7 +396,16 @@ output), and Binutils.")
 (define-public clang-10
   (clang-from-llvm llvm-10 clang-runtime-10
                    "08fbxa2a0kr3ni35ckppj0kyvlcyaywrhpqwcdrdy0z900mhcnw8"
-                   #:patches '("clang-10.0-libc-search-path.patch")))
+                   #:patches '("clang-10.0-libc-search-path.patch")
+                   #:tools-extra
+                   #f
+                   #;(origin
+                     (method url-fetch)
+                     (uri (llvm-download-uri "clang-tools-extra"
+                                             (package-version llvm-10)))
+                     (sha256
+                      (base32
+                       "074ija5s2jsdn0k035r2dzmryjmqxdnyg4xwvaqych2bazv8rpxc")))))
 
 (define-public clang-toolchain-10
   (make-clang-toolchain clang-10))

Re: Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

[Nikita Gillmann]

Ludovic Courtès transcribed 4.9K bytes:
> Hello!
> 
> The patch below produces a ‘clang’ package that contains
> ‘clang-tools-extra’ commands¹.
> 
[...]
> Any idea how to best package it?
> 
> We could of course have a ‘clang-full’ package, but it seems wasteful.
> We could also have a separate output for the extra commands, but it’s
> inconvenient.  It would be ideal to have a ‘clang-tools-extra’ package
> that depends on ‘clang’, but building them separately appears to be
> impossible.

Is it? The way we build clang and clang-tools-extra in pkgsrc is to
have them separate.
Refer to lang/clang-tools-extra and lang/clang to see how.
The gist is, BUILD_TARGET for clang-tools-extra gets overridden
and appended with targets to build.
That'S just after skipping through the clang-tools-extra package
for a minute.
I'm not sure if this is applicable to guix in any way.

> Thanks in advance!  :-)
> 
> Ludo’.
> 
> ¹ https://releases.llvm.org/10.0.0/tools/clang/tools/extra/docs/index.html
> 

> diff --git a/gnu/packages/llvm.scm b/gnu/packages/llvm.scm
> index d6c519bcbd..a3bcd75190 100644
> --- a/gnu/packages/llvm.scm
> +++ b/gnu/packages/llvm.scm
> @@ -166,7 +166,11 @@ compiler.  In LLVM this library is called \"compiler-rt\".")
>      (supported-systems (delete "mips64el-linux" %supported-systems))))
>  
>  (define* (clang-from-llvm llvm clang-runtime hash
> -                          #:key (patches '()))
> +                          #:key (patches '()) tools-extra)
> +  "Produce Clang with dependencies on LLVM and CLANG-RUNTIME, and applying the
> +given PATCHES.  When TOOLS-EXTRA is given, it must point to the
> +'clang-tools-extra' tarball, which contains code for 'clang-tidy', 'pp-trace',
> +'modularize', and other tools."
>    (package
>      (name "clang")
>      (version (package-version llvm))
> @@ -187,7 +191,10 @@ compiler.  In LLVM this library is called \"compiler-rt\".")
>      (inputs
>       `(("libxml2" ,libxml2)
>         ("gcc-lib" ,gcc "lib")
> -       ,@(package-inputs llvm)))
> +       ,@(package-inputs llvm)
> +       ,@(if tools-extra
> +             `(("clang-tools-extra" ,tools-extra))
> +             '())))
>      (propagated-inputs
>       `(("llvm" ,llvm)
>         ("clang-runtime" ,clang-runtime)))
> @@ -208,6 +215,19 @@ compiler.  In LLVM this library is called \"compiler-rt\".")
>         #:build-type "Release"
>  
>         #:phases (modify-phases %standard-phases
> +                  ,@(if tools-extra
> +                        `((add-after 'unpack 'add-tools-extra
> +                            (lambda* (#:key inputs #:allow-other-keys)
> +                              (let ((extra (assoc-ref inputs
> +                                                      "clang-tools-extra")))
> +                                (invoke "tar" "xf" extra)
> +                                (rename-file ,(string-append
> +                                               "clang-tools-extra-"
> +                                               (package-version llvm)
> +                                               ".src")
> +                                             "tools/extra")
> +                                #t))))
> +                        '())
>                    (add-after 'unpack 'add-missing-triplets
>                      (lambda _
>                        ;; Clang iterates through known triplets to search for
> @@ -376,7 +396,16 @@ output), and Binutils.")
>  (define-public clang-10
>    (clang-from-llvm llvm-10 clang-runtime-10
>                     "08fbxa2a0kr3ni35ckppj0kyvlcyaywrhpqwcdrdy0z900mhcnw8"
> -                   #:patches '("clang-10.0-libc-search-path.patch")))
> +                   #:patches '("clang-10.0-libc-search-path.patch")
> +                   #:tools-extra
> +                   #f
> +                   #;(origin
> +                     (method url-fetch)
> +                     (uri (llvm-download-uri "clang-tools-extra"
> +                                             (package-version llvm-10)))
> +                     (sha256
> +                      (base32
> +                       "074ija5s2jsdn0k035r2dzmryjmqxdnyg4xwvaqych2bazv8rpxc")))))
>  
>  (define-public clang-toolchain-10
>    (make-clang-toolchain clang-10))

Re: Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

[Ludovic Courtès]

Hi!

Nikita Gillmann <nikita@n0.is> skribis:

> Is it? The way we build clang and clang-tools-extra in pkgsrc is to
> have them separate.
> Refer to lang/clang-tools-extra and lang/clang to see how.
> The gist is, BUILD_TARGET for clang-tools-extra gets overridden
> and appended with targets to build.

Nice.  Do you have the URL of the build recipe?

Thanks,
Ludo’.

Re: Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

[Efraim Flashner]

[-- Attachment #1: Type: text/plain, Size: 843 bytes --]

On Mon, Apr 27, 2020 at 09:31:01AM +0200, Ludovic Courtès wrote:
> Hi!
> 
> Nikita Gillmann <nikita@n0.is> skribis:
> 
> > Is it? The way we build clang and clang-tools-extra in pkgsrc is to
> > have them separate.
> > Refer to lang/clang-tools-extra and lang/clang to see how.
> > The gist is, BUILD_TARGET for clang-tools-extra gets overridden
> > and appended with targets to build.
> 
> Nice.  Do you have the URL of the build recipe?
> 
> Thanks,
> Ludo’.
> 

Here's a link to the Github mirror of pkgsrc for clang-tools-extra
https://github.com/NetBSD/pkgsrc/blob/trunk/lang/clang-tools-extra/Makefile

-- 
Efraim Flashner   <efraim@flashner.co.il>   אפרים פלשנר
GPG key = A28B F40C 3E55 1372 662D  14F7 41AA E7DC CA3D 8351
Confidentiality cannot be guaranteed on emails sent or received unencrypted

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

[Nikita Gillmann]

Hi,

Efraim Flashner transcribed 1.9K bytes:
> On Mon, Apr 27, 2020 at 09:31:01AM +0200, Ludovic Courtès wrote:
> > Hi!
> > 
> > Nikita Gillmann <nikita@n0.is> skribis:
> > 
> > > Is it? The way we build clang and clang-tools-extra in pkgsrc is to
> > > have them separate.
> > > Refer to lang/clang-tools-extra and lang/clang to see how.
> > > The gist is, BUILD_TARGET for clang-tools-extra gets overridden
> > > and appended with targets to build.
> > 
> > Nice.  Do you have the URL of the build recipe?
> > 
> > Thanks,
> > Ludo’.
> > 
> 
> Here's a link to the Github mirror of pkgsrc for clang-tools-extra
> https://github.com/NetBSD/pkgsrc/blob/trunk/lang/clang-tools-extra/Makefile

http://cvsweb.netbsd.org/bsdweb.cgi/pkgsrc/lang/clang-tools-extra/#dirlist
http://cvsweb.netbsd.org/bsdweb.cgi/pkgsrc/lang/clang/

or navigating the (improve worthy) http://pkgsrc.se

But yeah github mirror should work out, or

https://anonhg.netbsd.org/pkgsrc/file/tip/lang/clang
https://anonhg.netbsd.org/pkgsrc/file/tip/lang/clang-tools-extra
 
> -- 
> Efraim Flashner   <efraim@flashner.co.il>   אפרים פלשנר
> GPG key = A28B F40C 3E55 1372 662D  14F7 41AA E7DC CA3D 8351
> Confidentiality cannot be guaranteed on emails sent or received unencrypted

Re: Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

[Ludovic Courtès]

[-- Attachment #1: Type: text/plain, Size: 526 bytes --]

Hello!

Clang-tools-extra is really meant to built within the Clang tree.
Attach is an attempt to build it out-of-tree but so far it fails with
undefined references for Clang libraries and the obvious solution of
adding ‘-l’ flags until it’s happy doesn’t work.

In my original attempt, the ‘clang-tidy’ binary is 40 MiB, and 32 MiB
are text.  This prolly comes from the fact that many Clang libraries
(libclang[A-Z]*.a) are statically linked in each of these programs.

Ideas welcome!

Ludo’.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: Type: text/x-patch, Size: 4041 bytes --]

diff --git a/gnu/packages/llvm.scm b/gnu/packages/llvm.scm
index 33d863ae11..896b7afb47 100644
--- a/gnu/packages/llvm.scm
+++ b/gnu/packages/llvm.scm
@@ -382,6 +382,52 @@ output), and Binutils.")
 (define-public clang-toolchain-10
   (make-clang-toolchain clang-10))
 
+(define (make-clang-tools-extra clang)
+  ;; XXX: An attempt to build clang-tools-extra outside the Clang tree.
+  (package
+    (inherit clang)
+    (name "clang-tools-extra")
+    (source (origin
+              (method url-fetch)
+              (uri (llvm-download-uri "clang-tools-extra"
+                                      (package-version llvm-10)))
+              (sha256
+               (base32
+                "074ija5s2jsdn0k035r2dzmryjmqxdnyg4xwvaqych2bazv8rpxc"))
+              (patches (search-patches "clang-tools-extra-cmake.patch"))))
+    (arguments
+     '(#:build-type "Release"
+       #:phases (modify-phases %standard-phases
+                  (add-before 'configure 'copy-clang-cmake-module
+                    (lambda* (#:key inputs #:allow-other-keys)
+                      (let ((clang-source (assoc-ref inputs "clang-source")))
+                        (mkdir "../clang-source")
+                        (with-directory-excursion "../clang-source"
+                          (invoke "tar" "xf" clang-source))
+
+                        ;; 'cmake/modules' is in the search path of the
+                        ;; top-level 'CMakeLists.txt', so copy it there.
+                        (install-file (car (find-files "../clang-source"
+                                                       "^AddClang\\.cmake$"))
+                                      "cmake/modules")
+                        #t)))
+                  (add-before 'build 'augment-header-path
+                    (lambda _
+                      (let ((context-h (car (find-files "../clang-source"
+                                                        "^Context\\.h$"))))
+                        (setenv "CPATH"
+                                (string-append (dirname
+                                                (canonicalize-path context-h))
+                                               ":" (getenv "CPATH")))
+                        #t))))))
+    (inputs
+     `(("clang" ,clang)
+       ("clang-source", (package-source clang))
+       ,@(package-inputs clang)))))
+
+(define-public clang-tools-extra-10
+  (make-clang-tools-extra clang-10))
+
 (define-public llvm-9
   (package
     (inherit llvm-10)
diff --git a/gnu/packages/patches/clang-tools-extra-cmake.patch b/gnu/packages/patches/clang-tools-extra-cmake.patch
new file mode 100644
index 0000000000..96672bb2dd
--- /dev/null
+++ b/gnu/packages/patches/clang-tools-extra-cmake.patch
@@ -0,0 +1,38 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 57bb970..6e5de7c 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1,3 +1,33 @@
++cmake_minimum_required(VERSION 3.4.3)
++
++project(Clang-Tools-Extra)
++
++find_package(LLVM REQUIRED HINTS "${LLVM_CMAKE_PATH}")
++list(APPEND CMAKE_MODULE_PATH ${LLVM_DIR})
++
++# This is the place where we put Clang's 'AddClang.cmake'.
++list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
++
++set(LLVM_TOOLS_BINARY_DIR ${TOOLS_BINARY_DIR} CACHE PATH "Path to llvm/bin")
++set(LLVM_LIBRARY_DIR ${LIBRARY_DIR} CACHE PATH "Path to llvm/lib")
++set(LLVM_MAIN_INCLUDE_DIR ${INCLUDE_DIR} CACHE PATH "Path to llvm/include")
++set(LLVM_BINARY_DIR ${LLVM_OBJ_ROOT} CACHE PATH "Path to LLVM build tree")
++set(LLVM_MAIN_SRC_DIR ${MAIN_SRC_DIR} CACHE PATH "Path to LLVM source tree")
++
++set(PACKAGE_VERSION "${LLVM_PACKAGE_VERSION}")
++
++include(AddClang)
++include(AddLLVM)
++include(TableGen)
++include(HandleLLVMOptions)
++include(VersionFromVCS)
++include(LLVMDistributionSupport)
++
++# Dummy targets to appease CMake.  These are normally found in the
++# Clang tree.
++add_custom_target(clang-resource-headers)
++add_custom_target(ClangSACheckers)
++
+ include(CMakeDependentOption)
+ 
+ add_subdirectory(clang-apply-replacements)

Re: Packaging ‘clang-tools-extra’ (‘clang-tidy’, etc.)

[Ludovic Courtès]

Hi,

Ludovic Courtès <ludovic.courtes@inria.fr> skribis:

> The patch below produces a ‘clang’ package that contains
> ‘clang-tools-extra’ commands¹.
>
> The problem is the size.  Before:
>
> $ guix size /gnu/store/qxdpxbvfdxfy5dnz4haql4xlxpmb5r6b-clang-10.0.0 |tail -1
> total: 995.2 MiB
> $ guix size /gnu/store/qxdpxbvfdxfy5dnz4haql4xlxpmb5r6b-clang-10.0.0 |head -2
> store item                                                       total    self
> /gnu/store/qxdpxbvfdxfy5dnz4haql4xlxpmb5r6b-clang-10.0.0           995.2   456.5  45.9%
>
>
> After:
>
> $ guix size /gnu/store/5h3xgpg33wip2b8ccri690jp6ikbq16s-clang-10.0.0 |tail -1
> total: 1525.4 MiB
> $ guix size /gnu/store/5h3xgpg33wip2b8ccri690jp6ikbq16s-clang-10.0.0 |head -2
> store item                                                       total    self
> /gnu/store/5h3xgpg33wip2b8ccri690jp6ikbq16s-clang-10.0.0          1525.4   986.8  64.7%
>
> (How they manage to fill that much disk space, I wonder.)
>
> Any idea how to best package it?
>
> We could of course have a ‘clang-full’ package, but it seems wasteful.
> We could also have a separate output for the extra commands, but it’s
> inconvenient.  It would be ideal to have a ‘clang-tools-extra’ package
> that depends on ‘clang’, but building them separately appears to be
> impossible.

After spending an unreasonable amount of time trying to build
‘clang-tools-extra’ as a separate package, I gave up and went for the
multiple output solution in commit
77a87ad4aceed9d89d615540e0fd147e3a8b2f64.  One can now get ‘clang-tidy’
and its friends with:

  guix install clang:extra

Ludo’.