Re: Follow-up on julia import script

[Nicolas Graves via "Development of GNU Guix and the GNU System distribution." <guix-devel@gnu.org>]

What we get from juliahub is really not bad, since it's really close to
what we have in other importers.

I've been working on that this wkend, nearly finished a first version,
but there's still a few points I'm not sure of :

- we don't have a `hash` field in the json, tough we might be able to
  get it by parsing Versions.toml additionally to the json. This I can
  do with a very simple parser.

- there's a difficulty for defining guix-name->julia-name with the
  simple guix/import/utils.scm snake-case. We should define proper
  functions to convert without ambiguity from CamelCase to snake-case
  and back, and that also encompasses cases like JuMP and OCReract. I
  had one option in mind but haven't tested it to get the name in the
  url. Or maybe we should have a #:julia-name field in arguments or
  properties. WDYT ?

- We only have direct and indirect dependencies in juliahub's json. We
  can't separate native-inputs (as is done in jvb1's guix.jl) or
  propagated-inputs (as discussed with zimoun) without additionally
  parsing Project.toml. This I can't do with my very simple parser.

- readme is full of html junk, I don't know if we can process it to get
  pure text output easily in Guile ?

It looks like that as of now :

(define-module (guix scripts juliahub)
  #:use-module (ice-9 textual-ports)
  #:use-module (ice-9 regex)
  #:use-module (ice-9 match)
  #:use-module (ice-9 string-fun)
  #:use-module (srfi srfi-9)
  #:use-module (srfi srfi-1)
  #:use-module (guix http-client)
  #:use-module (guix import utils)
  #:use-module (guix import json)
  #:use-module (guix base16)
  #:use-module (guix base32)
  #:use-module (guix packages)
  #:use-module (json)
  #:use-module ((guix licenses) #:prefix license:)

  #:export (juliahub->guix-package))

(define (juliahub-uri name)
  (let* ((url (string-append "https://docs.juliahub.com/" name "/"))
         (port (http-fetch url #:text? #t))
         (_ (get-line port))
         (meta (get-line port))
         (regex "url=[a-zA-Z0-9]{5}\\/[0-9\\.]*")
         (redirect (match:substring (string-match regex meta))))
    (close-port port)
    (string-drop redirect 4)))

(define (juliahub-url name)
  (let* ((url (string-append "https://docs.juliahub.com/" name "/"))
         (uri (juliahub-uri name)))
    (string-append url uri "/")))

(define (juliahub-slug-version name)
  (let* ((uri (juliahub-uri name))
         (slug (string-take uri 5))
         (latest-version (string-drop uri 6)))
    `(,slug ,latest-version)))

(define (json->juliahub-direct-dependencies vector)
  (if (vector? vector)
      (filter-map
       (lambda (el)
         (let ((dep (json->juliahub-dependency el)))
           (if (juliahub-dependency-direct? dep)
               dep
               #f)))
       (vector->list vector))))

;; Julia package.
(define-json-mapping <juliahub-package> make-juliahub-package juliahub-package?
  json->juliahub-package
  (homepage juliahub-package-homepage) ;string
  (readme juliahub-package-readme) ;string
  ;; (slug juliahub-package-slug) ;string
  (version juliahub-package-version) ;string
  (description juliahub-package-description) ;string
  (dependencies
   juliahub-package-dependencies "deps"
   json->juliahub-direct-dependencies) ;list of <juliahub-dependency>
   ;; (lambda (vector)
     ;; (map json->juliahub-dependency (vector->list vector))))
  (url juliahub-package-url) ;string
  (uuid juliahub-package-uuid) ;string
  (license juliahub-package-license)) ;string

(define-json-mapping <juliahub-dependency>
  make-juliahub-dependency juliahub-dependency?
  json->juliahub-dependency
  (direct? juliahub-dependency-direct? "direct") ;boolean
  (name juliahub-dependency-name) ;string
  (uuid juliahub-dependency-uuid) ;string
  (versions juliahub-dependency-versions "versions" vector->list)) ;list of strings
  ;; (slug juliahub-dependency-slug) ;string

(define (julia-name->guix-name name)
  (string-append "julia-" (snake-case name)))

(define* (juliahub-fetch name #:key (version #f))
  "Return a <juliahub-package> record for package NAME, or #f on failure."
  (and=> (json-fetch (string-append (juliahub-url name) "pkg.json"))
         json->juliahub-package))

(define (make-julia-sexp name version uri hash home-page synopsis description
                         dependencies licenses)
  "Return the `package' s-expression for a Julia package with the given NAME,
VERSION, URI, HASH, HOME-PAGE, DESCRIPTION, DEPENDENCIES, and LICENSES."
  `(package
     (name ,(julia-name->guix-name name))
     (version ,version)
     (source (origin
               (method url-fetch)
               (uri ,uri)
               (sha256
                (base32
                 "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"
                 ;; ,(bytevector->nix-base32-string hash)
                 ))))
     (build-system julia-build-system)
     ,@(if (null? dependencies)
           '()
           `((inputs
              (list ,@(map (compose julia-name->guix-name juliahub-dependency-name)
                           dependencies)))))
     (synopsis ,synopsis)
     (description ,description)
     (home-page ,home-page)
     (license ,(match licenses
                 (() #f)
                 ((license) (license->symbol license))
                 (_ `(list ,@(map license->symbol licenses)))))))

(define* (juliahub->guix-package package-name
                                 #:key version #:allow-other-keys)
  "Fetch the metadata for PACKAGE-NAME from juliahub.org, and return the
`package' s-expression corresponding to that package, or #f on failure.
Optionally include a VERSION string to fetch a specific version juliahub."
  (let ((package (if version
                      (juliahub-fetch package-name version)
                      (juliahub-fetch package-name))))
    (if package
        (let* ((dependencies-names
                (map juliahub-dependency-name
                     (juliahub-package-dependencies package)))
               (licenses
                (map spdx-string->license
                     (list (juliahub-package-license package)))))
          (values (make-julia-sexp
                   package-name
                   (juliahub-package-version package)
                   (juliahub-package-url package)
                   "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"
                   (juliahub-package-homepage package)
                   (juliahub-package-description package)
                   (beautify-description (juliahub-package-readme package))
                   (juliahub-package-dependencies package)
                   licenses)
                  dependencies-names))
        (values #f '()))))


-- 
Best regards,
Nicolas Graves