;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2022 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix extensions index)
  #:use-module ((guix i18n) #:select (G_))
  #:use-module ((guix ui) #:select (show-version-and-exit
                                    show-bug-report-information
                                    with-error-handling
                                    string->number*))
  #:use-module (guix scripts)
  #:use-module (sqlite3)
  #:use-module (ice-9 match)
  #:use-module (ice-9 format)
  #:use-module (ice-9 getopt-long)
  #:use-module (guix describe)
  #:use-module (guix store)
  #:use-module (guix monads)
  #:autoload   (guix combinators) (fold2)
  #:autoload   (guix grafts) (%graft?)
  #:autoload   (guix store roots) (gc-roots)
  #:use-module (guix derivations)
  #:use-module (guix packages)
  #:use-module (guix profiles)
  #:use-module ((guix progress) #:select (progress-reporter/bar
                                          call-with-progress-reporter))
  #:use-module (guix sets)
  #:use-module ((guix utils) #:select (cache-directory))
  #:autoload   (guix build utils) (find-files)
  #:autoload   (gnu packages) (fold-packages)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-9)
  #:use-module (srfi srfi-37)  ;; option
  #:use-module (srfi srfi-71)
  #:export     (guix-index))

(define debug #f)

(define application-version 2)

;; The following schema is the full schema at the `application-version`.  It
;; should be modified according to the development required. If the schema
;; needs modification across time, those should be changed directly in the
;; full-schema and the incremental changes should be referenced below
;; as migration step (for the existing dbs) below.
(define schema-full
  "
create table if not exists SchemaVersion (
  version integer primary key not null,
  date date,
  unique (version)
);

create table if not exists Packages (
  id        integer primary key autoincrement not null,
  name      text not null,
  version   text not null,
  unique    (name, version) -- add uniqueness constraint
);

create table if not exists Directories (
  id        integer primary key autoincrement not null,
  name      text not null,
  package   integer not null,
  foreign key (package) references Packages(id) on delete cascade,
  unique (name, package) -- add uniqueness constraint
);

create table if not exists Files (
  name      text not null,
  basename  text not null,
  directory integer not null,
  foreign key (directory) references Directories(id) on delete cascade
  unique (name, basename, directory) -- add uniqueness constraint
);

create index if not exists IndexFiles on Files(basename);")

;; List of tuple ((version . sqlite schema migration script)).  There should
;; be as much version increments with each step needed to migrate the db.
(define schema-to-migrate '((1 . "
create table if not exists SchemaVersion (
  version integer primary key not null,
  unique (version)
);
")
                            (2 . "
alter table SchemaVersion
add column date date;
")))

(define (call-with-database file proc)
  (let ((db (sqlite-open file)))
    (dynamic-wind
      (lambda () #t)
      (lambda () (proc db))
      (lambda () (sqlite-close db)))))

(define (insert-version db version)
  "Insert application VERSION into the DB."
  (define stmt-insert-version
    (sqlite-prepare db "\
INSERT OR IGNORE INTO SchemaVersion(version, date)
VALUES (:version, CURRENT_TIMESTAMP);"
                    #:cache? #t))
  (sqlite-exec db "begin immediate;")
  (sqlite-bind-arguments stmt-insert-version #:version version)
  (sqlite-fold (const #t) #t stmt-insert-version)
  (sqlite-exec db "commit;"))

(define (read-version db)
  "Read the current application version from the DB."

  (define stmt-select-version (sqlite-prepare db "\
SELECT version FROM SchemaVersion ORDER BY version DESC LIMIT 1;"
                                              #:cache? #f))
  (match (sqlite-fold cons '() stmt-select-version)
    ((#(version))
     version)))

(define (insert-files db package version directories)
    "Insert files from DIRECTORIES as belonging to PACKAGE at VERSION."
    (define stmt-select-package
      (sqlite-prepare db "\
SELECT id FROM Packages WHERE name = :name AND version = :version;"
                      #:cache? #t))

    (define stmt-insert-package
      (sqlite-prepare db "\
INSERT OR IGNORE INTO Packages(name, version) -- to avoid spurious writes
VALUES (:name, :version);"
                      #:cache? #t))

    (define stmt-select-directory
      (sqlite-prepare db "\
SELECT id FROM Directories WHERE name = :name AND package = :package;"
                      #:cache? #t))

    (define stmt-insert-directory
      (sqlite-prepare db "\
INSERT OR IGNORE INTO Directories(name, package) -- to avoid spurious writes
VALUES (:name, :package);"
                      #:cache? #t))

    (define stmt-insert-file
      (sqlite-prepare db "\
INSERT OR IGNORE INTO Files(name, basename, directory)
VALUES (:name, :basename, :directory);"
                      #:cache? #t))

    (sqlite-exec db "begin immediate;")
    (sqlite-bind-arguments stmt-insert-package
                           #:name package
                           #:version version)
    (sqlite-fold (const #t) #t stmt-insert-package)

    (sqlite-bind-arguments stmt-select-package
                           #:name package
                           #:version version)
    (match (sqlite-fold cons '() stmt-select-package)
      ((#(package-id))
       (when debug
         (format #t "(pkg, version, pkg-id): (~a, ~a, ~a)"
                 package version package-id)
         (pk 'package package-id package))
       (for-each (lambda (directory)
                   (define (strip file)
                     (string-drop file (+ (string-length directory) 1)))

                   (sqlite-reset stmt-insert-directory)
                   (sqlite-bind-arguments stmt-insert-directory
                                          #:name directory
                                          #:package package-id)
                   (sqlite-fold (const #t) #t stmt-insert-directory)

                   (sqlite-reset stmt-select-directory)
                   (sqlite-bind-arguments stmt-select-directory
                                          #:name directory
                                          #:package package-id)
                   (match (sqlite-fold cons '() stmt-select-directory)
                     ((#(directory-id))
                      (when debug
                        (format #t "(name, package, dir-id): (~a, ~a, ~a)\n"
                                directory package-id directory-id))
                      (for-each (lambda (file)
                                  ;; If DIRECTORY is a symlink, (find-files
                                  ;; DIRECTORY) returns the DIRECTORY singleton.
                                  (unless (string=? file directory)
                                    (sqlite-reset stmt-insert-file)
                                    (sqlite-bind-arguments stmt-insert-file
                                                           #:name (strip file)
                                                           #:basename
                                                           (basename file)
                                                           #:directory
                                                           directory-id)
                                    (sqlite-fold (const #t) #t stmt-insert-file)))
                                (find-files directory)))))
                 directories)))
    (sqlite-exec db "commit;"))

\f
;;;
;;; Indexing from local packages.
;;;

(define (insert-package db package)
  "Insert all the files of PACKAGE into DB."
  (mlet %store-monad ((drv (package->derivation package #:graft? #f)))
    (match (derivation->output-paths drv)
      (((labels . directories) ...)
       (when (every file-exists? directories)
         (insert-files db (package-name package) (package-version package)
                       directories))
       (return #t)))))

(define* (index-packages-from-store db)
  "Insert all current packages from the local store into the DB."
  (with-store store
    (parameterize ((%graft? #f))
      (let* ((packages   (fold-packages
                          (lambda (package result)
                            (cons package result))
                          '()
                          #:select? (lambda (package)
                                      (and (not (hidden-package? package))
                                           (not (package-superseded package))
                                           (supported-package? package)))))
             (nb-entries (length packages))
             (prefix     (format #f "Registering ~a packages" nb-entries))
             (progress   (progress-reporter/bar nb-entries prefix)))
        (call-with-progress-reporter progress
          (lambda (report)
            (for-each (lambda (package)
                        (run-with-store store
                            (insert-package db package))
                        (report))
                      packages)))))))

\f
;;;
;;; Indexing from local profiles.
;;;

(define (all-profiles)
  "Return the list of system profiles."
  (delete-duplicates
   (filter-map (lambda (root)
                 (if (file-exists? (string-append root "/manifest"))
                     root
                     (let ((root (string-append root "/profile")))
                       (and (file-exists? (string-append root "/manifest"))
                            root))))
               (gc-roots))))

(define (profiles->manifest-entries profiles)
  "Return deduplicated manifest entries across all PROFILES."
  (let loop ((visited (set))
             (profiles profiles)
             (entries '()))
    (match profiles
      (()
       entries)
      ((profile . rest)
       (let* ((manifest (profile-manifest profile))
              (entries visited
                       (fold2 (lambda (entry lst visited)
                                (let ((item (manifest-entry-item entry)))
                                  (if (set-contains? visited item)
                                      (values lst visited)
                                      (values (cons entry lst)
                                              (set-insert item
                                                          visited)))))
                              entries
                              visited
                              (manifest-transitive-entries manifest))))
         (loop visited rest entries))))))

(define (insert-manifest-entry db entry)
  "Insert a manifest ENTRY into DB."
  (insert-files db (manifest-entry-name entry)
                (manifest-entry-version entry)
                (list (manifest-entry-item entry)))) ;FIXME: outputs?

(define (index-packages-from-manifests-with-db db-pathname)
  "Index packages entries into DB-PATHNAME from the system manifests."
  (call-with-database db-pathname
    (lambda (db)
      (let* ((profiles   (all-profiles))
             (entries    (profiles->manifest-entries profiles))
             (nb-entries (length entries))
             (prefix     (format #f "Registering ~a packages" nb-entries))
             (progress   (progress-reporter/bar nb-entries prefix)))
        (call-with-progress-reporter progress
          (lambda (report)
            (for-each (lambda (entry)
                        (insert-manifest-entry db entry)
                        (report))
                      entries)))))))

\f
;;;
;;; Search.
;;;

(define-record-type <package-match>
  (package-match name version file)
  package-match?
  (name      package-match-name)
  (version   package-match-version)
  (file      package-match-file))

(define (matching-packages db file)
  "Return unique <package-match> corresponding to packages containing FILE."
  (define lookup-stmt
    (sqlite-prepare db "\
SELECT Packages.name, Packages.version, Directories.name, Files.name
FROM Packages
INNER JOIN Files, Directories
ON files.basename = :file
  AND directories.id = files.directory
  AND packages.id = directories.package;"))

  (sqlite-bind-arguments lookup-stmt #:file file)
  (sqlite-fold (lambda (result lst)
                 (match result
                   (#(package version directory file)
                    (cons (package-match package version
                                         (string-append directory "/" file))
                          lst))))
               '() lookup-stmt))

\f

;;;
;;; CLI
;;;

(define (index-packages-from-store-with-db db-pathname)
  "Index packages using db at location DB-PATHNAME."
  (call-with-database db-pathname
    (lambda (db)
      (index-packages-from-store db))))

(define (matching-packages-with-db db-pathname file)
  "Compute list of packages referencing FILE using db at DB-PATHNAME."
  (call-with-database db-pathname
    (lambda (db)
      (matching-packages db file))))

(define (read-version-from-db db-pathname)
  (call-with-database db-pathname
    (lambda (db) (read-version db))))

(define (migrate-schema-to-version db-pathname)
  (call-with-database db-pathname
    (lambda (db)
      (catch #t
        (lambda ()
          ;; Migrate from the current version to the full migrated schema
          ;; This can raise sqlite-error if the db is not properly configured yet
          (let* ((current-db-version (read-version db))
                 (next-db-version (+ 1 current-db-version)))
            (when (< current-db-version application-version)
              ;; when the current db version is older than the current application
              (let ((schema-migration-at-version (assoc-ref schema-to-migrate next-db-version)))
                (when schema-migration-at-version
                  ;; migrate the schema to the next version (if it exists)
                  (sqlite-exec db schema-migration-at-version)
                  ;; insert current version
                  (insert-version db next-db-version)
                  ;; iterate over the next migration if any
                  (migrate-schema-to-version db))))))
        (lambda (key . arg)
          ;; exception handler in case failure to read an inexisting db
          ;; Fallback to boostrap the schema
          (sqlite-exec db schema-full)
          (insert-version db application-version))))))

(define (print-matching-results matches)
  "Print the MATCHES matching results."
  (for-each (lambda (result)
              (format #t "~20a ~a~%"
                      (string-append (package-match-name result)
                                     "@" (package-match-version result))
                      (package-match-file result)))
            matches))

(define default-db-path
  (string-append (cache-directory #:ensure? #f)
                 "/index/db.sqlite"))

(define (show-help)
  (display (G_ "Usage: guix index [OPTIONS...] [search FILE...]
Without argument, indexes (package, file) relationships from the machine.
This allows indexation with 2 methods:

- out of the local manifests. This is the fastest implementation but this
indexes less packages. That'd be typically the use case for user local
indexation.

- out of the local store. This is slower due to implementation details (it
discusses with the store daemon for one). That'd be typically the use case for
building the largest db in one of the build farm node.

With 'search FILE', search for packages installing FILE.\n
Note: Internal cache is located at ~/.cache/guix/index/db.sqlite by default.
See --db-path for customization.\n"))
  (newline)
  (display (G_ "The valid values for OPTIONS are:"))
  (newline)
  (display (G_ "
  -h, --help          Display this help and exit"))
  (display (G_ "
  -V, --version       Display version information and exit"))
  (display (G_ "
  --db-path=DIR       Change default location of the cache db"))
  (newline)
  (display (G_ "
  --with-method=METH  Change default indexation method. By default it uses the
                      local \"manifests\" (faster). It can also uses the local
                      \"store\" (slower, typically on the farm build ci)."))
  (newline)
  (display (G_ "The valid values for ARGS are:"))
  (newline)
  (display (G_ "
  search FILE     Search for packages installing the FILE (from cache db)"))
  (newline)
  (display (G_ "
  <EMPTY>         Without any argument, it index packages. This fills in the
                  db cache using whatever indexation method is defined."))
  (show-bug-report-information))

(define %options
  (list
   (option '(#\h "help") #f #f
           (lambda args (show-help) (exit 0)))
   (option '(#\V "version") #f #f
           (lambda (opt name arg result)
             (catch 'sqlite-error
               (lambda ()
                 (let ((db-path (assoc-ref result 'db-path)))
                   (simple-format
                    #t
                    "Extension local cache database:\n- path: ~a\n- version: ~a\n\n"
                    db-path (read-version-from-db db-path))
                   ))
               (lambda (key . arg) 'no-db-yet-so-nothing-to-display))
             (show-version-and-exit "guix index")))
   ;; index data out of the method (store or package)
   (option '(#\d "db-path") #f #t
           (lambda (opt name arg result)
             (when debug
               (format #t "%options: --with-method: opt ~a\n" opt)
               (format #t "%options: --with-method: name ~a\n" name)
               (format #t "%options: --with-method: arg ~a\n" arg)
               (format #t "%options: --with-method: result ~a\n" result))
             (alist-cons 'db-path arg
                         (alist-delete 'db-path result))))

   ;; index data out of the method (store or package)
   (option '(#\m "with-method") #f #t
           (lambda (opt name arg result)
             (when debug
               (format #t "%options: --with-method: opt ~a\n" opt)
               (format #t "%options: --with-method: name ~a\n" name)
               (format #t "%options: --with-method: arg ~a\n" arg)
               (format #t "%options: --with-method: result ~a\n" result))
             (match arg
               ((or "manifests" "store")
                (alist-cons 'with-method arg
                            (alist-delete 'with-method result)))
               (_
                (G_ "guix index: Wrong indexation method, either manifests
 (fast) or store (slow)~%")))))))

(define %default-options
  `((db-path . ,default-db-path)
    (with-method . "manifests")))

(define-command (guix-index . args)
  (category extension)
  (synopsis "Index packages to search package for a given filename")

  (define (parse-sub-command arg result)
    ;; Parse sub-command ARG and augment RESULT accordingly.
    (when debug
      (format #t "parse-sub-command: arg: ~a\n" arg)
      (format #t "parse-sub-command: result: ~a\n" result)
      (format #t "parse-sub-command: (assoc-ref result 'action): ~a\n" (assoc-ref result 'action))
      (format #t "parse-sub-command: (assoc-ref result 'argument): ~a\n" (assoc-ref result 'argument)))
    (if (assoc-ref result 'action)
        (alist-cons 'argument arg result)
        (let ((action (string->symbol arg)))
          (case action
            ((search)
             (alist-cons 'action action result))
            (else (leave (G_ "~a: unknown action~%") action))))))

  (define (match-pair car)
    ;; Return a procedure that matches a pair with CAR.
    (match-lambda
      ((head . tail)
       (and (eq? car head) tail))
      (_ #f)))

  (define (option-arguments opts)
    ;; Extract the plain arguments from OPTS.
    (let* ((args   (reverse (filter-map (match-pair 'argument) opts)))
           (count  (length args))
           (action (or (assoc-ref opts 'action) 'index)))

      (when debug
        (format #t "option-arguments: args: ~a\n" args)
        (format #t "option-arguments: count: ~a\n" count)
        (format #t "option-arguments: action: ~a\n" action))

      (define (fail)
        (leave (G_ "wrong number of arguments for action '~a'~%")
               action))

      (unless action
        (format (current-error-port)
                (G_ "guix index: missing command name~%"))
        (format (current-error-port)
                (G_ "Try 'guix index --help' for more information.~%"))
        (exit 1))
      (alist-cons 'argument (string-concatenate args)
                  (alist-delete 'argument
                                (alist-cons 'action action
                                            (alist-delete 'action opts))))))

  (with-error-handling
    (let* ((opts        (parse-command-line args %options
                                            (list %default-options)
                                            #:argument-handler
                                            parse-sub-command))
           (args        (option-arguments opts))
           (action      (assoc-ref args 'action))
           (db-path     (assoc-ref args 'db-path))
           (with-method (assoc-ref args 'with-method)))
      (when debug
        (format #t "main: opts: ~a\n" opts)
        (format #t "main: args: ~a\n" args)
        (format #t "main: action: ~a\n" action)
        (format #t "main: db-path: ~a\n" db-path)
        (format #t "main: with-method: ~a\n" with-method))

      (match action
        ('search
         (unless (file-exists? db-path)
           (format (current-error-port)
                   (G_ "guix index: The local cache db does not exist yet.
You need to index packages first.\nTry 'guix index --help' for more information.~%"))
           (exit 1))
         (let* ((file (assoc-ref args 'argument))
                (matches (matching-packages-with-db db-path file)))
           (print-matching-results matches)
           (exit (pair? matches))))
        ('index
         (let ((db-dirpath (dirname db-path)))
           (unless (file-exists? db-dirpath)
             (mkdir db-dirpath)))
         ;; Migrate/initialize db to schema at version application-version
         (migrate-schema-to-version db-path)
         ;; Finally index packages
         (if (string= with-method "manifests")
             (index-packages-from-manifests-with-db db-path)
             (index-packages-from-store-with-db db-path)))))))