Re: neon: git for structured data [Was: Functional database]

[Roel Janssen <roel@gnu.org>]

[-- Attachment #1: rdf.scm --]
[-- Type: application/octet-stream, Size: 13033 bytes --]

;;; Copyright © 2018  Roel Janssen <roel@gnu.org>
;;;
;;; This program is free software: you can redistribute it and/or
;;; modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation, either version 3 of
;;; the License, or (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;;; Affero General Public License for more details.
;;;
;;; You should have received a copy of the GNU Affero General Public
;;; License along with this program.  If not, see
;;; <http://www.gnu.org/licenses/>.

;;
;; This is an interface to raptor2 and librdf.  See http://www.librdf.org
;; for the API documentation.
;;
;; Some wrapping has been done:
;; - The *-free functions reset the argument's pointer to 0x0.
;; - String arguments are converted to pointer using pointer->string,
;;   so that regular strings in Scheme can be used as C-string inputs.
;;

(define-module (redland rdf)
  #:use-module (system foreign)
  #:export (rdf-link-library
            rdf-unlink-library

            rdf-world-new
            rdf-world-open
            rdf-world-free
            rdf-world-get-raptor

            raptor-world-open
            raptor-world-free
            raptor-new-iostream-from-file-handle
            raptor-new-iostream-from-filename
            raptor-iostream-free

            rdf-storage-new
            rdf-storage-free
            rdf-storage-open
            rdf-storage-close
            rdf-storage-size

            rdf-model-new
            rdf-model-free
            rdf-model-transaction-start
            rdf-model-transaction-commit
            rdf-model-transaction-rollback
            rdf-model-write
            rdf-model-add
            rdf-model-size

            rdf-model-add-statement
            rdf-statement-new-from-nodes
            rdf-statement-free

            rdf-uri-new
            rdf-uri-free

            rdf-node-new-from-uri-local-name

            rdf-serializer-new
            rdf-serializer-free
            rdf-serializer-serialize-model-to-file
            rdf-serializer-serialize-model-to-string))

;;
;; LIBRARY HANDLE
;; ----------------------------------------------------------------------------

;; TODO: Integrate this with the build system.
(define %library-path "/home/roel/.guix-profile/lib")

(define %rdf-handle %null-pointer)
(define %raptor-handle %null-pointer)

(define (rdf-link-library)
  (set! %rdf-handle (dynamic-link (string-append %library-path "/librdf")))
  (set! %raptor-handle (dynamic-link (string-append %library-path "/libraptor2")))
  (and (dynamic-object? %rdf-handle)
       (dynamic-object? %raptor-handle)))

(define (rdf-unlink-library)
  (dynamic-unlink %rdf-handle)
  (dynamic-unlink %raptor-handle))

(rdf-link-library)

;;
;; CONVENIENCE
;; ----------------------------------------------------------------------------

(define-syntax-rule
  (define-foreign-function symbol handle return-type c-function args)
  (define symbol
    (pointer->procedure return-type (dynamic-func c-function handle) args)))

;;
;; WORLD FUNCTIONS
;; ----------------------------------------------------------------------------

;; Returns a newly allocated librdf_world object.
(define-foreign-function rdf-world-new
  %rdf-handle '* "librdf_new_world" '())

;; Opens a whole new world.
(define-foreign-function rdf-world-open
  %rdf-handle void "librdf_world_open" '(*))

;; Deallocates memory of the world.  This function takes one argument: A
;; librdf_world object.
(define-foreign-function rdf-world-free-raw
  %rdf-handle void "librdf_free_world" '(*))

(define-syntax-rule
  (rdf-world-free world)
  (begin
    (rdf-world-free-raw world)
    (set! world %null-pointer)))

;; Takes two arguments: librdf_world, and raptor_world.
(define-foreign-function rdf-world-set-raptor
  %rdf-handle void "librdf_world_set_raptor" '(* *))

;; Returns a raptor_world object for the specified librdf_world object.
(define-foreign-function rdf-world-get-raptor
  %rdf-handle '* "librdf_world_get_raptor" '(*))

;;
;; RAPTOR FUNCTIONS
;; ----------------------------------------------------------------------------

;(define raptor-world-new
;  (pointer->procedure '* (dynamic-func "raptor_new_world" %raptor-handle) '()))

(define-foreign-function raptor-world-free-raw
  %raptor-handle void "raptor_free_world" '())

(define-syntax-rule
  (raptor-world-free raptor)
  (begin
    (raptor-world-free-raw raptor)
    (set! raptor %null-pointer)))

;; Returns a raptor_iostream object.  Takes a raptor_world object, and a
;; FILE pointer.
(define-foreign-function raptor-new-iostream-from-file-handle
  %raptor-handle '* "raptor_new_iostream_from_file_handle" '(* *))

;; Returns a raptor_iostream object.  Takes a raptor_world object, and a
;; filename (string).
(define-foreign-function raptor-new-iostream-from-filename-raw
  %raptor-handle '* "raptor_new_iostream_from_filename" '(* *))

(define-syntax-rule
  (raptor-new-iostream-from-filename world filename)
  (raptor-new-iostream-from-filename-raw world (string->pointer filename)))

(define-foreign-function raptor-world-open
  %raptor-handle int "raptor_world_open" '(*))

(define-foreign-function raptor-world-free
  %raptor-handle void "raptor_free_world" '(*))

(define-foreign-function raptor-iostream-free
  %raptor-handle void "raptor_free_iostream" '(*))

;;
;; STORAGE FUNCTIONS
;; ----------------------------------------------------------------------------

;; Returns a newly allocated librdf_storage object.  This function takes four
;; arguments: A world object, storage name (string), name (string), and options
;; (string).
(define-foreign-function rdf-storage-new-raw
  %rdf-handle '* "librdf_new_storage" '(* * * *))

(define-syntax-rule
  (rdf-storage-new world storage-name name options)
  (rdf-storage-new-raw world
                       (string->pointer storage-name)
                       (string->pointer name)
                       (if (eq? options %null-pointer)
                           options
                           (string->pointer options))))

;; Deallocates memory of a librdf_storage object.  This function takes one
;; argument: A librdf_storage object.
(define-foreign-function rdf-storage-free-raw
  %rdf-handle void "librdf_free_storage" '(*))

(define-syntax-rule
  (rdf-storage-free storage)
  (begin
    (rdf-storage-free-raw storage)
    (set! storage %null-pointer)))

(define-foreign-function rdf-storage-size-raw
  %rdf-handle int "librdf_storage_size" '(*))

(define-syntax-rule
  (rdf-storage-size storage)
  (let ((size (rdf-storage-size-raw storage)))
    (if (< size 0)
        (if #f #t) ; Returns #<unspecified>
        size)))

;; Takes two arguments: librdf_storage, and librdf_model.
(define-foreign-function rdf-storage-open-raw
  %rdf-handle int "librdf_storage_open" '(* *))

(define-syntax-rule
  (rdf-storage-open storage model)
  (eq? (rdf-storage-open-raw storage model) 0))

(define-foreign-function rdf-storage-close-raw
  %rdf-handle int "librdf_storage_close" '(*))

(define-syntax-rule
  (rdf-storage-close storage)
  (begin
    (rdf-storage-close-raw storage)
    (set! storage %null-pointer)))

;;
;; MODEL FUNCTIONS
;; ----------------------------------------------------------------------------

;; Returns a newly allocated librdf_model object. This function takes three
;; arguments: A librdf_world object, a librdf_storage object, and an options
;; string.
(define-foreign-function rdf-model-new-raw
  %rdf-handle '* "librdf_new_model" '(* * *))

(define-syntax-rule
  (rdf-model-new world storage options)
  (rdf-model-new-raw world storage
                     (if (eq? options %null-pointer)
                         options
                         (string->pointer options))))

;; Deallocates memory of a librdf_model object.  This function takes one
;; argument: A librdf_model object.
(define-foreign-function rdf-model-free-raw
  %rdf-handle void "librdf_free_model" '(*))

(define-syntax-rule
  (rdf-model-free model)
  (begin
    (rdf-model-free-raw model)
    (set! model %null-pointer)))

;; Starts a transaction on a librdf_model object.  This function takes one
;; argument: A librdf_model object.
(define-foreign-function rdf-model-transaction-start
  %rdf-handle int "librdf_model_transaction_start" '(*))

;; Commits a transaction on a librdf_model object.  This function takes one
;; argument: A librdf_model object.
(define-foreign-function rdf-model-transaction-commit
  %rdf-handle int "librdf_model_transaction_start" '(*))

;; Rolls a transaction back on a librdf_model object.  This function takes one
;; argument: A librdf_model object.
(define-foreign-function rdf-model-transaction-rollback
  %rdf-handle int "librdf_model_transaction_start" '(*))

;; Writes a librdf_model object to a raptor_iostream.
(define-foreign-function rdf-model-write
  %rdf-handle int "librdf_model_write" '(* *))

;; Adds a triplet to an librdf_model object.  This function takes four
;; arguments: A librdf_model object, and three librdf_node objects.
(define-foreign-function rdf-model-add
  %rdf-handle int "librdf_model_add" '(* * * *))

(define-foreign-function rdf-model-size-raw
  %rdf-handle int "librdf_model_size" '(*))

(define-syntax-rule
  (rdf-model-size model)
  (let ((size (rdf-model-size-raw model)))
    (if (< size 0)
        (if #f #t) ; Returns #<unspecified>
        size)))

(define-foreign-function rdf-model-add-statement
  %rdf-handle int "librdf_model_add_statement" '(* *))

;;
;; STATEMENT FUNCTIONS
;; ----------------------------------------------------------------------------


(define-foreign-function rdf-statement-new-from-nodes
  %rdf-handle '* "librdf_new_statement_from_nodes" '(* * * *))

;; Deallocates memory of a librdf_statement object.  This function takes one
;; argument: A librdf_statement object.
(define-foreign-function rdf-statement-free-raw
  %rdf-handle void "librdf_free_statement" '(*))

(define-syntax-rule
  (rdf-statement-free statement)
  (begin
    (rdf-statement-free-raw statement)
    (set! statement %null-pointer)))

;;
;; URI FUNCTIONS
;; ----------------------------------------------------------------------------

;; Returns a newly allocated librdf_uri object.  This function takes two
;; arguments: A librdf_world object, and a URI (string).
(define-foreign-function rdf-uri-new-raw
  %rdf-handle '* "librdf_new_uri" '(* *))

(define-syntax-rule
  (rdf-uri-new world uri)
  (rdf-uri-new-raw world (string->pointer uri)))

;; Deallocates memory of a librdf_uri object.  This function takes one
;; argument: A librdf_uri object.
(define-foreign-function rdf-uri-free-raw
  %rdf-handle void "librdf_free_uri" '(*))

(define-syntax-rule
  (rdf-uri-free uri)
  (begin
    (rdf-uri-free-raw uri)
    (set! uri %null-pointer)))

;;
;; NODE FUNCTIONS
;; ----------------------------------------------------------------------------

;; This function takes three arguments: A librdf_world object,
;; a librdf_uri object, and string.
(define-foreign-function rdf-node-new-from-uri-local-name-raw
  %rdf-handle '* "librdf_new_node_from_uri_local_name" '(* * *))

(define-syntax-rule
  (rdf-node-new-from-uri-local-name world uri local-name)
  (rdf-node-new-from-uri-local-name-raw world uri (string->pointer local-name)))

;;
;; SERIALIZER FUNCTIONS
;; ----------------------------------------------------------------------------

;; Arguments: librdf_world*, name (string), mime_type (string), librdf_uri*.
;; Returns: librdf_serializer*
(define-foreign-function rdf-serializer-new-raw
  %rdf-handle '* "librdf_new_serializer" '(* * * *))

(define-syntax-rule
  (rdf-serializer-new world name mime-type uri)
  (rdf-serializer-new-raw world (if (eq? %null-pointer name)
                                    name
                                    (string->pointer name))
                          (if (eq? %null-pointer mime-type)
                              mime-type
                              (string->pointer mime-type)) uri))

;; Arguments: librdf_serializer*
(define-foreign-function rdf-serializer-free-raw
  %rdf-handle void "librdf_free_serializer" '(*))

(define-syntax-rule
  (rdf-serializer-free serializer)
  (begin 
    (rdf-serializer-free-raw serializer)
    (set! serializer %null-pointer)))

;; Arguments: librdf_serializer, name (string), librdf_uri, librdf_model.
;; Returns: integer
(define-foreign-function rdf-serializer-serialize-model-to-file-raw
  %rdf-handle int "librdf_serializer_serialize_model_to_file" '(* * * *))

(define-syntax-rule
  (rdf-serializer-serialize-model-to-file a b c d)
  (rdf-serializer-serialize-model-to-file-raw a (string->pointer b) c d))

;; Arguments: librdf_serializer, librdf_uri, librdf_model.
;; Returns: unsigned char (string)
(define-foreign-function rdf-serializer-serialize-model-to-string
  %rdf-handle '* "librdf_serializer_serialize_model_to_string" '(* * *))

[-- Attachment #2: Type: text/plain, Size: 4288 bytes --]

Dear Amirouche,

I'm not exactly sure if this fits in with your plans, but nevertheless
I'd like to share this code with you.

I recently looked into using triple stores (actually quad stores)
and wrote an interface to Redland librdf for Guile.

I attached the source code of the interface.
With this interface, you can write something like this:

--8<---------------cut here---------------start------------->8---
(use-modules (redland rdf) ; The attached module.
             (system foreign))

(define world (rdf-world-new))
(rdf-world-open world)

(define store (rdf-storage-new
               world
               "hashes"
               "redland"
               "new=true,hash-type='bdb',dir='path/to/triplestore'"))

(define model (rdf-model-new world store %null-pointer))

(define local-uri (rdf-uri-new world "http://localhost:5000/Redland/"))
(define s (rdf-node-new-from-uri-local-name world local-uri "Test"))
(define p (rdf-node-new-from-uri-local-name world local-uri "TestPredicate"))
(define o (rdf-node-new-from-uri-local-name world local-uri "TestObject"))

(define statement (rdf-statement-new-from-nodes world s p o))
(rdf-model-add-statement model statement)
(rdf-statement-free statement)

(rdf-model-size model)
(rdf-storage-size store)

;; Example mime-type: application/rdf+xml
(define serializer (rdf-serializer-new world %null-pointer "text/turtle" %null-pointer))
(define serialized (rdf-serializer-serialize-model-to-string serializer local-uri model))
(format #t "Serialized: ~s~%" (pointer->string serialized))

(rdf-uri-free local-uri)
(rdf-model-free model)
(rdf-storage-free store)
(rdf-world-free world)
--8<---------------cut here---------------end--------------->8---

Kind regards,
Roel Janssen


Amirouche Boubekki writes:

> I tried chez scheme and I think GNU Guile
> is a better platform for what I am trying
> to achieve, so I am back.
>
> I also know better what I want to achieve.
> I will create a triple store that comply
> with semantic web standard that is
> a RDF triple store. At [0] and [1] you will
> find a primer on what is RDF in the former
> and the concepts in the latter.
>
> [0] https://www.w3.org/TR/rdf11-primer/
> [1] https://www.w3.org/TR/rdf11-concepts/
>
> It will also be branch-able etc... like git.
>
> Also, I also plan to implement sparql.
> If you find sparql difficult I recommend
> the tutorial at data.world [2] in the mean time.
> It's not very difficult and looks like SQL.
> Hence I also plan to implement sparql [3].
>
> [2] https://docs.data.world/tutorials/sparql/
> [3] https://www.w3.org/TR/sparql11-overview/
>
> What I want to do is something similar to data.world,
> that is a gitlab-like platform for data and replace
> the use of git in projects like datahub.io [4].
>
> [4] http://datahub.io/core/registry
>
> Enough talking, what is the status? Well I finished
> porting what I had in chez and can now run the following
> scenario:
>
> - In master branch, I commit two triples
>
> - In other branch, that is orphan branch, I commit
>    two triples among where one of them overlaps with
>    master.
>
> - I can query both branch
>
> - In a merge commit, I fix the conflict between both
>    branch.
>
> - I can query the resulting branch and get the expected
>    result.
>
> The code might be easier to read [5]
>
> [5] https://github.com/amirouche/neon/blob/master/guile/neon.scm
>
> What is missing, in order of difficulty:
>
> - microkanren package
>    https://framagit.org/a-guile-mind/microkanren
>
> - wiredtiger 3 package
>
> - Turtle aka. .ttl format parser https://www.w3.org/TR/turtle/
>
> - sparql queries parser https://www.w3.org/TR/rdf-sparql-query/
>
> - I am not sure of the status of guile-squee yet
>    https://notabug.org/cwebber/guile-squee/
>
> - pluggable backends
>
> If you want to work one of this item, send me an email.
>
> What I plan to work on next:
>
>    There is a semantic difference between neon
>    and RDF triple stores. In a triple store you
>    can have as many times as you want the same
>    attribute given a subject. That is (ref subject)
>    doesn't return a proper alist.
>
> There is two other links that remain to be cited
>
> - https://www.w3.org/TR/rdf11-mt/
>
> - https://www.w3.org/TR/2014/NOTE-rdf11-datasets-20140225/
>
> Happy hacking,